diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml
index 540c45fa..11897a38 100644
--- a/.mvn/extensions.xml
+++ b/.mvn/extensions.xml
@@ -4,7 +4,7 @@
me.qoomon
maven-git-versioning-extension
- 9.7.0
+ 9.12.3
diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/common/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/common/RestApi.java
index 316c7276..bbfbf8a9 100644
--- a/server/src/main/java/au/org/aodn/ogcapi/server/common/RestApi.java
+++ b/server/src/main/java/au/org/aodn/ogcapi/server/common/RestApi.java
@@ -30,6 +30,7 @@
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
+import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
import java.math.BigDecimal;
import java.net.URI;
@@ -58,13 +59,13 @@ public class RestApi implements ApiApi, DefaultApi, ConformanceApi {
@Override
public ResponseEntity apiGet(String f) {
switch(OGCMediaTypeMapper.valueOf(f.toLowerCase())) {
- case json: {
+ case json -> {
return ResponseEntity
.status(HttpStatus.TEMPORARY_REDIRECT)
.location(URI.create("/api/v1/ogc/api-docs/v3"))
.build();
}
- default: {
+ default -> {
return ResponseEntity
.status(HttpStatus.TEMPORARY_REDIRECT)
.location(URI.create("/api/v1/ogc/swagger-ui/index.html"))
@@ -134,7 +135,7 @@ public ResponseEntity getConformanceDeclaration(String f) {
// "text/html"
},
method = RequestMethod.GET)
- public ResponseEntity> getCollections(
+ public ResponseEntity getCollections(
@Parameter(in = ParameterIn.QUERY, description = "Property to be return" ,schema=@Schema())
@Valid @RequestParam(value = "properties", required = false) List properties,
@Parameter(in = ParameterIn.QUERY, description = "Only records that have a geometry that intersects the bounding box are selected. The bounding box is provided as four or six numbers, depending on whether the coordinate reference system includes a vertical axis (height or depth): * Lower left corner, coordinate axis 1 * Lower left corner, coordinate axis 2 * Minimum value, coordinate axis 3 (optional) * Upper right corner, coordinate axis 1 * Upper right corner, coordinate axis 2 * Maximum value, coordinate axis 3 (optional) The coordinate reference system of the values is WGS 84 long/lat (http://www.opengis.net/def/crs/OGC/1.3/CRS84) unless a different coordinate reference system is specified in the parameter `bbox-crs`. For WGS 84 longitude/latitude the values are in most cases the sequence of minimum longitude, minimum latitude, maximum longitude and maximum latitude. However, in cases where the box spans the antimeridian the first value (west-most box edge) is larger than the third value (east-most box edge). If the vertical axis is included, the third and the sixth number are the bottom and the top of the 3-dimensional bounding box. If a record has multiple spatial geometry properties, it is the decision of the server whether only a single spatial geometry property is used to determine the extent or all relevant geometries." ,schema=@Schema())
@@ -162,14 +163,13 @@ public ResponseEntity> getCollections(
// the same, we append the bbox parameter to the filter in case user use this parameter
filter = OGCApiService.processBBoxParameter(CQLFields.geometry.name(), bbox, filter);
}
- return commonService.getCollectionList(
+ return commonService.writeCollectionList(
q,
filter,
properties,
sortBy,
- OGCMediaTypeMapper.json,
CQLCrsType.convertFromUrl(crs),
- stacToCollection::convert);
+ stacToCollection::convertOne);
}
else {
List reasons = new ArrayList<>();
diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java
index 435b7800..99028588 100644
--- a/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java
+++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java
@@ -23,7 +23,7 @@ public abstract class StacToCollections implements Converter model, Filter filter) {
List collections = model.getCollections().stream()
- .map(m -> getCollection(m, filter, hostname))
+ .map(m -> convertOne(m, filter))
.toList();
ExtendedCollections result = new ExtendedCollections();
@@ -36,4 +36,8 @@ public Collections convert(ElasticSearch.SearchResult model
return result;
}
+
+ public Collection convertOne(StacCollectionModel model, Filter filter) {
+ return getCollection(model, filter, hostname);
+ }
}
diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CollectionsResponseWriter.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CollectionsResponseWriter.java
new file mode 100644
index 00000000..12d1aec7
--- /dev/null
+++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CollectionsResponseWriter.java
@@ -0,0 +1,46 @@
+package au.org.aodn.ogcapi.server.core.service;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.List;
+
+/**
+ * Writes the same JSON object as {@code ExtendedCollections}, one collection at a time.
+ * Field order is {@code links}, {@code collections}, {@code total}, {@code search_after}.
+ */
+public final class CollectionsResponseWriter {
+
+ private CollectionsResponseWriter() {
+ }
+
+ /**
+ * Opens {@code {"links":[],"collections":[} and leaves the generator inside the array.
+ */
+ public static JsonGenerator open(OutputStream output, ObjectMapper mapper) throws IOException {
+ JsonGenerator generator = mapper.getFactory().createGenerator(output);
+ generator.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET);
+ generator.setCodec(mapper);
+ generator.writeStartObject();
+ generator.writeFieldName("links");
+ generator.writeStartArray();
+ generator.writeEndArray();
+ generator.writeFieldName("collections");
+ generator.writeStartArray();
+ return generator;
+ }
+
+ public static void finish(JsonGenerator generator, Long total, List searchAfter) throws IOException {
+ generator.writeEndArray();
+ if (total != null) {
+ generator.writeNumberField("total", total);
+ }
+ if (searchAfter != null) {
+ generator.writeFieldName("search_after");
+ generator.writeObject(searchAfter);
+ }
+ generator.writeEndObject();
+ }
+}
diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java
index 868b477e..46c839df 100644
--- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java
+++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java
@@ -28,6 +28,7 @@
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import java.util.function.Consumer;
import java.util.function.Supplier;
import static au.org.aodn.ogcapi.server.core.configuration.CacheConfig.ELASTIC_SEARCH_UUID_ONLY;
@@ -574,6 +575,53 @@ protected Supplier buildParameterSearchRequestSupplier(
maxSize);
}
+ @Override
+ public void validateByParameters(List keywords, String cql, List properties, String sortBy, CQLCrsType coor) throws CQLException {
+ buildParameterSearchRequestSupplier(keywords, cql, properties, sortBy, coor);
+ if (cql != null) {
+ readPageSize(cql, coor);
+ }
+ }
+
+ @Override
+ public ElasticSearchBase.SearchResult visitByParameters(
+ List keywords,
+ String cql,
+ List properties,
+ String sortBy,
+ CQLCrsType coor,
+ Consumer consumer) throws CQLException {
+
+ boolean unfiltered = (keywords == null || keywords.isEmpty()) && cql == null;
+ return visitCollectionBy(
+ buildParameterSearchRequestSupplier(keywords, cql, properties, sortBy, coor),
+ unfiltered ? null : properties,
+ unfiltered ? null : readPageSize(cql, coor),
+ consumer);
+ }
+
+ /**
+ * CQL {@code page_size} cap for {@code pageableSearch}. The request supplier applies the same cap
+ * to each batch; the iterator still needs the total cap when it is larger than one batch.
+ */
+ private Long readPageSize(String cql, CQLCrsType coor) throws CQLException {
+ if (cql == null) {
+ return null;
+ }
+ CQLToElasticFilterFactory factory = new CQLToElasticFilterFactory<>(coor, CQLFields.class);
+ CompilerUtil.parseFilter(Language.ECQL, cql, factory);
+ String raw = factory.getQuerySetting().get(CQLElasticSetting.page_size);
+ if (raw == null || raw.isBlank()) {
+ return null;
+ }
+ try {
+ return Long.parseLong(raw);
+ }
+ catch (NumberFormatException pe) {
+ return null;
+ }
+ }
+
@Override
public ElasticSearchBase.SearchResult searchByParameters(List keywords, String cql, List properties, String sortBy, CQLCrsType coor) throws CQLException {
diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java
index 42fab9db..b695f030 100644
--- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java
+++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearchBase.java
@@ -39,6 +39,7 @@
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiFunction;
+import java.util.function.Consumer;
import java.util.function.Supplier;
/**
@@ -357,23 +358,36 @@ protected SearchResult searchCollectionBy(final List
queries, should, filters, properties, searchAfter, sortOptions, score, maxSize
);
+ List collections = new ArrayList<>();
+ SearchResult result = visitCollectionBy(builderSupplier, properties, maxSize, collections::add);
+ result.setCollections(collections);
+ return result;
+ }
+
+ /**
+ * Same search as {@link #searchCollectionBy}, but each model is passed to {@code consumer}
+ * and then dropped. The returned result has {@code total} and {@code sortValues} only.
+ */
+ protected SearchResult visitCollectionBy(Supplier builderSupplier,
+ List properties,
+ Long maxSize,
+ Consumer consumer) {
try {
log.info("Start search {} {}", ZonedDateTime.now(), Thread.currentThread().getName());
Iterable> response = pageableSearch(builderSupplier, ObjectNode.class, maxSize);
SearchResult result = new SearchResult<>();
- result.collections = new ArrayList<>();
result.total = countRecordsHit(builderSupplier);
List lastSortValue = null;
- for(Hit i : response) {
- if(i != null) {
+ for (Hit i : response) {
+ if (i != null) {
StacCollectionModel model = this.formatResult(i.source());
// Use cache value of noland_geometry if user request centroid
- if(properties != null && properties.contains(CQLFields.centroid.name())) {
+ if (properties != null && properties.contains(CQLFields.centroid.name())) {
StacCollectionModel cache = cacheNoLandGeometry.getAllNoLandGeometry().get(model.getUuid());
- if(cache != null && cache.getSummaries() != null) {
- if(model.getSummaries() == null) {
+ if (cache != null && cache.getSummaries() != null) {
+ if (model.getSummaries() == null) {
model.setSummaries(cache.getSummaries());
}
else {
@@ -381,36 +395,39 @@ protected SearchResult searchCollectionBy(final List
}
}
}
- result.collections.add(model);
+ consumer.accept(model);
lastSortValue = i.sort();
}
}
log.info("End search {} {}", ZonedDateTime.now(), Thread.currentThread().getName());
- // Return the last sort value if exist
- if(lastSortValue != null && !lastSortValue.isEmpty()) {
- List