Do not hold whole STAC object in memory before pushing result to client

.mvn/extensions.xml

@@ -4,7 +4,7 @@
     <extension>
         <groupId>me.qoomon</groupId>
         <artifactId>maven-git-versioning-extension</artifactId>
-        <version>9.7.0</version>
+        <version>9.12.3</version>
     </extension>
 
 </extensions>

server/src/main/java/au/org/aodn/ogcapi/server/common/RestApi.java

@@ -30,6 +30,7 @@
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
+import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
 
 import java.math.BigDecimal;
 import java.net.URI;
@@ -58,13 +59,13 @@ public class RestApi implements ApiApi, DefaultApi, ConformanceApi {
     @Override
     public ResponseEntity<Void> apiGet(String f) {
         switch(OGCMediaTypeMapper.valueOf(f.toLowerCase())) {
-            case json: {
+            case json -> {
                 return ResponseEntity
                         .status(HttpStatus.TEMPORARY_REDIRECT)
                         .location(URI.create("/api/v1/ogc/api-docs/v3"))
                         .build();
             }
-            default: {
+            default -> {
                 return ResponseEntity
                         .status(HttpStatus.TEMPORARY_REDIRECT)
                         .location(URI.create("/api/v1/ogc/swagger-ui/index.html"))
@@ -134,7 +135,7 @@ public ResponseEntity<ConfClasses> getConformanceDeclaration(String f) {
             //        "text/html"
             },
             method = RequestMethod.GET)
-    public ResponseEntity<?> getCollections(
+    public ResponseEntity<StreamingResponseBody> getCollections(
             @Parameter(in = ParameterIn.QUERY, description = "Property to be return" ,schema=@Schema())
                 @Valid @RequestParam(value = "properties", required = false) List<String> properties,
             @Parameter(in = ParameterIn.QUERY, description = "Only records that have a geometry that intersects the bounding box are selected. The bounding box is provided as four or six numbers, depending on whether the coordinate reference system includes a vertical axis (height or depth):  * Lower left corner, coordinate axis 1 * Lower left corner, coordinate axis 2 * Minimum value, coordinate axis 3 (optional) * Upper right corner, coordinate axis 1 * Upper right corner, coordinate axis 2 * Maximum value, coordinate axis 3 (optional)  The coordinate reference system of the values is WGS 84 long/lat (http://www.opengis.net/def/crs/OGC/1.3/CRS84) unless a different coordinate reference system is specified in the parameter `bbox-crs`.  For WGS 84 longitude/latitude the values are in most cases the sequence of minimum longitude, minimum latitude, maximum longitude and maximum latitude.  However, in cases where the box spans the antimeridian the first value (west-most box edge) is larger than the third value (east-most box edge).  If the vertical axis is included, the third and the sixth number are the bottom and the top of the 3-dimensional bounding box.  If a record has multiple spatial geometry properties, it is the decision of the server whether only a single spatial geometry property is used to determine the extent or all relevant geometries." ,schema=@Schema())
@@ -162,14 +163,13 @@ public ResponseEntity<?> getCollections(
                 // the same, we append the bbox parameter to the filter in case user use this parameter
                 filter = OGCApiService.processBBoxParameter(CQLFields.geometry.name(), bbox, filter);
             }
-            return commonService.getCollectionList(
+            return commonService.writeCollectionList(
                     q,
                     filter,
                     properties,
                     sortBy,
-                    OGCMediaTypeMapper.json,
                     CQLCrsType.convertFromUrl(crs),
-                    stacToCollection::convert);
+                    stacToCollection::convertOne);
         }
         else {
             List<String> reasons = new ArrayList<>();

server/src/main/java/au/org/aodn/ogcapi/server/core/mapper/StacToCollections.java

@@ -23,7 +23,7 @@ public abstract class StacToCollections implements Converter<ElasticSearch.Searc
     public Collections convert(ElasticSearch.SearchResult<StacCollectionModel> model, Filter filter) {
 
         List<Collection> collections = model.getCollections().stream()
-                .map(m -> getCollection(m, filter, hostname))
+                .map(m -> convertOne(m, filter))
                 .toList();
 
         ExtendedCollections result = new ExtendedCollections();
@@ -36,4 +36,8 @@ public Collections convert(ElasticSearch.SearchResult<StacCollectionModel> model
 
         return result;
     }
+
+    public Collection convertOne(StacCollectionModel model, Filter filter) {
+        return getCollection(model, filter, hostname);
+    }
 }

server/src/main/java/au/org/aodn/ogcapi/server/core/service/CollectionsResponseWriter.java

@@ -0,0 +1,46 @@
+package au.org.aodn.ogcapi.server.core.service;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.List;
+
+/**
+ * Writes the same JSON object as {@code ExtendedCollections}, one collection at a time.
+ * Field order is {@code links}, {@code collections}, {@code total}, {@code search_after}.
+ */
+public final class CollectionsResponseWriter {
+
+    private CollectionsResponseWriter() {
+    }
+
+    /**
+     * Opens {@code {"links":[],"collections":[} and leaves the generator inside the array.
+     */
+    public static JsonGenerator open(OutputStream output, ObjectMapper mapper) throws IOException {
+        JsonGenerator generator = mapper.getFactory().createGenerator(output);
+        generator.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET);
+        generator.setCodec(mapper);
+        generator.writeStartObject();
+        generator.writeFieldName("links");
+        generator.writeStartArray();
+        generator.writeEndArray();
+        generator.writeFieldName("collections");
+        generator.writeStartArray();
+        return generator;
+    }
+
+    public static void finish(JsonGenerator generator, Long total, List<String> searchAfter) throws IOException {
+        generator.writeEndArray();
+        if (total != null) {
+            generator.writeNumberField("total", total);
+        }
+        if (searchAfter != null) {
+            generator.writeFieldName("search_after");
+            generator.writeObject(searchAfter);
+        }
+        generator.writeEndObject();
+    }
+}

server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java

@@ -28,6 +28,7 @@
 import java.util.*;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import java.util.function.Consumer;
 import java.util.function.Supplier;
 
 import static au.org.aodn.ogcapi.server.core.configuration.CacheConfig.ELASTIC_SEARCH_UUID_ONLY;
@@ -574,6 +575,53 @@ protected Supplier<SearchRequest.Builder> buildParameterSearchRequestSupplier(
                 maxSize);
     }
 
+    @Override
+    public void validateByParameters(List<String> keywords, String cql, List<String> properties, String sortBy, CQLCrsType coor) throws CQLException {
+        buildParameterSearchRequestSupplier(keywords, cql, properties, sortBy, coor);
+        if (cql != null) {
+            readPageSize(cql, coor);
+        }
+    }
+
+    @Override
+    public ElasticSearchBase.SearchResult<StacCollectionModel> visitByParameters(
+            List<String> keywords,
+            String cql,
+            List<String> properties,
+            String sortBy,
+            CQLCrsType coor,
+            Consumer<StacCollectionModel> consumer) throws CQLException {
+
+        boolean unfiltered = (keywords == null || keywords.isEmpty()) && cql == null;
+        return visitCollectionBy(
+                buildParameterSearchRequestSupplier(keywords, cql, properties, sortBy, coor),
+                unfiltered ? null : properties,
+                unfiltered ? null : readPageSize(cql, coor),
+                consumer);
+    }
+
+    /**
+     * CQL {@code page_size} cap for {@code pageableSearch}. The request supplier applies the same cap
+     * to each batch; the iterator still needs the total cap when it is larger than one batch.
+     */
+    private Long readPageSize(String cql, CQLCrsType coor) throws CQLException {
+        if (cql == null) {
+            return null;
+        }
+        CQLToElasticFilterFactory<CQLFields> factory = new CQLToElasticFilterFactory<>(coor, CQLFields.class);
+        CompilerUtil.parseFilter(Language.ECQL, cql, factory);
+        String raw = factory.getQuerySetting().get(CQLElasticSetting.page_size);
+        if (raw == null || raw.isBlank()) {
+            return null;
+        }
+        try {
+            return Long.parseLong(raw);
+        }
+        catch (NumberFormatException pe) {
+            return null;
+        }
+    }
+
     @Override
     public ElasticSearchBase.SearchResult<StacCollectionModel> searchByParameters(List<String> keywords, String cql, List<String> properties, String sortBy, CQLCrsType coor) throws CQLException {