diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java index 239a8381..84520908 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java @@ -145,17 +145,27 @@ public enum CQLFields implements CQLFieldsInterface { " .getAsLong()" + " }")) .order(order))), + // Phrase match on the field and its synonyms sub-field (search-time acronym expansion), fuzziness is not allowed + // for type phrase. Also used for quoted keywords, see fuzzy_title for why best of the two fields is scored. title( StacBasicField.Title.searchField, StacBasicField.Title.displayField, - null, + (literal) -> MultiMatchQuery.of(m -> m + .type(TextQueryType.Phrase) + .fields(StacBasicField.Title.searchField + "^2", StacBasicField.Title.searchField + ".synonyms^2") + .tieBreaker(0.1) + .query(literal))._toQuery(), null, (order) -> new SortOptions.Builder() .field(f -> f.field(StacBasicField.Title.sortField).order(order))), description( StacBasicField.Description.searchField, StacBasicField.Description.displayField, - null, + (literal) -> MultiMatchQuery.of(m -> m + .type(TextQueryType.Phrase) + .fields(StacBasicField.Description.searchField, StacBasicField.Description.searchField + ".synonyms") + .tieBreaker(0.1) + .query(literal))._toQuery(), null, null), providers( @@ -266,15 +276,18 @@ public enum CQLFields implements CQLFieldsInterface { null, (order) -> new SortOptions.Builder() .field(f -> f.field(StacSummeries.Score.sortField).order(order))), + // Fuzzy match on the field and its synonyms sub-field (search-time acronym expansion, e.g. "SOOP" -> "ships of opportunity"). + // best_fields scores the better field plus tie_breaker times the other. Plain words pass the synonym analyzer + // unchanged and would otherwise be scored twice. Title fields are boosted 2, description fields 1, as before. fuzzy_title( null, StacBasicField.Title.displayField, - (literal) -> MatchQuery.of(m -> m + (literal) -> MultiMatchQuery.of(m -> m + .type(TextQueryType.BestFields) + .fields(StacBasicField.Title.searchField + "^2", StacBasicField.Title.searchField + ".synonyms^2") + .tieBreaker(0.1) .fuzziness("AUTO") - .field(StacBasicField.Title.searchField) .prefixLength(4)// Use 4 to deal with NRMN short form may match NRM records - // Increase the relevance of matches in title - .boost(2.0F) .operator(Operator.And)// ensure all terms are matched with fuzziness .query(literal))._toQuery(), null, @@ -282,34 +295,16 @@ public enum CQLFields implements CQLFieldsInterface { fuzzy_desc( null, StacBasicField.Description.displayField, - (literal) -> MatchQuery.of(m -> m + (literal) -> MultiMatchQuery.of(m -> m + .type(TextQueryType.BestFields) + .fields(StacBasicField.Description.searchField, StacBasicField.Description.searchField + ".synonyms") + .tieBreaker(0.1) .fuzziness("AUTO") - .field(StacBasicField.Description.searchField) .prefixLength(4)// Use 4 to deal with NRMN short form may match NRM records .operator(Operator.And)// ensure all terms are matched with fuzziness .query(literal))._toQuery(), null, null), - // Acronym match on the synonyms sub-fields (search-time expansion), e.g. "SOOP" -> "ships of opportunity". - acronym_title( - StacBasicField.Title.searchField + ".synonyms", - StacBasicField.Title.displayField, - (literal) -> MatchQuery.of(m -> m - .field(StacBasicField.Title.searchField + ".synonyms") - .operator(Operator.And)// all expanded terms must match - .boost(2.0F)// align with fuzzy_title weighting - .query(literal))._toQuery(), - null, - null), - acronym_desc( - StacBasicField.Description.searchField + ".synonyms", - StacBasicField.Description.displayField, - (literal) -> MatchQuery.of(m -> m - .field(StacBasicField.Description.searchField + ".synonyms") - .operator(Operator.And) - .query(literal))._toQuery(), - null, - null), // Contains cloud-optimized data assets_summary( StacBasicField.AssetsSummary.searchField, diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 46c839df..e7d42d77 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -449,9 +449,6 @@ private static List createKeywordShouldClauses(List keywords) { should.add(CQLFields.organisation_vocabs.getPropertyEqualToQuery(term)); should.add(CQLFields.platform_vocabs.getPropertyEqualToQuery(term)); should.add(CQLFields.id.getPropertyEqualToQuery(term)); - // Acronym match on the *.synonyms sub-fields, e.g. "SOOP" -> "ships of opportunity". - should.add(CQLFields.acronym_title.getPropertyEqualToQuery(term)); - should.add(CQLFields.acronym_desc.getPropertyEqualToQuery(term)); // credit_contains uses match query by default, exact match is not applied here should.add(CQLFields.credit_contains.getPropertyEqualToQuery(term)); } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java index 6a6b6ec3..15350dec 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java @@ -38,6 +38,12 @@ public class ExplainSimplifier { protected static final String SYNONYM_PREFIX = "Synonym("; + // A dis_max (disjunction max) explanation includes every match, but it is scored as its winning child plus + // tie_breaker times the others, e.g. the best_fields multi_match on title and title.synonyms (see CQLFields.fuzzy_title). + // Lucene writes "max of:" for tie_breaker 0, otherwise "max plus 0.1 times others of:". + protected static final String MAX_OF_PREFIX = "max of:"; + protected static final String MAX_PLUS_PREFIX = "max plus "; + protected static final String RELEVANCE_DESCRIPTION_PREFIX = "_score:"; protected static final Comparator BY_SCORE_DESC = @@ -214,6 +220,19 @@ protected static void collectScoreParts(List details, for (ExplanationDetail detail : details) { String description = detail.description(); + // report only the winner, the tie_breaker share of the others is small and would list the same word twice + if (description != null + && (description.startsWith(MAX_OF_PREFIX) || description.startsWith(MAX_PLUS_PREFIX))) { + if (detail.details() != null) { + // the highest child, not one equal to the parent, so float rounding cannot lose the winner; a tie keeps the first + detail.details().stream() + .max(Comparator.comparingDouble(ExplanationDetail::value)) + .ifPresent(winner -> collectScoreParts( + List.of(winner), terms, filters)); + } + continue; + } + // most nodes are idf/tf breakdowns, skip the regex for them if (description != null && description.startsWith(WEIGHT_PREFIX)) { Matcher matcher = WEIGHT_PATTERN.matcher(description); diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestAdminApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestAdminApiTest.java index 8294865d..20d993ba 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestAdminApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestAdminApiTest.java @@ -187,14 +187,16 @@ public void explainSimplifiedFormatReportsMultiWordMatches() throws IOException assertFalse(term.path("term").asText().isBlank()); } - // both words of the query are reported, and separately for each field they hit, - // rather than being collapsed into one entry per field + // both words of the query are reported, rather than being collapsed into one entry per field List reported = StreamSupport.stream(terms.spliterator(), false) .map(term -> term.path("field").asText() + ":" + term.path("term").asText()) .toList(); - assertTrue(reported.contains("title:ocean")); - assertTrue(reported.contains("title:acidification")); - assertTrue(reported.size() > 2, "a multi word query hits more than one field"); + for (String word : List.of("ocean", "acidification")) { + long titleHits = reported.stream() + .filter(r -> r.equals("title:" + word) || r.equals("title.synonyms:" + word)) + .count(); + assertEquals(1, titleHits, "'" + word + "' must be reported once for title, got: " + reported); + } } @Test @@ -226,8 +228,8 @@ public void explainSimplifiedFormatReportsTheDatasetGroupPriorityInTheSortValues public void explainSimplifiedFormatReportsQuotedPhraseMatches() throws IOException { insertRecordsWithExplicitIds("7709f541-fc0c-4318-b5b9-9053aa474e0e.json"); - // a quoted query routes title and description through MatchPhraseQuery, which elastic - // search renders as a single weight(title:"ocean acidification" in N) leaf + // a quoted query routes title and description through a phrase multi_match on the field and its + // synonyms sub-field, each child renders as a single weight(title:"ocean acidification" in N) leaf URI simpleUri = explainUri("q", "\"ocean acidification\"", "format", "simple"); ResponseEntity response = testRestTemplate.getForEntity(simpleUri, JsonNode.class); @@ -237,9 +239,11 @@ public void explainSimplifiedFormatReportsQuotedPhraseMatches() throws IOExcepti .path("hits").path(0).path("matched_terms"); // the phrase is reported as one term with its quotes stripped, a term only - // extraction would drop this leaf entirely + // extraction would drop this leaf entirely. A plain phrase scores the same on title and + // title.synonyms, so either can win the dis_max assertTrue(StreamSupport.stream(terms.spliterator(), false) - .anyMatch(term -> "title".equals(term.path("field").asText()) + .anyMatch(term -> ("title".equals(term.path("field").asText()) + || "title.synonyms".equals(term.path("field").asText())) && "ocean acidification".equals(term.path("term").asText())), "a quoted phrase must be reported as a single matched term"); } diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java index 43db66ee..5510da71 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java @@ -5,10 +5,12 @@ import co.elastic.clients.elasticsearch._types.ScriptSortType; import co.elastic.clients.elasticsearch._types.SortOptions; import co.elastic.clients.elasticsearch._types.SortOrder; +import co.elastic.clients.elasticsearch.core.explain.ExplanationDetail; import co.elastic.clients.elasticsearch.core.search.Hit; import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -86,6 +88,68 @@ public void sortValuesAreLeftOutWhenTheHitCarriesNone() { assertNull(simplified.getSortValues()); } + @Test + public void disMaxReportsOnlyOneWinnerWhenPlainAndSynonymScoresTie() { + List terms = collectDisMax("max of:", 12.1f, + detail(12.1f, "weight(title:wave in 1) [BM25], result of:"), + detail(12.1f, "weight(title.synonyms:wave in 1) [BM25], result of:")); + + assertEquals(1, terms.size()); + assertEquals("title", terms.get(0).getField()); + assertEquals(12.1, terms.get(0).getScore(), 0.0001); + } + + @Test + public void disMaxReportsSynonymWinnerAsOneEntry() { + List terms = collectDisMax("max of:", 12.1f, + detail(4.2f, "weight(title:soop in 1) [BM25], result of:"), + detail(12.1f, "weight(Synonym(title.synonyms:soop title.synonyms:ships " + + "title.synonyms:of title.synonyms:opportunity) in 1) [BM25], result of:")); + + assertEquals(1, terms.size()); + assertEquals("title.synonyms", terms.get(0).getField()); + assertEquals("soop ships of opportunity", terms.get(0).getTerm()); + assertEquals(12.1, terms.get(0).getScore(), 0.0001); + } + + @Test + public void disMaxReportsHighestChildWhenNoChildEqualsParent() { + List terms = collectDisMax("max of:", 12.1f, + detail(12.0999f, "weight(title:wave in 1) [BM25], result of:"), + detail(4.0f, "weight(title.synonyms:wave in 1) [BM25], result of:")); + + assertEquals(1, terms.size()); + assertEquals("title", terms.get(0).getField()); + } + + @Test + public void disMaxWithTieBreakerReportsOnlyTheWinner() { + // best_fields multi_match with tie_breaker 0.1: 36.3 + 0.1 * 12.1 + List terms = collectDisMax("max plus 0.1 times others of:", 37.51f, + detail(36.3f, "weight(title:wave in 1) [BM25], result of:"), + detail(12.1f, "weight(title.synonyms:wave in 1) [BM25], result of:")); + + assertEquals(1, terms.size()); + assertEquals("title", terms.get(0).getField()); + assertEquals(36.3, terms.get(0).getScore(), 0.0001); + } + + private static List collectDisMax(String description, float value, + ExplanationDetail... children) { + List terms = new ArrayList<>(); + ExplainSimplifier.collectScoreParts( + List.of(detail(value, description, children)), terms, new ArrayList<>()); + return terms; + } + + private static ExplanationDetail detail(float value, String description, + ExplanationDetail... children) { + return ExplanationDetail.of(d -> d + .value(value) + .description(description) + .details(List.of(children))); + } + private static ExplainSimplifiedResponse.SortValue sortValue(String field, Object value) { return ExplainSimplifiedResponse.SortValue.builder() .field(field) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 5ea97cf7..ba34d103 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -357,7 +357,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { // Call rest api directly and get query result with search on "dataset" ResponseEntity collections = testRestTemplate.exchange( - getBasePath() + "/collections?q=dataset&filter=page_size=1 AND score>=1.3", + getBasePath() + "/collections?q=dataset&filter=page_size=1 AND score>=0.75", HttpMethod.GET, null, new ParameterizedTypeReference<>() { @@ -386,7 +386,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { // Now the same search, same page but search_after the actual cursor returned above collections = testRestTemplate.exchange( - getBasePath() + "/collections?q=dataset&filter=page_size=6 AND score>=1.3 AND search_after=" + + getBasePath() + "/collections?q=dataset&filter=page_size=6 AND score>=0.75 AND search_after=" + String.format("'%s|| %s || %s || %s'", collections.getBody().getSearchAfter().get(0), collections.getBody().getSearchAfter().get(1), @@ -401,7 +401,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { assertEquals(HttpStatus.OK, collections.getStatusCode(), "Get status OK"); log.info("{}", collections.getBody()); - // Remaining docs that clear min_score=1.3 after the first batch; the exact count is + // Remaining docs that clear min_score=0.75 after the first batch; the exact count is // BM25-dependent and varies by env, so accept any non-empty result up to the remaining total. int returnedSize = Objects.requireNonNull(collections.getBody()).getCollections().size(); assertTrue(returnedSize >= 1 && returnedSize <= 3, diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java index f5a262b7..f6c51eca 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java @@ -5,7 +5,10 @@ import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.FieldValue; import co.elastic.clients.elasticsearch._types.SortOptions; +import co.elastic.clients.elasticsearch._types.query_dsl.MultiMatchQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.Operator; import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import co.elastic.clients.elasticsearch._types.query_dsl.TextQueryType; import co.elastic.clients.elasticsearch.core.SearchRequest; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -37,10 +40,22 @@ public void searchByParametersWithDoubleQuote() throws Exception { "-score,-rank", CQLCrsType.EPSG4326); - assertEquals(9, capturingSearch.should.size(), - "Exact match should produce 9 queries (title + description + other fields, no dataset_group)"); - assertTrue(capturingSearch.should.get(0).isMatchPhrase(), "Title query should be MatchPhraseQuery"); - assertTrue(capturingSearch.should.get(1).isMatchPhrase(), "Description query should be MatchPhraseQuery"); + assertEquals(7, capturingSearch.should.size(), + "Exact match should produce 7 queries (title + description + other fields, no dataset_group)"); + + MultiMatchQuery title = capturingSearch.should.get(0).multiMatch(); + assertEquals(TextQueryType.Phrase, title.type(), "Title query should be a phrase multi_match"); + assertEquals(List.of("title^2", "title.synonyms^2"), title.fields()); + assertEquals(0.1, title.tieBreaker()); + assertEquals("ocean temperature", title.query(), "quotes are removed"); + assertNull(title.fuzziness(), "fuzziness is not allowed for a phrase"); + + MultiMatchQuery description = capturingSearch.should.get(1).multiMatch(); + assertEquals(TextQueryType.Phrase, description.type(), "Description query should be a phrase multi_match"); + assertEquals(List.of("description", "description.synonyms"), description.fields()); + assertEquals(0.1, description.tieBreaker()); + + assertSynonymsOnlyInsideMultiMatch(capturingSearch.should); assertTrue(capturingSearch.arguments.sortOptions().get(0).isScript(), "dataset_group priority sort should be the first sort key"); } @@ -56,12 +71,40 @@ public void searchByParametersWithoutDoubleQuote() throws Exception { "-score,-rank", CQLCrsType.EPSG4326); - assertEquals(9, capturingSearch.should.size(), "Fuzzy match should produce 9 queries"); - assertTrue(capturingSearch.should.get(0).isMatch(), "fuzzy_title should be MatchQuery"); + assertEquals(7, capturingSearch.should.size(), "Fuzzy match should produce 7 queries"); + + MultiMatchQuery title = capturingSearch.should.get(0).multiMatch(); + assertEquals(TextQueryType.BestFields, title.type(), "fuzzy_title should be a best_fields multi_match"); + assertEquals(List.of("title^2", "title.synonyms^2"), title.fields()); + assertEquals(0.1, title.tieBreaker()); + assertEquals("AUTO", title.fuzziness()); + assertEquals(4, title.prefixLength()); + assertEquals(Operator.And, title.operator()); + + MultiMatchQuery description = capturingSearch.should.get(1).multiMatch(); + assertEquals(TextQueryType.BestFields, description.type(), "fuzzy_desc should be a best_fields multi_match"); + assertEquals(List.of("description", "description.synonyms"), description.fields()); + assertEquals(0.1, description.tieBreaker()); + assertEquals("AUTO", description.fuzziness()); + assertEquals(4, description.prefixLength()); + assertEquals(Operator.And, description.operator()); + + assertSynonymsOnlyInsideMultiMatch(capturingSearch.should); assertTrue(capturingSearch.arguments.sortOptions().get(0).isScript(), "dataset_group priority sort should be the first sort key"); } + /** + * The synonyms sub-fields must only be scored inside the title/description multi_match, a separate clause on them + * would score plain words twice + */ + private static void assertSynonymsOnlyInsideMultiMatch(List should) { + should.stream() + .filter(q -> !q.isMultiMatch()) + .forEach(q -> assertFalse(q.toString().contains(".synonyms"), + "unexpected clause on a synonyms field: " + q)); + } + // The portal SEO pipeline requests these in bulk to build Dataset JSON-LD @Test public void seoPropertiesMapToStacFields() throws Exception { @@ -122,7 +165,7 @@ public void explainByParametersUsesScriptScoreRequestForKeywords() throws Except "title-only _source is lightweight so the larger search_after batch is used"); assertNotNull(capturingSearch.explainRequest.query()); assertTrue(capturingSearch.explainRequest.query().isScriptScore()); - assertEquals(9, capturingSearch.explainRequest.query().scriptScore() + assertEquals(7, capturingSearch.explainRequest.query().scriptScore() .query().bool().should().size()); assertNotNull(capturingSearch.explainRequest.source()); assertTrue(capturingSearch.explainRequest.source().isFilter());