From 9b8634fe157c49b4597f1a66bab7f0acdd781400 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Tue, 6 Oct 2026 14:20:30 +1100 Subject: [PATCH 01/12] Make sure title/description and their sycronym contribute to relevance calculation only once --- .../server/core/service/ElasticSearch.java | 21 ++++++++++++------- .../server/core/util/ExplainSimplifier.java | 14 +++++++++++++ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 868b477e..bcf909f1 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -437,26 +437,33 @@ private static List createKeywordShouldClauses(List keywords) { if (isExact) { // Match phrase in original title and description, not use fuzzy fields - should.add(CQLFields.title.getPropertyEqualToQuery(term)); - should.add(CQLFields.description.getPropertyEqualToQuery(term)); + should.add(bestOf(CQLFields.title, CQLFields.acronym_title, term)); + should.add(bestOf(CQLFields.description, CQLFields.acronym_desc, term)); } else { - should.add(CQLFields.fuzzy_title.getPropertyEqualToQuery(term)); - should.add(CQLFields.fuzzy_desc.getPropertyEqualToQuery(term)); + should.add(bestOf(CQLFields.fuzzy_title, CQLFields.acronym_title, term)); + should.add(bestOf(CQLFields.fuzzy_desc, CQLFields.acronym_desc, term)); } should.add(CQLFields.parameter_vocabs.getPropertyEqualToQuery(term)); should.add(CQLFields.organisation_vocabs.getPropertyEqualToQuery(term)); should.add(CQLFields.platform_vocabs.getPropertyEqualToQuery(term)); should.add(CQLFields.id.getPropertyEqualToQuery(term)); - // Acronym match on the *.synonyms sub-fields, e.g. "SOOP" -> "ships of opportunity". - should.add(CQLFields.acronym_title.getPropertyEqualToQuery(term)); - should.add(CQLFields.acronym_desc.getPropertyEqualToQuery(term)); // credit_contains uses match query by default, exact match is not applied here should.add(CQLFields.credit_contains.getPropertyEqualToQuery(term)); } return should; } + /** + * Scores only the better of a field's plain-text match and its synonym match, because words + * that pass through the synonym analyzer unchanged would otherwise be scored twice. + */ + private static Query bestOf(CQLFields text, CQLFields synonyms, String term) { + return DisMaxQuery.of(d -> d + .queries(text.getPropertyEqualToQuery(term), synonyms.getPropertyEqualToQuery(term)) + .tieBreaker(0.0))._toQuery(); + } + /** * Dataset-group candidate values for the priority sort, expanded from the search keywords with the same exact/quoted handling as the should clauses. */ diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java index 6a6b6ec3..acdabacd 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java @@ -38,6 +38,9 @@ public class ExplainSimplifier { protected static final String SYNONYM_PREFIX = "Synonym("; + // A dis_max (disjunction max) explanation includes every match, but only its winning child contributes. + protected static final String MAX_OF_PREFIX = "max of:"; + protected static final String RELEVANCE_DESCRIPTION_PREFIX = "_score:"; protected static final Comparator BY_SCORE_DESC = @@ -214,6 +217,17 @@ protected static void collectScoreParts(List details, for (ExplanationDetail detail : details) { String description = detail.description(); + if (description != null && description.startsWith(MAX_OF_PREFIX)) { + if (detail.details() != null) { + // the highest child, not one equal to the parent, so float rounding cannot lose the winner; a tie keeps the first + detail.details().stream() + .max(Comparator.comparingDouble(ExplanationDetail::value)) + .ifPresent(winner -> collectScoreParts( + List.of(winner), terms, filters)); + } + continue; + } + // most nodes are idf/tf breakdowns, skip the regex for them if (description != null && description.startsWith(WEIGHT_PREFIX)) { Matcher matcher = WEIGHT_PATTERN.matcher(description); From 56d134a3c58582e4e463dd0d2114207286b9aa03 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Tue, 6 Oct 2026 14:20:37 +1100 Subject: [PATCH 02/12] add tests --- .../core/util/ExplainSimplifierTest.java | 57 +++++++++++++++++++ .../server/service/ElasticSearchTest.java | 36 +++++++++--- 2 files changed, 86 insertions(+), 7 deletions(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java index 43db66ee..e79bfff4 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java @@ -5,10 +5,12 @@ import co.elastic.clients.elasticsearch._types.ScriptSortType; import co.elastic.clients.elasticsearch._types.SortOptions; import co.elastic.clients.elasticsearch._types.SortOrder; +import co.elastic.clients.elasticsearch.core.explain.ExplanationDetail; import co.elastic.clients.elasticsearch.core.search.Hit; import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -86,6 +88,61 @@ public void sortValuesAreLeftOutWhenTheHitCarriesNone() { assertNull(simplified.getSortValues()); } + @Test + public void disMaxReportsOnlyOneWinnerWhenPlainAndSynonymScoresTie() { + ExplanationDetail plain = detail(12.1f, "weight(title:wave in 1) [BM25], result of:"); + ExplanationDetail synonym = detail( + 12.1f, "weight(title.synonyms:wave in 1) [BM25], result of:"); + ExplanationDetail disMax = detail(12.1f, "max of:", plain, synonym); + List terms = new ArrayList<>(); + + ExplainSimplifier.collectScoreParts(List.of(disMax), terms, new ArrayList<>()); + + assertEquals(1, terms.size()); + assertEquals("title", terms.get(0).getField()); + assertEquals(12.1, terms.stream().mapToDouble( + ExplainSimplifiedResponse.MatchedTerm::getScore).sum(), 0.0001); + } + + @Test + public void disMaxReportsSynonymWinnerAsOneEntry() { + ExplanationDetail plain = detail(4.2f, "weight(title:soop in 1) [BM25], result of:"); + ExplanationDetail synonym = detail(12.1f, + "weight(Synonym(title.synonyms:soop title.synonyms:ships " + + "title.synonyms:of title.synonyms:opportunity) in 1) [BM25], result of:"); + ExplanationDetail disMax = detail(12.1f, "max of:", plain, synonym); + List terms = new ArrayList<>(); + + ExplainSimplifier.collectScoreParts(List.of(disMax), terms, new ArrayList<>()); + + assertEquals(1, terms.size()); + assertEquals("title.synonyms", terms.get(0).getField()); + assertEquals("soop ships of opportunity", terms.get(0).getTerm()); + assertEquals(12.1, terms.stream().mapToDouble( + ExplainSimplifiedResponse.MatchedTerm::getScore).sum(), 0.0001); + } + + @Test + public void disMaxReportsHighestChildWhenNoChildEqualsParent() { + ExplanationDetail plain = detail(12.0999f, "weight(title:wave in 1) [BM25], result of:"); + ExplanationDetail synonym = detail(4.0f, "weight(title.synonyms:wave in 1) [BM25], result of:"); + ExplanationDetail disMax = detail(12.1f, "max of:", plain, synonym); + List terms = new ArrayList<>(); + + ExplainSimplifier.collectScoreParts(List.of(disMax), terms, new ArrayList<>()); + + assertEquals(1, terms.size()); + assertEquals("title", terms.get(0).getField()); + } + + private static ExplanationDetail detail(float value, String description, + ExplanationDetail... children) { + return ExplanationDetail.of(d -> d + .value(value) + .description(description) + .details(List.of(children))); + } + private static ExplainSimplifiedResponse.SortValue sortValue(String field, Object value) { return ExplainSimplifiedResponse.SortValue.builder() .field(field) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java index f5a262b7..452f3609 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java @@ -37,10 +37,10 @@ public void searchByParametersWithDoubleQuote() throws Exception { "-score,-rank", CQLCrsType.EPSG4326); - assertEquals(9, capturingSearch.should.size(), - "Exact match should produce 9 queries (title + description + other fields, no dataset_group)"); - assertTrue(capturingSearch.should.get(0).isMatchPhrase(), "Title query should be MatchPhraseQuery"); - assertTrue(capturingSearch.should.get(1).isMatchPhrase(), "Description query should be MatchPhraseQuery"); + assertEquals(7, capturingSearch.should.size(), + "Exact match should produce 7 queries (two dis_max groups + other fields)"); + assertDisMax(capturingSearch.should.get(0), "title", true, "title.synonyms"); + assertDisMax(capturingSearch.should.get(1), "description", true, "description.synonyms"); assertTrue(capturingSearch.arguments.sortOptions().get(0).isScript(), "dataset_group priority sort should be the first sort key"); } @@ -56,8 +56,9 @@ public void searchByParametersWithoutDoubleQuote() throws Exception { "-score,-rank", CQLCrsType.EPSG4326); - assertEquals(9, capturingSearch.should.size(), "Fuzzy match should produce 9 queries"); - assertTrue(capturingSearch.should.get(0).isMatch(), "fuzzy_title should be MatchQuery"); + assertEquals(7, capturingSearch.should.size(), "Fuzzy match should produce 7 queries"); + assertDisMax(capturingSearch.should.get(0), "title", false, "title.synonyms"); + assertDisMax(capturingSearch.should.get(1), "description", false, "description.synonyms"); assertTrue(capturingSearch.arguments.sortOptions().get(0).isScript(), "dataset_group priority sort should be the first sort key"); } @@ -122,13 +123,34 @@ public void explainByParametersUsesScriptScoreRequestForKeywords() throws Except "title-only _source is lightweight so the larger search_after batch is used"); assertNotNull(capturingSearch.explainRequest.query()); assertTrue(capturingSearch.explainRequest.query().isScriptScore()); - assertEquals(9, capturingSearch.explainRequest.query().scriptScore() + assertEquals(7, capturingSearch.explainRequest.query().scriptScore() .query().bool().should().size()); assertNotNull(capturingSearch.explainRequest.source()); assertTrue(capturingSearch.explainRequest.source().isFilter()); assertFalse(capturingSearch.explainRequest.source().filter().includes().isEmpty()); } + private static void assertDisMax(Query query, String plainField, boolean phrase, + String synonymField) { + assertTrue(query.isDisMax()); + assertEquals(0.0, query.disMax().tieBreaker()); + assertEquals(2, query.disMax().queries().size()); + + Query plain = query.disMax().queries().get(0); + if (phrase) { + assertTrue(plain.isMatchPhrase()); + assertEquals(plainField, plain.matchPhrase().field()); + } + else { + assertTrue(plain.isMatch()); + assertEquals(plainField, plain.match().field()); + } + + Query synonym = query.disMax().queries().get(1); + assertTrue(synonym.isMatch()); + assertEquals(synonymField, synonym.match().field()); + } + @Test public void explainByParametersUsesCollectionQueryForEmptySearch() throws Exception { CapturingElasticSearch capturingSearch = new CapturingElasticSearch(mockClient); From 9bd21078fb149347a1ad8d97e8d682a52928b561 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Tue, 6 Oct 2026 14:29:33 +1100 Subject: [PATCH 03/12] better explain simplifier for disjunction max query --- .../server/core/service/ElasticSearch.java | 12 ++--- .../server/core/util/ExplainSimplifier.java | 1 + .../core/util/ExplainSimplifierTest.java | 45 +++++++++---------- .../server/service/ElasticSearchTest.java | 13 +++--- 4 files changed, 30 insertions(+), 41 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index bcf909f1..8ab109b6 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -435,15 +435,9 @@ private static List createKeywordShouldClauses(List keywords) { // If search text with double quote, remove quotee, otherwise keeps same String term = isExact ? t.substring(1, t.length() - 1) : t; - if (isExact) { - // Match phrase in original title and description, not use fuzzy fields - should.add(bestOf(CQLFields.title, CQLFields.acronym_title, term)); - should.add(bestOf(CQLFields.description, CQLFields.acronym_desc, term)); - } - else { - should.add(bestOf(CQLFields.fuzzy_title, CQLFields.acronym_title, term)); - should.add(bestOf(CQLFields.fuzzy_desc, CQLFields.acronym_desc, term)); - } + // Exact: match phrase in original title and description, not use fuzzy fields + should.add(bestOf(isExact ? CQLFields.title : CQLFields.fuzzy_title, CQLFields.acronym_title, term)); + should.add(bestOf(isExact ? CQLFields.description : CQLFields.fuzzy_desc, CQLFields.acronym_desc, term)); should.add(CQLFields.parameter_vocabs.getPropertyEqualToQuery(term)); should.add(CQLFields.organisation_vocabs.getPropertyEqualToQuery(term)); should.add(CQLFields.platform_vocabs.getPropertyEqualToQuery(term)); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java index acdabacd..85d5a86d 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java @@ -39,6 +39,7 @@ public class ExplainSimplifier { protected static final String SYNONYM_PREFIX = "Synonym("; // A dis_max (disjunction max) explanation includes every match, but only its winning child contributes. + // Lucene writes "max of:" only for tieBreaker 0 (see ElasticSearch.bestOf), otherwise "max plus ... of:". protected static final String MAX_OF_PREFIX = "max of:"; protected static final String RELEVANCE_DESCRIPTION_PREFIX = "_score:"; diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java index e79bfff4..2248a809 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java @@ -90,51 +90,46 @@ public void sortValuesAreLeftOutWhenTheHitCarriesNone() { @Test public void disMaxReportsOnlyOneWinnerWhenPlainAndSynonymScoresTie() { - ExplanationDetail plain = detail(12.1f, "weight(title:wave in 1) [BM25], result of:"); - ExplanationDetail synonym = detail( - 12.1f, "weight(title.synonyms:wave in 1) [BM25], result of:"); - ExplanationDetail disMax = detail(12.1f, "max of:", plain, synonym); - List terms = new ArrayList<>(); - - ExplainSimplifier.collectScoreParts(List.of(disMax), terms, new ArrayList<>()); + List terms = collectDisMax(12.1f, + detail(12.1f, "weight(title:wave in 1) [BM25], result of:"), + detail(12.1f, "weight(title.synonyms:wave in 1) [BM25], result of:")); assertEquals(1, terms.size()); assertEquals("title", terms.get(0).getField()); - assertEquals(12.1, terms.stream().mapToDouble( - ExplainSimplifiedResponse.MatchedTerm::getScore).sum(), 0.0001); + assertEquals(12.1, terms.get(0).getScore(), 0.0001); } @Test public void disMaxReportsSynonymWinnerAsOneEntry() { - ExplanationDetail plain = detail(4.2f, "weight(title:soop in 1) [BM25], result of:"); - ExplanationDetail synonym = detail(12.1f, - "weight(Synonym(title.synonyms:soop title.synonyms:ships " - + "title.synonyms:of title.synonyms:opportunity) in 1) [BM25], result of:"); - ExplanationDetail disMax = detail(12.1f, "max of:", plain, synonym); - List terms = new ArrayList<>(); - - ExplainSimplifier.collectScoreParts(List.of(disMax), terms, new ArrayList<>()); + List terms = collectDisMax(12.1f, + detail(4.2f, "weight(title:soop in 1) [BM25], result of:"), + detail(12.1f, "weight(Synonym(title.synonyms:soop title.synonyms:ships " + + "title.synonyms:of title.synonyms:opportunity) in 1) [BM25], result of:")); assertEquals(1, terms.size()); assertEquals("title.synonyms", terms.get(0).getField()); assertEquals("soop ships of opportunity", terms.get(0).getTerm()); - assertEquals(12.1, terms.stream().mapToDouble( - ExplainSimplifiedResponse.MatchedTerm::getScore).sum(), 0.0001); + assertEquals(12.1, terms.get(0).getScore(), 0.0001); } @Test public void disMaxReportsHighestChildWhenNoChildEqualsParent() { - ExplanationDetail plain = detail(12.0999f, "weight(title:wave in 1) [BM25], result of:"); - ExplanationDetail synonym = detail(4.0f, "weight(title.synonyms:wave in 1) [BM25], result of:"); - ExplanationDetail disMax = detail(12.1f, "max of:", plain, synonym); - List terms = new ArrayList<>(); - - ExplainSimplifier.collectScoreParts(List.of(disMax), terms, new ArrayList<>()); + List terms = collectDisMax(12.1f, + detail(12.0999f, "weight(title:wave in 1) [BM25], result of:"), + detail(4.0f, "weight(title.synonyms:wave in 1) [BM25], result of:")); assertEquals(1, terms.size()); assertEquals("title", terms.get(0).getField()); } + private static List collectDisMax(float value, + ExplanationDetail... children) { + List terms = new ArrayList<>(); + ExplainSimplifier.collectScoreParts( + List.of(detail(value, "max of:", children)), terms, new ArrayList<>()); + return terms; + } + private static ExplanationDetail detail(float value, String description, ExplanationDetail... children) { return ExplanationDetail.of(d -> d diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java index 452f3609..27c4627c 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java @@ -39,8 +39,8 @@ public void searchByParametersWithDoubleQuote() throws Exception { assertEquals(7, capturingSearch.should.size(), "Exact match should produce 7 queries (two dis_max groups + other fields)"); - assertDisMax(capturingSearch.should.get(0), "title", true, "title.synonyms"); - assertDisMax(capturingSearch.should.get(1), "description", true, "description.synonyms"); + assertDisMax(capturingSearch.should.get(0), "title", true); + assertDisMax(capturingSearch.should.get(1), "description", true); assertTrue(capturingSearch.arguments.sortOptions().get(0).isScript(), "dataset_group priority sort should be the first sort key"); } @@ -57,8 +57,8 @@ public void searchByParametersWithoutDoubleQuote() throws Exception { CQLCrsType.EPSG4326); assertEquals(7, capturingSearch.should.size(), "Fuzzy match should produce 7 queries"); - assertDisMax(capturingSearch.should.get(0), "title", false, "title.synonyms"); - assertDisMax(capturingSearch.should.get(1), "description", false, "description.synonyms"); + assertDisMax(capturingSearch.should.get(0), "title", false); + assertDisMax(capturingSearch.should.get(1), "description", false); assertTrue(capturingSearch.arguments.sortOptions().get(0).isScript(), "dataset_group priority sort should be the first sort key"); } @@ -130,8 +130,7 @@ public void explainByParametersUsesScriptScoreRequestForKeywords() throws Except assertFalse(capturingSearch.explainRequest.source().filter().includes().isEmpty()); } - private static void assertDisMax(Query query, String plainField, boolean phrase, - String synonymField) { + private static void assertDisMax(Query query, String plainField, boolean phrase) { assertTrue(query.isDisMax()); assertEquals(0.0, query.disMax().tieBreaker()); assertEquals(2, query.disMax().queries().size()); @@ -148,7 +147,7 @@ private static void assertDisMax(Query query, String plainField, boolean phrase, Query synonym = query.disMax().queries().get(1); assertTrue(synonym.isMatch()); - assertEquals(synonymField, synonym.match().field()); + assertEquals(plainField + ".synonyms", synonym.match().field()); } @Test From c934ef15cc5ebc8b8d41a5985042e20397cde46b Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Wed, 7 Oct 2026 11:24:03 +1100 Subject: [PATCH 04/12] store acronym set in cache for search lookup decision --- .../configuration/ElasticSearchConfig.java | 12 +++++ .../core/model/enumeration/CQLFields.java | 1 + .../server/core/service/ElasticSearch.java | 21 ++++++-- .../src/main/resources/application-edge.yaml | 2 + server/src/main/resources/application.yaml | 3 ++ .../server/service/ElasticSearchTest.java | 54 +++++++++++++++++++ 6 files changed, 88 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java index 10934756..a664854b 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java @@ -1,5 +1,6 @@ package au.org.aodn.ogcapi.server.core.configuration; +import au.org.aodn.ogcapi.server.core.service.AcronymLookup; import au.org.aodn.ogcapi.server.core.service.CacheNoLandGeometry; import au.org.aodn.ogcapi.server.core.service.ElasticSearch; import au.org.aodn.ogcapi.server.core.service.Search; @@ -70,4 +71,15 @@ public Search createElasticSearch(ElasticsearchClient client, return new ElasticSearch(client, cacheNoLandGeometry, mapper, indexName, pageSize, lightweightPageSize, searchAsYouTypeSize); } + /** + * The acronyms es-indexer pushed into the ES synonyms set, used to decide whether a keyword needs the synonyms fields + * @param client - The elastic search client + * @param synonymSetName - The ES synonyms set es-indexer writes for this environment + * @return The acronym lookup + */ + @Bean + public AcronymLookup createAcronymLookup(ElasticsearchClient client, + @Value("${elasticsearch.acronyms.name}") String synonymSetName) { + return new AcronymLookup(client, synonymSetName); + } } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java index 239a8381..7c06fba5 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java @@ -291,6 +291,7 @@ public enum CQLFields implements CQLFieldsInterface { null, null), // Acronym match on the synonyms sub-fields (search-time expansion), e.g. "SOOP" -> "ships of opportunity". + // Only queried when AcronymLookup finds an acronym in the keyword, plain words would match the same text twice. acronym_title( StacBasicField.Title.searchField + ".synonyms", StacBasicField.Title.displayField, diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 8ab109b6..e50811c0 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -19,6 +19,7 @@ import org.geotools.filter.text.commons.Language; import org.geotools.filter.text.cql2.CQLException; import org.opengis.filter.Filter; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.annotation.Cacheable; import org.springframework.http.HttpStatus; @@ -74,6 +75,10 @@ public class ElasticSearch extends ElasticSearchBase implements Search { @Value("${elasticsearch.semantic.max_suggestions:5}") protected Integer semanticMaxSuggestions; + // null when not wired, then every keyword is also scored against the synonyms fields + @Autowired(required = false) + protected AcronymLookup acronymLookup; + public ElasticSearch(ElasticsearchClient client, CacheNoLandGeometry cacheNoLandGeometry, ObjectMapper mapper, @@ -427,17 +432,22 @@ public ElasticSearchBase.SearchResult searchAllCollections( * Builds the relevance should clauses for each search keyword. Shared by searchByParameters (search) and buildParameterSearchRequestSupplier (explain) so the two align with each other. * These should clauses contribute to the Elasticsearch BM25 relevance score. */ - private static List createKeywordShouldClauses(List keywords) { + protected List createKeywordShouldClauses(List keywords) { List should = new ArrayList<>(); for (String t : keywords) { // If user's input (keywords) wrapped with double quot", and the text is not empty, treat the user intend to search with the exact term, so fuzzy matching not applied on title and description boolean isExact = t.startsWith("\"") && t.endsWith("\"") && t.length() > 2; // If search text with double quote, remove quotee, otherwise keeps same String term = isExact ? t.substring(1, t.length() - 1) : t; + // Only a keyword with an acronym needs the synonyms fields, e.g. "soop temperature" is searched as + // "ships of opportunity temperature". Plain words would just match the same text twice. + boolean expandAcronym = acronymLookup == null || acronymLookup.mayExpand(term); // Exact: match phrase in original title and description, not use fuzzy fields - should.add(bestOf(isExact ? CQLFields.title : CQLFields.fuzzy_title, CQLFields.acronym_title, term)); - should.add(bestOf(isExact ? CQLFields.description : CQLFields.fuzzy_desc, CQLFields.acronym_desc, term)); + CQLFields title = isExact ? CQLFields.title : CQLFields.fuzzy_title; + CQLFields description = isExact ? CQLFields.description : CQLFields.fuzzy_desc; + should.add(expandAcronym ? bestOf(title, CQLFields.acronym_title, term) : title.getPropertyEqualToQuery(term)); + should.add(expandAcronym ? bestOf(description, CQLFields.acronym_desc, term) : description.getPropertyEqualToQuery(term)); should.add(CQLFields.parameter_vocabs.getPropertyEqualToQuery(term)); should.add(CQLFields.organisation_vocabs.getPropertyEqualToQuery(term)); should.add(CQLFields.platform_vocabs.getPropertyEqualToQuery(term)); @@ -449,8 +459,9 @@ private static List createKeywordShouldClauses(List keywords) { } /** - * Scores only the better of a field's plain-text match and its synonym match, because words - * that pass through the synonym analyzer unchanged would otherwise be scored twice. + * Scores only the better of a field's plain-text match and its synonym match, used when the keyword contains an acronym. + * The non-acronym words pass through the synonym analyzer unchanged, so a record with both the acronym and its full + * name would otherwise be scored twice. */ private static Query bestOf(CQLFields text, CQLFields synonyms, String term) { return DisMaxQuery.of(d -> d diff --git a/server/src/main/resources/application-edge.yaml b/server/src/main/resources/application-edge.yaml index c8f57fb1..f5e4459e 100644 --- a/server/src/main/resources/application-edge.yaml +++ b/server/src/main/resources/application-edge.yaml @@ -15,5 +15,7 @@ ogcapi: elasticsearch-explain-enabled: true elasticsearch: + acronyms: + name: portal-acronyms-edge semantic: enabled: true diff --git a/server/src/main/resources/application.yaml b/server/src/main/resources/application.yaml index 9238a48a..48add408 100644 --- a/server/src/main/resources/application.yaml +++ b/server/src/main/resources/application.yaml @@ -29,6 +29,9 @@ elasticsearch: name: vocabs_index cloud_optimized_index: name: es-coindexer-dev + # The ES synonyms set es-indexer pushes acronym rules into, e.g. "soop => ships of opportunity" + acronyms: + name: portal-acronyms-dev serverUrl: http://localhost:9200 apiKey: search_as_you_type: diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java index 27c4627c..e3b4c1e7 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java @@ -1,6 +1,7 @@ package au.org.aodn.ogcapi.server.service; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType; +import au.org.aodn.ogcapi.server.core.service.AcronymLookup; import au.org.aodn.ogcapi.server.core.service.ElasticSearch; import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.FieldValue; @@ -63,6 +64,49 @@ public void searchByParametersWithoutDoubleQuote() throws Exception { "dataset_group priority sort should be the first sort key"); } + @Test + public void searchWithoutAcronymSkipsSynonymFields() throws Exception { + AcronymLookup lookup = mock(AcronymLookup.class); + CapturingElasticSearch capturingSearch = new CapturingElasticSearch(mockClient, lookup); + + // Ordinary words and an acronym's full name are both plain text, so title/description are matched once only + for (String keyword : List.of("coastal wave data", "ships of opportunity")) { + capturingSearch.searchByParameters( + List.of(keyword), + null, + null, + "-score,-rank", + CQLCrsType.EPSG4326); + + assertEquals(7, capturingSearch.should.size()); + assertPlainMatch(capturingSearch.should.get(0), "title"); + assertPlainMatch(capturingSearch.should.get(1), "description"); + assertFalse(capturingSearch.should.toString().contains(".synonyms"), + "'" + keyword + "' has no acronym, so the synonyms fields must not add to the score"); + } + } + + @Test + public void searchWithAcronymAndOtherWordsUsesSynonymFields() throws Exception { + AcronymLookup lookup = mock(AcronymLookup.class); + when(lookup.mayExpand("soop temperature")).thenReturn(true); + CapturingElasticSearch capturingSearch = new CapturingElasticSearch(mockClient, lookup); + + capturingSearch.searchByParameters( + List.of("soop temperature"), + null, + null, + "-score,-rank", + CQLCrsType.EPSG4326); + + assertEquals(7, capturingSearch.should.size()); + // ES expands the whole keyword on the synonyms field, so it is searched as "ships of opportunity temperature" + assertDisMax(capturingSearch.should.get(0), "title", false); + assertDisMax(capturingSearch.should.get(1), "description", false); + assertEquals("soop temperature", + capturingSearch.should.get(0).disMax().queries().get(1).match().query().stringValue()); + } + // The portal SEO pipeline requests these in bulk to build Dataset JSON-LD @Test public void seoPropertiesMapToStacFields() throws Exception { @@ -150,6 +194,11 @@ private static void assertDisMax(Query query, String plainField, boolean phrase) assertEquals(plainField + ".synonyms", synonym.match().field()); } + private static void assertPlainMatch(Query query, String plainField) { + assertTrue(query.isMatch(), "expected a plain match, not a dis_max with the synonyms field"); + assertEquals(plainField, query.match().field()); + } + @Test public void explainByParametersUsesCollectionQueryForEmptySearch() throws Exception { CapturingElasticSearch capturingSearch = new CapturingElasticSearch(mockClient); @@ -294,6 +343,11 @@ private CapturingElasticSearch(ElasticsearchClient client) { this.searchAfterSplitRegex = "\\|\\|"; } + private CapturingElasticSearch(ElasticsearchClient client, AcronymLookup acronymLookup) { + this(client); + this.acronymLookup = acronymLookup; + } + @Override protected SearchResult searchCollectionBy( List queries, From fb6cf0cc9419036f15457e1ff359c7b380b41aba Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Wed, 7 Oct 2026 11:24:34 +1100 Subject: [PATCH 05/12] store acronym set in cache for search lookup decision --- .../server/core/service/AcronymLookup.java | 98 +++++++++++++++ .../core/service/AcronymLookupTest.java | 116 ++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/service/AcronymLookup.java create mode 100644 server/src/test/java/au/org/aodn/ogcapi/server/core/service/AcronymLookupTest.java diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AcronymLookup.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AcronymLookup.java new file mode 100644 index 00000000..2d544542 --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AcronymLookup.java @@ -0,0 +1,98 @@ +package au.org.aodn.ogcapi.server.core.service; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch.synonyms.GetSynonymRequest; +import co.elastic.clients.elasticsearch.synonyms.SynonymRuleRead; +import jakarta.annotation.PostConstruct; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; + +import java.util.Arrays; +import java.util.Locale; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Knows which acronyms es-indexer pushed into the ES synonyms set (e.g. "soop => ships of opportunity"), so a + * search keyword is only scored against the *.synonyms sub-fields when it actually contains an acronym. + * Without this gate, plain words pass through the synonym analyser unchanged and are scored twice. + * The expansion itself is still done by ES at search time; this class only answers "does the keyword contain one?". + */ +@Slf4j +public class AcronymLookup { + + /** GET _synonyms defaults to 10 rules per page; a set is capped at 10000, so ask for them all. */ + protected static final int MAX_SYNONYM_RULES = 10000; + + protected final ElasticsearchClient client; + protected final String synonymSetName; + + // null means unknown (never loaded), in which case every keyword may expand + protected volatile Set acronyms; + + public AcronymLookup(ElasticsearchClient client, String synonymSetName) { + this.client = client; + this.synonymSetName = synonymSetName; + } + + /** + * Load at startup so the first search already has the list, then reload every hour. The rules only change + * when es-indexer reindexes or syncs acronyms, so a new acronym is not gated in for up to an hour. + */ + @PostConstruct + @Scheduled(initialDelay = 60 * 60 * 1000, fixedRate = 60 * 60 * 1000) + public void refresh() { + try { + acronyms = client.synonyms() + .getSynonym(GetSynonymRequest.of(g -> g.id(synonymSetName).size(MAX_SYNONYM_RULES))) + .synonymsSet() + .stream() + .map(SynonymRuleRead::synonyms) + .filter(rule -> rule.contains("=>")) + // "aad, aadx => australian antarctic division" -> ["aad", "aadx"] + .flatMap(rule -> Arrays.stream(rule.substring(0, rule.indexOf("=>")).split(","))) + .map(AcronymLookup::normalise) + .filter(acronym -> !acronym.isEmpty()) + .collect(Collectors.toUnmodifiableSet()); + log.info("Loaded {} acronyms from synonyms set '{}'", acronyms.size(), synonymSetName); + } + catch (ElasticsearchException e) { + if (e.status() == 404) { + // The set does not exist yet, so nothing can expand + acronyms = Set.of(); + log.info("Synonyms set '{}' does not exist, no acronym will expand", synonymSetName); + } + else { + log.warn("Could not load acronyms from synonyms set '{}', keeping previous list: {}", synonymSetName, e.getMessage()); + } + } + catch (Exception e) { + log.warn("Could not load acronyms from synonyms set '{}', keeping previous list: {}", synonymSetName, e.getMessage()); + } + } + + /** + * @param term - A search keyword, e.g. "soop temperature" + * @return true if the keyword contains a known acronym, or if the acronym list could not be loaded + */ + public boolean mayExpand(String term) { + Set known = acronyms; + if (known == null) { + return true; + } + String padded = " " + normalise(term) + " "; + return known.stream().anyMatch(acronym -> padded.contains(" " + acronym + " ")); + } + + /** + * Close enough to the ES standard tokenizer plus lowercase for acronym matching, so a multi-token acronym + * still matches as a sequence. + * in: "SOOP's (AAD) data" -> "soop s aad data" + */ + protected static String normalise(String text) { + return Arrays.stream(text.toLowerCase(Locale.ROOT).split("[^\\p{L}\\p{N}]+")) + .filter(token -> !token.isEmpty()) + .collect(Collectors.joining(" ")); + } +} diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/AcronymLookupTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/AcronymLookupTest.java new file mode 100644 index 00000000..2b6de468 --- /dev/null +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/AcronymLookupTest.java @@ -0,0 +1,116 @@ +package au.org.aodn.ogcapi.server.core.service; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch._types.ErrorResponse; +import co.elastic.clients.elasticsearch.synonyms.ElasticsearchSynonymsClient; +import co.elastic.clients.elasticsearch.synonyms.GetSynonymRequest; +import co.elastic.clients.elasticsearch.synonyms.GetSynonymResponse; +import co.elastic.clients.elasticsearch.synonyms.SynonymRuleRead; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +public class AcronymLookupTest { + + private ElasticsearchSynonymsClient synonymsClient; + private AcronymLookup lookup; + + @BeforeEach + public void setUp() { + ElasticsearchClient client = mock(ElasticsearchClient.class); + synonymsClient = mock(ElasticsearchSynonymsClient.class); + when(client.synonyms()).thenReturn(synonymsClient); + lookup = new AcronymLookup(client, "portal-acronyms-test"); + } + + @Test + public void ruleLeftSidesBecomeAcronyms() throws IOException { + givenRules("soop => ships of opportunity", "aad, AADX => australian antarctic division"); + lookup.refresh(); + + assertEquals(3, lookup.acronyms.size()); + assertTrue(lookup.acronyms.containsAll(List.of("soop", "aad", "aadx"))); + } + + @Test + public void keywordWithAcronymMayExpand() throws IOException { + givenRules("soop => ships of opportunity", "aad => australian antarctic division"); + lookup.refresh(); + + assertTrue(lookup.mayExpand("soop")); + assertTrue(lookup.mayExpand("SOOP temperature"), "an acronym among other words still expands"); + assertTrue(lookup.mayExpand("soop's data"), "a possessive does not hide the acronym"); + assertTrue(lookup.mayExpand("(AAD)")); + } + + @Test + public void keywordWithoutAcronymDoesNotExpand() throws IOException { + givenRules("soop => ships of opportunity"); + lookup.refresh(); + + assertFalse(lookup.mayExpand("ships of opportunity"), "the full name is plain text"); + assertFalse(lookup.mayExpand("coastal wave data")); + assertFalse(lookup.mayExpand("soopy"), "only whole tokens are acronyms"); + } + + @Test + public void missingSynonymsSetMeansNoAcronym() throws IOException { + when(synonymsClient.getSynonym(any(GetSynonymRequest.class))).thenThrow(error(404)); + lookup.refresh(); + + assertFalse(lookup.mayExpand("soop")); + } + + @Test + public void failedLoadKeepsEveryKeywordExpandable() throws IOException { + when(synonymsClient.getSynonym(any(GetSynonymRequest.class))) + .thenThrow(error(403)) + .thenThrow(new IOException("connection refused")); + + lookup.refresh(); + assertTrue(lookup.mayExpand("coastal wave data"), "unknown list must not drop acronym expansion"); + + lookup.refresh(); + assertTrue(lookup.mayExpand("coastal wave data")); + } + + @Test + public void failedRefreshKeepsPreviousList() throws IOException { + givenRules("soop => ships of opportunity"); + lookup.refresh(); + + when(synonymsClient.getSynonym(any(GetSynonymRequest.class))).thenThrow(error(503)); + lookup.refresh(); + + assertTrue(lookup.mayExpand("soop")); + assertFalse(lookup.mayExpand("coastal wave data")); + } + + @Test + public void normaliseFollowsStandardTokenizer() { + assertEquals("soop s aad data", AcronymLookup.normalise("SOOP's (AAD) data")); + assertEquals("", AcronymLookup.normalise(" , ")); + } + + private void givenRules(String... rules) throws IOException { + List set = Arrays.stream(rules) + .map(rule -> new SynonymRuleRead.Builder().id(rule).synonyms(rule).build()) + .toList(); + when(synonymsClient.getSynonym(any(GetSynonymRequest.class))) + .thenReturn(GetSynonymResponse.of(r -> r.count(set.size()).synonymsSet(set))); + } + + private static ElasticsearchException error(int status) { + return new ElasticsearchException("synonyms.get_synonym", ErrorResponse.of(e -> e + .status(status) + .error(c -> c.type("error").reason("status " + status)))); + } +} From 3d70fdc5e82ed23c048b5941d78ad492cf89419c Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Wed, 7 Oct 2026 11:24:52 +1100 Subject: [PATCH 06/12] add acronyms for test config --- server/src/test/resources/application-test.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/test/resources/application-test.yaml b/server/src/test/resources/application-test.yaml index 49f3a6df..89924333 100644 --- a/server/src/test/resources/application-test.yaml +++ b/server/src/test/resources/application-test.yaml @@ -13,6 +13,10 @@ elasticsearch: vocabs_index: name: test_vocabs_index + # the set is never created in tests, so no keyword expands acronyms + acronyms: + name: portal-acronyms-test + search_as_you_type: search_suggestions: path: search_suggestions From 680fbb118d9b0ab05c0713f1afdb9bc7d920a223 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Wed, 7 Oct 2026 12:09:26 +1100 Subject: [PATCH 07/12] fix failed tests --- .../org/aodn/ogcapi/server/common/RestAdminApiTest.java | 4 +++- .../au/org/aodn/ogcapi/server/features/RestApiTest.java | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestAdminApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestAdminApiTest.java index 8294865d..93a3409f 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestAdminApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestAdminApiTest.java @@ -194,7 +194,9 @@ public void explainSimplifiedFormatReportsMultiWordMatches() throws IOException .toList(); assertTrue(reported.contains("title:ocean")); assertTrue(reported.contains("title:acidification")); - assertTrue(reported.size() > 2, "a multi word query hits more than one field"); + // no acronym in the query, so the same words must not be scored again on the synonyms fields + assertTrue(reported.stream().noneMatch(r -> r.contains(".synonyms")), + "plain words must not match the synonyms fields, got: " + reported); } @Test diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 5ea97cf7..74416f55 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -356,8 +356,10 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json"); // Call rest api directly and get query result with search on "dataset" + // 0.65 is half the old 1.3, which was set when a plain word scored on both title/description and their + // synonyms fields. Total is a count without min_score, so it stays 4 whatever the threshold. ResponseEntity collections = testRestTemplate.exchange( - getBasePath() + "/collections?q=dataset&filter=page_size=1 AND score>=1.3", + getBasePath() + "/collections?q=dataset&filter=page_size=1 AND score>=0.65", HttpMethod.GET, null, new ParameterizedTypeReference<>() { @@ -386,7 +388,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { // Now the same search, same page but search_after the actual cursor returned above collections = testRestTemplate.exchange( - getBasePath() + "/collections?q=dataset&filter=page_size=6 AND score>=1.3 AND search_after=" + + getBasePath() + "/collections?q=dataset&filter=page_size=6 AND score>=0.65 AND search_after=" + String.format("'%s|| %s || %s || %s'", collections.getBody().getSearchAfter().get(0), collections.getBody().getSearchAfter().get(1), @@ -401,7 +403,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { assertEquals(HttpStatus.OK, collections.getStatusCode(), "Get status OK"); log.info("{}", collections.getBody()); - // Remaining docs that clear min_score=1.3 after the first batch; the exact count is + // Remaining docs that clear min_score=0.65 after the first batch; the exact count is // BM25-dependent and varies by env, so accept any non-empty result up to the remaining total. int returnedSize = Objects.requireNonNull(collections.getBody()).getCollections().size(); assertTrue(returnedSize >= 1 && returnedSize <= 3, From 0afa9a2a070ef88a7f31e9f9bd5020958d5acfe3 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Wed, 7 Oct 2026 12:11:20 +1100 Subject: [PATCH 08/12] better javadoc --- .../java/au/org/aodn/ogcapi/server/features/RestApiTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 74416f55..5d585078 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -356,8 +356,6 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { "bf287dfe-9ce4-4969-9c59-51c39ea4d011.json"); // Call rest api directly and get query result with search on "dataset" - // 0.65 is half the old 1.3, which was set when a plain word scored on both title/description and their - // synonyms fields. Total is a count without min_score, so it stays 4 whatever the threshold. ResponseEntity collections = testRestTemplate.exchange( getBasePath() + "/collections?q=dataset&filter=page_size=1 AND score>=0.65", HttpMethod.GET, From e4f1b68e625bf8a244e1f823293f333cf4933297 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Wed, 7 Oct 2026 12:16:51 +1100 Subject: [PATCH 09/12] refresh acronym every day --- .../org/aodn/ogcapi/server/core/service/AcronymLookup.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AcronymLookup.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AcronymLookup.java index 2d544542..3944d118 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AcronymLookup.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AcronymLookup.java @@ -37,11 +37,11 @@ public AcronymLookup(ElasticsearchClient client, String synonymSetName) { } /** - * Load at startup so the first search already has the list, then reload every hour. The rules only change - * when es-indexer reindexes or syncs acronyms, so a new acronym is not gated in for up to an hour. + * Load at startup so the first search already has the list, then reload every day. The rules only change + * when es-indexer reindexes or syncs acronyms, so a new acronym is not gated in for up to a day. */ @PostConstruct - @Scheduled(initialDelay = 60 * 60 * 1000, fixedRate = 60 * 60 * 1000) + @Scheduled(initialDelay = 24 * 60 * 60 * 1000, fixedRate = 24 * 60 * 60 * 1000) public void refresh() { try { acronyms = client.synonyms() From 898558b0f4ff30db6908a346d8fa07a03d291833 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Thu, 8 Oct 2026 12:05:34 +1100 Subject: [PATCH 10/12] revert AcronymLookup approach for #9450 Co-Authored-By: Claude Opus 5.5 --- .../configuration/ElasticSearchConfig.java | 12 -- .../core/model/enumeration/CQLFields.java | 1 - .../server/core/service/AcronymLookup.java | 98 --------------- .../server/core/service/ElasticSearch.java | 40 +++--- .../server/core/util/ExplainSimplifier.java | 15 --- .../src/main/resources/application-edge.yaml | 2 - server/src/main/resources/application.yaml | 3 - .../server/common/RestAdminApiTest.java | 4 +- .../core/service/AcronymLookupTest.java | 116 ------------------ .../core/util/ExplainSimplifierTest.java | 52 -------- .../ogcapi/server/features/RestApiTest.java | 6 +- .../server/service/ElasticSearchTest.java | 89 ++------------ .../src/test/resources/application-test.yaml | 4 - 13 files changed, 25 insertions(+), 417 deletions(-) delete mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/service/AcronymLookup.java delete mode 100644 server/src/test/java/au/org/aodn/ogcapi/server/core/service/AcronymLookupTest.java diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java index a664854b..10934756 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java @@ -1,6 +1,5 @@ package au.org.aodn.ogcapi.server.core.configuration; -import au.org.aodn.ogcapi.server.core.service.AcronymLookup; import au.org.aodn.ogcapi.server.core.service.CacheNoLandGeometry; import au.org.aodn.ogcapi.server.core.service.ElasticSearch; import au.org.aodn.ogcapi.server.core.service.Search; @@ -71,15 +70,4 @@ public Search createElasticSearch(ElasticsearchClient client, return new ElasticSearch(client, cacheNoLandGeometry, mapper, indexName, pageSize, lightweightPageSize, searchAsYouTypeSize); } - /** - * The acronyms es-indexer pushed into the ES synonyms set, used to decide whether a keyword needs the synonyms fields - * @param client - The elastic search client - * @param synonymSetName - The ES synonyms set es-indexer writes for this environment - * @return The acronym lookup - */ - @Bean - public AcronymLookup createAcronymLookup(ElasticsearchClient client, - @Value("${elasticsearch.acronyms.name}") String synonymSetName) { - return new AcronymLookup(client, synonymSetName); - } } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java index 7c06fba5..239a8381 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java @@ -291,7 +291,6 @@ public enum CQLFields implements CQLFieldsInterface { null, null), // Acronym match on the synonyms sub-fields (search-time expansion), e.g. "SOOP" -> "ships of opportunity". - // Only queried when AcronymLookup finds an acronym in the keyword, plain words would match the same text twice. acronym_title( StacBasicField.Title.searchField + ".synonyms", StacBasicField.Title.displayField, diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AcronymLookup.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AcronymLookup.java deleted file mode 100644 index 3944d118..00000000 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/AcronymLookup.java +++ /dev/null @@ -1,98 +0,0 @@ -package au.org.aodn.ogcapi.server.core.service; - -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.elasticsearch._types.ElasticsearchException; -import co.elastic.clients.elasticsearch.synonyms.GetSynonymRequest; -import co.elastic.clients.elasticsearch.synonyms.SynonymRuleRead; -import jakarta.annotation.PostConstruct; -import lombok.extern.slf4j.Slf4j; -import org.springframework.scheduling.annotation.Scheduled; - -import java.util.Arrays; -import java.util.Locale; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Knows which acronyms es-indexer pushed into the ES synonyms set (e.g. "soop => ships of opportunity"), so a - * search keyword is only scored against the *.synonyms sub-fields when it actually contains an acronym. - * Without this gate, plain words pass through the synonym analyser unchanged and are scored twice. - * The expansion itself is still done by ES at search time; this class only answers "does the keyword contain one?". - */ -@Slf4j -public class AcronymLookup { - - /** GET _synonyms defaults to 10 rules per page; a set is capped at 10000, so ask for them all. */ - protected static final int MAX_SYNONYM_RULES = 10000; - - protected final ElasticsearchClient client; - protected final String synonymSetName; - - // null means unknown (never loaded), in which case every keyword may expand - protected volatile Set acronyms; - - public AcronymLookup(ElasticsearchClient client, String synonymSetName) { - this.client = client; - this.synonymSetName = synonymSetName; - } - - /** - * Load at startup so the first search already has the list, then reload every day. The rules only change - * when es-indexer reindexes or syncs acronyms, so a new acronym is not gated in for up to a day. - */ - @PostConstruct - @Scheduled(initialDelay = 24 * 60 * 60 * 1000, fixedRate = 24 * 60 * 60 * 1000) - public void refresh() { - try { - acronyms = client.synonyms() - .getSynonym(GetSynonymRequest.of(g -> g.id(synonymSetName).size(MAX_SYNONYM_RULES))) - .synonymsSet() - .stream() - .map(SynonymRuleRead::synonyms) - .filter(rule -> rule.contains("=>")) - // "aad, aadx => australian antarctic division" -> ["aad", "aadx"] - .flatMap(rule -> Arrays.stream(rule.substring(0, rule.indexOf("=>")).split(","))) - .map(AcronymLookup::normalise) - .filter(acronym -> !acronym.isEmpty()) - .collect(Collectors.toUnmodifiableSet()); - log.info("Loaded {} acronyms from synonyms set '{}'", acronyms.size(), synonymSetName); - } - catch (ElasticsearchException e) { - if (e.status() == 404) { - // The set does not exist yet, so nothing can expand - acronyms = Set.of(); - log.info("Synonyms set '{}' does not exist, no acronym will expand", synonymSetName); - } - else { - log.warn("Could not load acronyms from synonyms set '{}', keeping previous list: {}", synonymSetName, e.getMessage()); - } - } - catch (Exception e) { - log.warn("Could not load acronyms from synonyms set '{}', keeping previous list: {}", synonymSetName, e.getMessage()); - } - } - - /** - * @param term - A search keyword, e.g. "soop temperature" - * @return true if the keyword contains a known acronym, or if the acronym list could not be loaded - */ - public boolean mayExpand(String term) { - Set known = acronyms; - if (known == null) { - return true; - } - String padded = " " + normalise(term) + " "; - return known.stream().anyMatch(acronym -> padded.contains(" " + acronym + " ")); - } - - /** - * Close enough to the ES standard tokenizer plus lowercase for acronym matching, so a multi-token acronym - * still matches as a sequence. - * in: "SOOP's (AAD) data" -> "soop s aad data" - */ - protected static String normalise(String text) { - return Arrays.stream(text.toLowerCase(Locale.ROOT).split("[^\\p{L}\\p{N}]+")) - .filter(token -> !token.isEmpty()) - .collect(Collectors.joining(" ")); - } -} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index e50811c0..868b477e 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -19,7 +19,6 @@ import org.geotools.filter.text.commons.Language; import org.geotools.filter.text.cql2.CQLException; import org.opengis.filter.Filter; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.annotation.Cacheable; import org.springframework.http.HttpStatus; @@ -75,10 +74,6 @@ public class ElasticSearch extends ElasticSearchBase implements Search { @Value("${elasticsearch.semantic.max_suggestions:5}") protected Integer semanticMaxSuggestions; - // null when not wired, then every keyword is also scored against the synonyms fields - @Autowired(required = false) - protected AcronymLookup acronymLookup; - public ElasticSearch(ElasticsearchClient client, CacheNoLandGeometry cacheNoLandGeometry, ObjectMapper mapper, @@ -432,43 +427,36 @@ public ElasticSearchBase.SearchResult searchAllCollections( * Builds the relevance should clauses for each search keyword. Shared by searchByParameters (search) and buildParameterSearchRequestSupplier (explain) so the two align with each other. * These should clauses contribute to the Elasticsearch BM25 relevance score. */ - protected List createKeywordShouldClauses(List keywords) { + private static List createKeywordShouldClauses(List keywords) { List should = new ArrayList<>(); for (String t : keywords) { // If user's input (keywords) wrapped with double quot", and the text is not empty, treat the user intend to search with the exact term, so fuzzy matching not applied on title and description boolean isExact = t.startsWith("\"") && t.endsWith("\"") && t.length() > 2; // If search text with double quote, remove quotee, otherwise keeps same String term = isExact ? t.substring(1, t.length() - 1) : t; - // Only a keyword with an acronym needs the synonyms fields, e.g. "soop temperature" is searched as - // "ships of opportunity temperature". Plain words would just match the same text twice. - boolean expandAcronym = acronymLookup == null || acronymLookup.mayExpand(term); - - // Exact: match phrase in original title and description, not use fuzzy fields - CQLFields title = isExact ? CQLFields.title : CQLFields.fuzzy_title; - CQLFields description = isExact ? CQLFields.description : CQLFields.fuzzy_desc; - should.add(expandAcronym ? bestOf(title, CQLFields.acronym_title, term) : title.getPropertyEqualToQuery(term)); - should.add(expandAcronym ? bestOf(description, CQLFields.acronym_desc, term) : description.getPropertyEqualToQuery(term)); + + if (isExact) { + // Match phrase in original title and description, not use fuzzy fields + should.add(CQLFields.title.getPropertyEqualToQuery(term)); + should.add(CQLFields.description.getPropertyEqualToQuery(term)); + } + else { + should.add(CQLFields.fuzzy_title.getPropertyEqualToQuery(term)); + should.add(CQLFields.fuzzy_desc.getPropertyEqualToQuery(term)); + } should.add(CQLFields.parameter_vocabs.getPropertyEqualToQuery(term)); should.add(CQLFields.organisation_vocabs.getPropertyEqualToQuery(term)); should.add(CQLFields.platform_vocabs.getPropertyEqualToQuery(term)); should.add(CQLFields.id.getPropertyEqualToQuery(term)); + // Acronym match on the *.synonyms sub-fields, e.g. "SOOP" -> "ships of opportunity". + should.add(CQLFields.acronym_title.getPropertyEqualToQuery(term)); + should.add(CQLFields.acronym_desc.getPropertyEqualToQuery(term)); // credit_contains uses match query by default, exact match is not applied here should.add(CQLFields.credit_contains.getPropertyEqualToQuery(term)); } return should; } - /** - * Scores only the better of a field's plain-text match and its synonym match, used when the keyword contains an acronym. - * The non-acronym words pass through the synonym analyzer unchanged, so a record with both the acronym and its full - * name would otherwise be scored twice. - */ - private static Query bestOf(CQLFields text, CQLFields synonyms, String term) { - return DisMaxQuery.of(d -> d - .queries(text.getPropertyEqualToQuery(term), synonyms.getPropertyEqualToQuery(term)) - .tieBreaker(0.0))._toQuery(); - } - /** * Dataset-group candidate values for the priority sort, expanded from the search keywords with the same exact/quoted handling as the should clauses. */ diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java index 85d5a86d..6a6b6ec3 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java @@ -38,10 +38,6 @@ public class ExplainSimplifier { protected static final String SYNONYM_PREFIX = "Synonym("; - // A dis_max (disjunction max) explanation includes every match, but only its winning child contributes. - // Lucene writes "max of:" only for tieBreaker 0 (see ElasticSearch.bestOf), otherwise "max plus ... of:". - protected static final String MAX_OF_PREFIX = "max of:"; - protected static final String RELEVANCE_DESCRIPTION_PREFIX = "_score:"; protected static final Comparator BY_SCORE_DESC = @@ -218,17 +214,6 @@ protected static void collectScoreParts(List details, for (ExplanationDetail detail : details) { String description = detail.description(); - if (description != null && description.startsWith(MAX_OF_PREFIX)) { - if (detail.details() != null) { - // the highest child, not one equal to the parent, so float rounding cannot lose the winner; a tie keeps the first - detail.details().stream() - .max(Comparator.comparingDouble(ExplanationDetail::value)) - .ifPresent(winner -> collectScoreParts( - List.of(winner), terms, filters)); - } - continue; - } - // most nodes are idf/tf breakdowns, skip the regex for them if (description != null && description.startsWith(WEIGHT_PREFIX)) { Matcher matcher = WEIGHT_PATTERN.matcher(description); diff --git a/server/src/main/resources/application-edge.yaml b/server/src/main/resources/application-edge.yaml index f5e4459e..c8f57fb1 100644 --- a/server/src/main/resources/application-edge.yaml +++ b/server/src/main/resources/application-edge.yaml @@ -15,7 +15,5 @@ ogcapi: elasticsearch-explain-enabled: true elasticsearch: - acronyms: - name: portal-acronyms-edge semantic: enabled: true diff --git a/server/src/main/resources/application.yaml b/server/src/main/resources/application.yaml index 48add408..9238a48a 100644 --- a/server/src/main/resources/application.yaml +++ b/server/src/main/resources/application.yaml @@ -29,9 +29,6 @@ elasticsearch: name: vocabs_index cloud_optimized_index: name: es-coindexer-dev - # The ES synonyms set es-indexer pushes acronym rules into, e.g. "soop => ships of opportunity" - acronyms: - name: portal-acronyms-dev serverUrl: http://localhost:9200 apiKey: search_as_you_type: diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestAdminApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestAdminApiTest.java index 93a3409f..8294865d 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestAdminApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestAdminApiTest.java @@ -194,9 +194,7 @@ public void explainSimplifiedFormatReportsMultiWordMatches() throws IOException .toList(); assertTrue(reported.contains("title:ocean")); assertTrue(reported.contains("title:acidification")); - // no acronym in the query, so the same words must not be scored again on the synonyms fields - assertTrue(reported.stream().noneMatch(r -> r.contains(".synonyms")), - "plain words must not match the synonyms fields, got: " + reported); + assertTrue(reported.size() > 2, "a multi word query hits more than one field"); } @Test diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/AcronymLookupTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/AcronymLookupTest.java deleted file mode 100644 index 2b6de468..00000000 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/AcronymLookupTest.java +++ /dev/null @@ -1,116 +0,0 @@ -package au.org.aodn.ogcapi.server.core.service; - -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.elasticsearch._types.ElasticsearchException; -import co.elastic.clients.elasticsearch._types.ErrorResponse; -import co.elastic.clients.elasticsearch.synonyms.ElasticsearchSynonymsClient; -import co.elastic.clients.elasticsearch.synonyms.GetSynonymRequest; -import co.elastic.clients.elasticsearch.synonyms.GetSynonymResponse; -import co.elastic.clients.elasticsearch.synonyms.SynonymRuleRead; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; - -public class AcronymLookupTest { - - private ElasticsearchSynonymsClient synonymsClient; - private AcronymLookup lookup; - - @BeforeEach - public void setUp() { - ElasticsearchClient client = mock(ElasticsearchClient.class); - synonymsClient = mock(ElasticsearchSynonymsClient.class); - when(client.synonyms()).thenReturn(synonymsClient); - lookup = new AcronymLookup(client, "portal-acronyms-test"); - } - - @Test - public void ruleLeftSidesBecomeAcronyms() throws IOException { - givenRules("soop => ships of opportunity", "aad, AADX => australian antarctic division"); - lookup.refresh(); - - assertEquals(3, lookup.acronyms.size()); - assertTrue(lookup.acronyms.containsAll(List.of("soop", "aad", "aadx"))); - } - - @Test - public void keywordWithAcronymMayExpand() throws IOException { - givenRules("soop => ships of opportunity", "aad => australian antarctic division"); - lookup.refresh(); - - assertTrue(lookup.mayExpand("soop")); - assertTrue(lookup.mayExpand("SOOP temperature"), "an acronym among other words still expands"); - assertTrue(lookup.mayExpand("soop's data"), "a possessive does not hide the acronym"); - assertTrue(lookup.mayExpand("(AAD)")); - } - - @Test - public void keywordWithoutAcronymDoesNotExpand() throws IOException { - givenRules("soop => ships of opportunity"); - lookup.refresh(); - - assertFalse(lookup.mayExpand("ships of opportunity"), "the full name is plain text"); - assertFalse(lookup.mayExpand("coastal wave data")); - assertFalse(lookup.mayExpand("soopy"), "only whole tokens are acronyms"); - } - - @Test - public void missingSynonymsSetMeansNoAcronym() throws IOException { - when(synonymsClient.getSynonym(any(GetSynonymRequest.class))).thenThrow(error(404)); - lookup.refresh(); - - assertFalse(lookup.mayExpand("soop")); - } - - @Test - public void failedLoadKeepsEveryKeywordExpandable() throws IOException { - when(synonymsClient.getSynonym(any(GetSynonymRequest.class))) - .thenThrow(error(403)) - .thenThrow(new IOException("connection refused")); - - lookup.refresh(); - assertTrue(lookup.mayExpand("coastal wave data"), "unknown list must not drop acronym expansion"); - - lookup.refresh(); - assertTrue(lookup.mayExpand("coastal wave data")); - } - - @Test - public void failedRefreshKeepsPreviousList() throws IOException { - givenRules("soop => ships of opportunity"); - lookup.refresh(); - - when(synonymsClient.getSynonym(any(GetSynonymRequest.class))).thenThrow(error(503)); - lookup.refresh(); - - assertTrue(lookup.mayExpand("soop")); - assertFalse(lookup.mayExpand("coastal wave data")); - } - - @Test - public void normaliseFollowsStandardTokenizer() { - assertEquals("soop s aad data", AcronymLookup.normalise("SOOP's (AAD) data")); - assertEquals("", AcronymLookup.normalise(" , ")); - } - - private void givenRules(String... rules) throws IOException { - List set = Arrays.stream(rules) - .map(rule -> new SynonymRuleRead.Builder().id(rule).synonyms(rule).build()) - .toList(); - when(synonymsClient.getSynonym(any(GetSynonymRequest.class))) - .thenReturn(GetSynonymResponse.of(r -> r.count(set.size()).synonymsSet(set))); - } - - private static ElasticsearchException error(int status) { - return new ElasticsearchException("synonyms.get_synonym", ErrorResponse.of(e -> e - .status(status) - .error(c -> c.type("error").reason("status " + status)))); - } -} diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java index 2248a809..43db66ee 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java @@ -5,12 +5,10 @@ import co.elastic.clients.elasticsearch._types.ScriptSortType; import co.elastic.clients.elasticsearch._types.SortOptions; import co.elastic.clients.elasticsearch._types.SortOrder; -import co.elastic.clients.elasticsearch.core.explain.ExplanationDetail; import co.elastic.clients.elasticsearch.core.search.Hit; import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.Test; -import java.util.ArrayList; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -88,56 +86,6 @@ public void sortValuesAreLeftOutWhenTheHitCarriesNone() { assertNull(simplified.getSortValues()); } - @Test - public void disMaxReportsOnlyOneWinnerWhenPlainAndSynonymScoresTie() { - List terms = collectDisMax(12.1f, - detail(12.1f, "weight(title:wave in 1) [BM25], result of:"), - detail(12.1f, "weight(title.synonyms:wave in 1) [BM25], result of:")); - - assertEquals(1, terms.size()); - assertEquals("title", terms.get(0).getField()); - assertEquals(12.1, terms.get(0).getScore(), 0.0001); - } - - @Test - public void disMaxReportsSynonymWinnerAsOneEntry() { - List terms = collectDisMax(12.1f, - detail(4.2f, "weight(title:soop in 1) [BM25], result of:"), - detail(12.1f, "weight(Synonym(title.synonyms:soop title.synonyms:ships " - + "title.synonyms:of title.synonyms:opportunity) in 1) [BM25], result of:")); - - assertEquals(1, terms.size()); - assertEquals("title.synonyms", terms.get(0).getField()); - assertEquals("soop ships of opportunity", terms.get(0).getTerm()); - assertEquals(12.1, terms.get(0).getScore(), 0.0001); - } - - @Test - public void disMaxReportsHighestChildWhenNoChildEqualsParent() { - List terms = collectDisMax(12.1f, - detail(12.0999f, "weight(title:wave in 1) [BM25], result of:"), - detail(4.0f, "weight(title.synonyms:wave in 1) [BM25], result of:")); - - assertEquals(1, terms.size()); - assertEquals("title", terms.get(0).getField()); - } - - private static List collectDisMax(float value, - ExplanationDetail... children) { - List terms = new ArrayList<>(); - ExplainSimplifier.collectScoreParts( - List.of(detail(value, "max of:", children)), terms, new ArrayList<>()); - return terms; - } - - private static ExplanationDetail detail(float value, String description, - ExplanationDetail... children) { - return ExplanationDetail.of(d -> d - .value(value) - .description(description) - .details(List.of(children))); - } - private static ExplainSimplifiedResponse.SortValue sortValue(String field, Object value) { return ExplainSimplifiedResponse.SortValue.builder() .field(field) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java index 5d585078..5ea97cf7 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/features/RestApiTest.java @@ -357,7 +357,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { // Call rest api directly and get query result with search on "dataset" ResponseEntity collections = testRestTemplate.exchange( - getBasePath() + "/collections?q=dataset&filter=page_size=1 AND score>=0.65", + getBasePath() + "/collections?q=dataset&filter=page_size=1 AND score>=1.3", HttpMethod.GET, null, new ParameterizedTypeReference<>() { @@ -386,7 +386,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { // Now the same search, same page but search_after the actual cursor returned above collections = testRestTemplate.exchange( - getBasePath() + "/collections?q=dataset&filter=page_size=6 AND score>=0.65 AND search_after=" + + getBasePath() + "/collections?q=dataset&filter=page_size=6 AND score>=1.3 AND search_after=" + String.format("'%s|| %s || %s || %s'", collections.getBody().getSearchAfter().get(0), collections.getBody().getSearchAfter().get(1), @@ -401,7 +401,7 @@ public void verifyCorrectPageSizeAndScoreWithQuery() throws IOException { assertEquals(HttpStatus.OK, collections.getStatusCode(), "Get status OK"); log.info("{}", collections.getBody()); - // Remaining docs that clear min_score=0.65 after the first batch; the exact count is + // Remaining docs that clear min_score=1.3 after the first batch; the exact count is // BM25-dependent and varies by env, so accept any non-empty result up to the remaining total. int returnedSize = Objects.requireNonNull(collections.getBody()).getCollections().size(); assertTrue(returnedSize >= 1 && returnedSize <= 3, diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java index e3b4c1e7..f5a262b7 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java @@ -1,7 +1,6 @@ package au.org.aodn.ogcapi.server.service; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType; -import au.org.aodn.ogcapi.server.core.service.AcronymLookup; import au.org.aodn.ogcapi.server.core.service.ElasticSearch; import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.FieldValue; @@ -38,10 +37,10 @@ public void searchByParametersWithDoubleQuote() throws Exception { "-score,-rank", CQLCrsType.EPSG4326); - assertEquals(7, capturingSearch.should.size(), - "Exact match should produce 7 queries (two dis_max groups + other fields)"); - assertDisMax(capturingSearch.should.get(0), "title", true); - assertDisMax(capturingSearch.should.get(1), "description", true); + assertEquals(9, capturingSearch.should.size(), + "Exact match should produce 9 queries (title + description + other fields, no dataset_group)"); + assertTrue(capturingSearch.should.get(0).isMatchPhrase(), "Title query should be MatchPhraseQuery"); + assertTrue(capturingSearch.should.get(1).isMatchPhrase(), "Description query should be MatchPhraseQuery"); assertTrue(capturingSearch.arguments.sortOptions().get(0).isScript(), "dataset_group priority sort should be the first sort key"); } @@ -57,56 +56,12 @@ public void searchByParametersWithoutDoubleQuote() throws Exception { "-score,-rank", CQLCrsType.EPSG4326); - assertEquals(7, capturingSearch.should.size(), "Fuzzy match should produce 7 queries"); - assertDisMax(capturingSearch.should.get(0), "title", false); - assertDisMax(capturingSearch.should.get(1), "description", false); + assertEquals(9, capturingSearch.should.size(), "Fuzzy match should produce 9 queries"); + assertTrue(capturingSearch.should.get(0).isMatch(), "fuzzy_title should be MatchQuery"); assertTrue(capturingSearch.arguments.sortOptions().get(0).isScript(), "dataset_group priority sort should be the first sort key"); } - @Test - public void searchWithoutAcronymSkipsSynonymFields() throws Exception { - AcronymLookup lookup = mock(AcronymLookup.class); - CapturingElasticSearch capturingSearch = new CapturingElasticSearch(mockClient, lookup); - - // Ordinary words and an acronym's full name are both plain text, so title/description are matched once only - for (String keyword : List.of("coastal wave data", "ships of opportunity")) { - capturingSearch.searchByParameters( - List.of(keyword), - null, - null, - "-score,-rank", - CQLCrsType.EPSG4326); - - assertEquals(7, capturingSearch.should.size()); - assertPlainMatch(capturingSearch.should.get(0), "title"); - assertPlainMatch(capturingSearch.should.get(1), "description"); - assertFalse(capturingSearch.should.toString().contains(".synonyms"), - "'" + keyword + "' has no acronym, so the synonyms fields must not add to the score"); - } - } - - @Test - public void searchWithAcronymAndOtherWordsUsesSynonymFields() throws Exception { - AcronymLookup lookup = mock(AcronymLookup.class); - when(lookup.mayExpand("soop temperature")).thenReturn(true); - CapturingElasticSearch capturingSearch = new CapturingElasticSearch(mockClient, lookup); - - capturingSearch.searchByParameters( - List.of("soop temperature"), - null, - null, - "-score,-rank", - CQLCrsType.EPSG4326); - - assertEquals(7, capturingSearch.should.size()); - // ES expands the whole keyword on the synonyms field, so it is searched as "ships of opportunity temperature" - assertDisMax(capturingSearch.should.get(0), "title", false); - assertDisMax(capturingSearch.should.get(1), "description", false); - assertEquals("soop temperature", - capturingSearch.should.get(0).disMax().queries().get(1).match().query().stringValue()); - } - // The portal SEO pipeline requests these in bulk to build Dataset JSON-LD @Test public void seoPropertiesMapToStacFields() throws Exception { @@ -167,38 +122,13 @@ public void explainByParametersUsesScriptScoreRequestForKeywords() throws Except "title-only _source is lightweight so the larger search_after batch is used"); assertNotNull(capturingSearch.explainRequest.query()); assertTrue(capturingSearch.explainRequest.query().isScriptScore()); - assertEquals(7, capturingSearch.explainRequest.query().scriptScore() + assertEquals(9, capturingSearch.explainRequest.query().scriptScore() .query().bool().should().size()); assertNotNull(capturingSearch.explainRequest.source()); assertTrue(capturingSearch.explainRequest.source().isFilter()); assertFalse(capturingSearch.explainRequest.source().filter().includes().isEmpty()); } - private static void assertDisMax(Query query, String plainField, boolean phrase) { - assertTrue(query.isDisMax()); - assertEquals(0.0, query.disMax().tieBreaker()); - assertEquals(2, query.disMax().queries().size()); - - Query plain = query.disMax().queries().get(0); - if (phrase) { - assertTrue(plain.isMatchPhrase()); - assertEquals(plainField, plain.matchPhrase().field()); - } - else { - assertTrue(plain.isMatch()); - assertEquals(plainField, plain.match().field()); - } - - Query synonym = query.disMax().queries().get(1); - assertTrue(synonym.isMatch()); - assertEquals(plainField + ".synonyms", synonym.match().field()); - } - - private static void assertPlainMatch(Query query, String plainField) { - assertTrue(query.isMatch(), "expected a plain match, not a dis_max with the synonyms field"); - assertEquals(plainField, query.match().field()); - } - @Test public void explainByParametersUsesCollectionQueryForEmptySearch() throws Exception { CapturingElasticSearch capturingSearch = new CapturingElasticSearch(mockClient); @@ -343,11 +273,6 @@ private CapturingElasticSearch(ElasticsearchClient client) { this.searchAfterSplitRegex = "\\|\\|"; } - private CapturingElasticSearch(ElasticsearchClient client, AcronymLookup acronymLookup) { - this(client); - this.acronymLookup = acronymLookup; - } - @Override protected SearchResult searchCollectionBy( List queries, diff --git a/server/src/test/resources/application-test.yaml b/server/src/test/resources/application-test.yaml index 89924333..49f3a6df 100644 --- a/server/src/test/resources/application-test.yaml +++ b/server/src/test/resources/application-test.yaml @@ -13,10 +13,6 @@ elasticsearch: vocabs_index: name: test_vocabs_index - # the set is never created in tests, so no keyword expands acronyms - acronyms: - name: portal-acronyms-test - search_as_you_type: search_suggestions: path: search_suggestions From 1caeb324c4597b1ea346d4fc58b835a16a39253f Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Thu, 8 Oct 2026 12:15:52 +1100 Subject: [PATCH 11/12] use multimatch for titile and description and remove should clause for acronym_* --- .../core/model/enumeration/CQLFields.java | 51 +++++++++---------- .../server/core/service/ElasticSearch.java | 3 -- 2 files changed, 23 insertions(+), 31 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java index 239a8381..84520908 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java @@ -145,17 +145,27 @@ public enum CQLFields implements CQLFieldsInterface { " .getAsLong()" + " }")) .order(order))), + // Phrase match on the field and its synonyms sub-field (search-time acronym expansion), fuzziness is not allowed + // for type phrase. Also used for quoted keywords, see fuzzy_title for why best of the two fields is scored. title( StacBasicField.Title.searchField, StacBasicField.Title.displayField, - null, + (literal) -> MultiMatchQuery.of(m -> m + .type(TextQueryType.Phrase) + .fields(StacBasicField.Title.searchField + "^2", StacBasicField.Title.searchField + ".synonyms^2") + .tieBreaker(0.1) + .query(literal))._toQuery(), null, (order) -> new SortOptions.Builder() .field(f -> f.field(StacBasicField.Title.sortField).order(order))), description( StacBasicField.Description.searchField, StacBasicField.Description.displayField, - null, + (literal) -> MultiMatchQuery.of(m -> m + .type(TextQueryType.Phrase) + .fields(StacBasicField.Description.searchField, StacBasicField.Description.searchField + ".synonyms") + .tieBreaker(0.1) + .query(literal))._toQuery(), null, null), providers( @@ -266,15 +276,18 @@ public enum CQLFields implements CQLFieldsInterface { null, (order) -> new SortOptions.Builder() .field(f -> f.field(StacSummeries.Score.sortField).order(order))), + // Fuzzy match on the field and its synonyms sub-field (search-time acronym expansion, e.g. "SOOP" -> "ships of opportunity"). + // best_fields scores the better field plus tie_breaker times the other. Plain words pass the synonym analyzer + // unchanged and would otherwise be scored twice. Title fields are boosted 2, description fields 1, as before. fuzzy_title( null, StacBasicField.Title.displayField, - (literal) -> MatchQuery.of(m -> m + (literal) -> MultiMatchQuery.of(m -> m + .type(TextQueryType.BestFields) + .fields(StacBasicField.Title.searchField + "^2", StacBasicField.Title.searchField + ".synonyms^2") + .tieBreaker(0.1) .fuzziness("AUTO") - .field(StacBasicField.Title.searchField) .prefixLength(4)// Use 4 to deal with NRMN short form may match NRM records - // Increase the relevance of matches in title - .boost(2.0F) .operator(Operator.And)// ensure all terms are matched with fuzziness .query(literal))._toQuery(), null, @@ -282,34 +295,16 @@ public enum CQLFields implements CQLFieldsInterface { fuzzy_desc( null, StacBasicField.Description.displayField, - (literal) -> MatchQuery.of(m -> m + (literal) -> MultiMatchQuery.of(m -> m + .type(TextQueryType.BestFields) + .fields(StacBasicField.Description.searchField, StacBasicField.Description.searchField + ".synonyms") + .tieBreaker(0.1) .fuzziness("AUTO") - .field(StacBasicField.Description.searchField) .prefixLength(4)// Use 4 to deal with NRMN short form may match NRM records .operator(Operator.And)// ensure all terms are matched with fuzziness .query(literal))._toQuery(), null, null), - // Acronym match on the synonyms sub-fields (search-time expansion), e.g. "SOOP" -> "ships of opportunity". - acronym_title( - StacBasicField.Title.searchField + ".synonyms", - StacBasicField.Title.displayField, - (literal) -> MatchQuery.of(m -> m - .field(StacBasicField.Title.searchField + ".synonyms") - .operator(Operator.And)// all expanded terms must match - .boost(2.0F)// align with fuzzy_title weighting - .query(literal))._toQuery(), - null, - null), - acronym_desc( - StacBasicField.Description.searchField + ".synonyms", - StacBasicField.Description.displayField, - (literal) -> MatchQuery.of(m -> m - .field(StacBasicField.Description.searchField + ".synonyms") - .operator(Operator.And) - .query(literal))._toQuery(), - null, - null), // Contains cloud-optimized data assets_summary( StacBasicField.AssetsSummary.searchField, diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 868b477e..7a704e03 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -448,9 +448,6 @@ private static List createKeywordShouldClauses(List keywords) { should.add(CQLFields.organisation_vocabs.getPropertyEqualToQuery(term)); should.add(CQLFields.platform_vocabs.getPropertyEqualToQuery(term)); should.add(CQLFields.id.getPropertyEqualToQuery(term)); - // Acronym match on the *.synonyms sub-fields, e.g. "SOOP" -> "ships of opportunity". - should.add(CQLFields.acronym_title.getPropertyEqualToQuery(term)); - should.add(CQLFields.acronym_desc.getPropertyEqualToQuery(term)); // credit_contains uses match query by default, exact match is not applied here should.add(CQLFields.credit_contains.getPropertyEqualToQuery(term)); } From 0ac990fa5835412643390c2f1c0719795634804a Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Thu, 8 Oct 2026 12:15:59 +1100 Subject: [PATCH 12/12] add tests --- .../server/core/util/ExplainSimplifier.java | 19 ++++++ .../core/util/ExplainSimplifierTest.java | 64 +++++++++++++++++++ .../server/service/ElasticSearchTest.java | 55 ++++++++++++++-- 3 files changed, 132 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java index 6a6b6ec3..15350dec 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java @@ -38,6 +38,12 @@ public class ExplainSimplifier { protected static final String SYNONYM_PREFIX = "Synonym("; + // A dis_max (disjunction max) explanation includes every match, but it is scored as its winning child plus + // tie_breaker times the others, e.g. the best_fields multi_match on title and title.synonyms (see CQLFields.fuzzy_title). + // Lucene writes "max of:" for tie_breaker 0, otherwise "max plus 0.1 times others of:". + protected static final String MAX_OF_PREFIX = "max of:"; + protected static final String MAX_PLUS_PREFIX = "max plus "; + protected static final String RELEVANCE_DESCRIPTION_PREFIX = "_score:"; protected static final Comparator BY_SCORE_DESC = @@ -214,6 +220,19 @@ protected static void collectScoreParts(List details, for (ExplanationDetail detail : details) { String description = detail.description(); + // report only the winner, the tie_breaker share of the others is small and would list the same word twice + if (description != null + && (description.startsWith(MAX_OF_PREFIX) || description.startsWith(MAX_PLUS_PREFIX))) { + if (detail.details() != null) { + // the highest child, not one equal to the parent, so float rounding cannot lose the winner; a tie keeps the first + detail.details().stream() + .max(Comparator.comparingDouble(ExplanationDetail::value)) + .ifPresent(winner -> collectScoreParts( + List.of(winner), terms, filters)); + } + continue; + } + // most nodes are idf/tf breakdowns, skip the regex for them if (description != null && description.startsWith(WEIGHT_PREFIX)) { Matcher matcher = WEIGHT_PATTERN.matcher(description); diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java index 43db66ee..5510da71 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java @@ -5,10 +5,12 @@ import co.elastic.clients.elasticsearch._types.ScriptSortType; import co.elastic.clients.elasticsearch._types.SortOptions; import co.elastic.clients.elasticsearch._types.SortOrder; +import co.elastic.clients.elasticsearch.core.explain.ExplanationDetail; import co.elastic.clients.elasticsearch.core.search.Hit; import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -86,6 +88,68 @@ public void sortValuesAreLeftOutWhenTheHitCarriesNone() { assertNull(simplified.getSortValues()); } + @Test + public void disMaxReportsOnlyOneWinnerWhenPlainAndSynonymScoresTie() { + List terms = collectDisMax("max of:", 12.1f, + detail(12.1f, "weight(title:wave in 1) [BM25], result of:"), + detail(12.1f, "weight(title.synonyms:wave in 1) [BM25], result of:")); + + assertEquals(1, terms.size()); + assertEquals("title", terms.get(0).getField()); + assertEquals(12.1, terms.get(0).getScore(), 0.0001); + } + + @Test + public void disMaxReportsSynonymWinnerAsOneEntry() { + List terms = collectDisMax("max of:", 12.1f, + detail(4.2f, "weight(title:soop in 1) [BM25], result of:"), + detail(12.1f, "weight(Synonym(title.synonyms:soop title.synonyms:ships " + + "title.synonyms:of title.synonyms:opportunity) in 1) [BM25], result of:")); + + assertEquals(1, terms.size()); + assertEquals("title.synonyms", terms.get(0).getField()); + assertEquals("soop ships of opportunity", terms.get(0).getTerm()); + assertEquals(12.1, terms.get(0).getScore(), 0.0001); + } + + @Test + public void disMaxReportsHighestChildWhenNoChildEqualsParent() { + List terms = collectDisMax("max of:", 12.1f, + detail(12.0999f, "weight(title:wave in 1) [BM25], result of:"), + detail(4.0f, "weight(title.synonyms:wave in 1) [BM25], result of:")); + + assertEquals(1, terms.size()); + assertEquals("title", terms.get(0).getField()); + } + + @Test + public void disMaxWithTieBreakerReportsOnlyTheWinner() { + // best_fields multi_match with tie_breaker 0.1: 36.3 + 0.1 * 12.1 + List terms = collectDisMax("max plus 0.1 times others of:", 37.51f, + detail(36.3f, "weight(title:wave in 1) [BM25], result of:"), + detail(12.1f, "weight(title.synonyms:wave in 1) [BM25], result of:")); + + assertEquals(1, terms.size()); + assertEquals("title", terms.get(0).getField()); + assertEquals(36.3, terms.get(0).getScore(), 0.0001); + } + + private static List collectDisMax(String description, float value, + ExplanationDetail... children) { + List terms = new ArrayList<>(); + ExplainSimplifier.collectScoreParts( + List.of(detail(value, description, children)), terms, new ArrayList<>()); + return terms; + } + + private static ExplanationDetail detail(float value, String description, + ExplanationDetail... children) { + return ExplanationDetail.of(d -> d + .value(value) + .description(description) + .details(List.of(children))); + } + private static ExplainSimplifiedResponse.SortValue sortValue(String field, Object value) { return ExplainSimplifiedResponse.SortValue.builder() .field(field) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java index f5a262b7..c7c6dc40 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java @@ -5,7 +5,10 @@ import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.FieldValue; import co.elastic.clients.elasticsearch._types.SortOptions; +import co.elastic.clients.elasticsearch._types.query_dsl.MultiMatchQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.Operator; import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import co.elastic.clients.elasticsearch._types.query_dsl.TextQueryType; import co.elastic.clients.elasticsearch.core.SearchRequest; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -37,10 +40,22 @@ public void searchByParametersWithDoubleQuote() throws Exception { "-score,-rank", CQLCrsType.EPSG4326); - assertEquals(9, capturingSearch.should.size(), - "Exact match should produce 9 queries (title + description + other fields, no dataset_group)"); - assertTrue(capturingSearch.should.get(0).isMatchPhrase(), "Title query should be MatchPhraseQuery"); - assertTrue(capturingSearch.should.get(1).isMatchPhrase(), "Description query should be MatchPhraseQuery"); + assertEquals(7, capturingSearch.should.size(), + "Exact match should produce 7 queries (title + description + other fields, no dataset_group)"); + + MultiMatchQuery title = capturingSearch.should.get(0).multiMatch(); + assertEquals(TextQueryType.Phrase, title.type(), "Title query should be a phrase multi_match"); + assertEquals(List.of("title^2", "title.synonyms^2"), title.fields()); + assertEquals(0.1, title.tieBreaker()); + assertEquals("ocean temperature", title.query(), "quotes are removed"); + assertNull(title.fuzziness(), "fuzziness is not allowed for a phrase"); + + MultiMatchQuery description = capturingSearch.should.get(1).multiMatch(); + assertEquals(TextQueryType.Phrase, description.type(), "Description query should be a phrase multi_match"); + assertEquals(List.of("description", "description.synonyms"), description.fields()); + assertEquals(0.1, description.tieBreaker()); + + assertSynonymsOnlyInsideMultiMatch(capturingSearch.should); assertTrue(capturingSearch.arguments.sortOptions().get(0).isScript(), "dataset_group priority sort should be the first sort key"); } @@ -56,12 +71,40 @@ public void searchByParametersWithoutDoubleQuote() throws Exception { "-score,-rank", CQLCrsType.EPSG4326); - assertEquals(9, capturingSearch.should.size(), "Fuzzy match should produce 9 queries"); - assertTrue(capturingSearch.should.get(0).isMatch(), "fuzzy_title should be MatchQuery"); + assertEquals(7, capturingSearch.should.size(), "Fuzzy match should produce 7 queries"); + + MultiMatchQuery title = capturingSearch.should.get(0).multiMatch(); + assertEquals(TextQueryType.BestFields, title.type(), "fuzzy_title should be a best_fields multi_match"); + assertEquals(List.of("title^2", "title.synonyms^2"), title.fields()); + assertEquals(0.1, title.tieBreaker()); + assertEquals("AUTO", title.fuzziness()); + assertEquals(4, title.prefixLength()); + assertEquals(Operator.And, title.operator()); + + MultiMatchQuery description = capturingSearch.should.get(1).multiMatch(); + assertEquals(TextQueryType.BestFields, description.type(), "fuzzy_desc should be a best_fields multi_match"); + assertEquals(List.of("description", "description.synonyms"), description.fields()); + assertEquals(0.1, description.tieBreaker()); + assertEquals("AUTO", description.fuzziness()); + assertEquals(4, description.prefixLength()); + assertEquals(Operator.And, description.operator()); + + assertSynonymsOnlyInsideMultiMatch(capturingSearch.should); assertTrue(capturingSearch.arguments.sortOptions().get(0).isScript(), "dataset_group priority sort should be the first sort key"); } + /** + * The synonyms sub-fields must only be scored inside the title/description multi_match, a separate clause on them + * would score plain words twice + */ + private static void assertSynonymsOnlyInsideMultiMatch(List should) { + should.stream() + .filter(q -> !q.isMultiMatch()) + .forEach(q -> assertFalse(q.toString().contains(".synonyms"), + "unexpected clause on a synonyms field: " + q)); + } + // The portal SEO pipeline requests these in bulk to build Dataset JSON-LD @Test public void seoPropertiesMapToStacFields() throws Exception {