diff --git a/SeforimApp/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/HybridSearchEngine.kt b/SeforimApp/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/HybridSearchEngine.kt index 20b9ece1f..d9a74ee03 100644 --- a/SeforimApp/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/HybridSearchEngine.kt +++ b/SeforimApp/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/HybridSearchEngine.kt @@ -102,7 +102,12 @@ internal class HybridSearchEngine( override fun rangeSnippet( text: String, ranges: List, - ): String = lexical.rangeSnippet(text, ranges) + lineStart: Int, + lineEnd: Int, + ): String = lexical.rangeSnippet(text, ranges, lineStart, lineEnd) + + // A quoted phrase asks for those exact words: they alone are highlighted, as they alone are found + override fun highlightsByMeaning(query: String): Boolean = semanticConfigured && !SearchQueryParser.parse(query).hasExactPhrases override suspend fun passagesByMeaning( query: String, @@ -244,22 +249,29 @@ internal class HybridSearchEngine( val withSnippets = if (snippets) lexical.attachSnippets(lexicalHits, query, near) else lexicalHits val lexicalById = withSnippets.associateBy { it.lineId } return entries.mapNotNull { entry -> - // Dense-only: a lexical snippet first, refined to the passage closest in meaning (refinedSnippets) - val hit = lexicalById[entry.lineId] ?: resolveLine(entry.lineId, query) - hit?.copy(score = entry.score) + val hit = lexicalById[entry.lineId] ?: denseHit(entry.lineId) ?: return@mapNotNull null + // No word in bold: the passage closest in meaning is highlighted once ready (refinedSnippets) + val snippet = if (snippets) lexical.rangeSnippet(hit.rawText, emptyList(), hit.lineStart, hit.lineEnd) else hit.snippet + hit.copy(score = entry.score, snippet = snippet) } } + // A dense-only line, its text cleaned as a lexical snippet source + private suspend fun denseHit(lineId: Long): LineHit? { + val hit = resolveLine(lineId, query) ?: return null + val text = SnippetSources.clean(hit.rawText) + return hit.copy(rawText = text, lineStart = 0, lineEnd = text.length) + } + override suspend fun refinedSnippets(hits: List): Map { - val lexicalIds = fused.orEmpty().mapNotNullTo(HashSet()) { entry -> entry.lexical?.lineId } - val denseOnly = hits.filter { it.lineId !in lexicalIds && it.rawText.isNotBlank() } - if (denseOnly.isEmpty()) return emptyMap() - // Snippet text, as the lexical snippets: a line the query's words show in keeps its lexical snippet - val texts = denseOnly.map { SnippetSources.clean(it.rawText) } - val highlights = highlights(texts, query) + val shown = hits.filter { it.rawText.isNotBlank() && it.lineEnd > it.lineStart } + if (shown.isEmpty()) return emptyMap() + // The found line alone, not the neighbors its snippet shows around it + val highlights = highlights(shown.map { it.rawText.substring(it.lineStart, it.lineEnd) }, query) return buildMap { - denseOnly.forEachIndexed { i, hit -> - if (highlights[i].byMeaning) put(hit.lineId, lexical.rangeSnippet(texts[i], highlights[i].ranges)) + shown.forEachIndexed { i, hit -> + val ranges = highlights[i].ranges.map { (it.first + hit.lineStart)..(it.last + hit.lineStart) } + if (ranges.isNotEmpty()) put(hit.lineId, lexical.rangeSnippet(hit.rawText, ranges, hit.lineStart, hit.lineEnd)) } } } diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/ui/panels/bookcontent/views/BookContentView.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/ui/panels/bookcontent/views/BookContentView.kt index cf3761c99..ac6ffaa3f 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/ui/panels/bookcontent/views/BookContentView.kt +++ b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/ui/panels/bookcontent/views/BookContentView.kt @@ -1523,11 +1523,8 @@ private fun LineItem( smartQuery?.let { query -> runSuspendCatching { searchEngine - .highlights( - listOf(annotated.text), - query, - alwaysByMeaning = true, - ).single() + .highlights(listOf(annotated.text), query) + .single() .ranges }.getOrNull() } diff --git a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchResultScreen.kt b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchResultScreen.kt index ca086ebf3..dc0b8f20a 100644 --- a/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchResultScreen.kt +++ b/SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchResultScreen.kt @@ -874,13 +874,14 @@ private fun PassagePreview( // The whole width, as the book's text modifier = Modifier.fillMaxWidth().padding(horizontal = 24.dp, vertical = 18.dp), ) { + val byMeaning = remember(query) { searchEngine.highlightsByMeaning(query) } lines.forEach { line -> val found = line.id == hit.lineId val annotated = remember(line.id, textSize) { buildAnnotatedFromHtml(line.content, textSize) } - // The search's own highlighting: the query's words, and in the found line, when none is there, - // the passage closest in meaning (as its snippet) + // The search's own highlighting (SearchEngine.highlights): by meaning, the passage closest in + // meaning in the found line alone; else the query's words in every line val ranges by produceState( - searchEngine.highlightRanges(annotated.text, query), + if (byMeaning) emptyList() else searchEngine.highlightRanges(annotated.text, query), annotated, query, found, diff --git a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchBench.kt b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchBench.kt index 2cebc71e8..8ee87c446 100644 --- a/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchBench.kt +++ b/SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchBench.kt @@ -40,6 +40,8 @@ fun main(args: Array) { engine.warmUp() println("warmUp ${ms(w)} ms") } + var refineNanos = 0L + var refinedHits = 0 repeat(rounds) { round -> println("--- round $round") for (q in System.getenv("BENCH_QUERY")?.let { listOf(it) } ?: QUERIES) { @@ -58,6 +60,10 @@ fun main(args: Array) { val s3 = System.nanoTime() val refined = page?.let { session.refinedSnippets(it.hits) }.orEmpty() val tRefine = ms(s3) + if (round > 0) { + refineNanos += System.nanoTime() - s3 + refinedHits += page?.hits?.size ?: 0 + } session?.close() println( "%-34s total=%6d facets=%6d page=%6d refine=%5d(%2d) hits=%8d first=%s".format( @@ -73,10 +79,34 @@ fun main(args: Array) { ) } } + if (refinedHits > 0) println("refine %.2f ms/result over %d results".format(refineNanos / 1e6 / refinedHits, refinedHits)) } + println(memory()) engine.close() } +/** + * Resident memory: all of it (with the mapped index pages, which the page cache lends and takes back), the process's own + * (anonymous: heap, native ONNX arenas), and the live heap after a GC. + */ +private fun memory(): String { + repeat(3) { System.gc() } + val heap = + java.lang.management.ManagementFactory + .getMemoryMXBean() + .heapMemoryUsage.used / 1_000_000 + // Linux only: elsewhere, the heap alone + val status = runCatching { java.io.File("/proc/self/status").readLines() }.getOrDefault(emptyList()) + + fun kb(key: String) = + status + .firstOrNull { it.startsWith(key) } + ?.filter(Char::isDigit) + ?.toLong() + ?.div(1000) + return "memory: rss=${kb("VmRSS:")} MB anon=${kb("RssAnon:")} MB heap=$heap MB" +} + private fun ms(start: Long) = (System.nanoTime() - start) / 1_000_000 private fun buildEngine( diff --git a/SeforimLibrary b/SeforimLibrary index 381fbea2a..2265fcb20 160000 --- a/SeforimLibrary +++ b/SeforimLibrary @@ -1 +1 @@ -Subproject commit 381fbea2a0997a91a1100db20db9904839e78b96 +Subproject commit 2265fcb20f95a81d505cb38bf92d2208180c5834 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 991bd3999..99aa877cd 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -39,7 +39,7 @@ sqlDelight = "2.3.2" adaptiveNavigation = "1.2.0" kosherkotlin = "3.0.5" seforimSiddur = "0.2.2" -seforimSemantic = "0.4.2" +seforimSemantic = "0.5.0" zstdJni = "1.5.7-10" metro = "1.4.5" lucene = "10.5.2"