[#25] Move the site search to DocSearch 3 and prepare the Algolia Crawler

docsearch/crawler-config.js

@@ -0,0 +1,100 @@
+/*
+ * The contents of this file are subject to the terms of the Common Development and
+ * Distribution License (the License). You may not use this file except in compliance with the
+ * License.
+ *
+ * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+ * specific language governing permission and limitations under the License.
+ *
+ * When distributing Covered Software, include this CDDL Header Notice in each file and include
+ * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+ * Header, with the fields enclosed by brackets [] replaced by your own identifying
+ * information: "Portions copyright [year] [name of copyright owner]".
+ *
+ * Copyright 2026 3A Systems, LLC.
+ */
+
+// Algolia Crawler configuration for doc.openidentityplatform.org, to replace the legacy
+// docsearch-scraper run by .github/workflows/publish.yml (config.json). It is pasted into the
+// Crawler dashboard, which runs it; see readme.md. It mirrors config.json: records only from the
+// product pages (the site home page is crawled for links only), same exclusions, same selectors for
+// the hierarchy and the content. Not carried over, because none of it changes the results: the
+// version in lvl0 and the desc(version) ranking (every component has version ~), the component
+// attribute (nothing reads it) and min_indexed_level. It writes a new index, so that it can be
+// checked before the site switches to it.
+
+new Crawler({
+  appId: 'X0ME9NKL6F',
+  // keep the key the Crawler dashboard generated; never put a write key in this repository
+  apiKey: '<crawler API key>',
+  rateLimit: 8,
+  maxDepth: 10,
+  startUrls: ['https://doc.openidentityplatform.org/'],
+  sitemaps: ['https://doc.openidentityplatform.org/sitemap.xml'],
+  discoveryPatterns: ['https://doc.openidentityplatform.org/**'],
+  exclusionPatterns: [
+    // API docs
+    'https://doc.openidentityplatform.org/*/apidocs/**',
+    'https://doc.openidentityplatform.org/openicf/**',
+    'https://doc.openidentityplatform.org/commons/**',
+    // older versions, e.g. /openam/15.2/
+    'https://doc.openidentityplatform.org/*/[0-9]*.[0-9]*/**',
+    // generated references that would flood the results
+    'https://doc.openidentityplatform.org/opendj/reference/dsconfig-subcommands-ref',
+    'https://doc.openidentityplatform.org/opendj/reference/appendix-log-messages',
+    'https://doc.openidentityplatform.org/openam/reference/chap-log-messages',
+  ],
+  actions: [
+    {
+      indexName: 'doc_openidentityplatform_v3',
+      // the products only, as the start_urls of config.json
+      pathsToMatch: ['https://doc.openidentityplatform.org/{openam,opendj,openidm,openig}/**'],
+      recordExtractor: ({ helpers }) =>
+        helpers.docsearch({
+          recordProps: {
+            lvl0: { selectors: '.nav-panel-explore .context .title', defaultValue: 'Open Identity Platform' },
+            lvl1: '.doc > h1.page',
+            lvl2: '.doc .sect1 > h2',
+            lvl3: '.doc .sect2 > h3',
+            lvl4: '.doc .sect3 > h4',
+            lvl5: '.doc .sidebarblock > .content > .title',
+            content: '.doc p, .doc dt, .doc td.content, .doc th.tableblock',
+          },
+          indexHeadings: true,
+          aggregateContent: true,
+          recordVersion: 'v3',
+        }),
+    },
+  ],
+  initialIndexSettings: {
+    doc_openidentityplatform_v3: {
+      attributesForFaceting: ['type', 'lang'],
+      attributesToRetrieve: ['hierarchy', 'content', 'anchor', 'url', 'url_without_anchor', 'type'],
+      attributesToHighlight: ['hierarchy', 'content'],
+      attributesToSnippet: ['content:10'],
+      camelCaseAttributes: ['hierarchy', 'content'],
+      searchableAttributes: [
+        'unordered(hierarchy.lvl0)',
+        'unordered(hierarchy.lvl1)',
+        'unordered(hierarchy.lvl2)',
+        'unordered(hierarchy.lvl3)',
+        'unordered(hierarchy.lvl4)',
+        'unordered(hierarchy.lvl5)',
+        'unordered(hierarchy.lvl6)',
+        'content',
+      ],
+      distinct: true,
+      attributeForDistinct: 'url',
+      customRanking: ['desc(weight.pageRank)', 'desc(weight.level)', 'asc(weight.position)'],
+      ranking: ['words', 'filters', 'typo', 'attribute', 'proximity', 'exact', 'custom'],
+      minWordSizefor1Typo: 3,
+      minWordSizefor2Typos: 7,
+      allowTyposOnNumericTokens: false,
+      minProximity: 1,
+      ignorePlurals: true,
+      advancedSyntax: true,
+      attributeCriteriaComputedByMinProximity: true,
+      removeWordsIfNoResults: 'allOptional',
+    },
+  },
+})

docsearch/readme.md

@@ -1,3 +1,44 @@
-Update Algolia index instructions:
+<!--
+The contents of this file are subject to the terms of the Common Development and
+Distribution License (the License). You may not use this file except in compliance with the
+License.
 
-https://gitlab.com/antora/antora-ui-default/-/issues/44#note_579313948
+You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+specific language governing permission and limitations under the License.
+
+When distributing Covered Software, include this CDDL Header Notice in each file and include
+the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+Header, with the fields enclosed by brackets [] replaced by your own identifying
+information: "Portions copyright [year] [name of copyright owner]".
+
+Copyright 2025-2026 3A Systems, LLC.
+-->
+# Search
+
+The site search is [DocSearch](https://docsearch.algolia.com/) on the Algolia application
+`X0ME9NKL6F`, index `doc_openidentityplatform`.
+
+- **Front end**: `@docsearch/js` 3.9.0, vendored in `supplemental-ui/js/vendor/docsearch.min.js` and
+  `supplemental-ui/css/vendor/docsearch.min.css`, set up in `supplemental-ui/partials/footer-scripts.hbs`.
+  It is rendered only when the build has both `ALGOLIA_SEARCH_API_KEY` and `ALGOLIA_APP_ID` in the environment.
+- **Crawler**: the legacy [docsearch-scraper](https://github.com/algolia/docsearch-scraper)
+  (`algolia/docsearch-scraper:v1.12.0`, no longer maintained), run with `config.json` by the "Reindex docs"
+  step of `.github/workflows/publish.yml` after every deployment. How the index was set up for it:
+  https://gitlab.com/antora/antora-ui-default/-/issues/44#note_579313948
+
+## Moving the crawler to the Algolia Crawler
+
+`crawler-config.js` is the same crawl for the [Algolia Crawler](https://www.algolia.com/doc/tools/crawler/),
+which replaces the legacy scraper. It has not been run yet: it needs Crawler access on the Algolia application,
+which an open-source documentation site gets through the [DocSearch program](https://docsearch.algolia.com/apply).
+
+1. Get Crawler access for the application `X0ME9NKL6F` (or the DocSearch program).
+2. Create a crawler in the Crawler dashboard and paste `crawler-config.js` over the generated configuration,
+   keeping the `apiKey` line the dashboard generated. Check with its URL tester that a product start page
+   (e.g. `/openam/`) gives records and the site home page does not.
+3. Run it. It writes the new index `doc_openidentityplatform_v3`, so the live search keeps using the old one.
+4. Check the results, e.g. by building the site with `indexName: 'doc_openidentityplatform_v3'` in
+   `footer-scripts.hbs`, and schedule the crawler (or trigger it after each deployment).
+5. In one pull request: switch `indexName` in `footer-scripts.hbs` to `doc_openidentityplatform_v3`, remove
+   the "Reindex docs" step from `publish.yml`, and remove `config.json`. The `ALGOLIA_API_KEY` secret
+   (write key of the legacy scraper) is then no longer needed.

supplemental-ui/partials/footer-scripts.hbs

@@ -1,26 +1,27 @@
 <script id="site-script" src="{{{uiRootPath}}}/js/site.js" data-ui-root-path="{{{uiRootPath}}}"></script>
 <script async src="{{{uiRootPath}}}/js/vendor/highlight.js"></script>
-{{#if env.ALGOLIA_SEARCH_API_KEY}}
+{{#if (and env.ALGOLIA_SEARCH_API_KEY env.ALGOLIA_APP_ID)}}
 <script src="{{uiRootPath}}/js/vendor/docsearch.min.js"></script>
-<!-- fetched from https://cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.js -->
+<!-- @docsearch/js 3.9.0, fetched from https://cdn.jsdelivr.net/npm/@docsearch/js@3.9.0/dist/umd/index.js -->
 <script>
-var search = docsearch({
-  {{#with env.ALGOLIA_APP_ID}}
-  appId: '{{this}}',
-  {{/with}}
+docsearch({
+  container: '#docsearch',
+  appId: '{{env.ALGOLIA_APP_ID}}',
   apiKey: '{{env.ALGOLIA_SEARCH_API_KEY}}',
   indexName: 'doc_openidentityplatform',
-  inputSelector: '#search-input',
-  autocompleteOptions: { hint: false, keyboardShortcuts: ['s'] },
-  algoliaOptions: { hitsPerPage: 5 }
-}).autocomplete
-search.on('autocomplete:closed', function () { search.autocomplete.setVal() })
+  placeholder: 'Search the docs',
+  // the legacy scraper anchors page titles, and the text before the first section, to the first id
+  // on the page, the cookie notice of head-scripts.hbs
+  transformItems: function (items) {
+    return items.map(function (item) {
+      return Object.assign({}, item, { url: item.url.replace(/#cookie-notice$/, '') })
+    })
+  }
+})
 </script>
 {{else if env.SITE_SEARCH_PROVIDER}}
 {{> search-scripts}}
-{{/if}}
 {{#if page.home}}
-{{#if (or env.ALGOLIA_SEARCH_API_KEY env.SITE_SEARCH_PROVIDER)}}
 <script>
 window.addEventListener('load', function focusSearchInput () {
   window.removeEventListener('load', focusSearchInput)

supplemental-ui/partials/head-styles.hbs

@@ -1,6 +1,6 @@
  <link rel="stylesheet" href="{{{uiRootPath}}}/css/site.css">
  <link rel="stylesheet" href="{{{uiRootPath}}}/css/site-extra.css">
- {{#if env.ALGOLIA_SEARCH_API_KEY}}
+ {{#if (and env.ALGOLIA_SEARCH_API_KEY env.ALGOLIA_APP_ID)}}
 <link rel="stylesheet" href="{{uiRootPath}}/css/vendor/docsearch.min.css">
-<!-- fetched from https://cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.css -->
+<!-- @docsearch/css 3.9.0, fetched from https://cdn.jsdelivr.net/npm/@docsearch/css@3.9.0/dist/style.css -->
 {{/if}}

supplemental-ui/partials/header-content.hbs

@@ -16,7 +16,11 @@
       </button>
     </div>
     <div id="topbar-nav" class="navbar-menu">
-      {{#if (or env.ALGOLIA_SEARCH_API_KEY env.SITE_SEARCH_PROVIDER)}}
+      {{#if (and env.ALGOLIA_SEARCH_API_KEY env.ALGOLIA_APP_ID)}}
+      <div class="navbar-end navbar-item search hide-for-print">
+        <div id="docsearch"></div>
+      </div>
+      {{else if env.SITE_SEARCH_PROVIDER}}
       <div class="navbar-end navbar-item search hide-for-print">
         <input id="search-input" type="text" placeholder="Search the docs"{{#if page.home}} autofocus{{/if}}>
       </div>