1.8.0: engine hardening — precision, badge = popup, 18 new technologies, real-site proof

.github/workflows/ci.yml

@@ -0,0 +1,73 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  unit:
+    name: Unit + corpus (Node ${{ matrix.node }})
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [20, 22]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+          cache: npm
+      - run: npm ci
+      - run: npm run lint
+      - run: npm run typecheck
+      - run: npm test
+      - run: npm run audit:manifest
+      - run: npm run audit:local
+
+  coverage:
+    name: Coverage (≥ 80% lines)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: npm
+      - run: npm ci
+      - run: npm run test:coverage
+
+  browser:
+    name: Real Chromium, real extension
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: npm
+      - run: npm ci
+      - run: npx playwright install --with-deps chromium
+      - run: npm run test:browser
+        env:
+          HEADLESS: '1'
+
+  package:
+    name: Store package
+    runs-on: ubuntu-latest
+    needs: [unit, browser]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+      - run: node scripts/check-version.mjs
+      - run: npm run pack
+      - uses: actions/upload-artifact@v4
+        with:
+          name: whatstack-extension
+          path: dist/*.zip
+          if-no-files-found: error

CONTRIBUTING.md

@@ -17,9 +17,19 @@ Before adding a signature:
 1. Prefer **asset URLs** and **runtime globals** over page text
 2. Never match marketing copy (e.g. “we use React”)
 3. For React-like hooks, require real registration (see DevTools renderer rule)
-4. Add a **positive fixture** and a **non-flaky prose-only** case when relevant
-5. Run `npm test`
-
-## Publishing (later)
-
-Chrome Web Store packaging is out of scope until the product owner is satisfied. Do not commit private keys (`.pem`) or store credentials.
+4. Match URLs on host / package boundaries — rules see the URL **without** its
+   query string (set `matchQuery: true` only if the query is the evidence)
+5. Globals need a shape check in the probe (`typeof x.method === 'function'`):
+   any element with an `id` becomes a window property
+6. Add a case to `POSITIVE` in `tests/signature-coverage.test.js` — the gate
+   also runs a prose / file-name / search-query case against every rule
+7. Add a brand mark in `shared/brand-icons.js`
+8. Run `npm run check`; if the golden corpus snapshot changes on purpose,
+   review it and run `npm run corpus:update`
+
+## Releases
+
+Bump `manifest.json`, `package.json` and `package-lock.json` together
+(`npm version <x.y.z> --no-git-tag-version`, then the manifest) and add
+`docs/RELEASE-<x.y.z>.md`; `scripts/check-version.mjs` enforces it. Do not
+commit private keys (`.pem`) or store credentials.

README.md

@@ -2,7 +2,7 @@
 
 Chrome extension (Manifest V3) that answers: **what stack is this page using?**
 
-Fully **local** detection — frameworks, architecture (microfrontends), build tools, state/data, UI, auth, payments, observability, analytics, and hosting hints. No cloud matching, no page upload.
+Fully **local** detection of 82 technologies — frameworks, CMS / site builders, architecture (microfrontends), build tools, state/data, UI, auth, payments, observability, analytics, and hosting hints. No cloud matching, no page upload.
 
 ## Load unpacked (development)
 
@@ -13,8 +13,11 @@ Fully **local** detection — frameworks, architecture (microfrontends), build t
 
 ## Features
 
-- Hybrid scan: light pass on navigate (badge) + deep pass when the popup opens
+- Hybrid scan: light pass on navigate, then a background MAIN-world probe, so the
+  **badge counts what the popup shows**; the popup always runs a fresh deep scan
 - Confidence tiers + expandable evidence (why it matched)
+- Evidence from asset URLs (host + path, never the query), runtime objects with the
+  right shape, and probed DOM flags — never page text, data islands or file names
 - Core stack headline + Copy / Markdown / JSON export
 - React detection aligned with React DevTools (renderer registered, not bare hook)
 - Microfrontend platforms: Module Federation, single-spa, qiankun, SystemJS
@@ -23,17 +26,26 @@ Fully **local** detection — frameworks, architecture (microfrontends), build t
 ## Develop / test
 
 ```bash
-npm test                 # unit + integration suites (node:test, no deps)
-npm run test:browser     # loads the real unpacked extension into Chrome
-npm run audit:manifest
-npm run audit:local
+npm ci
+npm test                 # unit, coverage gate, golden corpus (node:test, offline)
+npm run check            # lint + type check + tests + audits
+npm run test:coverage    # fails under 80% lines
+npx playwright install chromium
+npm run test:browser     # real unpacked extension in real Chromium
+HEADLESS=1 npm run test:browser   # same, headless (CI)
+npm run corpus:capture   # re-capture real sites into tests/corpus (network)
 ```
 
-`npm run test:browser` needs `npm install` (Playwright) and a Chrome/Chromium
-build; point `CHROME_PATH` at one if you don't have the `chrome` channel
-installed. It serves `tests/browser/fixtures/` over localhost and checks that
-pages which merely *display* framework markup produce no detections, while pages
-that really use the framework still do.
+`npm run test:browser` uses Playwright's bundled Chromium — branded Chrome
+stable ignores `--load-extension`. Set `CHROME_PATH` to use another
+Chromium-family build. It serves `tests/browser/fixtures/` over localhost and
+checks that pages which merely *display* or *mention* a framework produce no
+detections, that real ones do, and that the badge matches the popup.
+
+**Golden corpus.** `tests/corpus/*.json` are real sites captured by
+`scripts/capture-corpus.mjs` (light signals + probe globals, query strings and
+non-evidence text removed). `npm test` replays them offline. When a change
+alters a snapshot on purpose, review it and run `npm run corpus:update`.
 
 Detection evidence comes from real `querySelector` probes only — never from the
 page's HTML as text. `tests/dom-evidence.test.js` is the guard for that.
@@ -42,15 +54,20 @@ page's HTML as text. `tests/dom-evidence.test.js` is the guard for that.
 
 ```
 manifest.json
-background/service-worker.js   # cache, badge, deep scan
-content/content-script.js      # light signals
+background/service-worker.js   # message adapter, tab lifecycle
+content/content-script.js      # light signals (isolated world)
 popup/                         # toolbar UI
-shared/detect.js               # pure detection engine
-shared/signatures.js           # local rule pack
-shared/result-shape.js         # popup shaping + exports
+shared/detect.js               # pure engine + MAIN-world probe source
+shared/signatures.js           # local rule pack (82 technologies)
+shared/signature-matcher.js    # evidence → confidence
+shared/architecture-classifier.js  # badge count
+shared/scan-orchestrator.js    # light/deep passes, cache, badge, timeouts
+shared/url-policy.js           # which pages can be scanned
+shared/result-shape.js         # popup shaping + exports + report link
 shared/brand-icons.js          # tech marks (local SVG)
 tests/                         # node:test suites + fixtures
-tests/browser/                 # real-Chrome end-to-end suite
+tests/corpus/                  # golden corpus of real captured sites
+tests/browser/                 # real-Chromium end-to-end suite
 scripts/                       # audits, packaging, store assets
 icons/                         # toolbar + brand assets
 store/                         # listing copy, screenshots, promo tile

background/service-worker.js

@@ -3,52 +3,87 @@
  */
 
 import { ScanOrchestrator } from '../shared/scan-orchestrator.js';
+import { isRestrictedUrl } from '../shared/url-policy.js';
 
 const orchestrator = new ScanOrchestrator();
 
-chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
-  (async () => {
-    if (!msg || !msg.type) {
-      sendResponse({ ok: false });
-      return;
-    }
+/**
+ * @param {any} msg
+ * @param {chrome.runtime.MessageSender} sender
+ */
+async function handle(msg, sender) {
+  if (!msg || !msg.type) return { ok: false };
 
-    if (msg.type === 'LIGHT_SCAN') {
-      const tabId = sender.tab && sender.tab.id;
-      if (typeof tabId === 'number') {
-        await orchestrator.lightScan(tabId, msg.signals || {});
-      }
-      sendResponse({ ok: true });
-      return;
-    }
+  if (msg.type === 'LIGHT_SCAN') {
+    const tabId = sender.tab && sender.tab.id;
+    // Top frame of the visible document only: a prerendering or bfcached
+    // document must not overwrite the badge of the page on screen.
+    if (typeof tabId !== 'number') return { ok: false };
+    if (typeof sender.frameId === 'number' && sender.frameId !== 0) return { ok: false };
+    if (sender.documentLifecycle && sender.documentLifecycle !== 'active') return { ok: false };
+    await orchestrator.lightScan(tabId, msg.signals || {});
+    return { ok: true };
+  }
 
-    if (msg.type === 'GET_RESULT') {
-      const result = await orchestrator.getResult({
-        tabId: msg.tabId,
-        url: msg.url || '',
-        forceDeep: msg.forceDeep,
-      });
-      sendResponse({ ok: true, result });
-      return;
-    }
+  if (msg.type === 'GET_RESULT') {
+    const result = await orchestrator.getResult({ tabId: msg.tabId, url: msg.url || '', forceDeep: !!msg.forceDeep });
+    return { ok: true, result };
+  }
+
+  if (msg.type === 'REQUEST_DEEP_SCAN') {
+    const result = await orchestrator.deepScan(msg.tabId, msg.url || '');
+    return { ok: true, result };
+  }
 
-    if (msg.type === 'REQUEST_DEEP_SCAN') {
-      const result = await orchestrator.deepScan(msg.tabId, msg.url || '');
-      sendResponse({ ok: true, result });
-      return;
+  // Tooling only (corpus capture, browser suite): raw signals + probe output.
+  // Extension pages only — never a content script.
+  if (msg.type === 'CAPTURE_SIGNALS') {
+    // Content scripts report the page's URL as sender.url; extension pages
+    // report a chrome-extension://<our id>/ URL.
+    const own = chrome.runtime.getURL('');
+    if (sender.id !== chrome.runtime.id || !String(sender.url || '').startsWith(own)) {
+      return { ok: false, error: 'forbidden' };
     }
+    return { ok: true, ...(await orchestrator.capture(msg.tabId)) };
+  }
+
+  return { ok: false, error: 'unknown_type' };
+}
 
-    sendResponse({ ok: false, error: 'unknown_type' });
-  })();
+chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
+  handle(msg, sender).then(sendResponse, (err) => {
+    // Always answer: an unanswered message leaves the popup on "Scanning…"
+    // or shows Chrome's raw "message port closed" error.
+    console.warn('[WhatStack]', err);
+    sendResponse({ ok: false, error: String((err && err.message) || err) });
+  });
   return true; // async
 });
 
 chrome.tabs.onRemoved.addListener((tabId) => {
   orchestrator.invalidateTab(tabId);
 });
 
-chrome.tabs.onUpdated.addListener((tabId, changeInfo) => {
-  if (changeInfo.status === 'loading' && changeInfo.url) {
-    orchestrator.invalidateTab(tabId, changeInfo.url);
+chrome.tabs.onUpdated.addListener((tabId, changeInfo, tab) => {
+  if (changeInfo.status === 'loading') {
+    // A new document (including a reload of the same URL) or an SPA route
+    // change. Drop what we knew; the next LIGHT_SCAN or onTabComplete rebuilds it.
+    orchestrator.invalidateTab(tabId);
+  }
+  if (changeInfo.status === 'complete' && tab && tab.url) {
+    orchestrator.onTabComplete(tabId, tab.url);
+  }
+});
+
+// Tabs that were open before install/update have no (live) content script.
+chrome.runtime.onInstalled.addListener(async () => {
+  try {
+    const tabs = await chrome.tabs.query({ url: ['http://*/*', 'https://*/*'] });
+    for (const tab of tabs) {
+      if (typeof tab.id !== 'number' || isRestrictedUrl(tab.url || '') || tab.discarded) continue;
+      chrome.scripting.executeScript({ target: { tabId: tab.id }, files: ['content/content-script.js'] }).catch(() => {});
+    }
+  } catch {
+    /* best effort */
   }
 });