From 64cb50c4475dc758b2eac24daeb71386d5fae09b Mon Sep 17 00:00:00 2001 From: Kadukuntla Sathvika Reddy Date: Tue, 4 Aug 2026 18:23:28 +0530 Subject: [PATCH 1/2] Add OONI backend integration --- backend/fetching/channels/ooni.js | 185 +++++++++ backend/fetching/hooks/postToReport.js | 2 +- backend/fetching/ooniAlerts.js | 90 +++++ backend/fetching/ooniApi.js | 63 +++ backend/fetching/sourceToChannel.js | 8 + backend/models/credentials.js | 3 +- backend/models/source.js | 2 +- docs/OONI.md | 61 +++ docs/everythingaboutooni.md | 367 ++++++++++++++++++ package.json | 1 + scripts/backtest-ooni-alerts.js | 118 ++++++ scripts/domain-leading-indicators.js | 194 +++++++++ setup-windows.ps1 | 276 +++++++++++++ test/backend/lib.fetching.ooni-alerts.test.js | 66 ++++ .../backend/lib.fetching.ooni-channel.test.js | 33 ++ 15 files changed, 1466 insertions(+), 3 deletions(-) create mode 100644 backend/fetching/channels/ooni.js create mode 100644 backend/fetching/ooniAlerts.js create mode 100644 backend/fetching/ooniApi.js create mode 100644 docs/OONI.md create mode 100644 docs/everythingaboutooni.md create mode 100644 scripts/backtest-ooni-alerts.js create mode 100644 scripts/domain-leading-indicators.js create mode 100644 setup-windows.ps1 create mode 100644 test/backend/lib.fetching.ooni-alerts.test.js create mode 100644 test/backend/lib.fetching.ooni-channel.test.js diff --git a/backend/fetching/channels/ooni.js b/backend/fetching/channels/ooni.js new file mode 100644 index 000000000..516f9bd82 --- /dev/null +++ b/backend/fetching/channels/ooni.js @@ -0,0 +1,185 @@ +const { PollChannel } = require('downstream'); +const Report = require('../../models/report'); +const { fetchDailyMeasurements, fetchMeasurements } = require('../ooniApi'); +const { normalizeDailyCounts, evaluateAlert } = require('../ooniAlerts'); + +const DAY_MS = 24 * 60 * 60 * 1000; +const ALERT_DELAY_HOURS = 6; +const EVIDENCE_LIMIT = 5; +const NETWORK_NAMES = { + 44244: 'IranCell', + 58224: 'MCCI', +}; + +function shiftDay(day, offset) { + return new Date(day.getTime() + offset * DAY_MS); +} + +function dayString(day) { + return day.toISOString().slice(0, 10); +} + +function alertDateFor(now) { + const date = new Date(now); + date.setUTCHours(0, 0, 0, 0); + if (now.getUTCHours() < ALERT_DELAY_HOURS) return shiftDay(date, -1); + return date; +} + +function alertGuid(asn, alertDate) { + return `ooni:${asn}:volume:${alertDate}`; +} + +function alertContent(asn, alerts) { + const network = NETWORK_NAMES[asn] || `AS${asn}`; + const messages = alerts.map((alert) => { + if (alert.type === 'zero_measurements') { + return `no web connectivity measurements were recorded on ${alert.measurementDay}`; + } + return `the 2-day measurement average declined ${(alert.declineFraction * 100).toFixed(1)}%`; + }); + return `OONI volume alert for ${network} (AS${asn}): ${messages.join('; ')}.`; +} + +function measurementEvidence(measurement) { + return { + input: measurement.input, + measurementStartTime: measurement.measurement_start_time, + measurementUid: measurement.measurement_uid, + measurementUrl: measurement.measurement_url, + anomaly: measurement.anomaly, + confirmed: measurement.confirmed, + failure: measurement.failure, + blockingType: measurement.scores && measurement.scores.analysis + ? measurement.scores.analysis.blocking_type + : null, + verificationStatus: measurement.verification_status, + }; +} + +class OONIChannel extends PollChannel { + static INTERVAL = 60 * 60 * 1000; + + constructor(options) { + const asns = String(options.asns || '') + .split(/[\s,]+/) + .filter(Boolean) + .map(Number); + if (asns.length === 0 || asns.some((asn) => !Number.isInteger(asn) || asn <= 0)) { + throw new Error('OONI sources require one or more valid ASNs.'); + } + + super({ + ...options, + namespace: options.namespace || `ooni-${asns.join('-')}`, + }); + this.asns = asns; + this.interval = options.interval || OONIChannel.INTERVAL; + this.fetchDailyMeasurements = options.fetchDailyMeasurements || fetchDailyMeasurements; + this.fetchMeasurements = options.fetchMeasurements || fetchMeasurements; + } + + async fetchEvidence(asn, alerts) { + const decline = alerts.find((alert) => alert.type === 'measurement_decline'); + if (!decline) { + return { + available: false, + reason: 'No measurements were available on the zero-measurement day.', + confirmed: [], + anomalous: [], + }; + } + + const query = { + asn, + since: decline.recentStart, + until: decline.alertDate, + }; + try { + const [confirmed, anomalous] = await Promise.all([ + this.fetchMeasurements({ ...query, confirmed: true, limit: EVIDENCE_LIMIT }), + this.fetchMeasurements({ + ...query, + anomaly: true, + confirmed: false, + limit: EVIDENCE_LIMIT, + }), + ]); + + return { + available: true, + windowStart: decline.recentStart, + windowEnd: decline.recentEnd, + confirmed: confirmed.slice(0, EVIDENCE_LIMIT).map(measurementEvidence), + anomalous: anomalous + .filter((measurement) => !measurement.confirmed) + .slice(0, EVIDENCE_LIMIT) + .map(measurementEvidence), + }; + } catch (error) { + return { + available: false, + reason: `Related measurements could not be loaded: ${error.message}`, + confirmed: [], + anomalous: [], + }; + } + } + + async fetch() { + const alertDate = alertDateFor(new Date()); + const since = dayString(shiftDay(alertDate, -16)); + const until = dayString(alertDate); + const posts = []; + + for (const asn of this.asns) { + const rows = await this.fetchDailyMeasurements({ asn, since, until }); + const dailyCounts = normalizeDailyCounts(rows, since, until); + const alerts = evaluateAlert(dailyCounts, dayString(alertDate)); + if (alerts.length === 0) continue; + + const guid = alertGuid(asn, dayString(alertDate)); + if (await Report.exists({ guid })) continue; + + const evidence = await this.fetchEvidence(asn, alerts); + const post = this.parse({ asn, alerts, evidence, guid, fetchedAt: new Date() }); + posts.push(post); + this.enqueue(post); + } + + return posts; + } + + parse(rawMessage) { + const { asn, alerts, evidence, guid, fetchedAt } = rawMessage; + const alertDate = alerts[0].alertDate; + const searchParams = new URLSearchParams({ + probe_cc: 'IR', + probe_asn: `AS${asn}`, + test_name: 'web_connectivity', + since: evidence.windowStart || alerts[0].measurementDay, + until: alertDate, + }); + + return { + authoredAt: new Date(`${alertDate}T00:00:00.000Z`), + fetchedAt, + author: `OONI AS${asn}`, + content: alertContent(asn, alerts), + url: `https://explorer.ooni.org/search?${searchParams}`, + platform: 'OONI', + platformID: guid, + raw: { + probeCC: 'IR', + probeASN: asn, + networkName: NETWORK_NAMES[asn] || null, + testName: 'web_connectivity', + alertDate, + triggers: alerts, + evidence, + }, + }; + } +} + +module.exports = OONIChannel; \ No newline at end of file diff --git a/backend/fetching/hooks/postToReport.js b/backend/fetching/hooks/postToReport.js index bc8e118eb..201c132b0 100644 --- a/backend/fetching/hooks/postToReport.js +++ b/backend/fetching/hooks/postToReport.js @@ -29,7 +29,7 @@ module.exports = async function postToReport(post, next) { let metadata; - if (platform === 'RSS') { + if (platform === 'RSS' || platform === 'OONI') { // What do we want here? metadata = { // title: raw.title || null, diff --git a/backend/fetching/ooniAlerts.js b/backend/fetching/ooniAlerts.js new file mode 100644 index 000000000..4ce02a372 --- /dev/null +++ b/backend/fetching/ooniAlerts.js @@ -0,0 +1,90 @@ +const DAY_MS = 24 * 60 * 60 * 1000; + +function toDay(value) { + return new Date(`${value.slice(0, 10)}T00:00:00.000Z`); +} + +function dayString(value) { + return value.toISOString().slice(0, 10); +} + +function shiftDay(value, offset) { + return new Date(value.getTime() + offset * DAY_MS); +} + +function mean(values) { + return values.reduce((total, value) => total + value, 0) / values.length; +} + +function normalizeDailyCounts(rows, since, until) { + const countsByDay = new Map(); + rows.forEach((row) => { + const day = (row.measurement_start_day || '').slice(0, 10); + if (day) countsByDay.set(day, Number(row.measurement_count) || 0); + }); + + const normalized = []; + for (let day = toDay(since); day < toDay(until); day = shiftDay(day, 1)) { + const key = dayString(day); + normalized.push({ + day: key, + measurementCount: countsByDay.get(key) || 0, + }); + } + return normalized; +} + +function evaluateAlert(dailyCounts, alertDate, declineThreshold = 0.3) { + const date = toDay(alertDate); + const countsByDay = new Map( + dailyCounts.map(({ day, measurementCount }) => [day, measurementCount]), + ); + const count = (offset) => countsByDay.get(dayString(shiftDay(date, offset))) || 0; + const alerts = []; + const latestDay = dayString(shiftDay(date, -1)); + const latestCount = count(-1); + + if (latestCount === 0) { + alerts.push({ + type: 'zero_measurements', + alertDate: dayString(date), + measurementDay: latestDay, + measurementCount: 0, + }); + } + + const recentCounts = [count(-2), count(-1)]; + const baselineCounts = []; + for (let offset = -16; offset <= -3; offset++) { + const value = count(offset); + if (value > 0) baselineCounts.push(value); + } + + if (baselineCounts.length > 0) { + const recentAverage = mean(recentCounts); + const baselineAverage = mean(baselineCounts); + const declineFraction = 1 - recentAverage / baselineAverage; + if (declineFraction >= declineThreshold) { + alerts.push({ + type: 'measurement_decline', + alertDate: dayString(date), + recentStart: dayString(shiftDay(date, -2)), + recentEnd: latestDay, + recentCounts, + recentAverage, + baselineStart: dayString(shiftDay(date, -16)), + baselineEnd: dayString(shiftDay(date, -3)), + baselineNonZeroDays: baselineCounts.length, + baselineAverage, + declineFraction, + }); + } + } + + return alerts; +} + +module.exports = { + normalizeDailyCounts, + evaluateAlert, +}; \ No newline at end of file diff --git a/backend/fetching/ooniApi.js b/backend/fetching/ooniApi.js new file mode 100644 index 000000000..ff4b57385 --- /dev/null +++ b/backend/fetching/ooniApi.js @@ -0,0 +1,63 @@ +const AGGREGATION_URL = 'https://api.ooni.org/api/v1/aggregation'; + +async function fetchDailyMeasurements({ asn, since, until, fetchImpl = fetch }) { + const params = new URLSearchParams({ + probe_cc: 'IR', + probe_asn: String(asn), + test_name: 'web_connectivity', + axis_x: 'measurement_start_day', + since, + until, + }); + const url = `${AGGREGATION_URL}?${params}`; + const response = await fetchImpl(url, { + headers: { accept: 'application/json' }, + }); + if (!response.ok) { + throw new Error(`OONI aggregation request failed (${response.status}): ${url}`); + } + + const payload = await response.json(); + const result = payload.result || []; + return Array.isArray(result) ? result : [result]; +} + +async function fetchMeasurements({ + asn, + since, + until, + confirmed, + anomaly, + limit = 20, + fetchImpl = fetch, +}) { + const params = new URLSearchParams({ + probe_cc: 'IR', + probe_asn: `AS${asn}`, + test_name: 'web_connectivity', + since, + until, + limit: String(limit), + order_by: 'measurement_start_time', + order: 'desc', + }); + if (confirmed != null) params.set('confirmed', String(confirmed)); + if (anomaly != null) params.set('anomaly', String(anomaly)); + + const url = `https://api.ooni.org/api/v1/measurements?${params}`; + const response = await fetchImpl(url, { + headers: { accept: 'application/json' }, + }); + if (!response.ok) { + throw new Error(`OONI measurements request failed (${response.status}): ${url}`); + } + + const payload = await response.json(); + return payload.results || []; +} + +module.exports = { + AGGREGATION_URL, + fetchDailyMeasurements, + fetchMeasurements, +}; \ No newline at end of file diff --git a/backend/fetching/sourceToChannel.js b/backend/fetching/sourceToChannel.js index 91b64ad26..d6a86a740 100644 --- a/backend/fetching/sourceToChannel.js +++ b/backend/fetching/sourceToChannel.js @@ -12,6 +12,7 @@ const downstream = require('./downstream'); const AggieCrowdTangleChannel = require('./channels/crowdtangle'); const TelegramChannel = require('./channels/telegram'); const RSSChannel = require('./channels/rss'); +const OONIChannel = require('./channels/ooni'); const { TwitterPageChannel, JunkipediaChannel } = builtin; @@ -187,6 +188,13 @@ function createChannel(source) { }; channel = new RSSChannel(options); break; + case 'ooni': + options = { + ...options, + asns: lists, + }; + channel = new OONIChannel(options); + break; default: } diff --git a/backend/models/credentials.js b/backend/models/credentials.js index 16d37c695..513667b22 100644 --- a/backend/models/credentials.js +++ b/backend/models/credentials.js @@ -15,7 +15,8 @@ const credentialsTypes = [ 'crowdtangle', 'telegram', 'junkipedia', - 'rss' + 'rss', + 'ooni' ]; // validates secrete based on their type diff --git a/backend/models/source.js b/backend/models/source.js index ab14b87c6..18be9296d 100644 --- a/backend/models/source.js +++ b/backend/models/source.js @@ -25,7 +25,7 @@ var urlValidator = function (url) { ) } -var mediaValues = ['facebook', 'instagram', 'comments', 'elmo', 'twitter', 'rss', 'dummy', 'smsgh', 'whatsapp', 'telegram', 'junkipedia', 'dummy-pull', 'dummy-fast']; +var mediaValues = ['facebook', 'instagram', 'comments', 'elmo', 'twitter', 'rss', 'ooni', 'dummy', 'smsgh', 'whatsapp', 'telegram', 'junkipedia', 'dummy-pull', 'dummy-fast']; var sourceSchema = new mongoose.Schema({ media: { type: String, enum: mediaValues }, diff --git a/docs/OONI.md b/docs/OONI.md new file mode 100644 index 000000000..bec5133c6 --- /dev/null +++ b/docs/OONI.md @@ -0,0 +1,61 @@ +# OONI integration + +Aggie polls OONI's public aggregation API for Iran `web_connectivity` +measurement volume on AS44244 (IranCell) and AS58224 (MCCI). Alerts enter the +normal Aggie report pipeline and use deterministic daily GUIDs, so repeated +polling does not create duplicate reports. When both rules trigger for an ASN +on the same date, they are grouped into one report. + +## Alert rules + +For an alert date `D`, only completed UTC days are evaluated: + +1. `zero_measurements`: the measurement count on `D-1` is zero. OONI omits + zero-count days from its response, so missing calendar dates are filled with + zero before evaluation. +2. `measurement_decline`: the mean for `D-2` through `D-1` is at least 30% + below the mean of non-zero counts from `D-16` through `D-3`. + +The two windows are disjoint. A day can produce both alert types. Production +polling waits until 06:00 UTC before evaluating the previous UTC day, reducing +false zero alerts while OONI finishes publishing daily aggregates. + +## Configure Aggie + +1. Open Settings, then Credentials. +2. Create an `ooni` credential. OONI is public, so no token is requested. +3. Open Settings, then Sources. +4. Create an `ooni` source and select the credential. +5. Keep the default ASN list `44244 58224`. +6. Enable the source and turn global fetching on. + +OONI alerts appear as normal reports with media type `OONI`. Report metadata +contains the ASN, alert type, daily counts, baseline average, recent average, +and decline fraction. For decline alerts, the detail view also includes up to +five confirmed and five non-confirmed anomalous measurements from the recent +two-day window. These measurements provide context and are not presented as the +cause of the volume decline. A zero-only alert has no same-day tests to attach. +The report link opens the corresponding OONI Explorer query. + +## Historical backtest + +Run the same evaluator used by the production channel: + +```powershell +npm run backtest:ooni -- 2026-07-30 +``` + +The optional date is the alert date through which to evaluate. Output is written +to the ignored `data/ooni-alert-backtest.json` and +`data/ooni-alert-backtest.csv` files. + +For December 1, 2025 through July 30, 2026, the grouped backtest produced: + +| ASN | Reports | Zero triggers | Decline triggers | +| --- | ---: | ---: | ---: | +| AS44244 (IranCell) | 124 | 89 | 115 | +| AS58224 (MCCI) | 109 | 98 | 68 | +| Total | 233 | 187 | 183 | + +Grouping produces 137 fewer reports than emitting each trigger separately, a +37% reduction. There is no additional cooldown or incident-level suppression. \ No newline at end of file diff --git a/docs/everythingaboutooni.md b/docs/everythingaboutooni.md new file mode 100644 index 000000000..f051d73b9 --- /dev/null +++ b/docs/everythingaboutooni.md @@ -0,0 +1,367 @@ +# Everything About the OONI Integration + +## Purpose + +This integration adds OONI `web_connectivity` measurement-volume monitoring to +Aggie for two Iranian mobile networks: + +| Network | ASN | +| --- | --- | +| IranCell | AS44244 | +| MCCI | AS58224 | + +The integration uses Aggie's existing source, polling, report, and MongoDB +infrastructure. It does not introduce a separate service or database. + +## Requested Alert Rules + +For each ASN, Aggie evaluates the OONI measurement volume once a day using only +completed UTC days. + +For alert date `D`: + +1. `zero_measurements`: the total `measurement_count` on `D-1` is zero. +2. `measurement_decline`: the average measurement count for `D-2` through + `D-1` is at least 30% below the average for `D-16` through `D-3`. + +Only non-zero values are included when calculating the 14-day baseline. The +recent two-day window and the preceding 14-day baseline do not overlap. + +These are measurement-volume alerts aggregated across all `web_connectivity` +tests for an ASN. They are not individual website-failure alerts. + +## Daily Timing + +The OONI channel polls hourly. It waits until 06:00 UTC before evaluating the +previous completed day. This delay gives OONI time to publish the previous day's +aggregates and reduces false zero-measurement alerts. + +## Grouped Reports + +If zero volume and a 30% decline trigger for the same ASN and alert date, Aggie +creates one report containing both triggers. + +The report GUID follows this format: + +```text +ooni::volume: +``` + +For example: + +```text +ooni:44244:volume:2026-01-10 +``` + +This deterministic GUID prevents duplicate reports when the hourly poll runs +again. + +## Supporting Website Evidence + +A decline report includes selected website-level OONI measurements from its +recent two-day window: + +- Up to five confirmed measurements. +- Up to five anomalous but non-confirmed measurements. + +Confirmed and anomalous entries do not overlap. + +These measurements provide context. They are not presented as proof that a +particular website caused the measurement-volume decline. + +A zero-only alert has no measurements for the zero day, so it displays: + +```text +No measurements were available on the zero-measurement day. +``` + +If the optional evidence request fails, the core volume alert is still stored. +The report displays the evidence-loading error instead of dropping the alert. + +### OONI Result Meanings + +- `confirmed: true`: OONI found stronger interference evidence, such as a known + block page or middlebox fingerprint. +- `anomaly: true`: the result needs attention and may indicate interference, but + it is not confirmed. +- `failure: true`: the test or control request failed. This does not + automatically mean censorship. + +## Example Stored Report + +```json +{ + "guid": "ooni:44244:volume:2026-01-10", + "authoredAt": "2026-01-10T00:00:00.000Z", + "author": "OONI AS44244", + "content": "OONI volume alert for IranCell (AS44244): no web connectivity measurements were recorded on 2026-01-09; the 2-day measurement average declined 100.0%.", + "_media": ["OONI"], + "metadata": { + "rawAPIResponse": { + "probeCC": "IR", + "probeASN": 44244, + "networkName": "IranCell", + "testName": "web_connectivity", + "alertDate": "2026-01-10", + "triggers": [ + { + "type": "zero_measurements", + "measurementDay": "2026-01-09", + "measurementCount": 0 + }, + { + "type": "measurement_decline", + "recentStart": "2026-01-08", + "recentEnd": "2026-01-09", + "recentCounts": [0, 0], + "recentAverage": 0, + "baselineStart": "2025-12-25", + "baselineEnd": "2026-01-07", + "baselineNonZeroDays": 14, + "baselineAverage": 1216.43, + "declineFraction": 1 + } + ], + "evidence": { + "available": true, + "windowStart": "2026-01-08", + "windowEnd": "2026-01-09", + "confirmed": [], + "anomalous": [] + } + } + } +} +``` + +## Architecture + +```mermaid +flowchart LR + A[OONI Aggregation API] --> B[OONI Poll Channel] + B --> C[Normalize Missing Days] + C --> D[Evaluate Alert Rules] + D --> E[Group Daily Triggers] + E --> F[Fetch Supporting Measurements] + F --> G[Aggie Post-to-Report Hook] + G --> H[MongoDB Reports Collection] + H --> I[OONI Report Detail View] +``` + +## Database Storage + +OONI alerts use the same MongoDB `reports` collection as other Aggie reports. +They contain: + +- Alert summary and timestamps. +- ASN and network name. +- Trigger details. +- Recent and baseline counts. +- Decline percentage. +- Selected confirmed and anomalous measurements. +- OONI measurement and Explorer links. +- Aggie source and media identifiers. +- A deterministic deduplication GUID. + +The OONI source and polling configuration use Aggie's existing `sources` and +`credentials` collections. OONI is public, so its credential record does not +contain an API token. + +## Backend Files + +### `backend/fetching/ooniApi.js` + +Calls the public OONI APIs: + +- Aggregation endpoint for daily measurement counts. +- Measurements endpoint for confirmed and anomalous evidence. + +The evidence endpoint is queried directly for at most five confirmed and five +anomalous, non-confirmed measurements. + +### `backend/fetching/ooniAlerts.js` + +Contains dependency-free alert logic: + +- Fills omitted calendar dates with zero measurements. +- Calculates the two-day recent average. +- Calculates the preceding 14-day non-zero baseline. +- Produces zero and decline trigger objects. + +Keeping this logic separate allows the real-time channel and historical +backtest to use the same calculations. + +### `backend/fetching/channels/ooni.js` + +Implements the native Aggie `PollChannel`. It: + +- Parses configured ASNs. +- Polls OONI hourly. +- Applies the 06:00 UTC publication delay. +- Fetches daily aggregate counts. +- Evaluates and groups triggers. +- Fetches bounded supporting evidence. +- Creates deterministic GUIDs. +- Enqueues reports through Aggie's normal fetching pipeline. +- Preserves the core alert when evidence lookup fails. + +### `backend/fetching/sourceToChannel.js` + +Creates an `OONIChannel` when Aggie loads a source with media type `ooni`. +The source's existing `lists` field stores space- or comma-separated ASNs. + +### `backend/fetching/hooks/postToReport.js` + +Converts OONI channel posts into standard Aggie reports and stores OONI data in +`metadata.rawAPIResponse`. + +### `backend/models/source.js` + +Adds `ooni` to the valid source media types. + +### `backend/models/credentials.js` + +Adds `ooni` as a valid credential type. The record exists to follow Aggie's +current source/credential relationship, but no secret is required. + +## Frontend Files + +### `src/pages/Settings/Credentials/CreateCredentialForm.tsx` + +Adds a token-free OONI credential form. + +### `src/pages/Settings/source/CreateEditSourceForm.tsx` + +Adds the OONI source form. Its default ASN list is: + +```text +44244 58224 +``` + +### `src/api/common.ts` + +Adds OONI to frontend media and credential options. + +### `src/objectTypes.d.ts` + +Adds `OONI` to the frontend media type declaration. + +### `src/components/SocialMediaPost/SocialMediaIcon.tsx` + +Displays a globe icon for OONI reports. + +### `src/components/SocialMediaPost/OONIPost.tsx` + +Renders the expanded OONI report view with: + +- Network, ASN, and alert date. +- Every triggered volume rule. +- Recent daily counts. +- Recent and baseline averages. +- Baseline period and non-zero day count. +- Decline percentage. +- Up to five confirmed measurements. +- Up to five anomalous measurements. +- Blocking type and verification status. +- Links to raw OONI measurements. +- Clear language distinguishing context from causation. + +### `src/components/SocialMediaPost/index.tsx` + +Routes reports with media type `OONI` to `OONIPost`. + +## Backtest and Tests + +### `scripts/backtest-ooni-alerts.js` + +Runs the same alert evaluator used by the production channel from December 1, +2025 through a specified end date. + +Run it with: + +```powershell +npm run backtest:ooni -- 2026-07-30 +``` + +It writes: + +```text +data/ooni-alert-backtest.json +data/ooni-alert-backtest.csv +``` + +The `data` directory is ignored by Git. + +### `test/backend/lib.fetching.ooni-alerts.test.js` + +Tests: + +- Missing aggregation days becoming zero counts. +- Zero-measurement triggers. +- The 30% decline calculation. +- Disjoint recent and baseline windows. +- Exclusion of zero values from the baseline. + +### `package.json` + +Adds the `backtest:ooni` command. + +## Historical Results + +Backtest period: December 1, 2025 through July 30, 2026. + +| ASN | Grouped reports | Zero triggers | Decline triggers | +| --- | ---: | ---: | ---: | +| AS44244, IranCell | 124 | 89 | 115 | +| AS58224, MCCI | 109 | 98 | 68 | +| Total | 233 | 187 | 183 | + +Emitting every trigger separately would have generated 370 reports. Grouping +both rules by ASN and date produces 233 reports, which is 137 fewer reports and +a 37% reduction. + +The trigger totals exceed the report total because one grouped report can +contain both rules. + +## Aggie Configuration + +1. Open Settings and then Credentials. +2. Create an `ooni` credential. No token is required. +3. Open Settings and then Sources. +4. Create an `ooni` source. +5. Select the OONI credential. +6. Keep or enter the ASN list `44244 58224`. +7. Enable the source. +8. Turn global fetching on. + +After configuration, future qualifying dips create reports automatically. + +## Validation Completed + +The following checks passed: + +- JavaScript syntax checks for all OONI backend files. +- Editor diagnostics for modified JavaScript and TypeScript files. +- Synthetic zero-day and 30% decline checks. +- Non-zero baseline exclusion check. +- Live OONI aggregation requests for both ASNs. +- Live OONI confirmed-measurement query. +- Live OONI anomalous and non-confirmed measurement query. +- Five-confirmed and five-anomalous evidence limits. +- Grouped JSON and CSV reconciliation at 233 reports. +- Git whitespace validation. + +A full npm build and full repository test run remain pending because this +machine's network policy currently rejects TLS connections to +`registry.npmjs.org`. The OONI API itself is reachable and the integration logic +has been validated against live data. + +## Current Limitations + +- There is no incident-level cooldown beyond grouping rules by ASN and day. +- Website evidence is attached only to decline alerts, because zero days have no + measurements. +- Evidence is a bounded sample, not every measurement in the period. +- The production channel must be enabled in Aggie before polling begins. +- End-to-end runtime verification requires the repository dependencies to be + installed. diff --git a/package.json b/package.json index 087216d1c..252b852a3 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "dev:frontend": "cross-env PORT=8000 react-scripts start", "dev:backend": "cross-env ENVIRONMENT=development nodemon app.js", "dev:all": "npm run dev", + "backtest:ooni": "node scripts/backtest-ooni-alerts.js", "build": "cross-env CI=false react-scripts build", "start": "ENVIRONMENT=production node app.js", "postinstall": "node install.js" diff --git a/scripts/backtest-ooni-alerts.js b/scripts/backtest-ooni-alerts.js new file mode 100644 index 000000000..ef6a05b3d --- /dev/null +++ b/scripts/backtest-ooni-alerts.js @@ -0,0 +1,118 @@ +const fs = require('fs'); +const path = require('path'); +const { fetchDailyMeasurements } = require('../backend/fetching/ooniApi'); +const { normalizeDailyCounts, evaluateAlert } = require('../backend/fetching/ooniAlerts'); + +const ASNS = [44244, 58224]; +const ALERT_SINCE = '2025-12-01'; +const NETWORK_NAMES = { 44244: 'IranCell', 58224: 'MCCI' }; + +function shiftDay(day, offset) { + const value = new Date(`${day}T00:00:00.000Z`); + value.setUTCDate(value.getUTCDate() + offset); + return value.toISOString().slice(0, 10); +} + +function csvValue(value) { + if (value == null) return ''; + const text = Array.isArray(value) ? value.join('|') : String(value); + return `"${text.replace(/"/g, '""')}"`; +} + +async function main() { + const alertUntil = process.argv[2] || new Date().toISOString().slice(0, 10); + const dataSince = shiftDay(ALERT_SINCE, -16); + const dataUntil = alertUntil; + const output = []; + + for (const asn of ASNS) { + const rows = await fetchDailyMeasurements({ asn, since: dataSince, until: dataUntil }); + const dailyCounts = normalizeDailyCounts(rows, dataSince, dataUntil); + for (let alertDate = ALERT_SINCE; alertDate <= alertUntil; alertDate = shiftDay(alertDate, 1)) { + const triggers = evaluateAlert(dailyCounts, alertDate); + if (triggers.length > 0) { + output.push({ + asn, + networkName: NETWORK_NAMES[asn], + alertDate, + triggers, + }); + } + } + } + + const outputDirectory = path.join(__dirname, '..', 'data'); + fs.mkdirSync(outputDirectory, { recursive: true }); + const outputPath = path.join(outputDirectory, 'ooni-alert-backtest.json'); + fs.writeFileSync(outputPath, `${JSON.stringify(output, null, 2)}\n`); + const csvFields = [ + 'asn', 'networkName', 'alertDate', 'triggerTypes', 'measurementDay', + 'measurementCount', 'recentStart', 'recentEnd', 'recentCounts', + 'recentAverage', 'baselineStart', 'baselineEnd', 'baselineNonZeroDays', + 'baselineAverage', 'declineFraction', + ]; + const csvRows = output.map((alert) => { + const zero = alert.triggers.find((trigger) => trigger.type === 'zero_measurements') || {}; + const decline = alert.triggers.find((trigger) => trigger.type === 'measurement_decline') || {}; + const row = { + ...alert, + triggerTypes: alert.triggers.map((trigger) => trigger.type), + ...zero, + ...decline, + }; + return csvFields.map((field) => csvValue(row[field])).join(','); + }); + const csvPath = path.join(outputDirectory, 'ooni-alert-backtest.csv'); + fs.writeFileSync(csvPath, `${csvFields.join(',')}\n${csvRows.join('\n')}\n`); + const triggerCsvFields = [ + 'asn', 'networkName', 'alertDate', 'triggerType', 'zeroMeasurementDay', + 'zeroMeasurementCount', 'recentStart', 'recentEnd', 'recentCounts', + 'recentAverage', 'baselineStart', 'baselineEnd', 'baselineNonZeroDays', + 'baselineAverage', 'declineFraction', 'declinePercent', + ]; + const triggerCsvRows = output.flatMap((alert) => { + // When both triggers fire on the same day, keep only the zero-measurement alert. + const hasZero = alert.triggers.some((trigger) => trigger.type === 'zero_measurements'); + const triggers = hasZero + ? alert.triggers.filter((trigger) => trigger.type === 'zero_measurements') + : alert.triggers; + return triggers.map((trigger) => { + const row = { + asn: alert.asn, + networkName: alert.networkName, + alertDate: alert.alertDate, + triggerType: trigger.type, + ...trigger, + zeroMeasurementDay: trigger.type === 'zero_measurements' + ? trigger.measurementDay + : null, + zeroMeasurementCount: trigger.type === 'zero_measurements' + ? trigger.measurementCount + : null, + declinePercent: trigger.declineFraction == null + ? null + : trigger.declineFraction * 100, + }; + return triggerCsvFields.map((field) => csvValue(row[field])).join(','); + }); + }); + const triggerCsvPath = path.join(outputDirectory, 'ooni-alert-triggers.csv'); + fs.writeFileSync( + triggerCsvPath, + `${triggerCsvFields.join(',')}\n${triggerCsvRows.join('\n')}\n`, + ); + console.log(`Wrote ${output.length} alerts to ${outputPath}`); + console.log(`Wrote CSV output to ${csvPath}`); + console.log(`Wrote ${triggerCsvRows.length} individual triggers to ${triggerCsvPath}`); + ASNS.forEach((asn) => { + const alerts = output.filter((alert) => alert.asn === asn); + const zeros = alerts.filter((alert) => alert.triggers.some((trigger) => trigger.type === 'zero_measurements')).length; + const declines = alerts.filter((alert) => alert.triggers.some((trigger) => trigger.type === 'measurement_decline')).length; + console.log(`AS${asn}: ${alerts.length} reports (${zeros} zero-measurement triggers, ${declines} decline triggers)`); + }); +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); \ No newline at end of file diff --git a/scripts/domain-leading-indicators.js b/scripts/domain-leading-indicators.js new file mode 100644 index 000000000..59ba0f638 --- /dev/null +++ b/scripts/domain-leading-indicators.js @@ -0,0 +1,194 @@ +// Investigates domain-level OONI web_connectivity signals leading up to shutdown +// dates to find domains whose anomalies or confirmed blocks rise beforehand. +// +// For each shutdown date the script compares a lead-up window (the 21 days +// immediately before the date) against a baseline window (the prior 21 days) +// per domain, then ranks domains by emerging confirmed blocks and by the rise +// in anomaly rate. Output is written per ASN/date as CSV plus a JSON summary. + +const fs = require('fs'); +const path = require('path'); + +const AGGREGATION_URL = 'https://api.ooni.org/api/v1/aggregation'; +const ASNS = [44244, 58224]; +const NETWORK_NAMES = { 44244: 'IranCell', 58224: 'MCCI' }; +const SHUTDOWN_DATES = ['2025-06-18', '2026-01-08', '2026-02-28']; +const LEADUP_DAYS = 21; +const BASELINE_DAYS = 21; +const MIN_LEADUP_MEASUREMENTS = 5; // ignore domains with too little coverage +const TOP_N = 25; + +function shiftDay(day, offset) { + const value = new Date(`${day}T00:00:00.000Z`); + value.setUTCDate(value.getUTCDate() + offset); + return value.toISOString().slice(0, 10); +} + +function csvValue(value) { + if (value == null) return ''; + const text = Array.isArray(value) ? value.join('|') : String(value); + return `"${text.replace(/"/g, '""')}"`; +} + +async function fetchDomainAggregation({ asn, since, until }) { + const params = new URLSearchParams({ + probe_cc: 'IR', + probe_asn: String(asn), + test_name: 'web_connectivity', + axis_x: 'input', + since, + until, + }); + const url = `${AGGREGATION_URL}?${params}`; + const response = await fetch(url, { headers: { accept: 'application/json' } }); + if (!response.ok) { + throw new Error(`OONI aggregation failed (${response.status}): ${url}`); + } + const payload = await response.json(); + const result = payload.result || []; + return Array.isArray(result) ? result : [result]; +} + +function indexByDomain(rows) { + const map = new Map(); + rows.forEach((row) => { + if (!row.input) return; + map.set(row.input, { + measurements: Number(row.measurement_count) || 0, + anomalies: Number(row.anomaly_count) || 0, + confirmed: Number(row.confirmed_count) || 0, + failures: Number(row.failure_count) || 0, + }); + }); + return map; +} + +function rate(part, total) { + return total > 0 ? part / total : 0; +} + +async function analyzeDate(asn, shutdownDate) { + const leadupSince = shiftDay(shutdownDate, -LEADUP_DAYS); + const leadupUntil = shutdownDate; // days before the shutdown + const baselineSince = shiftDay(shutdownDate, -(LEADUP_DAYS + BASELINE_DAYS)); + const baselineUntil = leadupSince; + + const [leadupRows, baselineRows] = await Promise.all([ + fetchDomainAggregation({ asn, since: leadupSince, until: leadupUntil }), + fetchDomainAggregation({ asn, since: baselineSince, until: baselineUntil }), + ]); + + const leadup = indexByDomain(leadupRows); + const baseline = indexByDomain(baselineRows); + + const domains = []; + for (const [domain, l] of leadup.entries()) { + if (l.measurements < MIN_LEADUP_MEASUREMENTS) continue; + const b = baseline.get(domain) || { + measurements: 0, anomalies: 0, confirmed: 0, failures: 0, + }; + const leadupAnomalyRate = rate(l.anomalies, l.measurements); + const baselineAnomalyRate = rate(b.anomalies, b.measurements); + domains.push({ + shutdownDate, + asn, + networkName: NETWORK_NAMES[asn], + domain, + leadupWindow: `${leadupSince} to ${leadupUntil}`, + leadupMeasurements: l.measurements, + leadupAnomalies: l.anomalies, + leadupConfirmed: l.confirmed, + leadupAnomalyRate, + baselineWindow: `${baselineSince} to ${baselineUntil}`, + baselineMeasurements: b.measurements, + baselineAnomalies: b.anomalies, + baselineConfirmed: b.confirmed, + baselineAnomalyRate, + anomalyRateDelta: leadupAnomalyRate - baselineAnomalyRate, + confirmedDelta: l.confirmed - b.confirmed, + newlyConfirmed: b.confirmed === 0 && l.confirmed > 0, + newlyAnomalous: baselineAnomalyRate === 0 && leadupAnomalyRate > 0, + }); + } + + // Leading indicators: confirmed blocks or a meaningful rise in anomaly rate. + domains.sort((a, b) => { + if (b.leadupConfirmed !== a.leadupConfirmed) return b.leadupConfirmed - a.leadupConfirmed; + if (b.anomalyRateDelta !== a.anomalyRateDelta) return b.anomalyRateDelta - a.anomalyRateDelta; + return b.leadupAnomalies - a.leadupAnomalies; + }); + + return domains; +} + +async function main() { + const outputDirectory = path.join(__dirname, '..', 'data', 'domain-leading-indicators'); + fs.mkdirSync(outputDirectory, { recursive: true }); + + const csvFields = [ + 'shutdownDate', 'asn', 'networkName', 'domain', 'leadupWindow', + 'leadupMeasurements', 'leadupAnomalies', 'leadupConfirmed', 'leadupAnomalyRate', + 'baselineWindow', 'baselineMeasurements', 'baselineAnomalies', 'baselineConfirmed', + 'baselineAnomalyRate', 'anomalyRateDelta', 'confirmedDelta', 'newlyConfirmed', + 'newlyAnomalous', + ]; + + const summary = []; + for (const shutdownDate of SHUTDOWN_DATES) { + for (const asn of ASNS) { + const domains = await analyzeDate(asn, shutdownDate); + const top = domains.slice(0, TOP_N); + const csvRows = top.map((row) => csvFields.map((field) => { + const value = row[field]; + if (typeof value === 'number' && field.endsWith('Rate')) return csvValue(value.toFixed(4)); + return csvValue(value); + }).join(',')); + const csvPath = path.join( + outputDirectory, + `${shutdownDate}_AS${asn}_${NETWORK_NAMES[asn]}.csv`, + ); + fs.writeFileSync(csvPath, `${csvFields.join(',')}\n${csvRows.join('\n')}\n`); + + const confirmedDomains = domains.filter((d) => d.leadupConfirmed > 0); + const newAnomalyDomains = domains.filter((d) => d.newlyAnomalous && d.leadupAnomalies >= 5); + summary.push({ + shutdownDate, + asn, + networkName: NETWORK_NAMES[asn], + domainsConsidered: domains.length, + domainsWithConfirmedBlocks: confirmedDomains.length, + domainsNewlyAnomalous: newAnomalyDomains.length, + topConfirmed: confirmedDomains.slice(0, 10).map((d) => ({ + domain: d.domain, + confirmed: d.leadupConfirmed, + anomalies: d.leadupAnomalies, + measurements: d.leadupMeasurements, + })), + topEmergingAnomalies: newAnomalyDomains + .sort((a, b) => b.leadupAnomalies - a.leadupAnomalies) + .slice(0, 10) + .map((d) => ({ + domain: d.domain, + leadupAnomalies: d.leadupAnomalies, + leadupMeasurements: d.leadupMeasurements, + leadupAnomalyRate: Number(d.leadupAnomalyRate.toFixed(3)), + })), + }); + console.log( + `${shutdownDate} AS${asn} ${NETWORK_NAMES[asn]}: ` + + `${domains.length} domains, ` + + `${confirmedDomains.length} with confirmed blocks, ` + + `${newAnomalyDomains.length} newly anomalous`, + ); + } + } + + const summaryPath = path.join(outputDirectory, 'summary.json'); + fs.writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`); + console.log(`\nWrote per-date CSVs and summary to ${outputDirectory}`); +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/setup-windows.ps1 b/setup-windows.ps1 new file mode 100644 index 000000000..e05cac6c1 --- /dev/null +++ b/setup-windows.ps1 @@ -0,0 +1,276 @@ +[CmdletBinding()] +param( + [string]$InstallPath = (Join-Path $HOME "source\Aggie"), + [string]$NpmRegistry = "https://registry.npmjs.org/", + [string]$HttpsProxy = "", + [switch]$DryRun +) + +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +function Write-Step { + param([string]$Message) + Write-Host "`n==> $Message" -ForegroundColor Cyan +} + +function Refresh-Path { + $machinePath = [Environment]::GetEnvironmentVariable("Path", "Machine") + $userPath = [Environment]::GetEnvironmentVariable("Path", "User") + $env:Path = "$machinePath;$userPath" +} + +function Invoke-CommandChecked { + param( + [Parameter(Mandatory = $true)][string]$Command, + [Parameter(Mandatory = $true)][string[]]$Arguments + ) + + & $Command @Arguments + if ($LASTEXITCODE -ne 0) { + throw "$Command failed with exit code $LASTEXITCODE." + } +} + +function Install-WingetPackage { + param( + [Parameter(Mandatory = $true)][string]$Id, + [Parameter(Mandatory = $true)][string]$DisplayName + ) + + if ($DryRun) { + Write-Host "Would install $DisplayName ($Id)." + return + } + + Write-Step "Installing $DisplayName" + Invoke-CommandChecked "winget" @( + "install", "--id", $Id, "--exact", "--silent", + "--accept-package-agreements", "--accept-source-agreements" + ) + Refresh-Path +} + +function New-RandomString { + param([int]$Length = 32) + + $characters = "abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789" + $bytes = New-Object byte[] $Length + $generator = [Security.Cryptography.RandomNumberGenerator]::Create() + try { + $generator.GetBytes($bytes) + } finally { + $generator.Dispose() + } + + return -join ($bytes | ForEach-Object { $characters[$_ % $characters.Length] }) +} + +function Test-DockerReady { + & docker info *> $null + return $LASTEXITCODE -eq 0 +} + +if ($env:OS -ne "Windows_NT") { + throw "This bootstrap script supports Windows 10 and Windows 11 only." +} + +Write-Step "Checking Windows prerequisites" +if (-not (Get-Command winget -ErrorAction SilentlyContinue)) { + throw "winget is required. Install 'App Installer' from Microsoft Store, then rerun this script." +} + +if (-not (Get-Command git -ErrorAction SilentlyContinue)) { + Install-WingetPackage -Id "Git.Git" -DisplayName "Git" +} +if (-not (Get-Command node -ErrorAction SilentlyContinue)) { + Install-WingetPackage -Id "OpenJS.NodeJS.18" -DisplayName "Node.js 18" +} +if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { + Install-WingetPackage -Id "Docker.DockerDesktop" -DisplayName "Docker Desktop" +} + +if ($DryRun) { + Write-Host "Would clone Aggie when the script is not already inside the repository." + Write-Host "Would create ignored local configuration when missing." + Write-Host "Would start MongoDB 6 in Docker and run npm ci." + Write-Host "Dry run complete. No changes were made." -ForegroundColor Green + exit 0 +} + +Refresh-Path + +foreach ($command in @("git", "node", "npm", "docker")) { + if (-not (Get-Command $command -ErrorAction SilentlyContinue)) { + throw "$command was installed but is not available yet. Restart Windows, then rerun this script." + } +} + +$nodeVersion = (& node --version).TrimStart("v") +$parsedNodeVersion = [Version]$nodeVersion +if ($parsedNodeVersion.Major -ne 18 -or $parsedNodeVersion.Minor -lt 20) { + throw "Aggie requires Node.js 18.20 or newer within major version 18. Found $nodeVersion." +} + +$scriptRepoPath = $PSScriptRoot +if (Test-Path (Join-Path $scriptRepoPath "package.json")) { + $repoPath = $scriptRepoPath +} else { + $repoPath = $InstallPath + if (-not (Test-Path (Join-Path $repoPath "package.json"))) { + if (Test-Path $repoPath) { + $existingFiles = @(Get-ChildItem $repoPath -Force) + if ($existingFiles.Count -gt 0) { + throw "$repoPath exists and is not an Aggie checkout. Choose another -InstallPath." + } + } else { + New-Item -ItemType Directory -Path (Split-Path $repoPath -Parent) -Force | Out-Null + } + + Write-Step "Cloning Aggie" + Invoke-CommandChecked "git" @( + "clone", "--branch", "develop", "https://github.com/InetIntel/Aggie.git", $repoPath + ) + } +} + +Set-Location $repoPath +Write-Host "Repository: $repoPath" +Write-Host "Node: $(& node --version)" +Write-Host "npm: $(& npm --version)" + +Write-Step "Starting Docker Desktop" +if (-not (Test-DockerReady)) { + $dockerDesktop = Join-Path $env:ProgramFiles "Docker\Docker\Docker Desktop.exe" + if (-not (Test-Path $dockerDesktop)) { + throw "Docker Desktop is installed but could not be found. Restart Windows, then rerun this script." + } + + Start-Process $dockerDesktop + $dockerReady = $false + for ($attempt = 0; $attempt -lt 60; $attempt++) { + Start-Sleep -Seconds 3 + if (Test-DockerReady) { + $dockerReady = $true + break + } + } + if (-not $dockerReady) { + throw "Docker Desktop did not become ready. Complete any Docker/WSL setup shown on screen, then rerun this script. Docker sign-in is not required." + } +} + +Write-Step "Starting local MongoDB" +$mongoContainer = & docker ps -a --filter "name=^/aggie-mongo$" --format "{{.Names}}" +if ($mongoContainer -eq "aggie-mongo") { + Invoke-CommandChecked "docker" @("start", "aggie-mongo") +} else { + Invoke-CommandChecked "docker" @( + "run", "--detach", "--name", "aggie-mongo", "--restart", "unless-stopped", + "--publish", "27017:27017", "--volume", "aggie-mongo-data:/data/db", "mongo:6.0" + ) +} + +$mongoReady = $false +for ($attempt = 0; $attempt -lt 30; $attempt++) { + & docker exec aggie-mongo mongosh --quiet --eval "db.adminCommand({ ping: 1 }).ok" *> $null + if ($LASTEXITCODE -eq 0) { + $mongoReady = $true + break + } + Start-Sleep -Seconds 2 +} +if (-not $mongoReady) { + throw "MongoDB did not become ready. Run 'docker logs aggie-mongo' for details." +} + +$createdAdminPassword = $null +if (-not (Test-Path ".env")) { + Write-Step "Creating local environment configuration" + $createdAdminPassword = New-RandomString -Length 20 + $appSecret = New-RandomString -Length 48 + @" +ENVIRONMENT=development +DATABASE_URL=mongodb://localhost:27017/ +DATABASE_NAME=aggie +REACT_APP_BASE_URL=http://localhost:3000/ +REACT_APP_PORT=8000 +SENDGRID_API_KEY= +SECRET=$appSecret +ADMIN_PARTY=false +ADMIN_EMAIL=admin@localhost +ADMIN_USERNAME=admin +ADMIN_PASSWORD=$createdAdminPassword +API_REQUEST_TIMEOUT=60000 +JWT_SESSION=false +DETECT_HATE_SPEECH=false +"@ | Set-Content ".env" -Encoding ASCII +} else { + Write-Host "Keeping existing .env file." +} + +$secretsPath = "backend\config\secrets.json" +if (-not (Test-Path $secretsPath)) { + Write-Step "Creating local service configuration" + $secrets = [ordered]@{ + twitter = [ordered]@{ API_key = ""; API_key_secret = ""; access_token = ""; access_token_secret = "" } + crowdtangle = [ordered]@{ count = 100; sortParam = "date"; language = "en"; useLanguage = $false; zawgyiProb = 0.9; interval = "1500" } + comments = [ordered]@{ username = ""; password = ""; baseUrl = ""; pageCount = 100 } + elmo = [ordered]@{ authToken = "" } + gplaces = [ordered]@{ key = "" } + "group map" = [ordered]@{ zoom = "7"; center = "Ghana"; latitude = ""; longitude = "" } + logger = [ordered]@{ + SES = [ordered]@{ disabled = $true; level = "error"; silent = $false } + Slack = [ordered]@{ disabled = $true; level = "error" } + file = [ordered]@{ disabled = $false; level = "warn"; filename = "logs/master.log" } + console = [ordered]@{ disabled = $false; level = "warn" } + api = [ordered]@{ log_requests = $true; log_responses = $true; log_user_activity = $false; filename = "logs/api.log" } + master = [ordered]@{ filename = "logs/master.log" } + fetching = [ordered]@{ filename = "logs/fetching.log" } + analytics = [ordered]@{ filename = "logs/analytics.log" } + } + api_request_timeout = 60 + fetching = $false + experiment = $false + detectHateSpeech = $false + } + $secrets | ConvertTo-Json -Depth 8 | Set-Content $secretsPath -Encoding ASCII +} else { + Write-Host "Keeping existing $secretsPath file." +} + +Write-Step "Checking npm registry access" +$env:NPM_CONFIG_REGISTRY = $NpmRegistry +if ($HttpsProxy) { + $env:HTTPS_PROXY = $HttpsProxy + $env:NPM_CONFIG_HTTPS_PROXY = $HttpsProxy +} +Invoke-CommandChecked "npm" @( + "ping", "--registry=$NpmRegistry", "--fetch-timeout=15000", "--fetch-retries=0" +) + +Write-Step "Installing Aggie dependencies and initializing the database" +Invoke-CommandChecked "npm" @("ci", "--no-audit", "--no-fund") + +Write-Step "Installing the development process runner" +Invoke-CommandChecked "npm" @("install", "--global", "stmux", "--no-audit", "--no-fund") + +Write-Step "Validating installation" +Invoke-CommandChecked "npm" @("ls", "--depth=0") +if (-not (Get-Command stmux -ErrorAction SilentlyContinue)) { + throw "stmux was installed but is not available on PATH. Open a new terminal, then rerun this script." +} +$adminCount = (& docker exec aggie-mongo mongosh aggie --quiet --eval 'db.users.countDocuments({ role: "admin" })').Trim() +if ([int]$adminCount -lt 1) { + throw "Dependencies installed, but the Aggie admin user was not created." +} + +Write-Host "`nAggie setup is complete." -ForegroundColor Green +Write-Host "Repository: $repoPath" +Write-Host "Start it with: npm run dev" +Write-Host "Frontend: http://localhost:8000" +if ($createdAdminPassword) { + Write-Host "Username: admin" + Write-Host "Password: $createdAdminPassword" + Write-Host "The password is also stored in the ignored .env file." +} \ No newline at end of file diff --git a/test/backend/lib.fetching.ooni-alerts.test.js b/test/backend/lib.fetching.ooni-alerts.test.js new file mode 100644 index 000000000..9517ccbd0 --- /dev/null +++ b/test/backend/lib.fetching.ooni-alerts.test.js @@ -0,0 +1,66 @@ +const expect = require('chai').expect; +const { normalizeDailyCounts, evaluateAlert } = require('../../backend/fetching/ooniAlerts'); + +describe('OONI alerts', function() { + it('fills omitted aggregation days with zero measurements', function() { + const rows = [ + { measurement_start_day: '2025-12-01', measurement_count: 70 }, + { measurement_start_day: '2025-12-03', measurement_count: 20 }, + ]; + + const daily = normalizeDailyCounts(rows, '2025-12-01', '2025-12-04'); + + expect(daily).to.deep.equal([ + { day: '2025-12-01', measurementCount: 70 }, + { day: '2025-12-02', measurementCount: 0 }, + { day: '2025-12-03', measurementCount: 20 }, + ]); + }); + + it('alerts when the completed day has zero measurements', function() { + const daily = [ + { day: '2025-12-15', measurementCount: 50 }, + { day: '2025-12-16', measurementCount: 0 }, + ]; + + const alerts = evaluateAlert(daily, '2025-12-17'); + + expect(alerts[0]).to.include({ + type: 'zero_measurements', + measurementDay: '2025-12-16', + measurementCount: 0, + }); + }); + + it('compares the latest two days with the prior 14 non-zero days', function() { + const daily = []; + for (let day = 1; day <= 16; day++) { + daily.push({ + day: `2025-12-${String(day).padStart(2, '0')}`, + measurementCount: day <= 14 ? 100 : 60, + }); + } + + const alerts = evaluateAlert(daily, '2025-12-17'); + + expect(alerts).to.have.length(1); + expect(alerts[0].type).to.equal('measurement_decline'); + expect(alerts[0].baselineNonZeroDays).to.equal(14); + expect(alerts[0].declineFraction).to.equal(0.4); + }); + + it('excludes zero values from the 14-day baseline', function() { + const daily = []; + for (let day = 1; day <= 16; day++) { + daily.push({ + day: `2025-12-${String(day).padStart(2, '0')}`, + measurementCount: day === 1 ? 0 : (day <= 14 ? 100 : 60), + }); + } + + const alerts = evaluateAlert(daily, '2025-12-17'); + + expect(alerts[0].baselineNonZeroDays).to.equal(13); + expect(alerts[0].baselineAverage).to.equal(100); + }); +}); \ No newline at end of file diff --git a/test/backend/lib.fetching.ooni-channel.test.js b/test/backend/lib.fetching.ooni-channel.test.js new file mode 100644 index 000000000..e727846c5 --- /dev/null +++ b/test/backend/lib.fetching.ooni-channel.test.js @@ -0,0 +1,33 @@ +const expect = require('chai').expect; +const OONIChannel = require('../../backend/fetching/channels/ooni'); +const Report = require('../../backend/models/report'); + +describe('OONI channel', function() { + it('creates a report from mocked OONI data', async function() { + const originalExists = Report.exists; + Report.exists = async () => false; + + const queued = []; + + const channel = new OONIChannel({ + asns: '44244', + fetchDailyMeasurements: async () => [ + { measurement_start_day: '2026-08-03', measurement_count: 0 }, + ], + fetchMeasurements: async () => [], + }); + + channel.enqueue = (post) => queued.push(post); + + try { + const posts = await channel.fetch(); + + expect(posts).to.have.length(1); + expect(queued).to.have.length(1); + expect(posts[0].platform).to.equal('OONI'); + expect(posts[0].platformID).to.match(/^ooni:44244:volume:/); + } finally { + Report.exists = originalExists; + } + }); +}); \ No newline at end of file From 2f0b9d58a60defe20e3ee35285015018f3eef476 Mon Sep 17 00:00:00 2001 From: Kadukuntla Sathvika Reddy Date: Tue, 4 Aug 2026 22:31:01 +0530 Subject: [PATCH 2/2] Use only zero-measurement OONI alerts --- backend/fetching/channels/ooni.js | 85 +--------- backend/fetching/ooniAlerts.js | 50 +----- backend/fetching/ooniApi.js | 35 ----- docs/OONI.md | 37 ++--- docs/everythingaboutooni.md | 146 +++--------------- scripts/backtest-ooni-alerts.js | 58 +------ test/backend/lib.fetching.ooni-alerts.test.js | 22 +-- .../backend/lib.fetching.ooni-channel.test.js | 1 - 8 files changed, 57 insertions(+), 377 deletions(-) diff --git a/backend/fetching/channels/ooni.js b/backend/fetching/channels/ooni.js index 516f9bd82..e09e1eaee 100644 --- a/backend/fetching/channels/ooni.js +++ b/backend/fetching/channels/ooni.js @@ -1,11 +1,10 @@ const { PollChannel } = require('downstream'); const Report = require('../../models/report'); -const { fetchDailyMeasurements, fetchMeasurements } = require('../ooniApi'); +const { fetchDailyMeasurements } = require('../ooniApi'); const { normalizeDailyCounts, evaluateAlert } = require('../ooniAlerts'); const DAY_MS = 24 * 60 * 60 * 1000; const ALERT_DELAY_HOURS = 6; -const EVIDENCE_LIMIT = 5; const NETWORK_NAMES = { 44244: 'IranCell', 58224: 'MCCI', @@ -32,29 +31,7 @@ function alertGuid(asn, alertDate) { function alertContent(asn, alerts) { const network = NETWORK_NAMES[asn] || `AS${asn}`; - const messages = alerts.map((alert) => { - if (alert.type === 'zero_measurements') { - return `no web connectivity measurements were recorded on ${alert.measurementDay}`; - } - return `the 2-day measurement average declined ${(alert.declineFraction * 100).toFixed(1)}%`; - }); - return `OONI volume alert for ${network} (AS${asn}): ${messages.join('; ')}.`; -} - -function measurementEvidence(measurement) { - return { - input: measurement.input, - measurementStartTime: measurement.measurement_start_time, - measurementUid: measurement.measurement_uid, - measurementUrl: measurement.measurement_url, - anomaly: measurement.anomaly, - confirmed: measurement.confirmed, - failure: measurement.failure, - blockingType: measurement.scores && measurement.scores.analysis - ? measurement.scores.analysis.blocking_type - : null, - verificationStatus: measurement.verification_status, - }; + return `OONI volume alert for ${network} (AS${asn}): no web connectivity measurements were recorded on ${alerts[0].measurementDay}.`; } class OONIChannel extends PollChannel { @@ -76,59 +53,11 @@ class OONIChannel extends PollChannel { this.asns = asns; this.interval = options.interval || OONIChannel.INTERVAL; this.fetchDailyMeasurements = options.fetchDailyMeasurements || fetchDailyMeasurements; - this.fetchMeasurements = options.fetchMeasurements || fetchMeasurements; - } - - async fetchEvidence(asn, alerts) { - const decline = alerts.find((alert) => alert.type === 'measurement_decline'); - if (!decline) { - return { - available: false, - reason: 'No measurements were available on the zero-measurement day.', - confirmed: [], - anomalous: [], - }; - } - - const query = { - asn, - since: decline.recentStart, - until: decline.alertDate, - }; - try { - const [confirmed, anomalous] = await Promise.all([ - this.fetchMeasurements({ ...query, confirmed: true, limit: EVIDENCE_LIMIT }), - this.fetchMeasurements({ - ...query, - anomaly: true, - confirmed: false, - limit: EVIDENCE_LIMIT, - }), - ]); - - return { - available: true, - windowStart: decline.recentStart, - windowEnd: decline.recentEnd, - confirmed: confirmed.slice(0, EVIDENCE_LIMIT).map(measurementEvidence), - anomalous: anomalous - .filter((measurement) => !measurement.confirmed) - .slice(0, EVIDENCE_LIMIT) - .map(measurementEvidence), - }; - } catch (error) { - return { - available: false, - reason: `Related measurements could not be loaded: ${error.message}`, - confirmed: [], - anomalous: [], - }; - } } async fetch() { const alertDate = alertDateFor(new Date()); - const since = dayString(shiftDay(alertDate, -16)); + const since = dayString(shiftDay(alertDate, -1)); const until = dayString(alertDate); const posts = []; @@ -141,8 +70,7 @@ class OONIChannel extends PollChannel { const guid = alertGuid(asn, dayString(alertDate)); if (await Report.exists({ guid })) continue; - const evidence = await this.fetchEvidence(asn, alerts); - const post = this.parse({ asn, alerts, evidence, guid, fetchedAt: new Date() }); + const post = this.parse({ asn, alerts, guid, fetchedAt: new Date() }); posts.push(post); this.enqueue(post); } @@ -151,13 +79,13 @@ class OONIChannel extends PollChannel { } parse(rawMessage) { - const { asn, alerts, evidence, guid, fetchedAt } = rawMessage; + const { asn, alerts, guid, fetchedAt } = rawMessage; const alertDate = alerts[0].alertDate; const searchParams = new URLSearchParams({ probe_cc: 'IR', probe_asn: `AS${asn}`, test_name: 'web_connectivity', - since: evidence.windowStart || alerts[0].measurementDay, + since: alerts[0].measurementDay, until: alertDate, }); @@ -176,7 +104,6 @@ class OONIChannel extends PollChannel { testName: 'web_connectivity', alertDate, triggers: alerts, - evidence, }, }; } diff --git a/backend/fetching/ooniAlerts.js b/backend/fetching/ooniAlerts.js index 4ce02a372..d1a301df0 100644 --- a/backend/fetching/ooniAlerts.js +++ b/backend/fetching/ooniAlerts.js @@ -12,10 +12,6 @@ function shiftDay(value, offset) { return new Date(value.getTime() + offset * DAY_MS); } -function mean(values) { - return values.reduce((total, value) => total + value, 0) / values.length; -} - function normalizeDailyCounts(rows, since, until) { const countsByDay = new Map(); rows.forEach((row) => { @@ -34,54 +30,24 @@ function normalizeDailyCounts(rows, since, until) { return normalized; } -function evaluateAlert(dailyCounts, alertDate, declineThreshold = 0.3) { +function evaluateAlert(dailyCounts, alertDate) { const date = toDay(alertDate); const countsByDay = new Map( dailyCounts.map(({ day, measurementCount }) => [day, measurementCount]), ); - const count = (offset) => countsByDay.get(dayString(shiftDay(date, offset))) || 0; - const alerts = []; const latestDay = dayString(shiftDay(date, -1)); - const latestCount = count(-1); + const latestCount = countsByDay.get(latestDay) || 0; + + if (latestCount !== 0) return []; - if (latestCount === 0) { - alerts.push({ + return [ + { type: 'zero_measurements', alertDate: dayString(date), measurementDay: latestDay, measurementCount: 0, - }); - } - - const recentCounts = [count(-2), count(-1)]; - const baselineCounts = []; - for (let offset = -16; offset <= -3; offset++) { - const value = count(offset); - if (value > 0) baselineCounts.push(value); - } - - if (baselineCounts.length > 0) { - const recentAverage = mean(recentCounts); - const baselineAverage = mean(baselineCounts); - const declineFraction = 1 - recentAverage / baselineAverage; - if (declineFraction >= declineThreshold) { - alerts.push({ - type: 'measurement_decline', - alertDate: dayString(date), - recentStart: dayString(shiftDay(date, -2)), - recentEnd: latestDay, - recentCounts, - recentAverage, - baselineStart: dayString(shiftDay(date, -16)), - baselineEnd: dayString(shiftDay(date, -3)), - baselineNonZeroDays: baselineCounts.length, - baselineAverage, - declineFraction, - }); - } - } - - return alerts; + }, + ]; } module.exports = { diff --git a/backend/fetching/ooniApi.js b/backend/fetching/ooniApi.js index ff4b57385..707afa1aa 100644 --- a/backend/fetching/ooniApi.js +++ b/backend/fetching/ooniApi.js @@ -22,42 +22,7 @@ async function fetchDailyMeasurements({ asn, since, until, fetchImpl = fetch }) return Array.isArray(result) ? result : [result]; } -async function fetchMeasurements({ - asn, - since, - until, - confirmed, - anomaly, - limit = 20, - fetchImpl = fetch, -}) { - const params = new URLSearchParams({ - probe_cc: 'IR', - probe_asn: `AS${asn}`, - test_name: 'web_connectivity', - since, - until, - limit: String(limit), - order_by: 'measurement_start_time', - order: 'desc', - }); - if (confirmed != null) params.set('confirmed', String(confirmed)); - if (anomaly != null) params.set('anomaly', String(anomaly)); - - const url = `https://api.ooni.org/api/v1/measurements?${params}`; - const response = await fetchImpl(url, { - headers: { accept: 'application/json' }, - }); - if (!response.ok) { - throw new Error(`OONI measurements request failed (${response.status}): ${url}`); - } - - const payload = await response.json(); - return payload.results || []; -} - module.exports = { AGGREGATION_URL, fetchDailyMeasurements, - fetchMeasurements, }; \ No newline at end of file diff --git a/docs/OONI.md b/docs/OONI.md index bec5133c6..8294683e0 100644 --- a/docs/OONI.md +++ b/docs/OONI.md @@ -3,22 +3,17 @@ Aggie polls OONI's public aggregation API for Iran `web_connectivity` measurement volume on AS44244 (IranCell) and AS58224 (MCCI). Alerts enter the normal Aggie report pipeline and use deterministic daily GUIDs, so repeated -polling does not create duplicate reports. When both rules trigger for an ASN -on the same date, they are grouped into one report. +polling does not create duplicate reports. ## Alert rules For an alert date `D`, only completed UTC days are evaluated: -1. `zero_measurements`: the measurement count on `D-1` is zero. OONI omits - zero-count days from its response, so missing calendar dates are filled with - zero before evaluation. -2. `measurement_decline`: the mean for `D-2` through `D-1` is at least 30% - below the mean of non-zero counts from `D-16` through `D-3`. - -The two windows are disjoint. A day can produce both alert types. Production -polling waits until 06:00 UTC before evaluating the previous UTC day, reducing -false zero alerts while OONI finishes publishing daily aggregates. +Aggie creates a `zero_measurements` alert when the measurement count on `D-1` +is zero. OONI omits zero-count days from its response, so missing calendar dates +are filled with zero before evaluation. Production polling waits until 06:00 UTC +before evaluating the previous UTC day, reducing false alerts while OONI +finishes publishing daily aggregates. ## Configure Aggie @@ -30,12 +25,8 @@ false zero alerts while OONI finishes publishing daily aggregates. 6. Enable the source and turn global fetching on. OONI alerts appear as normal reports with media type `OONI`. Report metadata -contains the ASN, alert type, daily counts, baseline average, recent average, -and decline fraction. For decline alerts, the detail view also includes up to -five confirmed and five non-confirmed anomalous measurements from the recent -two-day window. These measurements provide context and are not presented as the -cause of the volume decline. A zero-only alert has no same-day tests to attach. -The report link opens the corresponding OONI Explorer query. +contains the ASN, alert type, date, and zero measurement count. The report link +opens the corresponding OONI Explorer query. ## Historical backtest @@ -49,13 +40,5 @@ The optional date is the alert date through which to evaluate. Output is written to the ignored `data/ooni-alert-backtest.json` and `data/ooni-alert-backtest.csv` files. -For December 1, 2025 through July 30, 2026, the grouped backtest produced: - -| ASN | Reports | Zero triggers | Decline triggers | -| --- | ---: | ---: | ---: | -| AS44244 (IranCell) | 124 | 89 | 115 | -| AS58224 (MCCI) | 109 | 98 | 68 | -| Total | 233 | 187 | 183 | - -Grouping produces 137 fewer reports than emitting each trigger separately, a -37% reduction. There is no additional cooldown or incident-level suppression. \ No newline at end of file +The backtest emits only zero-measurement alerts. There is no additional cooldown +or incident-level suppression. \ No newline at end of file diff --git a/docs/everythingaboutooni.md b/docs/everythingaboutooni.md index f051d73b9..0d152d094 100644 --- a/docs/everythingaboutooni.md +++ b/docs/everythingaboutooni.md @@ -20,12 +20,8 @@ completed UTC days. For alert date `D`: -1. `zero_measurements`: the total `measurement_count` on `D-1` is zero. -2. `measurement_decline`: the average measurement count for `D-2` through - `D-1` is at least 30% below the average for `D-16` through `D-3`. - -Only non-zero values are included when calculating the 14-day baseline. The -recent two-day window and the preceding 14-day baseline do not overlap. +Aggie creates a `zero_measurements` alert when the total `measurement_count` on +`D-1` is zero. Nonzero measurement counts do not generate an alert. These are measurement-volume alerts aggregated across all `web_connectivity` tests for an ASN. They are not individual website-failure alerts. @@ -36,10 +32,7 @@ The OONI channel polls hourly. It waits until 06:00 UTC before evaluating the previous completed day. This delay gives OONI time to publish the previous day's aggregates and reduces false zero-measurement alerts. -## Grouped Reports - -If zero volume and a 30% decline trigger for the same ASN and alert date, Aggie -creates one report containing both triggers. +## Report Deduplication The report GUID follows this format: @@ -56,37 +49,6 @@ ooni:44244:volume:2026-01-10 This deterministic GUID prevents duplicate reports when the hourly poll runs again. -## Supporting Website Evidence - -A decline report includes selected website-level OONI measurements from its -recent two-day window: - -- Up to five confirmed measurements. -- Up to five anomalous but non-confirmed measurements. - -Confirmed and anomalous entries do not overlap. - -These measurements provide context. They are not presented as proof that a -particular website caused the measurement-volume decline. - -A zero-only alert has no measurements for the zero day, so it displays: - -```text -No measurements were available on the zero-measurement day. -``` - -If the optional evidence request fails, the core volume alert is still stored. -The report displays the evidence-loading error instead of dropping the alert. - -### OONI Result Meanings - -- `confirmed: true`: OONI found stronger interference evidence, such as a known - block page or middlebox fingerprint. -- `anomaly: true`: the result needs attention and may indicate interference, but - it is not confirmed. -- `failure: true`: the test or control request failed. This does not - automatically mean censorship. - ## Example Stored Report ```json @@ -94,7 +56,7 @@ The report displays the evidence-loading error instead of dropping the alert. "guid": "ooni:44244:volume:2026-01-10", "authoredAt": "2026-01-10T00:00:00.000Z", "author": "OONI AS44244", - "content": "OONI volume alert for IranCell (AS44244): no web connectivity measurements were recorded on 2026-01-09; the 2-day measurement average declined 100.0%.", + "content": "OONI volume alert for IranCell (AS44244): no web connectivity measurements were recorded on 2026-01-09.", "_media": ["OONI"], "metadata": { "rawAPIResponse": { @@ -106,29 +68,11 @@ The report displays the evidence-loading error instead of dropping the alert. "triggers": [ { "type": "zero_measurements", + "alertDate": "2026-01-10", "measurementDay": "2026-01-09", "measurementCount": 0 - }, - { - "type": "measurement_decline", - "recentStart": "2026-01-08", - "recentEnd": "2026-01-09", - "recentCounts": [0, 0], - "recentAverage": 0, - "baselineStart": "2025-12-25", - "baselineEnd": "2026-01-07", - "baselineNonZeroDays": 14, - "baselineAverage": 1216.43, - "declineFraction": 1 } - ], - "evidence": { - "available": true, - "windowStart": "2026-01-08", - "windowEnd": "2026-01-09", - "confirmed": [], - "anomalous": [] - } + ] } } } @@ -140,12 +84,10 @@ The report displays the evidence-loading error instead of dropping the alert. flowchart LR A[OONI Aggregation API] --> B[OONI Poll Channel] B --> C[Normalize Missing Days] - C --> D[Evaluate Alert Rules] - D --> E[Group Daily Triggers] - E --> F[Fetch Supporting Measurements] - F --> G[Aggie Post-to-Report Hook] - G --> H[MongoDB Reports Collection] - H --> I[OONI Report Detail View] + C --> D[Check Previous Day for Zero] + D --> E[Aggie Post-to-Report Hook] + E --> F[MongoDB Reports Collection] + F --> G[OONI Report Detail View] ``` ## Database Storage @@ -155,11 +97,8 @@ They contain: - Alert summary and timestamps. - ASN and network name. -- Trigger details. -- Recent and baseline counts. -- Decline percentage. -- Selected confirmed and anomalous measurements. -- OONI measurement and Explorer links. +- Zero-measurement trigger details. +- OONI Explorer link. - Aggie source and media identifiers. - A deterministic deduplication GUID. @@ -171,22 +110,14 @@ contain an API token. ### `backend/fetching/ooniApi.js` -Calls the public OONI APIs: - -- Aggregation endpoint for daily measurement counts. -- Measurements endpoint for confirmed and anomalous evidence. - -The evidence endpoint is queried directly for at most five confirmed and five -anomalous, non-confirmed measurements. +Calls the public OONI aggregation API for daily measurement counts. ### `backend/fetching/ooniAlerts.js` Contains dependency-free alert logic: - Fills omitted calendar dates with zero measurements. -- Calculates the two-day recent average. -- Calculates the preceding 14-day non-zero baseline. -- Produces zero and decline trigger objects. +- Produces a zero-measurement trigger when the previous completed day is zero. Keeping this logic separate allows the real-time channel and historical backtest to use the same calculations. @@ -199,11 +130,9 @@ Implements the native Aggie `PollChannel`. It: - Polls OONI hourly. - Applies the 06:00 UTC publication delay. - Fetches daily aggregate counts. -- Evaluates and groups triggers. -- Fetches bounded supporting evidence. +- Evaluates the previous completed day for zero measurements. - Creates deterministic GUIDs. - Enqueues reports through Aggie's normal fetching pipeline. -- Preserves the core alert when evidence lookup fails. ### `backend/fetching/sourceToChannel.js` @@ -255,16 +184,8 @@ Displays a globe icon for OONI reports. Renders the expanded OONI report view with: - Network, ASN, and alert date. -- Every triggered volume rule. -- Recent daily counts. -- Recent and baseline averages. -- Baseline period and non-zero day count. -- Decline percentage. -- Up to five confirmed measurements. -- Up to five anomalous measurements. -- Blocking type and verification status. -- Links to raw OONI measurements. -- Clear language distinguishing context from causation. +- The zero-measurement trigger. +- A link to the corresponding OONI Explorer query. ### `src/components/SocialMediaPost/index.tsx` @@ -298,9 +219,7 @@ Tests: - Missing aggregation days becoming zero counts. - Zero-measurement triggers. -- The 30% decline calculation. -- Disjoint recent and baseline windows. -- Exclusion of zero values from the baseline. +- Nonzero completed days producing no alert. ### `package.json` @@ -310,18 +229,8 @@ Adds the `backtest:ooni` command. Backtest period: December 1, 2025 through July 30, 2026. -| ASN | Grouped reports | Zero triggers | Decline triggers | -| --- | ---: | ---: | ---: | -| AS44244, IranCell | 124 | 89 | 115 | -| AS58224, MCCI | 109 | 98 | 68 | -| Total | 233 | 187 | 183 | - -Emitting every trigger separately would have generated 370 reports. Grouping -both rules by ASN and date produces 233 reports, which is 137 fewer reports and -a 37% reduction. - -The trigger totals exceed the report total because one grouped report can -contain both rules. +The backtest emits only dates where the previous completed day has zero +measurements. Re-run the backtest to generate current zero-only totals. ## Aggie Configuration @@ -334,7 +243,7 @@ contain both rules. 7. Enable the source. 8. Turn global fetching on. -After configuration, future qualifying dips create reports automatically. +After configuration, future zero-measurement days create reports automatically. ## Validation Completed @@ -342,13 +251,8 @@ The following checks passed: - JavaScript syntax checks for all OONI backend files. - Editor diagnostics for modified JavaScript and TypeScript files. -- Synthetic zero-day and 30% decline checks. -- Non-zero baseline exclusion check. +- Synthetic zero-day and nonzero-day checks. - Live OONI aggregation requests for both ASNs. -- Live OONI confirmed-measurement query. -- Live OONI anomalous and non-confirmed measurement query. -- Five-confirmed and five-anomalous evidence limits. -- Grouped JSON and CSV reconciliation at 233 reports. - Git whitespace validation. A full npm build and full repository test run remain pending because this @@ -358,10 +262,8 @@ has been validated against live data. ## Current Limitations -- There is no incident-level cooldown beyond grouping rules by ASN and day. -- Website evidence is attached only to decline alerts, because zero days have no - measurements. -- Evidence is a bounded sample, not every measurement in the period. +- There is no incident-level cooldown beyond one deterministic report per ASN + and alert date. - The production channel must be enabled in Aggie before polling begins. - End-to-end runtime verification requires the repository dependencies to be installed. diff --git a/scripts/backtest-ooni-alerts.js b/scripts/backtest-ooni-alerts.js index ef6a05b3d..42fec249c 100644 --- a/scripts/backtest-ooni-alerts.js +++ b/scripts/backtest-ooni-alerts.js @@ -21,7 +21,7 @@ function csvValue(value) { async function main() { const alertUntil = process.argv[2] || new Date().toISOString().slice(0, 10); - const dataSince = shiftDay(ALERT_SINCE, -16); + const dataSince = shiftDay(ALERT_SINCE, -1); const dataUntil = alertUntil; const output = []; @@ -46,69 +46,25 @@ async function main() { const outputPath = path.join(outputDirectory, 'ooni-alert-backtest.json'); fs.writeFileSync(outputPath, `${JSON.stringify(output, null, 2)}\n`); const csvFields = [ - 'asn', 'networkName', 'alertDate', 'triggerTypes', 'measurementDay', - 'measurementCount', 'recentStart', 'recentEnd', 'recentCounts', - 'recentAverage', 'baselineStart', 'baselineEnd', 'baselineNonZeroDays', - 'baselineAverage', 'declineFraction', + 'asn', 'networkName', 'alertDate', 'triggerType', 'measurementDay', + 'measurementCount', ]; const csvRows = output.map((alert) => { - const zero = alert.triggers.find((trigger) => trigger.type === 'zero_measurements') || {}; - const decline = alert.triggers.find((trigger) => trigger.type === 'measurement_decline') || {}; + const trigger = alert.triggers[0]; const row = { ...alert, - triggerTypes: alert.triggers.map((trigger) => trigger.type), - ...zero, - ...decline, + triggerType: trigger.type, + ...trigger, }; return csvFields.map((field) => csvValue(row[field])).join(','); }); const csvPath = path.join(outputDirectory, 'ooni-alert-backtest.csv'); fs.writeFileSync(csvPath, `${csvFields.join(',')}\n${csvRows.join('\n')}\n`); - const triggerCsvFields = [ - 'asn', 'networkName', 'alertDate', 'triggerType', 'zeroMeasurementDay', - 'zeroMeasurementCount', 'recentStart', 'recentEnd', 'recentCounts', - 'recentAverage', 'baselineStart', 'baselineEnd', 'baselineNonZeroDays', - 'baselineAverage', 'declineFraction', 'declinePercent', - ]; - const triggerCsvRows = output.flatMap((alert) => { - // When both triggers fire on the same day, keep only the zero-measurement alert. - const hasZero = alert.triggers.some((trigger) => trigger.type === 'zero_measurements'); - const triggers = hasZero - ? alert.triggers.filter((trigger) => trigger.type === 'zero_measurements') - : alert.triggers; - return triggers.map((trigger) => { - const row = { - asn: alert.asn, - networkName: alert.networkName, - alertDate: alert.alertDate, - triggerType: trigger.type, - ...trigger, - zeroMeasurementDay: trigger.type === 'zero_measurements' - ? trigger.measurementDay - : null, - zeroMeasurementCount: trigger.type === 'zero_measurements' - ? trigger.measurementCount - : null, - declinePercent: trigger.declineFraction == null - ? null - : trigger.declineFraction * 100, - }; - return triggerCsvFields.map((field) => csvValue(row[field])).join(','); - }); - }); - const triggerCsvPath = path.join(outputDirectory, 'ooni-alert-triggers.csv'); - fs.writeFileSync( - triggerCsvPath, - `${triggerCsvFields.join(',')}\n${triggerCsvRows.join('\n')}\n`, - ); console.log(`Wrote ${output.length} alerts to ${outputPath}`); console.log(`Wrote CSV output to ${csvPath}`); - console.log(`Wrote ${triggerCsvRows.length} individual triggers to ${triggerCsvPath}`); ASNS.forEach((asn) => { const alerts = output.filter((alert) => alert.asn === asn); - const zeros = alerts.filter((alert) => alert.triggers.some((trigger) => trigger.type === 'zero_measurements')).length; - const declines = alerts.filter((alert) => alert.triggers.some((trigger) => trigger.type === 'measurement_decline')).length; - console.log(`AS${asn}: ${alerts.length} reports (${zeros} zero-measurement triggers, ${declines} decline triggers)`); + console.log(`AS${asn}: ${alerts.length} zero-measurement reports`); }); } diff --git a/test/backend/lib.fetching.ooni-alerts.test.js b/test/backend/lib.fetching.ooni-alerts.test.js index 9517ccbd0..44c8145d0 100644 --- a/test/backend/lib.fetching.ooni-alerts.test.js +++ b/test/backend/lib.fetching.ooni-alerts.test.js @@ -32,7 +32,7 @@ describe('OONI alerts', function() { }); }); - it('compares the latest two days with the prior 14 non-zero days', function() { + it('does not alert when the completed day has measurements', function() { const daily = []; for (let day = 1; day <= 16; day++) { daily.push({ @@ -43,24 +43,6 @@ describe('OONI alerts', function() { const alerts = evaluateAlert(daily, '2025-12-17'); - expect(alerts).to.have.length(1); - expect(alerts[0].type).to.equal('measurement_decline'); - expect(alerts[0].baselineNonZeroDays).to.equal(14); - expect(alerts[0].declineFraction).to.equal(0.4); - }); - - it('excludes zero values from the 14-day baseline', function() { - const daily = []; - for (let day = 1; day <= 16; day++) { - daily.push({ - day: `2025-12-${String(day).padStart(2, '0')}`, - measurementCount: day === 1 ? 0 : (day <= 14 ? 100 : 60), - }); - } - - const alerts = evaluateAlert(daily, '2025-12-17'); - - expect(alerts[0].baselineNonZeroDays).to.equal(13); - expect(alerts[0].baselineAverage).to.equal(100); + expect(alerts).to.deep.equal([]); }); }); \ No newline at end of file diff --git a/test/backend/lib.fetching.ooni-channel.test.js b/test/backend/lib.fetching.ooni-channel.test.js index e727846c5..8ad2a4ee2 100644 --- a/test/backend/lib.fetching.ooni-channel.test.js +++ b/test/backend/lib.fetching.ooni-channel.test.js @@ -14,7 +14,6 @@ describe('OONI channel', function() { fetchDailyMeasurements: async () => [ { measurement_start_day: '2026-08-03', measurement_count: 0 }, ], - fetchMeasurements: async () => [], }); channel.enqueue = (post) => queued.push(post);