diff --git a/backend/fetching/channels/ooni.js b/backend/fetching/channels/ooni.js new file mode 100644 index 000000000..e09e1eaee --- /dev/null +++ b/backend/fetching/channels/ooni.js @@ -0,0 +1,112 @@ +const { PollChannel } = require('downstream'); +const Report = require('../../models/report'); +const { fetchDailyMeasurements } = require('../ooniApi'); +const { normalizeDailyCounts, evaluateAlert } = require('../ooniAlerts'); + +const DAY_MS = 24 * 60 * 60 * 1000; +const ALERT_DELAY_HOURS = 6; +const NETWORK_NAMES = { + 44244: 'IranCell', + 58224: 'MCCI', +}; + +function shiftDay(day, offset) { + return new Date(day.getTime() + offset * DAY_MS); +} + +function dayString(day) { + return day.toISOString().slice(0, 10); +} + +function alertDateFor(now) { + const date = new Date(now); + date.setUTCHours(0, 0, 0, 0); + if (now.getUTCHours() < ALERT_DELAY_HOURS) return shiftDay(date, -1); + return date; +} + +function alertGuid(asn, alertDate) { + return `ooni:${asn}:volume:${alertDate}`; +} + +function alertContent(asn, alerts) { + const network = NETWORK_NAMES[asn] || `AS${asn}`; + return `OONI volume alert for ${network} (AS${asn}): no web connectivity measurements were recorded on ${alerts[0].measurementDay}.`; +} + +class OONIChannel extends PollChannel { + static INTERVAL = 60 * 60 * 1000; + + constructor(options) { + const asns = String(options.asns || '') + .split(/[\s,]+/) + .filter(Boolean) + .map(Number); + if (asns.length === 0 || asns.some((asn) => !Number.isInteger(asn) || asn <= 0)) { + throw new Error('OONI sources require one or more valid ASNs.'); + } + + super({ + ...options, + namespace: options.namespace || `ooni-${asns.join('-')}`, + }); + this.asns = asns; + this.interval = options.interval || OONIChannel.INTERVAL; + this.fetchDailyMeasurements = options.fetchDailyMeasurements || fetchDailyMeasurements; + } + + async fetch() { + const alertDate = alertDateFor(new Date()); + const since = dayString(shiftDay(alertDate, -1)); + const until = dayString(alertDate); + const posts = []; + + for (const asn of this.asns) { + const rows = await this.fetchDailyMeasurements({ asn, since, until }); + const dailyCounts = normalizeDailyCounts(rows, since, until); + const alerts = evaluateAlert(dailyCounts, dayString(alertDate)); + if (alerts.length === 0) continue; + + const guid = alertGuid(asn, dayString(alertDate)); + if (await Report.exists({ guid })) continue; + + const post = this.parse({ asn, alerts, guid, fetchedAt: new Date() }); + posts.push(post); + this.enqueue(post); + } + + return posts; + } + + parse(rawMessage) { + const { asn, alerts, guid, fetchedAt } = rawMessage; + const alertDate = alerts[0].alertDate; + const searchParams = new URLSearchParams({ + probe_cc: 'IR', + probe_asn: `AS${asn}`, + test_name: 'web_connectivity', + since: alerts[0].measurementDay, + until: alertDate, + }); + + return { + authoredAt: new Date(`${alertDate}T00:00:00.000Z`), + fetchedAt, + author: `OONI AS${asn}`, + content: alertContent(asn, alerts), + url: `https://explorer.ooni.org/search?${searchParams}`, + platform: 'OONI', + platformID: guid, + raw: { + probeCC: 'IR', + probeASN: asn, + networkName: NETWORK_NAMES[asn] || null, + testName: 'web_connectivity', + alertDate, + triggers: alerts, + }, + }; + } +} + +module.exports = OONIChannel; \ No newline at end of file diff --git a/backend/fetching/hooks/postToReport.js b/backend/fetching/hooks/postToReport.js index bc8e118eb..201c132b0 100644 --- a/backend/fetching/hooks/postToReport.js +++ b/backend/fetching/hooks/postToReport.js @@ -29,7 +29,7 @@ module.exports = async function postToReport(post, next) { let metadata; - if (platform === 'RSS') { + if (platform === 'RSS' || platform === 'OONI') { // What do we want here? metadata = { // title: raw.title || null, diff --git a/backend/fetching/ooniAlerts.js b/backend/fetching/ooniAlerts.js new file mode 100644 index 000000000..d1a301df0 --- /dev/null +++ b/backend/fetching/ooniAlerts.js @@ -0,0 +1,56 @@ +const DAY_MS = 24 * 60 * 60 * 1000; + +function toDay(value) { + return new Date(`${value.slice(0, 10)}T00:00:00.000Z`); +} + +function dayString(value) { + return value.toISOString().slice(0, 10); +} + +function shiftDay(value, offset) { + return new Date(value.getTime() + offset * DAY_MS); +} + +function normalizeDailyCounts(rows, since, until) { + const countsByDay = new Map(); + rows.forEach((row) => { + const day = (row.measurement_start_day || '').slice(0, 10); + if (day) countsByDay.set(day, Number(row.measurement_count) || 0); + }); + + const normalized = []; + for (let day = toDay(since); day < toDay(until); day = shiftDay(day, 1)) { + const key = dayString(day); + normalized.push({ + day: key, + measurementCount: countsByDay.get(key) || 0, + }); + } + return normalized; +} + +function evaluateAlert(dailyCounts, alertDate) { + const date = toDay(alertDate); + const countsByDay = new Map( + dailyCounts.map(({ day, measurementCount }) => [day, measurementCount]), + ); + const latestDay = dayString(shiftDay(date, -1)); + const latestCount = countsByDay.get(latestDay) || 0; + + if (latestCount !== 0) return []; + + return [ + { + type: 'zero_measurements', + alertDate: dayString(date), + measurementDay: latestDay, + measurementCount: 0, + }, + ]; +} + +module.exports = { + normalizeDailyCounts, + evaluateAlert, +}; \ No newline at end of file diff --git a/backend/fetching/ooniApi.js b/backend/fetching/ooniApi.js new file mode 100644 index 000000000..707afa1aa --- /dev/null +++ b/backend/fetching/ooniApi.js @@ -0,0 +1,28 @@ +const AGGREGATION_URL = 'https://api.ooni.org/api/v1/aggregation'; + +async function fetchDailyMeasurements({ asn, since, until, fetchImpl = fetch }) { + const params = new URLSearchParams({ + probe_cc: 'IR', + probe_asn: String(asn), + test_name: 'web_connectivity', + axis_x: 'measurement_start_day', + since, + until, + }); + const url = `${AGGREGATION_URL}?${params}`; + const response = await fetchImpl(url, { + headers: { accept: 'application/json' }, + }); + if (!response.ok) { + throw new Error(`OONI aggregation request failed (${response.status}): ${url}`); + } + + const payload = await response.json(); + const result = payload.result || []; + return Array.isArray(result) ? result : [result]; +} + +module.exports = { + AGGREGATION_URL, + fetchDailyMeasurements, +}; \ No newline at end of file diff --git a/backend/fetching/sourceToChannel.js b/backend/fetching/sourceToChannel.js index 91b64ad26..d6a86a740 100644 --- a/backend/fetching/sourceToChannel.js +++ b/backend/fetching/sourceToChannel.js @@ -12,6 +12,7 @@ const downstream = require('./downstream'); const AggieCrowdTangleChannel = require('./channels/crowdtangle'); const TelegramChannel = require('./channels/telegram'); const RSSChannel = require('./channels/rss'); +const OONIChannel = require('./channels/ooni'); const { TwitterPageChannel, JunkipediaChannel } = builtin; @@ -187,6 +188,13 @@ function createChannel(source) { }; channel = new RSSChannel(options); break; + case 'ooni': + options = { + ...options, + asns: lists, + }; + channel = new OONIChannel(options); + break; default: } diff --git a/backend/models/credentials.js b/backend/models/credentials.js index 16d37c695..513667b22 100644 --- a/backend/models/credentials.js +++ b/backend/models/credentials.js @@ -15,7 +15,8 @@ const credentialsTypes = [ 'crowdtangle', 'telegram', 'junkipedia', - 'rss' + 'rss', + 'ooni' ]; // validates secrete based on their type diff --git a/backend/models/source.js b/backend/models/source.js index ab14b87c6..18be9296d 100644 --- a/backend/models/source.js +++ b/backend/models/source.js @@ -25,7 +25,7 @@ var urlValidator = function (url) { ) } -var mediaValues = ['facebook', 'instagram', 'comments', 'elmo', 'twitter', 'rss', 'dummy', 'smsgh', 'whatsapp', 'telegram', 'junkipedia', 'dummy-pull', 'dummy-fast']; +var mediaValues = ['facebook', 'instagram', 'comments', 'elmo', 'twitter', 'rss', 'ooni', 'dummy', 'smsgh', 'whatsapp', 'telegram', 'junkipedia', 'dummy-pull', 'dummy-fast']; var sourceSchema = new mongoose.Schema({ media: { type: String, enum: mediaValues }, diff --git a/docs/OONI.md b/docs/OONI.md new file mode 100644 index 000000000..8294683e0 --- /dev/null +++ b/docs/OONI.md @@ -0,0 +1,44 @@ +# OONI integration + +Aggie polls OONI's public aggregation API for Iran `web_connectivity` +measurement volume on AS44244 (IranCell) and AS58224 (MCCI). Alerts enter the +normal Aggie report pipeline and use deterministic daily GUIDs, so repeated +polling does not create duplicate reports. + +## Alert rules + +For an alert date `D`, only completed UTC days are evaluated: + +Aggie creates a `zero_measurements` alert when the measurement count on `D-1` +is zero. OONI omits zero-count days from its response, so missing calendar dates +are filled with zero before evaluation. Production polling waits until 06:00 UTC +before evaluating the previous UTC day, reducing false alerts while OONI +finishes publishing daily aggregates. + +## Configure Aggie + +1. Open Settings, then Credentials. +2. Create an `ooni` credential. OONI is public, so no token is requested. +3. Open Settings, then Sources. +4. Create an `ooni` source and select the credential. +5. Keep the default ASN list `44244 58224`. +6. Enable the source and turn global fetching on. + +OONI alerts appear as normal reports with media type `OONI`. Report metadata +contains the ASN, alert type, date, and zero measurement count. The report link +opens the corresponding OONI Explorer query. + +## Historical backtest + +Run the same evaluator used by the production channel: + +```powershell +npm run backtest:ooni -- 2026-07-30 +``` + +The optional date is the alert date through which to evaluate. Output is written +to the ignored `data/ooni-alert-backtest.json` and +`data/ooni-alert-backtest.csv` files. + +The backtest emits only zero-measurement alerts. There is no additional cooldown +or incident-level suppression. \ No newline at end of file diff --git a/docs/everythingaboutooni.md b/docs/everythingaboutooni.md new file mode 100644 index 000000000..0d152d094 --- /dev/null +++ b/docs/everythingaboutooni.md @@ -0,0 +1,269 @@ +# Everything About the OONI Integration + +## Purpose + +This integration adds OONI `web_connectivity` measurement-volume monitoring to +Aggie for two Iranian mobile networks: + +| Network | ASN | +| --- | --- | +| IranCell | AS44244 | +| MCCI | AS58224 | + +The integration uses Aggie's existing source, polling, report, and MongoDB +infrastructure. It does not introduce a separate service or database. + +## Requested Alert Rules + +For each ASN, Aggie evaluates the OONI measurement volume once a day using only +completed UTC days. + +For alert date `D`: + +Aggie creates a `zero_measurements` alert when the total `measurement_count` on +`D-1` is zero. Nonzero measurement counts do not generate an alert. + +These are measurement-volume alerts aggregated across all `web_connectivity` +tests for an ASN. They are not individual website-failure alerts. + +## Daily Timing + +The OONI channel polls hourly. It waits until 06:00 UTC before evaluating the +previous completed day. This delay gives OONI time to publish the previous day's +aggregates and reduces false zero-measurement alerts. + +## Report Deduplication + +The report GUID follows this format: + +```text +ooni::volume: +``` + +For example: + +```text +ooni:44244:volume:2026-01-10 +``` + +This deterministic GUID prevents duplicate reports when the hourly poll runs +again. + +## Example Stored Report + +```json +{ + "guid": "ooni:44244:volume:2026-01-10", + "authoredAt": "2026-01-10T00:00:00.000Z", + "author": "OONI AS44244", + "content": "OONI volume alert for IranCell (AS44244): no web connectivity measurements were recorded on 2026-01-09.", + "_media": ["OONI"], + "metadata": { + "rawAPIResponse": { + "probeCC": "IR", + "probeASN": 44244, + "networkName": "IranCell", + "testName": "web_connectivity", + "alertDate": "2026-01-10", + "triggers": [ + { + "type": "zero_measurements", + "alertDate": "2026-01-10", + "measurementDay": "2026-01-09", + "measurementCount": 0 + } + ] + } + } +} +``` + +## Architecture + +```mermaid +flowchart LR + A[OONI Aggregation API] --> B[OONI Poll Channel] + B --> C[Normalize Missing Days] + C --> D[Check Previous Day for Zero] + D --> E[Aggie Post-to-Report Hook] + E --> F[MongoDB Reports Collection] + F --> G[OONI Report Detail View] +``` + +## Database Storage + +OONI alerts use the same MongoDB `reports` collection as other Aggie reports. +They contain: + +- Alert summary and timestamps. +- ASN and network name. +- Zero-measurement trigger details. +- OONI Explorer link. +- Aggie source and media identifiers. +- A deterministic deduplication GUID. + +The OONI source and polling configuration use Aggie's existing `sources` and +`credentials` collections. OONI is public, so its credential record does not +contain an API token. + +## Backend Files + +### `backend/fetching/ooniApi.js` + +Calls the public OONI aggregation API for daily measurement counts. + +### `backend/fetching/ooniAlerts.js` + +Contains dependency-free alert logic: + +- Fills omitted calendar dates with zero measurements. +- Produces a zero-measurement trigger when the previous completed day is zero. + +Keeping this logic separate allows the real-time channel and historical +backtest to use the same calculations. + +### `backend/fetching/channels/ooni.js` + +Implements the native Aggie `PollChannel`. It: + +- Parses configured ASNs. +- Polls OONI hourly. +- Applies the 06:00 UTC publication delay. +- Fetches daily aggregate counts. +- Evaluates the previous completed day for zero measurements. +- Creates deterministic GUIDs. +- Enqueues reports through Aggie's normal fetching pipeline. + +### `backend/fetching/sourceToChannel.js` + +Creates an `OONIChannel` when Aggie loads a source with media type `ooni`. +The source's existing `lists` field stores space- or comma-separated ASNs. + +### `backend/fetching/hooks/postToReport.js` + +Converts OONI channel posts into standard Aggie reports and stores OONI data in +`metadata.rawAPIResponse`. + +### `backend/models/source.js` + +Adds `ooni` to the valid source media types. + +### `backend/models/credentials.js` + +Adds `ooni` as a valid credential type. The record exists to follow Aggie's +current source/credential relationship, but no secret is required. + +## Frontend Files + +### `src/pages/Settings/Credentials/CreateCredentialForm.tsx` + +Adds a token-free OONI credential form. + +### `src/pages/Settings/source/CreateEditSourceForm.tsx` + +Adds the OONI source form. Its default ASN list is: + +```text +44244 58224 +``` + +### `src/api/common.ts` + +Adds OONI to frontend media and credential options. + +### `src/objectTypes.d.ts` + +Adds `OONI` to the frontend media type declaration. + +### `src/components/SocialMediaPost/SocialMediaIcon.tsx` + +Displays a globe icon for OONI reports. + +### `src/components/SocialMediaPost/OONIPost.tsx` + +Renders the expanded OONI report view with: + +- Network, ASN, and alert date. +- The zero-measurement trigger. +- A link to the corresponding OONI Explorer query. + +### `src/components/SocialMediaPost/index.tsx` + +Routes reports with media type `OONI` to `OONIPost`. + +## Backtest and Tests + +### `scripts/backtest-ooni-alerts.js` + +Runs the same alert evaluator used by the production channel from December 1, +2025 through a specified end date. + +Run it with: + +```powershell +npm run backtest:ooni -- 2026-07-30 +``` + +It writes: + +```text +data/ooni-alert-backtest.json +data/ooni-alert-backtest.csv +``` + +The `data` directory is ignored by Git. + +### `test/backend/lib.fetching.ooni-alerts.test.js` + +Tests: + +- Missing aggregation days becoming zero counts. +- Zero-measurement triggers. +- Nonzero completed days producing no alert. + +### `package.json` + +Adds the `backtest:ooni` command. + +## Historical Results + +Backtest period: December 1, 2025 through July 30, 2026. + +The backtest emits only dates where the previous completed day has zero +measurements. Re-run the backtest to generate current zero-only totals. + +## Aggie Configuration + +1. Open Settings and then Credentials. +2. Create an `ooni` credential. No token is required. +3. Open Settings and then Sources. +4. Create an `ooni` source. +5. Select the OONI credential. +6. Keep or enter the ASN list `44244 58224`. +7. Enable the source. +8. Turn global fetching on. + +After configuration, future zero-measurement days create reports automatically. + +## Validation Completed + +The following checks passed: + +- JavaScript syntax checks for all OONI backend files. +- Editor diagnostics for modified JavaScript and TypeScript files. +- Synthetic zero-day and nonzero-day checks. +- Live OONI aggregation requests for both ASNs. +- Git whitespace validation. + +A full npm build and full repository test run remain pending because this +machine's network policy currently rejects TLS connections to +`registry.npmjs.org`. The OONI API itself is reachable and the integration logic +has been validated against live data. + +## Current Limitations + +- There is no incident-level cooldown beyond one deterministic report per ASN + and alert date. +- The production channel must be enabled in Aggie before polling begins. +- End-to-end runtime verification requires the repository dependencies to be + installed. diff --git a/package.json b/package.json index 087216d1c..252b852a3 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "dev:frontend": "cross-env PORT=8000 react-scripts start", "dev:backend": "cross-env ENVIRONMENT=development nodemon app.js", "dev:all": "npm run dev", + "backtest:ooni": "node scripts/backtest-ooni-alerts.js", "build": "cross-env CI=false react-scripts build", "start": "ENVIRONMENT=production node app.js", "postinstall": "node install.js" diff --git a/scripts/backtest-ooni-alerts.js b/scripts/backtest-ooni-alerts.js new file mode 100644 index 000000000..42fec249c --- /dev/null +++ b/scripts/backtest-ooni-alerts.js @@ -0,0 +1,74 @@ +const fs = require('fs'); +const path = require('path'); +const { fetchDailyMeasurements } = require('../backend/fetching/ooniApi'); +const { normalizeDailyCounts, evaluateAlert } = require('../backend/fetching/ooniAlerts'); + +const ASNS = [44244, 58224]; +const ALERT_SINCE = '2025-12-01'; +const NETWORK_NAMES = { 44244: 'IranCell', 58224: 'MCCI' }; + +function shiftDay(day, offset) { + const value = new Date(`${day}T00:00:00.000Z`); + value.setUTCDate(value.getUTCDate() + offset); + return value.toISOString().slice(0, 10); +} + +function csvValue(value) { + if (value == null) return ''; + const text = Array.isArray(value) ? value.join('|') : String(value); + return `"${text.replace(/"/g, '""')}"`; +} + +async function main() { + const alertUntil = process.argv[2] || new Date().toISOString().slice(0, 10); + const dataSince = shiftDay(ALERT_SINCE, -1); + const dataUntil = alertUntil; + const output = []; + + for (const asn of ASNS) { + const rows = await fetchDailyMeasurements({ asn, since: dataSince, until: dataUntil }); + const dailyCounts = normalizeDailyCounts(rows, dataSince, dataUntil); + for (let alertDate = ALERT_SINCE; alertDate <= alertUntil; alertDate = shiftDay(alertDate, 1)) { + const triggers = evaluateAlert(dailyCounts, alertDate); + if (triggers.length > 0) { + output.push({ + asn, + networkName: NETWORK_NAMES[asn], + alertDate, + triggers, + }); + } + } + } + + const outputDirectory = path.join(__dirname, '..', 'data'); + fs.mkdirSync(outputDirectory, { recursive: true }); + const outputPath = path.join(outputDirectory, 'ooni-alert-backtest.json'); + fs.writeFileSync(outputPath, `${JSON.stringify(output, null, 2)}\n`); + const csvFields = [ + 'asn', 'networkName', 'alertDate', 'triggerType', 'measurementDay', + 'measurementCount', + ]; + const csvRows = output.map((alert) => { + const trigger = alert.triggers[0]; + const row = { + ...alert, + triggerType: trigger.type, + ...trigger, + }; + return csvFields.map((field) => csvValue(row[field])).join(','); + }); + const csvPath = path.join(outputDirectory, 'ooni-alert-backtest.csv'); + fs.writeFileSync(csvPath, `${csvFields.join(',')}\n${csvRows.join('\n')}\n`); + console.log(`Wrote ${output.length} alerts to ${outputPath}`); + console.log(`Wrote CSV output to ${csvPath}`); + ASNS.forEach((asn) => { + const alerts = output.filter((alert) => alert.asn === asn); + console.log(`AS${asn}: ${alerts.length} zero-measurement reports`); + }); +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); \ No newline at end of file diff --git a/scripts/domain-leading-indicators.js b/scripts/domain-leading-indicators.js new file mode 100644 index 000000000..59ba0f638 --- /dev/null +++ b/scripts/domain-leading-indicators.js @@ -0,0 +1,194 @@ +// Investigates domain-level OONI web_connectivity signals leading up to shutdown +// dates to find domains whose anomalies or confirmed blocks rise beforehand. +// +// For each shutdown date the script compares a lead-up window (the 21 days +// immediately before the date) against a baseline window (the prior 21 days) +// per domain, then ranks domains by emerging confirmed blocks and by the rise +// in anomaly rate. Output is written per ASN/date as CSV plus a JSON summary. + +const fs = require('fs'); +const path = require('path'); + +const AGGREGATION_URL = 'https://api.ooni.org/api/v1/aggregation'; +const ASNS = [44244, 58224]; +const NETWORK_NAMES = { 44244: 'IranCell', 58224: 'MCCI' }; +const SHUTDOWN_DATES = ['2025-06-18', '2026-01-08', '2026-02-28']; +const LEADUP_DAYS = 21; +const BASELINE_DAYS = 21; +const MIN_LEADUP_MEASUREMENTS = 5; // ignore domains with too little coverage +const TOP_N = 25; + +function shiftDay(day, offset) { + const value = new Date(`${day}T00:00:00.000Z`); + value.setUTCDate(value.getUTCDate() + offset); + return value.toISOString().slice(0, 10); +} + +function csvValue(value) { + if (value == null) return ''; + const text = Array.isArray(value) ? value.join('|') : String(value); + return `"${text.replace(/"/g, '""')}"`; +} + +async function fetchDomainAggregation({ asn, since, until }) { + const params = new URLSearchParams({ + probe_cc: 'IR', + probe_asn: String(asn), + test_name: 'web_connectivity', + axis_x: 'input', + since, + until, + }); + const url = `${AGGREGATION_URL}?${params}`; + const response = await fetch(url, { headers: { accept: 'application/json' } }); + if (!response.ok) { + throw new Error(`OONI aggregation failed (${response.status}): ${url}`); + } + const payload = await response.json(); + const result = payload.result || []; + return Array.isArray(result) ? result : [result]; +} + +function indexByDomain(rows) { + const map = new Map(); + rows.forEach((row) => { + if (!row.input) return; + map.set(row.input, { + measurements: Number(row.measurement_count) || 0, + anomalies: Number(row.anomaly_count) || 0, + confirmed: Number(row.confirmed_count) || 0, + failures: Number(row.failure_count) || 0, + }); + }); + return map; +} + +function rate(part, total) { + return total > 0 ? part / total : 0; +} + +async function analyzeDate(asn, shutdownDate) { + const leadupSince = shiftDay(shutdownDate, -LEADUP_DAYS); + const leadupUntil = shutdownDate; // days before the shutdown + const baselineSince = shiftDay(shutdownDate, -(LEADUP_DAYS + BASELINE_DAYS)); + const baselineUntil = leadupSince; + + const [leadupRows, baselineRows] = await Promise.all([ + fetchDomainAggregation({ asn, since: leadupSince, until: leadupUntil }), + fetchDomainAggregation({ asn, since: baselineSince, until: baselineUntil }), + ]); + + const leadup = indexByDomain(leadupRows); + const baseline = indexByDomain(baselineRows); + + const domains = []; + for (const [domain, l] of leadup.entries()) { + if (l.measurements < MIN_LEADUP_MEASUREMENTS) continue; + const b = baseline.get(domain) || { + measurements: 0, anomalies: 0, confirmed: 0, failures: 0, + }; + const leadupAnomalyRate = rate(l.anomalies, l.measurements); + const baselineAnomalyRate = rate(b.anomalies, b.measurements); + domains.push({ + shutdownDate, + asn, + networkName: NETWORK_NAMES[asn], + domain, + leadupWindow: `${leadupSince} to ${leadupUntil}`, + leadupMeasurements: l.measurements, + leadupAnomalies: l.anomalies, + leadupConfirmed: l.confirmed, + leadupAnomalyRate, + baselineWindow: `${baselineSince} to ${baselineUntil}`, + baselineMeasurements: b.measurements, + baselineAnomalies: b.anomalies, + baselineConfirmed: b.confirmed, + baselineAnomalyRate, + anomalyRateDelta: leadupAnomalyRate - baselineAnomalyRate, + confirmedDelta: l.confirmed - b.confirmed, + newlyConfirmed: b.confirmed === 0 && l.confirmed > 0, + newlyAnomalous: baselineAnomalyRate === 0 && leadupAnomalyRate > 0, + }); + } + + // Leading indicators: confirmed blocks or a meaningful rise in anomaly rate. + domains.sort((a, b) => { + if (b.leadupConfirmed !== a.leadupConfirmed) return b.leadupConfirmed - a.leadupConfirmed; + if (b.anomalyRateDelta !== a.anomalyRateDelta) return b.anomalyRateDelta - a.anomalyRateDelta; + return b.leadupAnomalies - a.leadupAnomalies; + }); + + return domains; +} + +async function main() { + const outputDirectory = path.join(__dirname, '..', 'data', 'domain-leading-indicators'); + fs.mkdirSync(outputDirectory, { recursive: true }); + + const csvFields = [ + 'shutdownDate', 'asn', 'networkName', 'domain', 'leadupWindow', + 'leadupMeasurements', 'leadupAnomalies', 'leadupConfirmed', 'leadupAnomalyRate', + 'baselineWindow', 'baselineMeasurements', 'baselineAnomalies', 'baselineConfirmed', + 'baselineAnomalyRate', 'anomalyRateDelta', 'confirmedDelta', 'newlyConfirmed', + 'newlyAnomalous', + ]; + + const summary = []; + for (const shutdownDate of SHUTDOWN_DATES) { + for (const asn of ASNS) { + const domains = await analyzeDate(asn, shutdownDate); + const top = domains.slice(0, TOP_N); + const csvRows = top.map((row) => csvFields.map((field) => { + const value = row[field]; + if (typeof value === 'number' && field.endsWith('Rate')) return csvValue(value.toFixed(4)); + return csvValue(value); + }).join(',')); + const csvPath = path.join( + outputDirectory, + `${shutdownDate}_AS${asn}_${NETWORK_NAMES[asn]}.csv`, + ); + fs.writeFileSync(csvPath, `${csvFields.join(',')}\n${csvRows.join('\n')}\n`); + + const confirmedDomains = domains.filter((d) => d.leadupConfirmed > 0); + const newAnomalyDomains = domains.filter((d) => d.newlyAnomalous && d.leadupAnomalies >= 5); + summary.push({ + shutdownDate, + asn, + networkName: NETWORK_NAMES[asn], + domainsConsidered: domains.length, + domainsWithConfirmedBlocks: confirmedDomains.length, + domainsNewlyAnomalous: newAnomalyDomains.length, + topConfirmed: confirmedDomains.slice(0, 10).map((d) => ({ + domain: d.domain, + confirmed: d.leadupConfirmed, + anomalies: d.leadupAnomalies, + measurements: d.leadupMeasurements, + })), + topEmergingAnomalies: newAnomalyDomains + .sort((a, b) => b.leadupAnomalies - a.leadupAnomalies) + .slice(0, 10) + .map((d) => ({ + domain: d.domain, + leadupAnomalies: d.leadupAnomalies, + leadupMeasurements: d.leadupMeasurements, + leadupAnomalyRate: Number(d.leadupAnomalyRate.toFixed(3)), + })), + }); + console.log( + `${shutdownDate} AS${asn} ${NETWORK_NAMES[asn]}: ` + + `${domains.length} domains, ` + + `${confirmedDomains.length} with confirmed blocks, ` + + `${newAnomalyDomains.length} newly anomalous`, + ); + } + } + + const summaryPath = path.join(outputDirectory, 'summary.json'); + fs.writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`); + console.log(`\nWrote per-date CSVs and summary to ${outputDirectory}`); +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/setup-windows.ps1 b/setup-windows.ps1 new file mode 100644 index 000000000..e05cac6c1 --- /dev/null +++ b/setup-windows.ps1 @@ -0,0 +1,276 @@ +[CmdletBinding()] +param( + [string]$InstallPath = (Join-Path $HOME "source\Aggie"), + [string]$NpmRegistry = "https://registry.npmjs.org/", + [string]$HttpsProxy = "", + [switch]$DryRun +) + +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +function Write-Step { + param([string]$Message) + Write-Host "`n==> $Message" -ForegroundColor Cyan +} + +function Refresh-Path { + $machinePath = [Environment]::GetEnvironmentVariable("Path", "Machine") + $userPath = [Environment]::GetEnvironmentVariable("Path", "User") + $env:Path = "$machinePath;$userPath" +} + +function Invoke-CommandChecked { + param( + [Parameter(Mandatory = $true)][string]$Command, + [Parameter(Mandatory = $true)][string[]]$Arguments + ) + + & $Command @Arguments + if ($LASTEXITCODE -ne 0) { + throw "$Command failed with exit code $LASTEXITCODE." + } +} + +function Install-WingetPackage { + param( + [Parameter(Mandatory = $true)][string]$Id, + [Parameter(Mandatory = $true)][string]$DisplayName + ) + + if ($DryRun) { + Write-Host "Would install $DisplayName ($Id)." + return + } + + Write-Step "Installing $DisplayName" + Invoke-CommandChecked "winget" @( + "install", "--id", $Id, "--exact", "--silent", + "--accept-package-agreements", "--accept-source-agreements" + ) + Refresh-Path +} + +function New-RandomString { + param([int]$Length = 32) + + $characters = "abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789" + $bytes = New-Object byte[] $Length + $generator = [Security.Cryptography.RandomNumberGenerator]::Create() + try { + $generator.GetBytes($bytes) + } finally { + $generator.Dispose() + } + + return -join ($bytes | ForEach-Object { $characters[$_ % $characters.Length] }) +} + +function Test-DockerReady { + & docker info *> $null + return $LASTEXITCODE -eq 0 +} + +if ($env:OS -ne "Windows_NT") { + throw "This bootstrap script supports Windows 10 and Windows 11 only." +} + +Write-Step "Checking Windows prerequisites" +if (-not (Get-Command winget -ErrorAction SilentlyContinue)) { + throw "winget is required. Install 'App Installer' from Microsoft Store, then rerun this script." +} + +if (-not (Get-Command git -ErrorAction SilentlyContinue)) { + Install-WingetPackage -Id "Git.Git" -DisplayName "Git" +} +if (-not (Get-Command node -ErrorAction SilentlyContinue)) { + Install-WingetPackage -Id "OpenJS.NodeJS.18" -DisplayName "Node.js 18" +} +if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { + Install-WingetPackage -Id "Docker.DockerDesktop" -DisplayName "Docker Desktop" +} + +if ($DryRun) { + Write-Host "Would clone Aggie when the script is not already inside the repository." + Write-Host "Would create ignored local configuration when missing." + Write-Host "Would start MongoDB 6 in Docker and run npm ci." + Write-Host "Dry run complete. No changes were made." -ForegroundColor Green + exit 0 +} + +Refresh-Path + +foreach ($command in @("git", "node", "npm", "docker")) { + if (-not (Get-Command $command -ErrorAction SilentlyContinue)) { + throw "$command was installed but is not available yet. Restart Windows, then rerun this script." + } +} + +$nodeVersion = (& node --version).TrimStart("v") +$parsedNodeVersion = [Version]$nodeVersion +if ($parsedNodeVersion.Major -ne 18 -or $parsedNodeVersion.Minor -lt 20) { + throw "Aggie requires Node.js 18.20 or newer within major version 18. Found $nodeVersion." +} + +$scriptRepoPath = $PSScriptRoot +if (Test-Path (Join-Path $scriptRepoPath "package.json")) { + $repoPath = $scriptRepoPath +} else { + $repoPath = $InstallPath + if (-not (Test-Path (Join-Path $repoPath "package.json"))) { + if (Test-Path $repoPath) { + $existingFiles = @(Get-ChildItem $repoPath -Force) + if ($existingFiles.Count -gt 0) { + throw "$repoPath exists and is not an Aggie checkout. Choose another -InstallPath." + } + } else { + New-Item -ItemType Directory -Path (Split-Path $repoPath -Parent) -Force | Out-Null + } + + Write-Step "Cloning Aggie" + Invoke-CommandChecked "git" @( + "clone", "--branch", "develop", "https://github.com/InetIntel/Aggie.git", $repoPath + ) + } +} + +Set-Location $repoPath +Write-Host "Repository: $repoPath" +Write-Host "Node: $(& node --version)" +Write-Host "npm: $(& npm --version)" + +Write-Step "Starting Docker Desktop" +if (-not (Test-DockerReady)) { + $dockerDesktop = Join-Path $env:ProgramFiles "Docker\Docker\Docker Desktop.exe" + if (-not (Test-Path $dockerDesktop)) { + throw "Docker Desktop is installed but could not be found. Restart Windows, then rerun this script." + } + + Start-Process $dockerDesktop + $dockerReady = $false + for ($attempt = 0; $attempt -lt 60; $attempt++) { + Start-Sleep -Seconds 3 + if (Test-DockerReady) { + $dockerReady = $true + break + } + } + if (-not $dockerReady) { + throw "Docker Desktop did not become ready. Complete any Docker/WSL setup shown on screen, then rerun this script. Docker sign-in is not required." + } +} + +Write-Step "Starting local MongoDB" +$mongoContainer = & docker ps -a --filter "name=^/aggie-mongo$" --format "{{.Names}}" +if ($mongoContainer -eq "aggie-mongo") { + Invoke-CommandChecked "docker" @("start", "aggie-mongo") +} else { + Invoke-CommandChecked "docker" @( + "run", "--detach", "--name", "aggie-mongo", "--restart", "unless-stopped", + "--publish", "27017:27017", "--volume", "aggie-mongo-data:/data/db", "mongo:6.0" + ) +} + +$mongoReady = $false +for ($attempt = 0; $attempt -lt 30; $attempt++) { + & docker exec aggie-mongo mongosh --quiet --eval "db.adminCommand({ ping: 1 }).ok" *> $null + if ($LASTEXITCODE -eq 0) { + $mongoReady = $true + break + } + Start-Sleep -Seconds 2 +} +if (-not $mongoReady) { + throw "MongoDB did not become ready. Run 'docker logs aggie-mongo' for details." +} + +$createdAdminPassword = $null +if (-not (Test-Path ".env")) { + Write-Step "Creating local environment configuration" + $createdAdminPassword = New-RandomString -Length 20 + $appSecret = New-RandomString -Length 48 + @" +ENVIRONMENT=development +DATABASE_URL=mongodb://localhost:27017/ +DATABASE_NAME=aggie +REACT_APP_BASE_URL=http://localhost:3000/ +REACT_APP_PORT=8000 +SENDGRID_API_KEY= +SECRET=$appSecret +ADMIN_PARTY=false +ADMIN_EMAIL=admin@localhost +ADMIN_USERNAME=admin +ADMIN_PASSWORD=$createdAdminPassword +API_REQUEST_TIMEOUT=60000 +JWT_SESSION=false +DETECT_HATE_SPEECH=false +"@ | Set-Content ".env" -Encoding ASCII +} else { + Write-Host "Keeping existing .env file." +} + +$secretsPath = "backend\config\secrets.json" +if (-not (Test-Path $secretsPath)) { + Write-Step "Creating local service configuration" + $secrets = [ordered]@{ + twitter = [ordered]@{ API_key = ""; API_key_secret = ""; access_token = ""; access_token_secret = "" } + crowdtangle = [ordered]@{ count = 100; sortParam = "date"; language = "en"; useLanguage = $false; zawgyiProb = 0.9; interval = "1500" } + comments = [ordered]@{ username = ""; password = ""; baseUrl = ""; pageCount = 100 } + elmo = [ordered]@{ authToken = "" } + gplaces = [ordered]@{ key = "" } + "group map" = [ordered]@{ zoom = "7"; center = "Ghana"; latitude = ""; longitude = "" } + logger = [ordered]@{ + SES = [ordered]@{ disabled = $true; level = "error"; silent = $false } + Slack = [ordered]@{ disabled = $true; level = "error" } + file = [ordered]@{ disabled = $false; level = "warn"; filename = "logs/master.log" } + console = [ordered]@{ disabled = $false; level = "warn" } + api = [ordered]@{ log_requests = $true; log_responses = $true; log_user_activity = $false; filename = "logs/api.log" } + master = [ordered]@{ filename = "logs/master.log" } + fetching = [ordered]@{ filename = "logs/fetching.log" } + analytics = [ordered]@{ filename = "logs/analytics.log" } + } + api_request_timeout = 60 + fetching = $false + experiment = $false + detectHateSpeech = $false + } + $secrets | ConvertTo-Json -Depth 8 | Set-Content $secretsPath -Encoding ASCII +} else { + Write-Host "Keeping existing $secretsPath file." +} + +Write-Step "Checking npm registry access" +$env:NPM_CONFIG_REGISTRY = $NpmRegistry +if ($HttpsProxy) { + $env:HTTPS_PROXY = $HttpsProxy + $env:NPM_CONFIG_HTTPS_PROXY = $HttpsProxy +} +Invoke-CommandChecked "npm" @( + "ping", "--registry=$NpmRegistry", "--fetch-timeout=15000", "--fetch-retries=0" +) + +Write-Step "Installing Aggie dependencies and initializing the database" +Invoke-CommandChecked "npm" @("ci", "--no-audit", "--no-fund") + +Write-Step "Installing the development process runner" +Invoke-CommandChecked "npm" @("install", "--global", "stmux", "--no-audit", "--no-fund") + +Write-Step "Validating installation" +Invoke-CommandChecked "npm" @("ls", "--depth=0") +if (-not (Get-Command stmux -ErrorAction SilentlyContinue)) { + throw "stmux was installed but is not available on PATH. Open a new terminal, then rerun this script." +} +$adminCount = (& docker exec aggie-mongo mongosh aggie --quiet --eval 'db.users.countDocuments({ role: "admin" })').Trim() +if ([int]$adminCount -lt 1) { + throw "Dependencies installed, but the Aggie admin user was not created." +} + +Write-Host "`nAggie setup is complete." -ForegroundColor Green +Write-Host "Repository: $repoPath" +Write-Host "Start it with: npm run dev" +Write-Host "Frontend: http://localhost:8000" +if ($createdAdminPassword) { + Write-Host "Username: admin" + Write-Host "Password: $createdAdminPassword" + Write-Host "The password is also stored in the ignored .env file." +} \ No newline at end of file diff --git a/test/backend/lib.fetching.ooni-alerts.test.js b/test/backend/lib.fetching.ooni-alerts.test.js new file mode 100644 index 000000000..44c8145d0 --- /dev/null +++ b/test/backend/lib.fetching.ooni-alerts.test.js @@ -0,0 +1,48 @@ +const expect = require('chai').expect; +const { normalizeDailyCounts, evaluateAlert } = require('../../backend/fetching/ooniAlerts'); + +describe('OONI alerts', function() { + it('fills omitted aggregation days with zero measurements', function() { + const rows = [ + { measurement_start_day: '2025-12-01', measurement_count: 70 }, + { measurement_start_day: '2025-12-03', measurement_count: 20 }, + ]; + + const daily = normalizeDailyCounts(rows, '2025-12-01', '2025-12-04'); + + expect(daily).to.deep.equal([ + { day: '2025-12-01', measurementCount: 70 }, + { day: '2025-12-02', measurementCount: 0 }, + { day: '2025-12-03', measurementCount: 20 }, + ]); + }); + + it('alerts when the completed day has zero measurements', function() { + const daily = [ + { day: '2025-12-15', measurementCount: 50 }, + { day: '2025-12-16', measurementCount: 0 }, + ]; + + const alerts = evaluateAlert(daily, '2025-12-17'); + + expect(alerts[0]).to.include({ + type: 'zero_measurements', + measurementDay: '2025-12-16', + measurementCount: 0, + }); + }); + + it('does not alert when the completed day has measurements', function() { + const daily = []; + for (let day = 1; day <= 16; day++) { + daily.push({ + day: `2025-12-${String(day).padStart(2, '0')}`, + measurementCount: day <= 14 ? 100 : 60, + }); + } + + const alerts = evaluateAlert(daily, '2025-12-17'); + + expect(alerts).to.deep.equal([]); + }); +}); \ No newline at end of file diff --git a/test/backend/lib.fetching.ooni-channel.test.js b/test/backend/lib.fetching.ooni-channel.test.js new file mode 100644 index 000000000..8ad2a4ee2 --- /dev/null +++ b/test/backend/lib.fetching.ooni-channel.test.js @@ -0,0 +1,32 @@ +const expect = require('chai').expect; +const OONIChannel = require('../../backend/fetching/channels/ooni'); +const Report = require('../../backend/models/report'); + +describe('OONI channel', function() { + it('creates a report from mocked OONI data', async function() { + const originalExists = Report.exists; + Report.exists = async () => false; + + const queued = []; + + const channel = new OONIChannel({ + asns: '44244', + fetchDailyMeasurements: async () => [ + { measurement_start_day: '2026-08-03', measurement_count: 0 }, + ], + }); + + channel.enqueue = (post) => queued.push(post); + + try { + const posts = await channel.fetch(); + + expect(posts).to.have.length(1); + expect(queued).to.have.length(1); + expect(posts[0].platform).to.equal('OONI'); + expect(posts[0].platformID).to.match(/^ooni:44244:volume:/); + } finally { + Report.exists = originalExists; + } + }); +}); \ No newline at end of file