Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ jobs:
steps:
- uses: actions/checkout@v7
with:
fetch-depth: 0
fetch-depth: 1
ref: ${{ inputs.commit_sha || github.sha }}
- uses: actions/setup-node@v6
with:
Expand Down
13 changes: 5 additions & 8 deletions .github/workflows/fetch-cbp.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,8 @@ jobs:
fetch:
runs-on: ubuntu-latest
steps:
# fetch-depth: 0 is load-bearing, not hygiene. This workflow validates a
# full site build, and `npm run build` runs build-aggregates.mjs, which
# reconstructs the 30-day window with `git log --since=... -- crossings.json`.
# Under the default depth-1 clone that log returns one commit, so every
# aggregate silently collapsed to ~2 samples while the pages kept claiming
# "30 days of CBP data" (found 2026-07-27; rankings.json had fallen to 1
# entry). deploy.yml already does this; it just stopped being the deployer.
# Full history is needed here to materialize the rolling 30-day snapshot
# artifact. Site builds consume the artifact and can use shallow clones.
- uses: actions/checkout@v7
with:
fetch-depth: 0
Expand All @@ -40,14 +35,16 @@ jobs:
run: node scripts/fetch-cbp.mjs
- name: Fetch USD/MXN exchange rate
run: node scripts/fetch-fx.mjs
- name: Materialize rolling snapshot history
run: npm run history:build -- --include-current
- name: Validate static site
run: npm run build
- name: Commit validated data snapshot
id: snapshot
run: |
git config user.name "borderpulse-bot"
git config user.email "borderpulse-bot@users.noreply.github.com"
git add public/data/crossings.json public/data/exchange-rate.json
git add public/data/crossings.json public/data/exchange-rate.json public/data/snapshot-history.json
if git diff --cached --quiet; then
echo "No data changes."
echo "changed=false" >> "$GITHUB_OUTPUT"
Expand Down
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,5 @@ drafts/blog/auto-a/queue.json
# (those are PR artifacts); the all-items raw feed is regenerated each run.
drafts/curation/latest-feed.json
.playwright-mcp/
.vercel
.env*.local
14 changes: 14 additions & 0 deletions DECISIONS.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,20 @@ Format: date · one-line decision · short why.

---

## 2026-09-29 · Evaluate Vercel with a public-app preview

Keep the existing GitHub Pages site and automatic ads while testing Vercel.
Vercel builds use the same Vite app plus a five-minute CDN-cached CBP endpoint;
the client retains the last successful reading and falls back to the published
static snapshot with its original timestamp when the endpoint fails. A compact
30-day snapshot artifact makes historical aggregates independent of build-host
Git history. Do not connect every data commit to a Vercel build: the scheduled
writer runs too often for that to be a cost-controlled default. A production
domain move needs a production-equivalent preview, usage review, and owner
approval first.

---

## 2026-08-20 · Freshness thresholds come from measured delivery, not the cron

Issue #58 asked for stale/fresh thresholds "derived from the configured
Expand Down
4 changes: 4 additions & 0 deletions ROADMAP.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ cross-border experience; it is a product hypothesis, not a claim of demand.

### Now: earn trust in the free entry point

- [~] Review the cost-controlled Vercel preview from the current public app:
verify automatic ads, live CBP endpoint, static fallback, 30-day history,
and measured usage before any domain move. The public site stays on Pages
until Sebastian approves a production cutover.
- Keep official northbound CBP data, source timestamps, honest stale states and
the static fallback reliable. Keep the free experience supported by ads.
- Measure whether people complete a crossing decision and return. Review
Expand Down
33 changes: 33 additions & 0 deletions api/public/crossings.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { CBP_URL, createCbpPayload } from '../../scripts/fetch-cbp.mjs';

// Serve one official snapshot from the CDN for five minutes. The frontend
// keeps its last good response and can fall back to the static Pages feed.
export const config = { maxDuration: 10 };

function reply(res, status, body, cache = false) {
res.status(status);
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.setHeader('Cache-Control', cache ? 'public, max-age=0, must-revalidate' : 'no-store');
if (cache) {
res.setHeader('Vercel-CDN-Cache-Control', 'public, s-maxage=300, stale-while-revalidate=600');
res.setHeader('CDN-Cache-Control', 'public, s-maxage=300, stale-while-revalidate=600');
}
res.end(JSON.stringify(body));
}

export default async function handler(req, res) {
if (req.method !== 'GET') return reply(res, 405, { error: 'Method not allowed' });

try {
const upstream = await fetch(CBP_URL, {
headers: { 'User-Agent': 'borderpulse.com/1.0 (live-data-cache)' },
signal: AbortSignal.timeout(8_000),
});
if (!upstream.ok) throw new Error(`CBP source returned ${upstream.status}`);
const payload = createCbpPayload(await upstream.json());
return reply(res, 200, payload, true);
} catch (error) {
console.error('[public-crossings]', error);
return reply(res, 502, { error: 'Official CBP data is temporarily unavailable' });
}
}
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
"fetch:cbp": "node scripts/fetch-cbp.mjs",
"fetch:fx": "node scripts/fetch-fx.mjs",
"fetch:sb": "node scripts/fetch-sb.mjs",
"history:build": "node scripts/build-snapshot-history.mjs",
"aggregates": "node scripts/build-aggregates.mjs",
"rankings": "node scripts/build-rankings.mjs",
"blog:index": "node scripts/build-blog-index.mjs",
Expand Down
1 change: 1 addition & 0 deletions public/data/snapshot-history.json

Large diffs are not rendered by default.

35 changes: 12 additions & 23 deletions scripts/build-aggregates.mjs
Original file line number Diff line number Diff line change
@@ -1,18 +1,14 @@
import fs from 'node:fs';
import path from 'node:path';
import { execSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const root = path.resolve(__dirname, '..');

const DATA_FILE = 'public/data/crossings.json';
const HISTORY_FILE = 'public/data/snapshot-history.json';
const LOOKBACK_DAYS = 30;

function sh(cmd) {
return execSync(cmd, { cwd: root, encoding: 'utf8', maxBuffer: 50 * 1024 * 1024 }).trim();
}

function median(arr) {
if (!arr.length) return null;
const sorted = [...arr].sort((a, b) => a - b);
Expand Down Expand Up @@ -53,15 +49,17 @@ async function main() {
return tzCache.get(key);
}

let log;
try {
log = sh(`git log --since="${since}" --pretty=format:%H --date=iso -- ${DATA_FILE}`);
} catch (e) {
console.warn('[aggregates] git log failed, skipping', e.message);
return;
const historyPath = path.resolve(root, HISTORY_FILE);
if (!fs.existsSync(historyPath)) {
throw new Error(`Missing ${HISTORY_FILE}. Run npm run history:build in the data-ingestion environment.`);
}
const shas = log.split('\n').map((x) => x.trim()).filter(Boolean);
console.log(`[aggregates] ${shas.length} snapshots over last ${LOOKBACK_DAYS} days`);
const history = JSON.parse(fs.readFileSync(historyPath, 'utf8'));
const snapshots = (history.snapshots || []).filter((snapshot) => {
const ts = new Date(snapshot.fetched_at);
return !Number.isNaN(ts.getTime()) && ts >= new Date(since);
});
if (!snapshots.length) throw new Error(`No snapshots in ${HISTORY_FILE} for the last ${LOOKBACK_DAYS} days`);
console.log(`[aggregates] ${snapshots.length} explicit snapshots over last ${LOOKBACK_DAYS} days`);

// portNumber -> day (0-6) -> hour (0-23) -> [waits]
const buckets = new Map();
Expand All @@ -76,16 +74,7 @@ async function main() {
byHour.get(hour).push(wait);
}

for (const sha of shas) {
let raw;
try {
raw = sh(`git show ${sha}:${DATA_FILE}`);
} catch {
continue;
}
let doc;
try { doc = JSON.parse(raw); } catch { continue; }

for (const doc of snapshots) {
const ts = doc.fetched_at ? new Date(doc.fetched_at) : null;
if (!ts || Number.isNaN(ts.getTime())) continue;
// Bucket by PORT-LOCAL day/hour. The original UTC bucketing shifted
Expand Down
89 changes: 89 additions & 0 deletions scripts/build-snapshot-history.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
#!/usr/bin/env node

import fs from 'node:fs';
import path from 'node:path';
import { execFileSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';

const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const DATA_FILE = 'public/data/crossings.json';
const OUTPUT_FILE = path.resolve(root, 'public/data/snapshot-history.json');
const LOOKBACK_DAYS = 30;

function git(args) {
return execFileSync('git', args, {
cwd: root,
encoding: 'utf8',
maxBuffer: 50 * 1024 * 1024,
}).trim();
}

function compactSnapshot(raw, source) {
let doc;
try {
doc = JSON.parse(raw);
} catch {
return null;
}
const fetchedAt = doc?.fetched_at;
const timestamp = fetchedAt ? new Date(fetchedAt) : null;
if (!timestamp || Number.isNaN(timestamp.getTime()) || !Array.isArray(doc.crossings)) return null;

return {
fetched_at: timestamp.toISOString(),
source,
crossings: doc.crossings.map((crossing) => ({
port_number: String(crossing.port_number),
current_wait_time: typeof crossing.current_wait_time === 'number'
? crossing.current_wait_time
: null,
})),
};
}

function main() {
const since = new Date(Date.now() - LOOKBACK_DAYS * 24 * 60 * 60 * 1000);
const sinceIso = since.toISOString();
const snapshots = new Map();
const log = git(['log', `--since=${sinceIso}`, '--format=%H', '--', DATA_FILE]);
const shas = log.split('\n').map((sha) => sha.trim()).filter(Boolean);

for (const sha of shas) {
try {
const snapshot = compactSnapshot(git(['show', `${sha}:${DATA_FILE}`]), `commit:${sha}`);
if (snapshot && new Date(snapshot.fetched_at) >= since) snapshots.set(snapshot.fetched_at, snapshot);
} catch {
// A malformed historical commit must not make a valid current artifact
// disappear. The live fetch and its validation remain the hard gate.
}
}

if (process.argv.includes('--include-current')) {
const current = compactSnapshot(
fs.readFileSync(path.resolve(root, DATA_FILE), 'utf8'),
'working-tree',
);
if (current && new Date(current.fetched_at) >= since) snapshots.set(current.fetched_at, current);
}

const ordered = [...snapshots.values()].sort((a, b) => a.fetched_at.localeCompare(b.fetched_at));
if (ordered.length === 0) {
throw new Error('No valid snapshots found for the explicit history artifact');
}

const artifact = {
source: DATA_FILE,
lookback_days: LOOKBACK_DAYS,
generated_at: new Date().toISOString(),
snapshots: ordered,
};
fs.writeFileSync(OUTPUT_FILE, `${JSON.stringify(artifact)}\n`);
console.log(`[history] wrote ${ordered.length} snapshots → ${OUTPUT_FILE}`);
}

try {
main();
} catch (error) {
console.error(`[history] ${error.message}`);
process.exit(1);
}
27 changes: 17 additions & 10 deletions scripts/fetch-cbp.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,13 @@
*/
import fs from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { canonicalPortNumber, pinIdentity, requiresPortCollapse } from '../src/lib/portIdentity.js';

// bwt.cbp.gov serves two endpoints with the same JSON shape:
// /api/waittimes — legacy, frozen on 3/26/2026 (do NOT use)
// /api/bwtpublicmod — current, used by the CBP Angular app
const CBP_URL = 'https://bwt.cbp.gov/api/bwtpublicmod';
export const CBP_URL = 'https://bwt.cbp.gov/api/bwtpublicmod';
const OUT_PATH = path.join(process.cwd(), 'public', 'data', 'crossings.json');

// Minimum healthy crossing count. CBP's Mexican-border feed yields ~43 distinct
Expand Down Expand Up @@ -256,11 +257,7 @@ const dedupe = (rows) => {
return keep;
};

async function main() {
console.log(`Fetching ${CBP_URL}...`);
const res = await fetch(CBP_URL, { headers: { 'User-Agent': 'borderpulse.com/1.0 (static-fetch)' } });
if (!res.ok) throw new Error(`CBP fetch failed: HTTP ${res.status}`);
const data = await res.json();
export function createCbpPayload(data, { fetchedAt = new Date().toISOString() } = {}) {
if (!Array.isArray(data)) throw new Error('CBP response was not an array');

const rawMexican = data.filter(isMexicanBorder);
Expand Down Expand Up @@ -297,17 +294,27 @@ async function main() {
seenPorts.add(c.port_number);
}

const payload = {
return {
source: 'U.S. Customs and Border Protection',
source_url: CBP_URL,
fetched_at: new Date().toISOString(),
fetched_at: fetchedAt,
count: mexican.length,
crossings: mexican,
};
}

async function main() {
console.log(`Fetching ${CBP_URL}...`);
const res = await fetch(CBP_URL, { headers: { 'User-Agent': 'borderpulse.com/1.0 (static-fetch)' } });
if (!res.ok) throw new Error(`CBP fetch failed: HTTP ${res.status}`);
const payload = createCbpPayload(await res.json());

await fs.mkdir(path.dirname(OUT_PATH), { recursive: true });
await fs.writeFile(OUT_PATH, JSON.stringify(payload, null, 2));
console.log(`Wrote ${mexican.length} Mexican-border crossings → ${OUT_PATH}`);
console.log(`Wrote ${payload.count} Mexican-border crossings → ${OUT_PATH}`);
}

main().catch((err) => { console.error(err); process.exit(1); });
// The Vercel public endpoint imports the parser without triggering a file write.
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
main().catch((err) => { console.error(err); process.exit(1); });
}
27 changes: 22 additions & 5 deletions src/components/utils/dataService.js
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
/**
* dataService - reads static JSON snapshots published by the GitHub Action
* fetch workflow (scripts/fetch-cbp.mjs). No auth, no entity API, no LLM.
* dataService - reads the cached official-data endpoint on Vercel, with the
* GitHub Action's static JSON snapshot as a fallback. Pages uses static data.
*/
import { buildSlugMap } from '../../lib/slugs.js';

const DATA_PATH = '/data/crossings.json';
const LIVE_DATA_PATH = '/api/public/crossings';
const FX_PATH = '/data/exchange-rate.json';
const SB_PATH = '/data/crossings-sb.json';

Expand Down Expand Up @@ -61,13 +62,29 @@ export class DataService {
return res.json();
}

async fetchCrossingsDoc() {
if (import.meta.env?.VITE_PUBLIC_CBP_API === 'true') {
try {
// No cache-busting query: Vercel's five-minute CDN cache is deliberate.
const res = await fetch(LIVE_DATA_PATH);
if (!res.ok) throw new Error(`HTTP ${res.status} for ${LIVE_DATA_PATH}`);
return { doc: await res.json(), fromFallback: false };
} catch (error) {
console.warn('[dataService] live CBP endpoint failed; using published snapshot', error);
return { doc: await this.fetchJson(DATA_PATH), fromFallback: true };
}
}
return { doc: await this.fetchJson(DATA_PATH), fromFallback: false };
}

async getBorderData() {
try {
const [crossingsDoc, fxDoc, sbDoc] = await Promise.all([
this.fetchJson(DATA_PATH),
const [crossingsResult, fxDoc, sbDoc] = await Promise.all([
this.fetchCrossingsDoc(),
this.fetchJson(FX_PATH).catch(() => null),
this.fetchJson(SB_PATH).catch(() => null),
]);
const { doc: crossingsDoc, fromFallback } = crossingsResult;
const crossings = mergeSouthbound(crossingsDoc, sbDoc);
const payload = {
success: true,
Expand All @@ -78,7 +95,7 @@ export class DataService {
southbound_source: sbDoc?.source || null,
southbound_timestamp: sbDoc?.fetched_at || null,
southbound_note: sbDoc?.note || null,
fromFallback: false,
fromFallback,
};
this.cache = payload;
this.notify(payload);
Expand Down
6 changes: 6 additions & 0 deletions vercel.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"$schema": "https://openapi.vercel.sh/vercel.json",
"framework": "vite",
"buildCommand": "VITE_PUBLIC_CBP_API=true npm run build",
"outputDirectory": "dist"
}
Loading