diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 32a41d56f..7f48864c3 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -91,6 +91,29 @@ jobs: - name: Run integration tests run: npm run test:integration + agent365-storage: + name: Agent365 storage (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + - name: Install dependencies + run: npm ci + - name: Build + run: npm run build + - name: Run Agent365 filesystem and exporter regressions + run: npm run test:unit -- test/internal/unit/a365/persistentStore.test.ts test/internal/unit/a365/durableDeliveryManager.test.ts test/internal/unit/a365/agent365Exporter.test.ts test/internal/unit/a365/windowsFileAccessControl.test.ts + - name: Verify cross-process persistence and replay + run: node --test test/integration/durable-storage.test.mjs + changelog: name: CHANGELOG updated runs-on: ubuntu-latest diff --git a/A365_DOCUMENTATION.md b/A365_DOCUMENTATION.md index 1ca1e6fbf..23381c1e9 100644 --- a/A365_DOCUMENTATION.md +++ b/A365_DOCUMENTATION.md @@ -262,8 +262,9 @@ When both `tokenResolver` and `contextualTokenResolver` are set, `contextualToke ## Durable Delivery -Durable delivery is enabled by default for the A365 HTTP exporter when retryable A365 HTTP exports -must survive process restarts. +Durable delivery is requested by default for the A365 HTTP exporter. **On Unix, including Linux +and macOS, it requires an explicitly configured, existing, absolute storage root.** Missing or +invalid configuration logs an error and disables only offline buffering/replay; online export continues. ```typescript import { useMicrosoftOpenTelemetry } from "@microsoft/opentelemetry"; @@ -282,7 +283,7 @@ useMicrosoftOpenTelemetry({ }); ``` -Durable delivery is enabled by default. Set `durableDelivery.enabled: false` to force legacy +Durable delivery is requested by default. Set `durableDelivery.enabled: false` to force legacy network-only delivery. It applies only to the A365 HTTP exporter, so set `enableObservabilityExporter: true` alongside `a365.enabled: true`. @@ -290,8 +291,8 @@ network-only delivery. It applies only to the A365 HTTP exporter, so set | Option | Default | Notes | | ------------------------------------ | ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | -| `enabled` | `true` | Durable delivery stays on unless you explicitly disable it | -| `storageDirectory` | auto | Uses the configured directory, or creates a secure platform-specific default root plus a stable per-application `app-` partition | +| `enabled` | `true` | Requests offline buffering/replay; `false` bypasses storage initialization entirely | +| `storageDirectory` | unset | Unix: required existing absolute customer-provisioned root. Windows: existing platform defaults. Stable `app-` cache partition | | `maxStorageBytes` | `50 * 1024 * 1024` | Bounds pending, quarantined, active leased, and non-stale temporary records within the current `app-` partition only | | `maxRecordAgeMilliseconds` | `2 * 24 * 60 * 60 * 1000` | Expired records are pruned before capacity eviction, within the current `app-` partition only | | `replayIntervalMilliseconds` | `2 * 60 * 1000` | Scheduled replay cadence | @@ -302,15 +303,19 @@ network-only delivery. It applies only to the A365 HTTP exporter, so set ### Operational Notes -- Durable records are stored as plaintext JSON files. On POSIX, the SDK creates owner-only durable - directories/files (`0700` / `0600`), rejects symlink roots, and requires the root to be owned by - the current user. Default POSIX storage probes `TMPDIR`, `/var/tmp`, then `/tmp` and atomically - creates one `a365-otel-durable-` leaf under each candidate; it does not create a - multi-directory SDK-owned tree under a shared temp directory. Windows defaults use the per-user - `Microsoft/A365/otel-durable` location under the selected candidate and also reject a symlinked - final root. On Windows, the SDK removes inherited ACLs and grants full control only to the current - Windows identity and built-in Administrators. ACL hardening failures cause durable storage - initialization to fail and the exporter to use network-only delivery. +- Durable records are plaintext JSON. On Unix the configured root is a customer-trusted security + boundary. The SDK checks configuration and directory existence, not confidentiality: it does not + inspect or rewrite ownership, permissions, ACLs, symlinks, or ancestors. Customers protect these + paths throughout the application's lifetime. Configuring `/tmp` explicitly is still unsafe. +- Unix never selects `TMPDIR`, `/var/tmp`, `/tmp`, or the current directory implicitly and never + creates the configured root. It creates only the normal `app-` cache beneath a valid root. + Existing file/directory creation modes (`0600`/`0700`, filtered by umask) are retained without + chmod/chown or permission repair. Customers must use a compatible service umask; the SDK does not + override it. Existing customer root/cache permissions are unchanged. +- Windows retains its existing `LOCALAPPDATA`, `TEMP`, and `os.tmpdir()` candidate behavior, + `Microsoft/A365/otel-durable` layout, final-root symlink rejection, and ACL helper. Inherited ACLs + are removed and full control granted to the current identity and Administrators as before this + issue. ACL initialization failures are logged and disable only offline buffering/replay. - Every explicit or default durable root is partitioned again under a stable `app-` child derived from the process identity, so applications that share a base directory do not replay one another's telemetry. All retention, capacity, and pruning behavior described below operates only @@ -319,11 +324,12 @@ network-only delivery. It applies only to the A365 HTTP exporter, so set executable upgrade), the previous partition becomes orphaned and is not age-pruned, capacity- pruned, or otherwise cleaned up automatically. Operators who need to reclaim that space should remove stale `app-` directories under the durable root out-of-band. -- Durable delivery is enabled by default when the A365 HTTP exporter is active. Set +- Durable delivery is requested by default when the A365 HTTP exporter is active. Set `durableDelivery.enabled: false` to force legacy network-only delivery. - If durable storage initialization fails, the exporter logs the error and continues in network-only - mode. Successful and non-retryable live sends still complete, but retryable responses that cannot - be persisted are reported as failures. + mode. Successful online sends still succeed; failed network sends that cannot be persisted are + reported as failures. Initialization is shared by startup, export, flush, and shutdown paths; + missing/invalid configuration does not start replay or trigger a hidden storage fallback. - Delivery is at-least-once. A retryable request can be replayed after a crash, timeout, or shutdown race, so downstream consumers must tolerate duplicates. - HTTP 401, 408, 429, and 5xx responses plus transport failures are retryable. Live delivery @@ -342,10 +348,34 @@ network-only delivery. It applies only to the A365 HTTP exporter, so set retryable response pauses both immediate sends and replay probes until the gate reopens. The effective delay is capped at one hour. - If records must survive container restarts, rescheduling, or host restarts, point - `storageDirectory` at a protected persistent volume. Default temp directories and container - filesystems are convenient for process restarts, but ephemeral storage can be lost when a + `storageDirectory` at a protected persistent volume. Ephemeral storage can be lost when a container is replaced and still counts against container ephemeral-storage limits. +### Storage provisioning and migration + +**Unix breaking change:** applications that previously relied on implicit temporary storage must +provision and configure a root or set `durableDelivery.enabled: false`. During deployment, create a +dedicated private directory as the application's service account, normally mode `0700`, for example: + +```sh +install -d -m 700 "$HOME/.local/state/my-agent/telemetry" +``` + +Review existing ownership, extended/inherited ACL grants, symlinks, and ancestor protection before +reuse. Do not chmod shared system directories. Configure the absolute path through +`a365.durableDelivery.storageDirectory` (or the exporter's `durableDelivery.storageDirectory`); +the example's `A365_DURABLE_STORAGE_DIRECTORY` is application-read configuration, not an automatic +SDK environment override. Root validation is not a security audit. + +Reusing the same explicit root, service identity, executable path, and application entry point +preserves `app-` partition identity and existing record replay. Retention limits and the record +format are unchanged. No old temporary-directory backlog is discovered or migrated automatically. +For a controlled migration, stop all writers/replayers, review the old cache and provenance, secure +the destination root and ACLs, and move the existing application partition intact. If executable or +application identity changes, its partition hash also changes; review the intended destination +partition before transferring records. Resume one instance first and confirm replay, allowing for +at-least-once duplicates. Retain a protected backup until migration is verified. + ## Shutdown Call `shutdownMicrosoftOpenTelemetry()` during graceful shutdown to flush pending telemetry and release resources: diff --git a/CHANGELOG.md b/CHANGELOG.md index 55d79c752..bd58530b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## [Unreleased] +### Bugs Fixed +- **Breaking change on Unix (Linux/macOS):** Agent365 offline buffering/replay now requires an explicitly configured, existing, absolute `durableDelivery.storageDirectory`. No automatic Unix temporary-directory fallback or root creation. Missing/invalid roots are logged and disable only disk buffering/replay; online export continues. Customers provision and maintain ownership, permissions (normally a private service-account-owned `0700` root), ACLs, and path-replacement protection; the SDK does not inspect or repair Unix access controls. Windows behavior, explicit-root cache identities, record compatibility, and retention limits are unchanged. Old temporary backlogs are not automatically migrated; see [storage provisioning and migration](A365_DOCUMENTATION.md#storage-provisioning-and-migration). + ### Features Added - Add privacy-safe Model Observation attributes to `InferenceScope`, including the canonical served response model and cache read/write input-token counts. Attributes can be supplied when the scope starts or recorded after the response is available; inference operations accept enum values or canonical strings and use canonical OpenTelemetry names for both operation attributes and span names. Cache-token semantic-convention constants are exported. [#261](https://github.com/microsoft/opentelemetry-distro-javascript/pull/261) diff --git a/README.md b/README.md index f426d9c43..c48502402 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ useMicrosoftOpenTelemetry({ }); ``` -When you enable the A365 HTTP exporter, durable delivery is enabled by default. Tune the durable +When you enable the A365 HTTP exporter, durable delivery is requested by default. Tune the durable store if you need a protected persistent volume or different bounds: ```typescript @@ -112,13 +112,15 @@ With durable delivery enabled, retryable A365 exporter payloads (including HTTP and 5xx responses) are stored as plaintext local files in a stable per-application partition and replayed with a freshly resolved token plus the exporter's current cluster/domain routing; durable files never store bearer tokens or authoritative route metadata. Delivery is at-least-once, so -duplicates are possible. The SDK restricts durable storage to the current user and Administrators -on Windows and uses owner-only `0700` directories and `0600` files on POSIX. Use a protected +duplicates are possible. Unix requires an existing absolute customer-provisioned root; the customer +maintains ownership, permissions, ACLs, and ancestor/path-replacement protection. The SDK does not +inspect or repair Unix access controls. Windows retains its current-user/Administrators ACL behavior. +Use a protected persistent volume if records must survive container or host restarts; ephemeral container storage only survives process restarts and still counts against the container's ephemeral-storage quota. -If durable storage permissions cannot be secured, the exporter logs the initialization failure and -degrades to network-only delivery; successful and non-retryable live sends still complete, but -retryable results that cannot be persisted fail. +Missing/invalid storage configuration or initialization failure is logged and disables only offline +buffering/replay. Online export continues; failed sends that cannot be persisted are reported as failures. +See [provisioning and migration](A365_DOCUMENTATION.md#storage-provisioning-and-migration). For A365 scenarios, scope APIs, baggage, hosting middleware, and official terminology alignment, see [A365_DOCUMENTATION.md](./A365_DOCUMENTATION.md). @@ -370,13 +372,16 @@ new ObservabilityHostingManager().configure(adapter as unknown as { use(...m: un #### A365 durable delivery Durable delivery applies only to the A365 HTTP exporter, so set both `a365.enabled: true` and -`a365.enableObservabilityExporter: true` when you use it. Durable delivery is enabled by default; -set `durableDelivery.enabled: false` to force legacy network-only delivery. +`a365.enableObservabilityExporter: true` when you use it. Durable delivery is requested by default; +set `durableDelivery.enabled: false` to force legacy network-only delivery and bypass storage initialization. +**Unix breaking change (Linux/macOS):** buffering requires an explicitly configured, existing, +absolute `durableDelivery.storageDirectory`; otherwise an error is logged and only offline +buffering/replay is disabled. Online export continues. | Option | Type | Default | Description | | ------------------------------------ | --------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `enabled` | `boolean` | `true` | Durable delivery stays on unless you explicitly disable it | -| `storageDirectory` | `string` | auto | Root directory for durable records. When omitted, the SDK probes `LOCALAPPDATA`, then `TEMP`, then `os.tmpdir()` on Windows and appends `Microsoft/A365/otel-durable`; on POSIX it probes `TMPDIR`, then `/var/tmp`, then `/tmp` and uses the single `a365-otel-durable-` leaf. Every base root then gets a stable per-application `app-` child | +| `enabled` | `boolean` | `true` | Requests buffering/replay; `false` bypasses storage initialization | +| `storageDirectory` | `string` | unset | Unix: required existing absolute customer-provisioned root; no fallback/root creation. Windows: unchanged `LOCALAPPDATA`, `TEMP`, `os.tmpdir()` defaults under `Microsoft/A365/otel-durable`. Stable `app-` cache child | | `maxStorageBytes` | `number` | `50 * 1024 * 1024` | Maximum total bytes retained for pending and quarantined durable records | | `maxRecordAgeMilliseconds` | `number` | `2 * 24 * 60 * 60 * 1000` | Maximum record age before expiry pruning | | `replayIntervalMilliseconds` | `number` | `2 * 60 * 1000` | Delay between scheduled replay passes | @@ -387,11 +392,17 @@ set `durableDelivery.enabled: false` to force legacy network-only delivery. Operational notes: -- Durable records are stored as plaintext JSON files. On POSIX, the SDK creates owner-only durable - directories/files (`0700` / `0600`). On Windows, the SDK removes inherited ACLs and grants full +- Durable records are plaintext JSON. Customers provision and maintain a private storage root, + normally owned by the service account with mode `0700`, and review ACLs and ancestors to prevent + path replacement. The SDK trusts configured Unix paths; it does not inspect or repair ownership, + permissions, ACLs, or symlinks. Explicitly configuring `/tmp` or another shared root is still + unsafe. No Unix temporary-directory fallback or automatic root creation occurs. Existing + explicitly configured cache identities and replay are preserved; old implicit backlogs are not + automatically migrated. See [provisioning and migration](./A365_DOCUMENTATION.md#storage-provisioning-and-migration). + On Windows, the SDK removes inherited ACLs and grants full control only to the current Windows identity and built-in Administrators. ACL hardening failures cause durable storage initialization to fail and the exporter to use network-only delivery. -- Durable delivery is enabled by default when the A365 HTTP exporter is active. Set +- Durable delivery is requested by default when the A365 HTTP exporter is active. Set `durableDelivery.enabled: false` to force legacy network-only delivery. - Durable delivery is at-least-once. If a retryable request succeeds immediately before a crash or after replay, duplicates are possible and receivers must be idempotent. @@ -399,8 +410,8 @@ Operational notes: current cluster/domain routing; durable files do not store bearer tokens or authoritative route metadata. - If durable storage cannot initialize, the exporter logs the error and continues in network-only - mode. Successful and non-retryable live sends still complete, but retryable results that cannot - be persisted are reported as failures. + mode. Successful online sends still succeed; failed sends that cannot be persisted are reported + as failures. - If token resolution returns no token, throws, or times out, live delivery attempts to persist the record for replay and replay releases the claim without extending the shared transmission backoff. - Storage is bounded by both `maxStorageBytes` and `maxRecordAgeMilliseconds`; expired records are diff --git a/src/a365/configuration/A365ConfigurationOptions.ts b/src/a365/configuration/A365ConfigurationOptions.ts index fa25e3576..7b773f15e 100644 --- a/src/a365/configuration/A365ConfigurationOptions.ts +++ b/src/a365/configuration/A365ConfigurationOptions.ts @@ -120,6 +120,6 @@ export interface A365Options { /** Maximum estimated payload size (bytes) per HTTP chunk. */ maxPayloadBytes?: number; - /** Durable delivery options for local spool-and-replay behavior. */ + /** Local spool/replay options. Unix requires a customer-provisioned existing absolute storageDirectory. */ durableDelivery?: Agent365DurableDeliveryOptions; } diff --git a/src/a365/exporter/Agent365ExporterOptions.ts b/src/a365/exporter/Agent365ExporterOptions.ts index 357de135e..20ed22d50 100644 --- a/src/a365/exporter/Agent365ExporterOptions.ts +++ b/src/a365/exporter/Agent365ExporterOptions.ts @@ -86,7 +86,7 @@ export interface Agent365ExporterOptions { /** Maximum estimated payload size (bytes) per HTTP chunk. @default 900 * 1024 (900KB) */ maxPayloadBytes?: number; - /** Durable delivery options for local spool-and-replay behavior. */ + /** Local spool/replay options. Unix requires a customer-provisioned existing absolute storageDirectory. */ durableDelivery?: Agent365DurableDeliveryOptions; } diff --git a/src/a365/exporter/durable/DurableDeliveryOptions.ts b/src/a365/exporter/durable/DurableDeliveryOptions.ts index 1c9c50983..de2e11f99 100644 --- a/src/a365/exporter/durable/DurableDeliveryOptions.ts +++ b/src/a365/exporter/durable/DurableDeliveryOptions.ts @@ -10,7 +10,17 @@ export const DEFAULT_DURABLE_SHUTDOWN_TIMEOUT_MS = 10_000; export const DEFAULT_DURABLE_TOKEN_TIMEOUT_MS = 30_000; export interface Agent365DurableDeliveryOptions { + /** Enables disk buffering/replay when storage can initialize. False bypasses storage initialization. */ enabled?: boolean; + /** + * Customer-provisioned storage root. Unix requires an existing absolute directory; + * missing/invalid roots disable only disk buffering/replay with an error log. + * Customers maintain ownership, permissions (normally service-account-owned 0700), + * ACLs, and protection against replacement of this path or its ancestors. + * The SDK does not inspect or repair Unix access controls; shared roots such as + * /tmp remain unsafe even when explicitly configured. Windows retains its defaults. + * Ignored when enabled is false. Reusing the same root preserves cache identity. + */ storageDirectory?: string; maxStorageBytes?: number; maxRecordAgeMilliseconds?: number; diff --git a/src/a365/exporter/durable/PersistentStore.ts b/src/a365/exporter/durable/PersistentStore.ts index 3c46e830d..1fc22d322 100644 --- a/src/a365/exporter/durable/PersistentStore.ts +++ b/src/a365/exporter/durable/PersistentStore.ts @@ -24,7 +24,6 @@ const TEMPORARY_SUFFIX = ".tmp"; // can still overflow this limit once that metadata is appended. const MAX_FILENAME_COMPONENT_BYTES = 255; const WINDOWS_PROBE_DIRECTORY = ["Microsoft", "A365", "otel-durable"] as const; -const POSIX_DEFAULT_LEAF_PREFIX = "a365-otel-durable"; const APPLICATION_PARTITION_PREFIX = "app-"; const PROCESS_START_CWD = process.cwd(); @@ -481,6 +480,19 @@ async function resolveStorageRoot( logger: ILogger, directoryAccessControl?: DirectoryAccessControl, ): Promise { + if (process.platform !== "win32") { + const root = options.storageDirectory; + if (typeof root !== "string" || root.trim().length === 0 || !isAbsolute(root)) { + throw new Error( + "Unix durable storage requires storageDirectory to specify an existing absolute directory provisioned by the customer", + ); + } + if (!(await fs.stat(root)).isDirectory()) { + throw new Error(`Durable storageDirectory must be an existing directory: ${root}`); + } + return initializeRoot(partitionedStorageRoot(root), false); + } + if (options.storageDirectory !== undefined) { return initializeExplicitRoot(options.storageDirectory, directoryAccessControl); } @@ -519,7 +531,6 @@ async function initializeRoot( throw error; } } - return verifyRoot(root, directoryAccessControl); } @@ -543,40 +554,25 @@ async function verifyRoot( root: string, directoryAccessControl?: DirectoryAccessControl, ): Promise { - let stats = await fs.lstat(root); - if (stats.isSymbolicLink() || !stats.isDirectory()) { - throw new Error(`Durable storage root must be a real directory: ${root}`); - } - if (process.platform === "win32") { + await verifyWindowsDirectory(root); await directoryAccessControl?.hardenDirectory(root); - stats = await fs.lstat(root); - if (stats.isSymbolicLink() || !stats.isDirectory()) { - throw new Error(`Durable storage root must be a real directory: ${root}`); - } - } else { - const uid = process.getuid?.(); - if (uid !== undefined && stats.uid !== uid) { - throw new Error(`Durable storage root must be owned by the current user: ${root}`); - } - - await fs.chmod(root, OWNER_DIRECTORY_MODE); - stats = await fs.lstat(root); - if ( - stats.isSymbolicLink() || - !stats.isDirectory() || - (uid !== undefined && stats.uid !== uid) - ) { - throw new Error( - `Durable storage root must be a real directory owned by the current user: ${root}`, - ); - } + await verifyWindowsDirectory(root); + } else if (!(await fs.stat(root)).isDirectory()) { + throw new Error(`Durable storage cache must be a directory: ${root}`); } await probeRoot(root); return root; } +async function verifyWindowsDirectory(directory: string): Promise { + const stats = await fs.lstat(directory); + if (stats.isSymbolicLink() || !stats.isDirectory()) { + throw new Error(`Durable storage root must be a real directory: ${directory}`); + } +} + let defaultDirectoryAccessControl: DirectoryAccessControl | undefined; function resolveDirectoryAccessControl( @@ -595,11 +591,7 @@ function resolveDirectoryAccessControl( } function defaultStorageBaseRoot(candidate: string): string { - if (process.platform === "win32") { - return join(candidate, ...WINDOWS_PROBE_DIRECTORY); - } - - return join(candidate, `${POSIX_DEFAULT_LEAF_PREFIX}-${process.getuid?.() ?? "unknown"}`); + return join(candidate, ...WINDOWS_PROBE_DIRECTORY); } export function applicationPartition(): string { @@ -626,10 +618,7 @@ function partitionedStorageRoot(root: string): string { } function storageRootCandidates(): string[] { - const candidates = - process.platform === "win32" - ? [process.env.LOCALAPPDATA, process.env.TEMP, os.tmpdir()] - : [process.env.TMPDIR, "/var/tmp", "/tmp"]; + const candidates = [process.env.LOCALAPPDATA, process.env.TEMP, os.tmpdir()]; return [...new Set(candidates.filter(isNonEmptyString))]; } diff --git a/test/integration/durable-storage.test.mjs b/test/integration/durable-storage.test.mjs new file mode 100644 index 000000000..af14a7711 --- /dev/null +++ b/test/integration/durable-storage.test.mjs @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); +const CHILD_SCRIPT = ` + import assert from "node:assert/strict"; + import { stat } from "node:fs/promises"; + import { dirname } from "node:path"; + import { + DURABLE_RECORD_VERSION, + PersistentStore, + ResolvedDurableDeliveryOptions, + } from "./dist/esm/a365/exporter/durable/index.js"; + + const store = await PersistentStore.create( + new ResolvedDurableDeliveryOptions({ + storageDirectory: process.env.DURABLE_STORAGE_DIRECTORY, + }), + { info() {}, warn() {}, error() {} }, + ); + let path; + if (process.env.DURABLE_STORAGE_ACTION === "persist") { + path = await store.persist({ + version: DURABLE_RECORD_VERSION, + id: "restart-record", + createdAt: Date.now(), + tenantId: "tenant", + agentId: "agent", + agenticUserId: "user", + useS2SEndpoint: false, + body: '{"resourceSpans":[]}', + }); + if (process.platform !== "win32") { + assert.equal((await stat(path)).mode & 0o777, 0o600); + } + } else { + const claims = await store.claimBatch(1); + assert.equal(claims.length, 1); + assert.equal(claims[0].record.id, "restart-record"); + assert.equal(claims[0].record.body, '{"resourceSpans":[]}'); + path = claims[0].leasePath; + await store.complete(claims[0]); + assert.deepEqual(await store.claimBatch(1), []); + } + const root = dirname(path); + if (process.platform !== "win32") { + assert.equal((await stat(root)).mode & 0o777, 0o700); + assert.equal((await stat(dirname(root))).mode & 0o777, 0o700); + } + console.log(root); +`; + +test("durable telemetry replays from the same private cache across process restarts", async (t) => { + const directory = await mkdtemp(join(tmpdir(), "a365-secure-restart-")); + t.after(() => rm(directory, { recursive: true, force: true })); + await mkdir(join(directory, "cache"), { mode: 0o700 }); + const roots = []; + + for (const action of ["persist", "replay"]) { + const { stdout } = await execFileAsync( + process.execPath, + ["--input-type=module", "--eval", CHILD_SCRIPT], + { + cwd: process.cwd(), + env: { + ...process.env, + DURABLE_STORAGE_DIRECTORY: join(directory, "cache"), + DURABLE_STORAGE_ACTION: action, + }, + timeout: 20_000, + }, + ); + roots.push(stdout.trim()); + } + + assert.equal(roots[0], roots[1]); +}); diff --git a/test/internal/unit/a365/agent365Exporter.test.ts b/test/internal/unit/a365/agent365Exporter.test.ts index e3c5563ac..67a90db69 100644 --- a/test/internal/unit/a365/agent365Exporter.test.ts +++ b/test/internal/unit/a365/agent365Exporter.test.ts @@ -12,6 +12,7 @@ import { Agent365Exporter } from "../../../../src/a365/exporter/Agent365Exporter import { applicationPartition } from "../../../../src/a365/exporter/durable/PersistentStore.js"; import { DURABLE_RECORD_VERSION, + DurableDeliveryManager, PersistentStore, } from "../../../../src/a365/exporter/durable/index.js"; import { @@ -140,7 +141,7 @@ async function seedDurablePendingRecord( fileName: string, contents: string, ): Promise { - await mkdir(durableStorageRoot(directory), { recursive: true }); + await mkdir(durableStorageRoot(directory), { recursive: true, mode: 0o700 }); await writeFile(join(durableStorageRoot(directory), fileName), contents, "utf8"); } @@ -1678,6 +1679,69 @@ describe("Agent365Exporter", () => { await exporter.shutdown(); }); + it + .skipIf(process.platform === "win32") + .each(["absent", "blank", "relative", "missing", "file"])( + "logs and disables only offline delivery for a Unix %s root, including subsequent exports and flush", + async (kind) => { + const directory = await createStorageDirectory(); + const file = join(directory, "file"); + await writeFile(file, "not a directory"); + const storageDirectory = + kind === "absent" + ? undefined + : kind === "blank" + ? " \t" + : kind === "relative" + ? "relative" + : kind === "missing" + ? join(directory, "missing") + : file; + const customLogger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + configureA365Logger({ logger: customLogger, logLevel: "info|warn|error" }); + const create = vi.spyOn(PersistentStore, "create"); + const replay = vi.spyOn(DurableDeliveryManager.prototype, "startReplay"); + fetchSpy.mockResolvedValue({ status: 200, headers: new Headers() }); + const exporter = new Agent365Exporter({ + tokenResolver: () => "token", + durableDelivery: { enabled: true, storageDirectory }, + }); + + try { + assert.strictEqual(await exportResult(exporter, [makeSpan()]), ExportResultCode.SUCCESS); + await exporter.forceFlush(); + assert.strictEqual(await exportResult(exporter, [makeSpan()]), ExportResultCode.SUCCESS); + expect(create).toHaveBeenCalledTimes(1); + expect(replay).not.toHaveBeenCalled(); + expect(fetchSpy).toHaveBeenCalledTimes(2); + assert.deepEqual(await readdir(directory), ["file"]); + assert.isTrue( + customLogger.error.mock.calls.some(([message]) => + String(message).includes("Durable delivery initialization failed"), + ), + ); + } finally { + await exporter.shutdown(); + } + }, + ); + + it("bypasses all storage initialization when explicitly disabled", async () => { + const create = vi.spyOn(PersistentStore, "create"); + const replay = vi.spyOn(DurableDeliveryManager.prototype, "startReplay"); + const exporter = new Agent365Exporter({ + tokenResolver: () => "token", + durableDelivery: { enabled: false, storageDirectory: "invalid-relative-root" }, + }); + try { + assert.strictEqual(await exportResult(exporter, [makeSpan()]), ExportResultCode.SUCCESS); + await exporter.forceFlush(); + } finally { + await exporter.shutdown(); + } + expect(create).not.toHaveBeenCalled(); + expect(replay).not.toHaveBeenCalled(); + }); it("fails retryable exports when durable initialization fails and records cannot be persisted", async () => { vi.spyOn(PersistentStore, "create").mockRejectedValue(new Error("disk unavailable")); fetchSpy.mockResolvedValue({ diff --git a/test/internal/unit/a365/durableDeliveryManager.test.ts b/test/internal/unit/a365/durableDeliveryManager.test.ts index a91bb77f8..ff86f463d 100644 --- a/test/internal/unit/a365/durableDeliveryManager.test.ts +++ b/test/internal/unit/a365/durableDeliveryManager.test.ts @@ -290,7 +290,7 @@ describe("DurableDeliveryManager", () => { "durable-manager-test-artifacts", randomUUID(), ); - await fs.mkdir(root, { recursive: true }); + await fs.mkdir(root, { recursive: true, mode: 0o700 }); try { const logger = makeLogger(); diff --git a/test/internal/unit/a365/persistentStore.test.ts b/test/internal/unit/a365/persistentStore.test.ts index f1b586a6e..dd1688458 100644 --- a/test/internal/unit/a365/persistentStore.test.ts +++ b/test/internal/unit/a365/persistentStore.test.ts @@ -94,7 +94,7 @@ describe("PersistentStore", () => { } }); - it("rejects symlink roots", async () => { + it.skipIf(process.platform !== "win32")("rejects Windows symlink roots", async () => { const target = join(scratchRoot, "target"); const link = join(scratchRoot, "link"); await fs.mkdir(target, { recursive: true }); @@ -105,6 +105,92 @@ describe("PersistentStore", () => { ); }); + it.skipIf(process.platform === "win32").each([undefined, "", " \t", "relative", "./relative"])( + "rejects invalid Unix configuration %s without attempting temporary storage", + async (storageDirectory) => { + process.env.TMPDIR = scratchRoot; + const mkdir = vi.spyOn(fs, "mkdir").mockRejectedValue(new Error("unexpected mkdir")); + const logger = makeLogger(); + await expect( + PersistentStore.create(new ResolvedDurableDeliveryOptions({ storageDirectory }), logger), + ).rejects.toThrow(/storageDirectory/); + expect(mkdir).not.toHaveBeenCalled(); + expect(logger.warn).not.toHaveBeenCalled(); + assert.deepEqual(await fs.readdir(scratchRoot), []); + }, + ); + + it.skipIf(process.platform === "win32")( + "rejects nonexistent and non-directory Unix roots without creating them or falling back", + async () => { + const missing = join(scratchRoot, "missing", "root"); + const file = join(scratchRoot, "file"); + await fs.writeFile(file, "not a directory"); + process.env.TMPDIR = scratchRoot; + for (const storageDirectory of [missing, file]) { + await expect( + PersistentStore.create( + new ResolvedDurableDeliveryOptions({ storageDirectory }), + makeLogger(), + ), + ).rejects.toThrow(); + } + assert.isFalse(await pathExists(dirname(missing))); + assert.deepEqual(await fs.readdir(scratchRoot), ["file"]); + }, + ); + + it.skipIf(process.platform === "win32").each([0o755, 0o770, 0o777, 0o1777])( + "trusts customer-managed Unix root and partition permissions without modifying mode %s", + async (mode) => { + const root = join(scratchRoot, "customer-root"); + const partition = explicitStorageRoot(root); + await fs.mkdir(partition, { recursive: true }); + await fs.chmod(root, mode); + await fs.chmod(partition, mode); + const chmod = vi.spyOn(fs, "chmod"); + const chown = vi.spyOn(fs, "chown"); + const accessControl = { hardenDirectory: vi.fn(async () => {}) }; + const store = await createStore(root, {}, makeLogger(), accessControl); + await store.persist(makeRecord()); + const restarted = await createStore(root); + const [claim] = await restarted.claimBatch(1); + await restarted.complete(claim); + assert.strictEqual((await fs.stat(root)).mode & 0o7777, mode); + assert.strictEqual((await fs.stat(partition)).mode & 0o7777, mode); + expect(chmod).not.toHaveBeenCalled(); + expect(chown).not.toHaveBeenCalled(); + expect(accessControl.hardenDirectory).not.toHaveBeenCalled(); + }, + ); + + it.skipIf(process.platform === "win32")( + "trusts customer-protected Unix symlinks and preserves the configured cache identity", + async () => { + const target = join(scratchRoot, "target"); + const link = join(scratchRoot, "link"); + await fs.mkdir(target); + await fs.symlink(target, link, "dir"); + const storedPath = await (await createStore(link)).persist(makeRecord()); + assert.strictEqual(dirname(dirname(storedPath)), link); + const restarted = await createStore(link); + assert.strictEqual((await restarted.claimBatch(1)).length, 1); + }, + ); + + it("reopens the same secure partition and replays persisted telemetry", async () => { + const root = join(scratchRoot, "restart-replay"); + const record = makeRecord(); + const storedPath = await (await createStore(root)).persist(record); + + const restartedStore = await createStore(root); + const [claim] = await restartedStore.claimBatch(1); + assert.deepEqual(claim.record, record); + assert.strictEqual(dirname(claim.leasePath), dirname(storedPath)); + await restartedStore.complete(claim); + assert.deepEqual(await listStoreFiles(root), []); + }); + it("surfaces explicit directory failures without fallback", async () => { const blockedParent = join(scratchRoot, "blocked-parent"); const fallbackBase = join(scratchRoot, "fallback-base"); @@ -180,54 +266,6 @@ describe("PersistentStore", () => { } }); - it("uses POSIX fallback candidates in exact de-duplicated order", async () => { - const restorePlatform = usePosixDefaultCandidates(nativePlatform); - const logger = makeLogger(); - const actualMkdir = fs.mkdir.bind(fs); - const expectedAttempts = [defaultStorageRoot("/tmp"), defaultStorageRoot("/var/tmp")]; - const blockedBaseRoots = [defaultStorageBaseRoot("/tmp"), defaultStorageBaseRoot("/var/tmp")]; - - tmpdirOverride.value = "/tmp"; - vi.spyOn(fs, "mkdir").mockImplementation(async (path, options) => { - const normalizedPath = String(path); - if ( - blockedBaseRoots.some( - (baseRoot) => - normalizedPath === baseRoot || - normalizedPath.startsWith(`${baseRoot}\\`) || - normalizedPath.startsWith(`${baseRoot}/`), - ) - ) { - const error = new Error("blocked default root") as NodeJS.ErrnoException; - error.code = "EACCES"; - throw error; - } - - return actualMkdir(path, options as never); - }); - - try { - process.env.TMPDIR = "/tmp"; - - await expect( - PersistentStore.create( - new ResolvedDurableDeliveryOptions({ - enabled: true, - }), - logger, - ), - ).rejects.toThrow(/blocked default root/); - - assert.deepEqual( - logger.warn.mock.calls.map((call) => call[1]), - expectedAttempts, - ); - assert.strictEqual(logger.warn.mock.calls.length, expectedAttempts.length); - } finally { - restorePlatform(); - } - }); - it("rejects a symlinked default root before falling back", async () => { const firstCandidate = join(scratchRoot, "symlinked-default-candidate"); const fallbackBase = join(scratchRoot, "fallback-base"); @@ -252,28 +290,6 @@ describe("PersistentStore", () => { } }); - it("uses a single owner-only per-user leaf for a POSIX default root", async () => { - if (nativePlatform === "win32") { - return; - } - - const candidate = join(scratchRoot, "posix-default-candidate"); - await fs.mkdir(candidate, { recursive: true }); - process.env.TMPDIR = candidate; - - const storedPath = await (await createDefaultStore()).persist(makeRecord()); - const root = defaultStorageRoot(candidate); - const baseRoot = defaultStorageBaseRoot(candidate); - - assert.ok(storedPath.startsWith(root)); - assert.strictEqual(dirname(root), baseRoot); - assert.strictEqual(dirname(baseRoot), candidate); - assert.match(basename(baseRoot), /^a365-otel-durable-\d+$/); - assert.match(basename(root), /^app-[0-9a-f]{16}$/); - assert.strictEqual((await fs.stat(root)).mode & 0o777, 0o700); - assert.strictEqual((await fs.stat(root)).uid, process.getuid!()); - }); - it("applies Windows ACL hardening to explicit durable roots before persisting records", async () => { const restorePlatform = useWindowsDefaultCandidates(nativePlatform); const root = join(scratchRoot, "windows-acl-store"); @@ -949,6 +965,9 @@ async function createStore( hardenDirectory(directory: string): Promise; }, ): Promise { + if (process.platform !== "win32") { + await fs.mkdir(storageDirectory, { recursive: true, mode: 0o700 }); + } return PersistentStore.create( new ResolvedDurableDeliveryOptions({ enabled: true, @@ -1101,17 +1120,3 @@ function useWindowsDefaultCandidates(nativePlatform: string): () => void { Object.defineProperty(process, "platform", { ...descriptor, value: "win32" }); return () => Object.defineProperty(process, "platform", descriptor); } - -function usePosixDefaultCandidates(nativePlatform: string): () => void { - if (nativePlatform !== "win32") { - return () => {}; - } - - const descriptor = Object.getOwnPropertyDescriptor(process, "platform"); - if (!descriptor) { - throw new Error("Unable to override process.platform for this test"); - } - - Object.defineProperty(process, "platform", { ...descriptor, value: "linux" }); - return () => Object.defineProperty(process, "platform", descriptor); -}