diff --git a/.github/workflows/prepare-release.yaml b/.github/workflows/prepare-release.yaml index 2df2802f..f18964db 100644 --- a/.github/workflows/prepare-release.yaml +++ b/.github/workflows/prepare-release.yaml @@ -4,7 +4,7 @@ # 1. Bumps the selected package's version in its own package.json # 2. Generates a changelog from git history since the package's last release tag # 3. Updates the package's changelog file and creates a release branch + package-scoped tag -# Packages are versioned and tagged independently (tag prefixes: v, azuremanaged-v, durable-functions-v). +# Packages are versioned and tagged independently (tag prefixes: v, azureblob-v, azuremanaged-v, durable-functions-v). # After the workflow completes, manually open a PR from the release branch to main, then publish # from the package directory using the npm command shown in the run summary (mind the dist-tag). @@ -19,6 +19,7 @@ on: type: choice options: - durabletask-js + - durabletask-js-azureblob - durabletask-js-azuremanaged - azure-functions-durable version: @@ -60,6 +61,12 @@ jobs: TAG_PREFIX="azuremanaged-v" CHANGELOG_PATH="packages/durabletask-js-azuremanaged/CHANGELOG.md" ;; + durabletask-js-azureblob) + PKG_DIR="packages/durabletask-js-azureblob" + NPM_NAME="@microsoft/durabletask-js-azureblob" + TAG_PREFIX="azureblob-v" + CHANGELOG_PATH="packages/durabletask-js-azureblob/CHANGELOG.md" + ;; azure-functions-durable) PKG_DIR="packages/azure-functions-durable" NPM_NAME="durable-functions" @@ -67,7 +74,7 @@ jobs: CHANGELOG_PATH="packages/azure-functions-durable/CHANGELOG.md" ;; *) - echo "::error::Unknown package '$PACKAGE'. Expected one of: durabletask-js, durabletask-js-azuremanaged, azure-functions-durable." + echo "::error::Unknown package '$PACKAGE'. Expected one of: durabletask-js, durabletask-js-azureblob, durabletask-js-azuremanaged, azure-functions-durable." exit 1 ;; esac diff --git a/CHANGELOG.md b/CHANGELOG.md index 91ac550a..ebc4008f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ### New +- Add opt-in `payloads` storage to gRPC clients/workers, with transparent UTF-8-size-based + externalization and awaited query/history/work-item hydration across orchestration, activity, + and entity V1/V2 payloads. Add the optional `@microsoft/durabletask-js-azureblob` storage package. +- Add .NET-aligned `setLargePayloadAutoPurge()`, reserved replay-safe cleanup tasks, and DTS + lifecycle transport. Storage configuration alone does not enable deletion; cleanup checks + blob ownership and ETags, preserves uncertain outcomes, and leaves legacy v1 blobs untouched. - Expose readonly `OrchestrationContext.sourceInstanceId` from the existing protobuf request properties on initial execution and replay, independently of parent identity. Strings are preserved verbatim; null or missing metadata returns `undefined`. @@ -43,6 +49,8 @@ ### Fixes +- Keep `PayloadStorageException` cause options structural so its public declarations remain + usable by ES2020 consumers without the global ES2022 `ErrorOptions` type. - Prevent shared or duplicate pending tasks from stranding `whenAll` orchestrations after all activities finish. Notify every dependent composite, preserve duplicate result positions, and detach completed `whenAny` groups from their remaining children. diff --git a/README.md b/README.md index 823f6dea..4b88e2c2 100644 --- a/README.md +++ b/README.md @@ -23,17 +23,30 @@ const entityResponseBytes = await worker.processEntityBatchRequest(entityBatchRe `TaskHubGrpcClient` already exposes orchestration start/query/event/terminate/suspend/resume/purge APIs and entity signal/read/query/clean APIs through its existing `hostAddress` and `metadataGenerator` options. Host integrations that need task-hub routing metadata should provide it through `metadataGenerator`, keeping host-specific metadata policy outside the core client. Azure-managed scheduler connection strings remain in `@microsoft/durabletask-js-azuremanaged`. +## Large payloads + +Clients and workers can opt into transparent external payload storage with the `payloads` +option. The optional [`@microsoft/durabletask-js-azureblob`](./packages/durabletask-js-azureblob) +package implements .NET-compatible Azure Blob tokens and UTF-8/gzip storage. The default +externalization threshold is 256 KiB (inclusive); the maximum payload size is 10 MiB. +Core users do not acquire Azure Storage dependencies, and disabled behavior is unchanged. + +Configure all participating clients and workers before storing externalized payloads. +Azure-managed builders expose `.useExternalizedPayloads({ store })`. Cleanup is separately +opt-in through `client.setLargePayloadAutoPurge(true)` and requires a supporting DTS backend. +See the package's [configuration, interoperability, and cleanup contract](./packages/durabletask-js-azureblob/README.md). + ## Long durable timers `createTimer(Date | seconds)` has no SDK-imposed total-duration cap. Like the Python SDK, core workers default to three-day backend segments, including durable retry delays. A ten-day timer uses 3 + 3 + 3 + 1 day segments but remains one logical `TimerTask`. -| Entry point | Timer behavior | -| --- | --- | -| Core `TaskHubGrpcWorker` / `TestOrchestrationWorker` | Three-day default | -| Azure-managed worker builder | Explicitly native timers; DTS supports long timers | -| `durable-functions` worker / `runOrchestrator` | Inherits the core three-day default | +| Entry point | Timer behavior | +| ---------------------------------------------------- | -------------------------------------------------- | +| Core `TaskHubGrpcWorker` / `TestOrchestrationWorker` | Three-day default | +| Azure-managed worker builder | Explicitly native timers; DTS supports long timers | +| `durable-functions` worker / `runOrchestrator` | Inherits the core three-day default | Core `TaskHubGrpcWorker({ maximumTimerIntervalMs })` and `TestOrchestrationWorker(backend, { maximumTimerIntervalMs })` accept the Python-equivalent @@ -344,13 +357,21 @@ const worker = new TaskHubGrpcWorker({ versioning: { defaultVersion: "v2" }, workItemFilters: "auto", }); -worker.addNamedOrchestrator("Order", async function* (ctx) { - yield ctx.callActivity("Price"); -}, "v1"); -worker.addNamedOrchestrator("Order", async function* (ctx) { - yield ctx.callActivity("Price"); - yield ctx.callSubOrchestrator("Audit"); -}, "v2"); +worker.addNamedOrchestrator( + "Order", + async function* (ctx) { + yield ctx.callActivity("Price"); + }, + "v1", +); +worker.addNamedOrchestrator( + "Order", + async function* (ctx) { + yield ctx.callActivity("Price"); + yield ctx.callSubOrchestrator("Audit"); + }, + "v2", +); worker.addNamedActivity("Price", () => 100, "v1"); worker.addNamedActivity("Price", () => 120, "v2"); worker.addNamedOrchestrator("Audit", () => "audited", "v2"); @@ -381,11 +402,11 @@ versioned registration for that name disables this fallback: unknown versions fa existing not-registered error (including the requested version), marked non-retriable. An unversioned request never selects a versioned implementation. -| Scheduled work | When `options.version` is omitted | Explicit `version: ""` | -| --- | --- | --- | -| Activity | Current orchestration instance's `ctx.version` | Unversioned | -| Sub-orchestration | Worker's `versioning.defaultVersion`, or unversioned if unset | Unversioned | -| Top-level orchestration | Client's `defaultVersion`, or unversioned if unset | Unversioned | +| Scheduled work | When `options.version` is omitted | Explicit `version: ""` | +| ----------------------- | ------------------------------------------------------------- | ---------------------- | +| Activity | Current orchestration instance's `ctx.version` | Unversioned | +| Sub-orchestration | Worker's `versioning.defaultVersion`, or unversioned if unset | Unversioned | +| Top-level orchestration | Client's `defaultVersion`, or unversioned if unset | Unversioned | An explicit nonempty version overrides these defaults. Activities do **not** inherit the worker default, and children do **not** inherit the parent's version. Both policy and handler retries @@ -431,10 +452,7 @@ Long-running orchestrations can restart with fresh history and optionally move t orchestration version: ```typescript -const eternalOrchestrator: TOrchestrator = async function* ( - ctx: OrchestrationContext, - iteration: number, -): any { +const eternalOrchestrator: TOrchestrator = async function* (ctx: OrchestrationContext, iteration: number): any { yield ctx.callActivity(processIteration, iteration); ctx.continueAsNew(iteration + 1, true, "2.0.0"); }; diff --git a/doc/release_process.md b/doc/release_process.md index 4bccad11..3306feb2 100644 --- a/doc/release_process.md +++ b/doc/release_process.md @@ -1,14 +1,15 @@ # Release Process -This document describes how to cut a new release of the three npm packages published from this monorepo: `@microsoft/durabletask-js` (core), `@microsoft/durabletask-js-azuremanaged`, and `durable-functions` (the Azure Functions v4 compatibility provider, whose source lives in `packages/azure-functions-durable`). +This document describes how to cut a new release of the core SDK, its Azure Blob and Azure-managed extensions, and `durable-functions` (the Azure Functions v4 compatibility provider, whose source lives in `packages/azure-functions-durable`). ## Overview -| Package (npm name) | Directory | Changelog | npm | -|---|---|---|---| -| `@microsoft/durabletask-js` (core) | `packages/durabletask-js` | `CHANGELOG.md` | [npmjs](https://www.npmjs.com/package/@microsoft/durabletask-js) | +| Package (npm name) | Directory | Changelog | npm | +| ---------------------------------------- | -------------------------------------- | --------------------------------------------------- | ----------------------------------------------------------------------------- | +| `@microsoft/durabletask-js` (core) | `packages/durabletask-js` | `CHANGELOG.md` | [npmjs](https://www.npmjs.com/package/@microsoft/durabletask-js) | +| `@microsoft/durabletask-js-azureblob` | `packages/durabletask-js-azureblob` | `packages/durabletask-js-azureblob/CHANGELOG.md` | [npmjs](https://www.npmjs.com/package/@microsoft/durabletask-js-azureblob) | | `@microsoft/durabletask-js-azuremanaged` | `packages/durabletask-js-azuremanaged` | `packages/durabletask-js-azuremanaged/CHANGELOG.md` | [npmjs](https://www.npmjs.com/package/@microsoft/durabletask-js-azuremanaged) | -| `durable-functions` | `packages/azure-functions-durable` | `packages/azure-functions-durable/CHANGELOG.md` | [npmjs](https://www.npmjs.com/package/durable-functions) | +| `durable-functions` | `packages/azure-functions-durable` | `packages/azure-functions-durable/CHANGELOG.md` | [npmjs](https://www.npmjs.com/package/durable-functions) | This is an **npm workspaces** monorepo. The official build (`eng/ci/official-build.yml`) produces signed `.tgz` artifacts, and the **sanctioned way to publish them to npm is the ESRP release pipeline** (`eng/ci/release.yml`, see Step 3 below). A manual `npm publish` from the package directory is documented as an alternative for maintainers who need it. @@ -16,10 +17,12 @@ This is an **npm workspaces** monorepo. The official build (`eng/ci/official-bui **Each package is versioned, changelogged, and tagged independently.** A release covers exactly one package; its version number, its changelog, and its git tag advance on their own and do **not** have to match the other packages. -There is one hard ordering constraint between packages: +Core must be published before a dependent package requiring that version: -- **`@microsoft/durabletask-js` (core) must be published before `durable-functions`.** The compat package declares an **exact** dependency on core (currently `"@microsoft/durabletask-js": "0.4.0"`). If `durable-functions` is published while that exact core version is not yet on npm, the published compat package is **uninstallable**. -- **`@microsoft/durabletask-js-azuremanaged` is unordered relative to `durable-functions`.** It depends on core only through a peer **floor** (`"@microsoft/durabletask-js": ">=0.3.0"`), which is already satisfied by the published core, so it can be released before or after the other packages. +- **`@microsoft/durabletask-js` (core) must be published before `durable-functions`.** The compat package declares an **exact** dependency on core (currently `"@microsoft/durabletask-js": "0.4.1"`). If `durable-functions` is published while that exact core version is not yet on npm, the published compat package is **uninstallable**. +- **Azure Blob and Azure-managed now require core `>=0.4.1`.** Core `0.4.1` is the first source version containing `PayloadStore` and the payload transport APIs; the already-published `0.4.0` cannot satisfy this floor. Publish core before either extension. The extensions remain unordered relative to each other and `durable-functions`, and their own version numbers remain independent. + +For the first payload-capable core release, explicitly select `version: 0.4.1` in **Prepare Release**. The source version is already reserved as `0.4.1`; leaving the version input empty would auto-increment it. No package is published by this feature change. ## Versioning Scheme @@ -29,14 +32,14 @@ We follow [semver](https://semver.org/) with optional pre-release tags: X.Y.Z-alpha.N → X.Y.Z-beta.N → X.Y.Z-rc.N → X.Y.Z (stable) ``` -| Version Type | Example | -|---|---| -| Alpha | `0.1.0-alpha.1` | -| Beta | `0.1.0-beta.1` | -| Release Candidate | `0.1.0-rc.1` | -| Stable | `0.1.0` | +| Version Type | Example | +| ----------------- | --------------- | +| Alpha | `0.1.0-alpha.1` | +| Beta | `0.1.0-beta.1` | +| Release Candidate | `0.1.0-rc.1` | +| Stable | `0.1.0` | -The scheme above describes the version **string**. Mapping a version to an **npm dist-tag** is a separate concern and does **not** follow a per-stage `beta`/`next` convention: the **Prepare Release** workflow's emitted **manual fallback** `npm publish` command tags every prerelease `preview` and omits `--tag` for a stable GA (so only a GA moves `latest`). Publishing a prerelease through the sanctioned ESRP path is **blocked pending B11** until owners confirm whether the ESRP task can set that tag. See *Quick Reference: npm Dist Tags* below for the authoritative rule. +The scheme above describes the version **string**. Mapping a version to an **npm dist-tag** is a separate concern and does **not** follow a per-stage `beta`/`next` convention: the **Prepare Release** workflow's emitted **manual fallback** `npm publish` command tags every prerelease `preview` and omits `--tag` for a stable GA (so only a GA moves `latest`). Publishing a prerelease through the sanctioned ESRP path is **blocked pending B11** until owners confirm whether the ESRP task can set that tag. See _Quick Reference: npm Dist Tags_ below for the authoritative rule. ## Automated Release Preparation (Recommended) @@ -50,7 +53,7 @@ The changelog step lists commits since the released package's **last package-sco 1. Go to **Actions** → **Prepare Release** in GitHub 2. Click **Run workflow** -3. **Select the `package` to release** (required) — one of `durabletask-js`, `durabletask-js-azuremanaged`, or `azure-functions-durable`. Each run releases exactly one package. +3. **Select the `package` to release** (required) — one of `durabletask-js`, `durabletask-js-azureblob`, `durabletask-js-azuremanaged`, or `azure-functions-durable`. Each run releases exactly one package. 4. Optionally specify a version (e.g., `0.2.0-beta.1` for core, or `4.0.0-beta.1` for `durable-functions`). Leave empty to auto-increment the selected package's current version (bumps the prerelease number for a prerelease, otherwise the patch). 5. Click **Run workflow** @@ -61,8 +64,8 @@ For the **one package you selected** (and only that package): 1. **Determines the next version**: uses the `version` input, or auto-increments the selected package's current version 2. **Generates a changelog**: lists commits since that package's last release tag, scoped to the package's directory (`git log ..HEAD -- `), so only commits that touched that package are included 3. **Bumps the version**: updates `version` in that package's own `package.json` -4. **Updates that package's changelog**: core writes `CHANGELOG.md`, Azure Managed writes `packages/durabletask-js-azuremanaged/CHANGELOG.md`, and `durable-functions` writes `packages/azure-functions-durable/CHANGELOG.md` -5. **Creates a release branch and a package-scoped tag**: tag `` and branch `release/`, where the prefix is `v` (core), `azuremanaged-v`, or `durable-functions-v` +4. **Updates that package's changelog**: core writes `CHANGELOG.md`; Azure Blob, Azure Managed, and `durable-functions` write their own package's `CHANGELOG.md` +5. **Creates a release branch and a package-scoped tag**: tag `` and branch `release/`, where the prefix is `v` (core), `azureblob-v`, `azuremanaged-v`, or `durable-functions-v` 6. **For `durable-functions` only**: verifies the exact-pinned `@microsoft/durabletask-js` version is already published on public npm, and fails the run if it is not (guards the uninstallable-dependency case) ### After the Workflow Completes @@ -98,8 +101,9 @@ Trigger the official build pipeline to produce signed `.tgz` artifacts. The offi 1. Click **Run pipeline** on `main` (or locate the `main` run that already contains the release commit) 2. Wait for it to complete -3. A single `main` build packs **all three** packages into one `drop` artifact; which single package publishes is decided later by the `package` parameter in the release pipeline (Step 3). Verify `drop` contains the correctly versioned `.tgz` files: +3. A single `main` build packs **all four** packages into one `drop` artifact; which single package publishes is decided later by the `package` parameter in the release pipeline (Step 3). Verify `drop` contains the correctly versioned `.tgz` files: - `buildoutputs/durabletask-js/microsoft-durabletask-js-X.Y.Z.tgz` + - `buildoutputs/durabletask-js-azureblob/microsoft-durabletask-js-azureblob-X.Y.Z.tgz` - `buildoutputs/durabletask-js-azuremanaged/microsoft-durabletask-js-azuremanaged-X.Y.Z.tgz` - `buildoutputs/azure-functions-durable/durable-functions-X.Y.Z.tgz` @@ -110,6 +114,7 @@ Never publish from the release branch — it exists only to carry the version/ch Trigger the release pipeline to publish one signed package to npm via ESRP. **This is the sanctioned publish path.** `eng/ci/release.yml` consumes the `durabletask-js.official` build artifact from **`main`** (its pipeline resource is pinned to `branch: main`), so its source is the `main` official build selected in Step 2 — not the release branch. The pipeline's `package` runtime parameter controls which one of these release stages is inserted into the compiled 1ES plan. It defaults to `durabletask-js`, so verify or change it before queueing: a wrong or unchanged default compiles the wrong package stage. ESRP approval remains the final safety gate before publishing. - `durabletask-js` inserts `release_durabletask_js` for core `@microsoft/durabletask-js`. +- `durabletask-js-azureblob` inserts `release_durabletask_js_azureblob` for `@microsoft/durabletask-js-azureblob`. - `durabletask-js-azuremanaged` inserts `release_durabletask_js_azuremanaged` for `@microsoft/durabletask-js-azuremanaged`. - `azure-functions-durable` inserts `release_durable_functions` for `durable-functions`. @@ -122,14 +127,15 @@ Trigger the release pipeline to publish one signed package to npm via ESRP. **Th This compile-time selection eliminates the failure demonstrated by run **294746**: de-selecting the core stage at queue time did not expose its result to the Azure Managed dependency condition as the expected literal `Skipped`, so the selected package's stage was skipped too. The selected package now has no cross-stage dependency because the other package stages do not exist in that run's compiled plan. -**Two runs for core + compat.** The compat **Prepare Release** run guards on core already being published to **public npm** — it runs `npm view @microsoft/durabletask-js@ --registry https://registry.npmjs.org/` and fails if that exact version is absent (`durable-functions` exact-pins core). Publish and verify core first; only then prepare `durable-functions` and publish it in a later release-pipeline run. (`azuremanaged` depends on core only through a peer floor that is already satisfied and has no ordering constraint.) +**Separate runs for core and dependents.** The compat **Prepare Release** run guards on core already being published to **public npm** — it runs `npm view @microsoft/durabletask-js@ --registry https://registry.npmjs.org/` and fails if that exact version is absent (`durable-functions` exact-pins core). Publish and verify core first; only then publish the payload extensions and prepare/publish `durable-functions` in later release-pipeline runs. -> **Open question (B11) — hard blocker for ESRP prereleases:** it is not yet confirmed whether the ESRP release task can set the npm **dist-tag** (e.g. publish a prerelease under `preview` rather than moving `latest`). Publishing a prerelease such as `durable-functions@4.0.0-beta.1` through ESRP is **blocked** until the ESRP / 1ES pipeline owners confirm how to apply the `preview` tag — do **not** assume ESRP applies a dist-tag. The `--tag` guidance in *Quick Reference: npm Dist Tags* applies to a manual `npm publish`. +> **Open question (B11) — hard blocker for ESRP prereleases:** it is not yet confirmed whether the ESRP release task can set the npm **dist-tag** (e.g. publish a prerelease under `preview` rather than moving `latest`). Publishing a prerelease such as `durable-functions@4.0.0-beta.1` through ESRP is **blocked** until the ESRP / 1ES pipeline owners confirm how to apply the `preview` tag — do **not** assume ESRP applies a dist-tag. The `--tag` guidance in _Quick Reference: npm Dist Tags_ applies to a manual `npm publish`. ### Step 4: Verify npm Publish ```bash npm view @microsoft/durabletask-js versions +npm view @microsoft/durabletask-js-azureblob versions npm view @microsoft/durabletask-js-azuremanaged versions npm view durable-functions versions @@ -160,13 +166,14 @@ Bump the `"version"` field in **only the package you are releasing**: ```bash # core: packages/durabletask-js/package.json → "version": "X.Y.Z" +# azureblob: packages/durabletask-js-azureblob/package.json → "version": "X.Y.Z" # azuremanaged: packages/durabletask-js-azuremanaged/package.json → "version": "X.Y.Z" # compat: packages/azure-functions-durable/package.json → "version": "X.Y.Z" ``` Then update the affected cross-package dependency **for the package you are releasing**: -- **`@microsoft/durabletask-js-azuremanaged`** declares a peer **floor** on core. Only raise it if this release actually requires a newer core — it does not have to equal the release version: +- **`@microsoft/durabletask-js-azureblob` and `@microsoft/durabletask-js-azuremanaged`** declare a peer **floor** on core. Raise it when the package requires a newer core API — it does not have to equal the extension's own release version. The payload API floor is `>=0.4.1`; never lower it to the incompatible published `0.4.0`: ```jsonc "peerDependencies": { @@ -186,11 +193,12 @@ Then update the affected cross-package dependency **for the package you are rele Move items from the `## Upcoming` section of the package's changelog into a new versioned section. The changelog is a **generated** list of the released package's commit messages / PR links — not a place for hand-authored breaking-change narrative; detailed preview and migration guidance lives in `packages/azure-functions-durable/README.md` (for `durable-functions`), not the changelog. To reproduce the tooling manually, promote any curated `## Upcoming` notes into the new section and add one `### Changes` entry per commit, mirroring `git log ..HEAD -- ` (each commit subject with its `(#NN)` linked). Update only the released package's changelog: -| Package | Changelog | -|---|---| -| `@microsoft/durabletask-js` | `CHANGELOG.md` | +| Package | Changelog | +| ---------------------------------------- | --------------------------------------------------- | +| `@microsoft/durabletask-js` | `CHANGELOG.md` | +| `@microsoft/durabletask-js-azureblob` | `packages/durabletask-js-azureblob/CHANGELOG.md` | | `@microsoft/durabletask-js-azuremanaged` | `packages/durabletask-js-azuremanaged/CHANGELOG.md` | -| `durable-functions` | `packages/azure-functions-durable/CHANGELOG.md` | +| `durable-functions` | `packages/azure-functions-durable/CHANGELOG.md` | ```markdown ## Upcoming @@ -202,6 +210,7 @@ Move items from the `## Upcoming` section of the package's changelog into a new ## vX.Y.Z (YYYY-MM-DD) ### Changes + - Some change ([#NN](https://github.com/microsoft/durabletask-js/pull/NN)) - Another change ([#NN](https://github.com/microsoft/durabletask-js/pull/NN)) ``` diff --git a/eng/ci/release.yml b/eng/ci/release.yml index 206d4c49..9270a82c 100644 --- a/eng/ci/release.yml +++ b/eng/ci/release.yml @@ -8,6 +8,7 @@ parameters: default: durabletask-js values: - durabletask-js + - durabletask-js-azureblob - durabletask-js-azuremanaged - azure-functions-durable @@ -101,6 +102,41 @@ extends: mainpublisher: 'durabletask-java' domaintenantid: '33e01921-4d64-4f8c-a055-5bdaffd5e33d' + - ${{ if eq(parameters.package, 'durabletask-js-azureblob') }}: + - stage: release_durabletask_js_azureblob + displayName: 'Release @microsoft/durabletask-js-azureblob' + jobs: + - job: durabletask_js_azureblob + displayName: 'Release @microsoft/durabletask-js-azureblob' + templateContext: + type: releaseJob + isProduction: true + inputs: + - input: pipelineArtifact + pipeline: DurableTaskJSBuildPipeline + artifactName: drop + targetPath: $(System.DefaultWorkingDirectory)/drop + + steps: + - task: SFP.release-tasks.custom-build-release-task.EsrpRelease@9 + displayName: 'ESRP Release @microsoft/durabletask-js-azureblob' + inputs: + connectedservicename: 'dtfx-internal-esrp-prod' + usemanagedidentity: true + keyvaultname: 'durable-esrp-akv' + signcertname: 'dts-esrp-cert' + clientid: '0b3ed1a4-0727-4a50-b82a-02c2bd9dec89' + intent: 'PackageDistribution' + contenttype: 'npm' + contentsource: 'Folder' + folderlocation: '$(System.DefaultWorkingDirectory)/drop/buildoutputs/durabletask-js-azureblob' + waitforreleasecompletion: true + owners: 'wangbill@microsoft.com' + approvers: 'kaibocai@microsoft.com' + serviceendpointurl: 'https://api.esrp.microsoft.com' + mainpublisher: 'durabletask-java' + domaintenantid: '33e01921-4d64-4f8c-a055-5bdaffd5e33d' + - ${{ if eq(parameters.package, 'azure-functions-durable') }}: - stage: release_durable_functions displayName: 'Release durable-functions' diff --git a/eng/templates/build.yml b/eng/templates/build.yml index 06c1ab26..a6110554 100644 --- a/eng/templates/build.yml +++ b/eng/templates/build.yml @@ -48,6 +48,12 @@ jobs: npm pack displayName: "pack @microsoft/durabletask-js" + # Pack @microsoft/durabletask-js-azureblob + - script: | + cd packages/durabletask-js-azureblob + npm pack + displayName: "pack @microsoft/durabletask-js-azureblob" + # Pack @microsoft/durabletask-js-azuremanaged - script: | cd packages/durabletask-js-azuremanaged @@ -73,6 +79,12 @@ jobs: SourceFolder: $(System.DefaultWorkingDirectory)/packages/durabletask-js-azuremanaged Contents: "*.tgz" TargetFolder: $(Build.ArtifactStagingDirectory)/buildoutputs/durabletask-js-azuremanaged + - task: CopyFiles@2 + displayName: "Copy durabletask-js-azureblob to staging" + inputs: + SourceFolder: $(System.DefaultWorkingDirectory)/packages/durabletask-js-azureblob + Contents: "*.tgz" + TargetFolder: $(Build.ArtifactStagingDirectory)/buildoutputs/durabletask-js-azureblob - task: CopyFiles@2 displayName: "Copy azure-functions-durable to staging" inputs: diff --git a/examples/azure-managed/README.md b/examples/azure-managed/README.md index 28cfedc8..13f7db87 100644 --- a/examples/azure-managed/README.md +++ b/examples/azure-managed/README.md @@ -4,17 +4,18 @@ Runnable samples demonstrating every major feature of the Durable Task JavaScrip ## Sample Index -| Sample | Scenario | Key Features | Emulator Required | -|--------|----------|-------------|-------------------| -| [hello-orchestrations](hello-orchestrations/) | Core patterns | Activity sequence, fan-out/fan-in, sub-orchestrations, `whenAny` | Yes | -| [retry-and-error-handling](retry-and-error-handling/) | Fault tolerance | `RetryPolicy`, `handleFailure`, `AsyncRetryHandler`, sub-orchestration retry, `raiseIfFailed()` | Yes | -| [human-interaction](human-interaction/) | Event-driven workflows | External events, timers, `whenAny` race, `sendEvent`, custom status | Yes | -| [lifecycle-management](lifecycle-management/) | Orchestration control | Terminate (recursive), suspend/resume, restart, continue-as-new, purge, tags | Yes | -| [query-and-history](query-and-history/) | Monitoring & debugging | `getAllInstances`, pagination, `listInstanceIds`, `getOrchestrationHistory`, typed events | Yes | -| [versioning](versioning/) | Safe deployments | Version match strategies, failure strategies, `ctx.version`, `ctx.compareVersionTo()` | Yes | -| [unit-testing](unit-testing/) | Testing without infra | `InMemoryOrchestrationBackend`, `TestOrchestrationClient`, `TestOrchestrationWorker`, `ReplaySafeLogger` | **No** | -| [basics](basics/) | Azure-managed basics | Connection strings, `DefaultAzureCredential`, `createAzureManagedClient` | Yes | -| [distributed-tracing](distributed-tracing/) | OpenTelemetry tracing | `NodeSDK`, OTLP export, Jaeger, `DurableTaskAzureManagedClientBuilder` | Yes | +| Sample | Scenario | Key Features | Emulator Required | +| ----------------------------------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------- | -------------------- | +| [hello-orchestrations](hello-orchestrations/) | Core patterns | Activity sequence, fan-out/fan-in, sub-orchestrations, `whenAny` | Yes | +| [retry-and-error-handling](retry-and-error-handling/) | Fault tolerance | `RetryPolicy`, `handleFailure`, `AsyncRetryHandler`, sub-orchestration retry, `raiseIfFailed()` | Yes | +| [human-interaction](human-interaction/) | Event-driven workflows | External events, timers, `whenAny` race, `sendEvent`, custom status | Yes | +| [lifecycle-management](lifecycle-management/) | Orchestration control | Terminate (recursive), suspend/resume, restart, continue-as-new, purge, tags | Yes | +| [query-and-history](query-and-history/) | Monitoring & debugging | `getAllInstances`, pagination, `listInstanceIds`, `getOrchestrationHistory`, typed events | Yes | +| [large-payloads](large-payloads/) | External payloads | Transparent UTF-8/gzip blob storage and large activity/orchestration inputs/results | DTS and Blob/Azurite | +| [versioning](versioning/) | Safe deployments | Version match strategies, failure strategies, `ctx.version`, `ctx.compareVersionTo()` | Yes | +| [unit-testing](unit-testing/) | Testing without infra | `InMemoryOrchestrationBackend`, `TestOrchestrationClient`, `TestOrchestrationWorker`, `ReplaySafeLogger` | **No** | +| [basics](basics/) | Azure-managed basics | Connection strings, `DefaultAzureCredential`, `createAzureManagedClient` | Yes | +| [distributed-tracing](distributed-tracing/) | OpenTelemetry tracing | `NodeSDK`, OTLP export, Jaeger, `DurableTaskAzureManagedClientBuilder` | Yes | ### Quick Start (Local Emulator) @@ -145,11 +146,11 @@ done ### Feature Coverage Map -| Feature | Sample(s) | -|---------|-----------| -| `ctx.callActivity()` | hello-orchestrations, retry-and-error-handling | -| `whenAll()` | hello-orchestrations, unit-testing | -| `whenAny()` | hello-orchestrations, human-interaction | +| Feature | Sample(s) | +| --------------------------- | -------------------------------------------------------------------- | +| `ctx.callActivity()` | hello-orchestrations, retry-and-error-handling | +| `whenAll()` | hello-orchestrations, unit-testing | +| `whenAny()` | hello-orchestrations, human-interaction | | `ctx.callSubOrchestrator()` | hello-orchestrations, retry-and-error-handling, lifecycle-management | | `ctx.waitForExternalEvent()` | human-interaction, unit-testing | @@ -195,13 +196,13 @@ This example demonstrates **OpenTelemetry distributed tracing** with the Durable When you run this example, the SDK automatically produces [W3C Trace Context](https://www.w3.org/TR/trace-context/) spans for every stage of a durable orchestration: -| Span | Kind | Description | -|------|------|-------------| -| `create_orchestration:` | PRODUCER | Client scheduling a new orchestration | -| `orchestration:` | SERVER | Worker executing the orchestration | -| `activity:` (scheduling) | CLIENT | Orchestrator scheduling an activity | -| `activity:` (execution) | SERVER | Worker executing the activity | -| `timer:` | INTERNAL | Timer created inside an orchestration | +| Span | Kind | Description | +| --------------------------------- | -------- | ------------------------------------- | +| `create_orchestration:` | PRODUCER | Client scheduling a new orchestration | +| `orchestration:` | SERVER | Worker executing the orchestration | +| `activity:` (scheduling) | CLIENT | Orchestrator scheduling an activity | +| `activity:` (execution) | SERVER | Worker executing the activity | +| `timer:` | INTERNAL | Timer created inside an orchestration | | `orchestration_event:` | PRODUCER | Event raised to another orchestration | All spans are linked via `traceparent` propagation, giving you a single end-to-end trace from the client all the way through parallel activity fan-out and back. @@ -240,14 +241,15 @@ docker compose up -d This starts: -| Service | Port | Purpose | -|---------|------|---------| -| **DTS Emulator** | `8080` | Local gRPC endpoint (no authentication) | -| **Jaeger UI** | `16686` | Trace visualization dashboard | -| **Jaeger OTLP (HTTP)** | `4318` | OpenTelemetry trace receiver | -| **Jaeger OTLP (gRPC)** | `4317` | OpenTelemetry trace receiver (gRPC) | +| Service | Port | Purpose | +| ---------------------- | ------- | --------------------------------------- | +| **DTS Emulator** | `8080` | Local gRPC endpoint (no authentication) | +| **Jaeger UI** | `16686` | Trace visualization dashboard | +| **Jaeger OTLP (HTTP)** | `4318` | OpenTelemetry trace receiver | +| **Jaeger OTLP (gRPC)** | `4317` | OpenTelemetry trace receiver (gRPC) | > **Tip:** You can also run just the DTS Emulator standalone: +> > ```bash > docker run -d --name dts-emulator -p 8080:8080 -p 8082:8082 \ > mcr.microsoft.com/dts/dts-emulator:latest @@ -378,12 +380,12 @@ const traceExporter = new AzureMonitorTraceExporter({ ## DTS Emulator Docker Image Reference -| Property | Value | -|----------|-------| -| **Image** | `should be mcr.microsoft.com/dts/dts-emulator:latest` | -| **gRPC Port** | `8080` | -| **Dashboard Port** | `8082` | -| **Authentication** | `None` (no credentials required) | +| Property | Value | +| --------------------- | -------------------------------------------------------------------- | +| **Image** | `should be mcr.microsoft.com/dts/dts-emulator:latest` | +| **gRPC Port** | `8080` | +| **Dashboard Port** | `8082` | +| **Authentication** | `None` (no credentials required) | | **Connection String** | `Endpoint=http://localhost:8080;Authentication=None;TaskHub=default` | ### Running the Emulator Standalone @@ -432,10 +434,10 @@ docker compose down ## Files in This Example -| File | Description | -|------|-------------| -| `distributed-tracing/` | Distributed tracing sample – OTel setup + orchestrations | -| `basics/` | Basic sample – connection strings, DefaultAzureCredential | -| `docker-compose.yml` | DTS Emulator + Jaeger stack | -| `.env.emulator` | Pre-configured env vars for the local emulator | -| `.env.example` | Template for Azure Managed DTS (cloud) | +| File | Description | +| ---------------------- | --------------------------------------------------------- | +| `distributed-tracing/` | Distributed tracing sample – OTel setup + orchestrations | +| `basics/` | Basic sample – connection strings, DefaultAzureCredential | +| `docker-compose.yml` | DTS Emulator + Jaeger stack | +| `.env.emulator` | Pre-configured env vars for the local emulator | +| `.env.example` | Template for Azure Managed DTS (cloud) | diff --git a/examples/azure-managed/large-payloads/README.md b/examples/azure-managed/large-payloads/README.md new file mode 100644 index 00000000..2a24df36 --- /dev/null +++ b/examples/azure-managed/large-payloads/README.md @@ -0,0 +1,33 @@ +# Large payloads + +This sample round-trips a 300 KiB input through an orchestration and an activity, +verifying that the client receives the original output rather than a blob token. +The standard 256 KiB threshold is measured on serialized UTF-8 payloads. + +Build the SDK, configure a DTS endpoint and a blob endpoint, then run from the repository root: + +```sh +npm install +npm run build +npm run example -- ./examples/azure-managed/large-payloads/index.ts +``` + +Set these in `examples/azure-managed/.env` or your environment: + +```text +DURABLE_TASK_SCHEDULER_CONNECTION_STRING=Endpoint=http://localhost:8080;Authentication=None;TaskHub=default +AZURE_STORAGE_CONNECTION_STRING=UseDevelopmentStorage=true +``` + +For local execution, start DTS using the parent directory's Docker Compose file and run +Azurite separately on its default blob port 10000. The sample can instead use existing +Azure resources; no account or role is provisioned. Connection strings must never be committed. +For identity-based storage authentication, see the +[payload package documentation](../../../packages/durabletask-js-azureblob/README.md). + +The sample leaves its instance intact and does **not** modify the task hub's auto-purge setting. +To opt into cleanup separately, configure a supporting DTS backend and payload-enabled workers, +call `client.setLargePayloadAutoPurge(true)`, then purge terminal instances through the usual +client API. Keep a payload-enabled worker running to process tombstones. Disabling with +`client.setLargePayloadAutoPurge(false)` pauses backend fetches without terminating the runner. +Legacy v1 blobs, soft-deleted data, and retained blob versions are not immediately reclaimed. diff --git a/examples/azure-managed/large-payloads/index.ts b/examples/azure-managed/large-payloads/index.ts new file mode 100644 index 00000000..04e576d5 --- /dev/null +++ b/examples/azure-managed/large-payloads/index.ts @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as dotenv from "dotenv"; +import * as path from "path"; +import { ActivityContext, OrchestrationContext } from "@microsoft/durabletask-js"; +import { BlobPayloadStore } from "@microsoft/durabletask-js-azureblob"; +import { + DurableTaskAzureManagedClientBuilder, + createAzureManagedWorkerBuilder, +} from "@microsoft/durabletask-js-azuremanaged"; + +dotenv.config({ path: path.join(__dirname, "..", ".env") }); + +async function echo(_context: ActivityContext, input: string): Promise { + return input; +} + +async function* largePayload(context: OrchestrationContext, input: string): AsyncGenerator { + context.setCustomStatus({ phase: "echo", inputBytes: input.length }); + return yield context.callActivity(echo, input); +} + +async function main(): Promise { + const scheduler = process.env.DURABLE_TASK_SCHEDULER_CONNECTION_STRING; + const storage = process.env.AZURE_STORAGE_CONNECTION_STRING; + if (!scheduler || !storage) { + throw new Error("Set DURABLE_TASK_SCHEDULER_CONNECTION_STRING and AZURE_STORAGE_CONNECTION_STRING."); + } + const store = new BlobPayloadStore({ connectionString: storage }); + const payloads = { store }; + const worker = createAzureManagedWorkerBuilder(scheduler) + .useExternalizedPayloads(payloads) + .addOrchestrator(largePayload) + .addActivity(echo) + .build(); + const client = new DurableTaskAzureManagedClientBuilder() + .connectionString(scheduler) + .useExternalizedPayloads(payloads) + .build(); + let started = false; + try { + await worker.start(); + started = true; + const input = "x".repeat(300 * 1024); + const id = await client.scheduleNewOrchestration(largePayload, input); + const state = await client.waitForOrchestrationCompletion(id, true, 60); + if (!state) throw new Error("The orchestration was not found."); + state.raiseIfFailed(); + if (state.serializedOutput !== JSON.stringify(input)) + throw new Error("The hydrated output did not match the input."); + console.log(`Round-tripped ${Buffer.byteLength(input, "utf8")} bytes through orchestration and activity payloads.`); + console.log(`Instance ${id} retained. Auto-purge is a separate, task-hub-wide opt-in; see the sample README.`); + } finally { + if (started) await worker.stop(); + await client.stop(); + } +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/examples/azure-managed/large-payloads/sample.json b/examples/azure-managed/large-payloads/sample.json new file mode 100644 index 00000000..a5490f47 --- /dev/null +++ b/examples/azure-managed/large-payloads/sample.json @@ -0,0 +1,7 @@ +{ + "name": "large-payloads", + "description": "Transparent orchestration/activity payload externalization and .NET-compatible Azure Blob storage", + "requiresEmulator": true, + "skipCi": true, + "skipCiReason": "Requires an independently configured Azure Blob/Azurite endpoint in addition to DTS." +} diff --git a/internal/protocol/protos/large_payload_purge.proto b/internal/protocol/protos/large_payload_purge.proto new file mode 100644 index 00000000..c615ffff --- /dev/null +++ b/internal/protocol/protos/large_payload_purge.proto @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// +// DTS-only contract from durabletask-dotnet 91739687ff647699079cdadd627daa00562982b7. + +syntax = "proto3"; + +package microsoft.durabletask.largepayloads; + +option csharp_namespace = "Microsoft.DurableTask.Protobuf.LargePayloads"; + +// The backend owns the tombstone ledger, task-hub setting, and retry scheduling. +// The credentialed worker performs ownership-checked blob deletes and reports outcomes. +service LargePayloadPurge { + rpc SetLargePayloadAutoPurge(SetLargePayloadAutoPurgeRequest) returns (SetLargePayloadAutoPurgeResponse); + rpc GetLargePayloadTombstones(GetLargePayloadTombstonesRequest) returns (GetLargePayloadTombstonesResponse); + rpc ReportLargePayloadPurgeResults(ReportLargePayloadPurgeResultsRequest) returns (ReportLargePayloadPurgeResultsResponse); +} + +message SetLargePayloadAutoPurgeRequest { + bool enabled = 1; +} + +message SetLargePayloadAutoPurgeResponse {} + +message LargePayloadTombstone { + // Opaque correlation token. Echo unchanged; never parse or reconstruct it. + string tombstone_token = 1; + string payload_token = 2; +} + +enum LargePayloadPurgeDisposition { + LARGE_PAYLOAD_PURGE_DISPOSITION_UNSPECIFIED = 0; + LARGE_PAYLOAD_PURGE_DISPOSITION_DELETED = 1; + LARGE_PAYLOAD_PURGE_DISPOSITION_RETRY = 2; + LARGE_PAYLOAD_PURGE_DISPOSITION_QUARANTINED = 3; +} + +message LargePayloadPurgeResult { + string tombstone_token = 1; + LargePayloadPurgeDisposition disposition = 2; +} + +message GetLargePayloadTombstonesRequest { + int32 limit = 1; +} + +message GetLargePayloadTombstonesResponse { + repeated LargePayloadTombstone tombstones = 1; +} + +message ReportLargePayloadPurgeResultsRequest { + repeated LargePayloadPurgeResult results = 1; +} + +message ReportLargePayloadPurgeResultsResponse {} diff --git a/jest.config.js b/jest.config.js index cc3ea143..d4f5c1dd 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,20 +1,21 @@ -/** @type {import('ts-jest').JestConfigWithTsJest} */ -module.exports = { - preset: "ts-jest", - testEnvironment: "node", - testMatch: ["**/tests/**/*.spec.ts", "**/test/**/*.spec.ts"], - moduleFileExtensions: ["ts", "tsx", "js", "jsx", "json", "node"], - moduleNameMapper: { - "^@microsoft/durabletask-js$": "/packages/durabletask-js/src/index.ts", - "^@microsoft/durabletask-js-azuremanaged$": "/packages/durabletask-js-azuremanaged/src/index.ts", - "^@microsoft/durabletask-js-export-history$": "/packages/durabletask-js-export-history/src/index.ts", - }, - transform: { - "^.+\\.tsx?$": [ - "ts-jest", - { - tsconfig: "tsconfig.base.json", - }, - ], - }, -}; +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: "ts-jest", + testEnvironment: "node", + testMatch: ["**/tests/**/*.spec.ts", "**/test/**/*.spec.ts"], + moduleFileExtensions: ["ts", "tsx", "js", "jsx", "json", "node"], + moduleNameMapper: { + "^@microsoft/durabletask-js$": "/packages/durabletask-js/src/index.ts", + "^@microsoft/durabletask-js-azuremanaged$": "/packages/durabletask-js-azuremanaged/src/index.ts", + "^@microsoft/durabletask-js-azureblob$": "/packages/durabletask-js-azureblob/src/index.ts", + "^@microsoft/durabletask-js-export-history$": "/packages/durabletask-js-export-history/src/index.ts", + }, + transform: { + "^.+\\.tsx?$": [ + "ts-jest", + { + tsconfig: "tsconfig.base.json", + }, + ], + }, +}; diff --git a/package-lock.json b/package-lock.json index 8ef8ff3e..78789512 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1630,6 +1630,10 @@ "resolved": "packages/durabletask-js", "link": true }, + "node_modules/@microsoft/durabletask-js-azureblob": { + "resolved": "packages/durabletask-js-azureblob", + "link": true + }, "node_modules/@microsoft/durabletask-js-azuremanaged": { "resolved": "packages/durabletask-js-azuremanaged", "link": true @@ -7600,7 +7604,7 @@ "dependencies": { "@azure/functions": "^4.16.1", "@grpc/grpc-js": "^1.14.4", - "@microsoft/durabletask-js": "0.4.0" + "@microsoft/durabletask-js": "0.4.1" }, "devDependencies": { "@types/jest": "^29.5.1", @@ -7623,7 +7627,7 @@ }, "packages/durabletask-js": { "name": "@microsoft/durabletask-js", - "version": "0.4.0", + "version": "0.4.1", "license": "MIT", "dependencies": { "@grpc/grpc-js": "^1.14.4", @@ -7651,6 +7655,21 @@ } } }, + "packages/durabletask-js-azureblob": { + "name": "@microsoft/durabletask-js-azureblob", + "version": "0.4.0", + "license": "MIT", + "dependencies": { + "@azure/core-auth": "^1.10.0", + "@azure/storage-blob": "^12.26.0" + }, + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "@microsoft/durabletask-js": ">=0.4.1" + } + }, "packages/durabletask-js-azuremanaged": { "name": "@microsoft/durabletask-js-azuremanaged", "version": "0.4.0", @@ -7671,7 +7690,7 @@ }, "peerDependencies": { "@grpc/grpc-js": "^1.14.4", - "@microsoft/durabletask-js": ">=0.3.0" + "@microsoft/durabletask-js": ">=0.4.1" } }, "packages/durabletask-js-azuremanaged/node_modules/@types/node": { diff --git a/package.json b/package.json index 28d397c7..194ba76f 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "build:azuremanaged": "npm run build -w @microsoft/durabletask-js-azuremanaged", "test": "npm run test --workspaces", "test:unit": "npm run test:unit --workspaces --if-present", + "test:payload-packages": "node scripts/test-payload-package-compatibility.js", "test:e2e": "./scripts/test-e2e.sh", "test:e2e:internal": "jest tests/e2e --runInBand --detectOpenHandles", "test:e2e:one": "jest tests/e2e --runInBand --detectOpenHandles --testNamePattern", diff --git a/packages/azure-functions-durable/package.json b/packages/azure-functions-durable/package.json index d4cf0059..20e54bec 100644 --- a/packages/azure-functions-durable/package.json +++ b/packages/azure-functions-durable/package.json @@ -68,7 +68,7 @@ "dependencies": { "@azure/functions": "^4.16.1", "@grpc/grpc-js": "^1.14.4", - "@microsoft/durabletask-js": "0.4.0" + "@microsoft/durabletask-js": "0.4.1" }, "peerDependencies": { "@azure/identity": "^4.0.0" diff --git a/packages/durabletask-js-azureblob/CHANGELOG.md b/packages/durabletask-js-azureblob/CHANGELOG.md new file mode 100644 index 00000000..5706c52b --- /dev/null +++ b/packages/durabletask-js-azureblob/CHANGELOG.md @@ -0,0 +1,10 @@ +## Upcoming + +### New + +- Require core SDK `>=0.4.1`, excluding published `0.4.0`, which does not export the payload APIs. +- Add opt-in Azure Blob payload storage, interoperable with durabletask-dotnet + `91739687ff647699079cdadd627daa00562982b7`: UTF-8/gzip blobs, credential-free v2 tokens, + legacy v1 reads, safe same-service and identity-only cross-account routing. +- Support ownership-checked, ETag-conditional deletion for DTS automatic payload cleanup, + including snapshots and confirmed-absence handling. diff --git a/packages/durabletask-js-azureblob/LICENSE b/packages/durabletask-js-azureblob/LICENSE new file mode 100644 index 00000000..22aed37e --- /dev/null +++ b/packages/durabletask-js-azureblob/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Microsoft Corporation. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/durabletask-js-azureblob/README.md b/packages/durabletask-js-azureblob/README.md new file mode 100644 index 00000000..8654a514 --- /dev/null +++ b/packages/durabletask-js-azureblob/README.md @@ -0,0 +1,159 @@ +# Azure Blob payload storage for Durable Task + +`@microsoft/durabletask-js-azureblob` adds opt-in external payload storage to the +Durable Task JavaScript SDK. Core users do not acquire Azure Storage dependencies. +Its tokens, UTF-8/gzip representation, ownership metadata, and auto-purge protocol +match [durabletask-dotnet at `91739687`](https://github.com/microsoft/durabletask-dotnet/tree/91739687ff647699079cdadd627daa00562982b7/src/Extensions/AzureBlobPayloads). + +## Configure clients and workers + +Requires `@microsoft/durabletask-js >=0.4.1`, the first core source version with +`PayloadStore` and the payload transport APIs. Published core `0.4.0` is incompatible; +npm rejects that peer pairing. Publish the payload-capable core release before this extension. + +```sh +npm install @microsoft/durabletask-js @microsoft/durabletask-js-azureblob +``` + +```typescript +import { TaskHubGrpcClient, TaskHubGrpcWorker } from "@microsoft/durabletask-js"; +import { BlobPayloadStore } from "@microsoft/durabletask-js-azureblob"; + +const store = new BlobPayloadStore({ + connectionString: process.env.AZURE_STORAGE_CONNECTION_STRING!, + // containerName: "durabletask-payloads", + // compressionEnabled: true, +}); +const payloads = { store, thresholdBytes: 256 * 1024, maxPayloadBytes: 10 * 1024 * 1024 }; +const client = new TaskHubGrpcClient({ hostAddress: "localhost:8080", payloads }); +const worker = new TaskHubGrpcWorker({ hostAddress: "localhost:8080", payloads }); +``` + +For identity authentication, install `@azure/identity` and replace the store configuration: + +```typescript +import { DefaultAzureCredential } from "@azure/identity"; + +const store = new BlobPayloadStore({ + accountUri: "https://myaccount.blob.core.windows.net", + credential: new DefaultAzureCredential(), +}); +``` + +The identity needs permission to create the private container and read/write its blobs, +and delete permission when auto-purge is enabled. The SDK does not provision accounts or roles. +Azure-managed scheduler builders expose `.useExternalizedPayloads(payloads)` on both +`DurableTaskAzureManagedClientBuilder` and `DurableTaskAzureManagedWorkerBuilder`. +Scheduler credentials and blob credentials are independent. + +Configure every participating client and worker, including .NET workers in a mixed-language +task hub, **before** writing externalized payloads. Workers advertise large-payload capability +only when enabled. Removing support while stored tokens remain makes those instances unreadable. +This feature targets the Durable Task Scheduler; it does not reconfigure a Durable Functions host. + +## Payload contract + +| Option | Default | Behavior | +| -------------------------- | ---------------------- | ----------------------------------------------------------------------------------------- | +| `payloads.thresholdBytes` | 262144 (256 KiB) | Externalize at **greater than or equal** serialized UTF-8 bytes; maximum threshold 1 MiB. | +| `payloads.maxPayloadBytes` | 10485760 (10 MiB) | Reject larger externalized payloads before uploading. | +| `containerName` | `durabletask-payloads` | Lazily create a private container, once per successful initialization. | +| `compressionEnabled` | `true` | Gzip payload bytes; record `Content-Encoding: gzip`. | + +Sizes refer to serialized values, including JSON quotes/escapes, not JavaScript string length +or compressed size. Absent and empty payloads remain inline. With the feature disabled, +existing transport and payload behavior are unchanged. + +The SDK transparently externalizes client start/event/entity-signal inputs and management +reasons/outputs, activity results, orchestration actions and custom status, continue-as-new +inputs, rewritten history, and entity state/results/outgoing operations. State queries, +waits, history streams, and orchestration/activity/entity V1/V2 work items are hydrated before +conversion or user execution. Each message runs at most eight payload operations concurrently; +failure/cancellation stops further dispatch and drains already-started operations. + +New blobs have 32-character GUID names and `managed_by=dts` metadata on compressed and +uncompressed writes. Tokens are `blob:v2:{absoluteBlobUrl}`, without account keys or SAS. +Readers also recognize `blob:v1:{container}:{blobName}` in the configured container. +Same-service v2 reads preserve the configured credential/pipeline across containers, +including path-style/Azurite endpoints. Another account or endpoint requires an explicit +`TokenCredential`; account keys and configured SAS credentials are never forwarded there. + +Storage uses Azure SDK exponential retries: eight retries after the initial request, +250 ms initial delay, 10-second maximum delay, and two-minute request timeout. Optional +`clientOptions` accepts Azure SDK pipeline/retry overrides. Missing blobs and store/size +failures surface as `PayloadStorageException`. Permanent activity/orchestrator response +upload failures become non-retriable task/orchestration failures; transient failures remain +retryable. Entity upload failures and download failures propagate rather than fabricating success. + +Custom stores can extend `PayloadStore` from the core package. They must honor `AbortSignal`, +recognize only their own tokens, and return confirmed `PayloadDeleteOutcome` values for cleanup. +Deletion is optional; a store without deletion support leaves cleanup work retryable. + +## Automatic cleanup + +```typescript +await worker.start(); +await client.setLargePayloadAutoPurge(true); // default batch size 500; range 1..1000 +await client.setLargePayloadAutoPurge(true, 250); // update through SetBatchSize +await client.setLargePayloadAutoPurge(false); // pause backend fetches; do not terminate the runner +``` + +This requires a DTS backend implementing `microsoft.durabletask.largepayloads.LargePayloadPurge`. +Merely configuring storage does **not** enable deletion. The last explicit setting wins; +coordinate calls from multiple clients. + +Enabling writes the backend setting first, schedules the unversioned singleton +`BlobPurgeJobOrchestrator` at `BlobPurgeJob-__dt_blob_payload_autopurge__`, verifies its identity +and Running status, then enqueues `SetBatchSize`. Live instances are deduplicated, not replaced. +Do not use these names/IDs for business tasks. The steps are not transactional: earlier successful +steps remain applied if a later step fails or is canceled. Completion acknowledges event enqueue, +not application of the configuration. The optional third argument is an `AbortSignal`. +Disabling ignores batch size and only writes the backend setting; already-fetched deletes may finish. + +Payload-enabled workers automatically register the runner and its three activities, include +them with wildcard versions in nonempty filters, and bypass business version policy for those +reserved tasks. All-empty/unfiltered subscriptions remain unfiltered. Purge RPCs reuse the +worker's current authenticated transport across reconnects. + +The runner fetches due tombstones, deletes chunks of 50 with at most four chunk activities +concurrent, and reports one positional outcome per attempted tombstone, preserving correlation +tokens unchanged. Each chunk limits deletion to eight concurrent blobs with a 60-second ceiling +per cooperative store call. Fetch/report RPCs have 60-second deadlines. Activities make up to three +attempts with 15/30-second retry delays, excluding unsupported backends. Empty/disabled fetches and +all-Retry results idle on a one-minute durable timer, interruptible by configuration. The runner +continues as new after five cycles, retaining configuration events, batch size, and diagnostic count. +Unsupported fetch/report RPCs wait for explicit configuration instead of polling indefinitely. + +Deletion checks the case-insensitive ownership marker, then uses that read's ETag and includes +snapshots. Unowned blobs are never deleted. Only confirmed `BlobNotFound`/`ContainerNotFound` +responses count as already absent. Malformed v2 tokens, legacy v1 tokens, and invalid storage +requests are quarantined; authorization, uncertain errors, ETag conflicts, unsupported stores, +and unknown token versions remain retryable. The backend owns retry scheduling and quarantined +evidence. Logs contain bounded classifications, not payload tokens or raw storage errors. + +Only tracked v2 references are eligible for automatic cleanup; v1 blobs remain in storage. +`Deleted` means storage accepted deletion or confirmed a terminal outcome, **not** immediate +physical reclamation. Soft-delete retention and blob versions can continue to consume storage. +The progress count can include duplicate/already-absent/unowned outcomes. + +## Local verification + +Unit tests use the real Azure SDK pipeline with an offline HTTP transport and checked-in +gzip vectors generated by .NET 10.0.13 using the pinned implementation's write/flush sequence. +Run `npm run test:unit -w @microsoft/durabletask-js-azureblob`. +The optional `test:azurite` suite requires a local-only +`DURABLETASK_BLOB_TEST_CONNECTION_STRING` and never provisions cloud resources. + +After `npm run build`, run the packed-consumer regression from the repository root: + +```sh +npm pack @microsoft/durabletask-js@0.4.0 --ignore-scripts --pack-destination +npm run test:payload-packages -- /microsoft-durabletask-js-0.4.0.tgz +``` + +This checks actual npm peer resolution against the published archive: both payload-enabled +extensions must reject core `0.4.0` with `ERESOLVE`. It then installs the new packed core and +extensions in an isolated consumer and loads their public APIs without contacting storage or DTS. +It also type-checks the packed public declarations with `lib: ES2020`, strict checking, and +`skipLibCheck: false`, without inheriting the ambient ES2022 `ErrorOptions` type. +All temporary consumer installations are removed afterward. diff --git a/packages/durabletask-js-azureblob/jest.config.js b/packages/durabletask-js-azureblob/jest.config.js new file mode 100644 index 00000000..2696e51a --- /dev/null +++ b/packages/durabletask-js-azureblob/jest.config.js @@ -0,0 +1,9 @@ +module.exports = { + preset: "ts-jest", + testEnvironment: "node", + roots: ["/test"], + testMatch: ["**/*.spec.ts"], + moduleNameMapper: { + "^@microsoft/durabletask-js$": "/../durabletask-js/src/index.ts", + }, +}; diff --git a/packages/durabletask-js-azureblob/package.json b/packages/durabletask-js-azureblob/package.json new file mode 100644 index 00000000..1faf4395 --- /dev/null +++ b/packages/durabletask-js-azureblob/package.json @@ -0,0 +1,44 @@ +{ + "name": "@microsoft/durabletask-js-azureblob", + "version": "0.4.0", + "description": "Azure Blob externalized payload storage for the Durable Task JavaScript SDK", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "require": "./dist/index.js", + "import": "./dist/index.js" + } + }, + "files": [ + "dist", + "CHANGELOG.md", + "LICENSE", + "README.md" + ], + "scripts": { + "clean": "node -e \"require('fs').rmSync('dist', {recursive:true, force:true})\"", + "build": "npm run clean && tsc -p tsconfig.build.json", + "test": "jest --runInBand --detectOpenHandles", + "test:unit": "jest test --runInBand --detectOpenHandles", + "test:azurite": "jest test/azurite.spec.ts --runInBand --detectOpenHandles", + "prepublishOnly": "npm run build && npm run test" + }, + "engines": { + "node": ">=22.0.0" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/microsoft/durabletask-js.git", + "directory": "packages/durabletask-js-azureblob" + }, + "license": "MIT", + "dependencies": { + "@azure/core-auth": "^1.10.0", + "@azure/storage-blob": "^12.26.0" + }, + "peerDependencies": { + "@microsoft/durabletask-js": ">=0.4.1" + } +} diff --git a/packages/durabletask-js-azureblob/src/blob-payload-store.ts b/packages/durabletask-js-azureblob/src/blob-payload-store.ts new file mode 100644 index 00000000..c28037c9 --- /dev/null +++ b/packages/durabletask-js-azureblob/src/blob-payload-store.ts @@ -0,0 +1,264 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { TokenCredential } from "@azure/core-auth"; +import { + BlobClient, + BlobServiceClient, + ContainerClient, + StoragePipelineOptions, + StorageRetryPolicyType, +} from "@azure/storage-blob"; +import { PayloadStore, PayloadStorageException, PayloadDeleteOutcome } from "@microsoft/durabletask-js"; +import { randomUUID } from "crypto"; +import { Writable } from "stream"; +import { pipeline } from "stream/promises"; +import { promisify } from "util"; +import { createGunzip, gzip } from "zlib"; + +const compress = promisify(gzip); + +export interface BlobPayloadStoreOptions { + connectionString?: string; + accountUri?: string | URL; + credential?: TokenCredential; + /** Defaults to durabletask-payloads. */ + containerName?: string; + /** Defaults to true. */ + compressionEnabled?: boolean; + /** Azure SDK HTTP, proxy, and retry configuration. */ + clientOptions?: StoragePipelineOptions; +} + +/** .NET-compatible blob:v2 writes and blob:v1/v2 reads, with ownership-checked conditional cleanup. */ +export class BlobPayloadStore extends PayloadStore { + private readonly _service: BlobServiceClient; + private readonly _container: ContainerClient; + private readonly _clientOptions: StoragePipelineOptions; + private _generation?: object; + private _initializing?: { promise: Promise; signal?: AbortSignal }; + + constructor(private readonly _options: BlobPayloadStoreOptions) { + super(); + const containerName = _options.containerName ?? "durabletask-payloads"; + if (!containerName) throw new TypeError("containerName must not be empty."); + this._clientOptions = { + ..._options.clientOptions, + retryOptions: { + retryPolicyType: StorageRetryPolicyType.EXPONENTIAL, + maxTries: 9, + retryDelayInMs: 250, + maxRetryDelayInMs: 10000, + tryTimeoutInMs: 120000, + ..._options.clientOptions?.retryOptions, + }, + }; + if (_options.accountUri && _options.credential) { + this._service = new BlobServiceClient(String(_options.accountUri), _options.credential, this._clientOptions); + } else if (_options.connectionString) { + this._service = BlobServiceClient.fromConnectionString(_options.connectionString, this._clientOptions); + } else { + throw new TypeError("Provide connectionString or both accountUri and credential."); + } + this._container = this._service.getContainerClient(containerName); + } + + isKnownPayloadToken(value: string): boolean { + return value.startsWith("blob:v1:") || value.startsWith("blob:v2:"); + } + + async upload(payload: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + const blob = this._container.getBlockBlobClient(randomUUID().replace(/-/g, "")); + const compressed = this._options.compressionEnabled ?? true; + const bytes = Buffer.from(payload, "utf8"); + const content = compressed ? await compress(bytes) : bytes; + signal?.throwIfAborted(); + for (let attempt = 0; ; attempt++) { + const generation = await this._ensureContainer(signal); + try { + await blob.upload(content, content.length, { + abortSignal: signal, + metadata: { managed_by: "dts" }, + blobHTTPHeaders: compressed ? { blobContentEncoding: "gzip" } : undefined, + }); + } catch (error) { + if (attempt === 0 && storageCode(error) === "ContainerNotFound") { + if (this._generation === generation) this._generation = undefined; + continue; + } + throw error; + } + const url = new URL(blob.url); + url.search = ""; + url.hash = ""; + url.username = ""; + url.password = ""; + return `blob:v2:${url.href}`; + } + } + + async download(token: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + const blob = this._blobClient(token); + try { + const response = await blob.download(0, undefined, { abortSignal: signal }); + const body = response.readableStreamBody; + if (!body) throw new PayloadStorageException("The payload blob response has no readable body."); + const chunks: Buffer[] = []; + const target = new Writable({ + write(chunk: Buffer, _encoding, callback) { + chunks.push(chunk); + callback(); + }, + }); + if (response.contentEncoding?.toLowerCase() === "gzip") { + await pipeline(body, createGunzip(), target, { signal }); + } else { + await pipeline(body, target, { signal }); + } + return Buffer.concat(chunks) + .toString("utf8") + .replace(/^\uFEFF/, ""); + } catch (error) { + if (storageStatus(error) === 404) { + throw new PayloadStorageException("The payload blob was not found. It may have been deleted.", { + cause: error, + }); + } + throw error; + } + } + + async delete(token: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + const blob = this._blobClient(token); + try { + const properties = await blob.getProperties({ abortSignal: signal }); + const owned = Object.entries(properties.metadata ?? {}).some( + ([key, value]) => key.toLowerCase() === "managed_by" && value === "dts", + ); + if (!owned) return PayloadDeleteOutcome.NotStoreOwned; + if (!properties.etag) throw new PayloadStorageException("The ownership response is missing its ETag."); + await blob.delete({ + abortSignal: signal, + deleteSnapshots: "include", + conditions: { ifMatch: properties.etag }, + }); + return PayloadDeleteOutcome.Deleted; + } catch (error) { + if ( + storageStatus(error) === 404 && + (storageCode(error) === "BlobNotFound" || storageCode(error) === "ContainerNotFound") + ) { + return PayloadDeleteOutcome.AlreadyAbsent; + } + throw error; + } + } + + private _blobClient(token: string): BlobClient { + if (token.startsWith("blob:v1:")) { + const rest = token.slice("blob:v1:".length); + const separator = rest.indexOf(":"); + if ( + separator < 1 || + separator === rest.length - 1 || + rest.slice(0, separator) !== this._container.containerName + ) { + throw new TypeError("Invalid external payload token or token container does not match."); + } + return this._container.getBlobClient(rest.slice(separator + 1)); + } + if (!token.startsWith("blob:v2:")) throw new TypeError("Invalid external payload token."); + let url: URL; + try { + url = new URL(token.slice("blob:v2:".length)); + } catch (error) { + throw new TypeError("Invalid external payload blob URL.", { cause: error }); + } + if ((url.protocol !== "https:" && url.protocol !== "http:") || url.username || url.password) { + throw new TypeError("Invalid external payload blob URL."); + } + const decoded = new BlobClient(url.href); + if (!decoded.containerName || !decoded.name) throw new TypeError("Invalid external payload blob URL."); + const container = this._service.getContainerClient(decoded.containerName); + const configuredUrl = new URL(container.url); + // Match the service/container boundary, not normalized blob-name escaping. The SDK + // preserves path-style account prefixes and the configured credential/pipeline. + if ( + url.origin === configuredUrl.origin && + url.pathname.startsWith(configuredUrl.pathname.replace(/\/+$/, "") + "/") + ) { + return container.getBlobClient(decoded.name); + } + if (this._options.credential) return new BlobClient(url.href, this._options.credential, this._clientOptions); + throw new PayloadStorageException( + "Cross-account or cross-endpoint payload access requires identity authentication.", + ); + } + + private async _ensureContainer(signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + if (this._generation) return this._generation; + if (!this._initializing) { + const initializing = { + signal, + promise: this._container.createIfNotExists({ abortSignal: signal }).then(() => { + this._generation = {}; + return this._generation; + }), + }; + this._initializing = initializing; + const clear = () => { + if (this._initializing === initializing) this._initializing = undefined; + }; + void initializing.promise.then(clear, clear); + } + const initializing = this._initializing; + try { + return await waitForInitialization(initializing.promise, signal); + } catch (error) { + if (initializing.signal?.aborted && !signal?.aborted) { + if (this._initializing === initializing) this._initializing = undefined; + return this._ensureContainer(signal); + } + throw error; + } + } +} + +async function waitForInitialization(promise: Promise, signal?: AbortSignal): Promise { + if (!signal) return promise; + signal.throwIfAborted(); + let onAbort = () => {}; + try { + return await new Promise((resolve, reject) => { + onAbort = () => reject(signal.reason); + signal.addEventListener("abort", onAbort, { once: true }); + promise.then(resolve, reject); + if (signal.aborted) onAbort(); + }); + } finally { + signal.removeEventListener("abort", onAbort); + } +} + +function storageCode(error: unknown): unknown { + if (error === null || typeof error !== "object") return undefined; + // HEAD failures have no response body; the Azure SDK exposes their service code in details. + if ( + "details" in error && + error.details !== null && + typeof error.details === "object" && + "errorCode" in error.details && + typeof error.details.errorCode === "string" + ) { + return error.details.errorCode; + } + return "code" in error ? error.code : undefined; +} + +function storageStatus(error: unknown): unknown { + return error !== null && typeof error === "object" && "statusCode" in error ? error.statusCode : undefined; +} diff --git a/packages/durabletask-js-azureblob/src/index.ts b/packages/durabletask-js-azureblob/src/index.ts new file mode 100644 index 00000000..1518fd48 --- /dev/null +++ b/packages/durabletask-js-azureblob/src/index.ts @@ -0,0 +1,4 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +export { BlobPayloadStore, BlobPayloadStoreOptions } from "./blob-payload-store"; diff --git a/packages/durabletask-js-azureblob/test/azurite.spec.ts b/packages/durabletask-js-azureblob/test/azurite.spec.ts new file mode 100644 index 00000000..54238a25 --- /dev/null +++ b/packages/durabletask-js-azureblob/test/azurite.spec.ts @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { randomUUID, createHash } from "crypto"; +import { BlobServiceClient, ContainerClient } from "@azure/storage-blob"; +import { PayloadDeleteOutcome } from "@microsoft/durabletask-js"; +import { BlobPayloadStore } from "../src"; +import fixtures from "./dotnet-payload-fixtures.json"; + +const connectionString = process.env.DURABLETASK_BLOB_TEST_CONNECTION_STRING; +const localOnly = connectionString?.match(/BlobEndpoint=([^;]+)/i)?.[1]; +if (connectionString && (!localOnly || !["127.0.0.1", "localhost", "[::1]"].includes(new URL(localOnly).hostname))) { + throw new Error("Azurite tests require an explicit local-only BlobEndpoint."); +} + +(connectionString ? describe : describe.skip)("BlobPayloadStore with local Azurite", () => { + let service: BlobServiceClient; + let container: ContainerClient; + let containerName: string; + beforeEach(() => { + service = BlobServiceClient.fromConnectionString(connectionString!); + containerName = `payloads-${randomUUID()}`; + container = service.getContainerClient(containerName); + }); + afterEach(async () => { + await container.deleteIfExists(); + }); + + it.each([true, false])("round-trips UTF-8 payloads and same-account tokens (gzip=%s)", async (compressionEnabled) => { + const store = new BlobPayloadStore({ connectionString, containerName, compressionEnabled }); + const value = "\u00e9\ud83d\ude80".repeat(100000); + const token = await store.upload(value); + expect(await store.download(token)).toBe(value); + const url = new URL(token.slice("blob:v2:".length)); + const blobName = url.pathname.split("/").at(-1)!; + expect(await store.download(`blob:v1:${containerName}:${blobName}`)).toBe(value); + const properties = await container.getBlobClient(blobName).getProperties(); + expect(properties.metadata).toMatchObject({ managed_by: "dts" }); + expect(properties.contentEncoding ?? "").toBe(compressionEnabled ? "gzip" : ""); + const rotated = new BlobPayloadStore({ connectionString, containerName: `rotated-${randomUUID()}` }); + expect(await rotated.download(token)).toBe(value); + }); + + it.each(fixtures.vectors)("hydrates .NET-produced compressed blob $id", async (fixture) => { + await container.create(); + const blob = container.getBlockBlobClient("dotnet"); + const bytes = Buffer.from(fixture.gzipBase64, "base64"); + await blob.upload(bytes, bytes.length, { + metadata: { managed_by: "dts" }, + blobHTTPHeaders: { blobContentEncoding: "gzip" }, + }); + const store = new BlobPayloadStore({ connectionString, containerName }); + const actual = await store.download(`blob:v2:${blob.url}`); + expect(Buffer.byteLength(actual, "utf8")).toBe(fixture.utf8Bytes); + expect(createHash("sha256").update(actual, "utf8").digest("hex")).toBe(fixture.sha256); + }); + + it("deletes owned blobs and snapshots but never deletes unowned blobs", async () => { + const store = new BlobPayloadStore({ connectionString, containerName }); + const token = await store.upload("owned"); + const owned = container.getBlobClient(new URL(token.slice("blob:v2:".length)).pathname.split("/").at(-1)!); + await owned.createSnapshot(); + expect(await store.delete(token)).toBe(PayloadDeleteOutcome.Deleted); + expect(await store.delete(token)).toBe(PayloadDeleteOutcome.AlreadyAbsent); + expect(await owned.exists()).toBe(false); + const customer = container.getBlockBlobClient("customer"); + await customer.upload("customer data", 13); + expect(await store.delete(`blob:v2:${customer.url}`)).toBe(PayloadDeleteOutcome.NotStoreOwned); + expect(await customer.exists()).toBe(true); + }); +}); diff --git a/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts new file mode 100644 index 00000000..dd0548d3 --- /dev/null +++ b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts @@ -0,0 +1,318 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { Readable } from "stream"; +import { gunzipSync, gzipSync } from "zlib"; +import { IHttpClient, HttpOperationResponse, WebResource } from "@azure/storage-blob"; +import { PayloadDeleteOutcome, PayloadStorageException } from "@microsoft/durabletask-js"; +import { BlobPayloadStore } from "../src"; +import fixtures from "./dotnet-payload-fixtures.json"; + +const accountKey = Buffer.alloc(64).toString("base64"); +const connectionString = `DefaultEndpointsProtocol=https;AccountName=account;AccountKey=${accountKey};BlobEndpoint=https://account.invalid`; + +class RecordingClient implements IHttpClient { + requests: WebResource[] = []; + bodies: Buffer[] = []; + content = Buffer.from("payload"); + encoding = ""; + metadata: Record = { managed_by: "dts" }; + error: { status: number; code: string } | undefined; + failFirstUpload = false; + async sendRequest(request: WebResource): Promise { + this.requests.push(request); + const headers = request.headers.clone(); + for (const header of headers.headersArray()) headers.remove(header.name); + headers.set("x-ms-request-id", "offline"); + headers.set("etag", '"owned-etag"'); + headers.set("last-modified", new Date(0).toUTCString()); + headers.set("x-ms-blob-type", "BlockBlob"); + for (const [key, value] of Object.entries(this.metadata)) headers.set(`x-ms-meta-${key}`, value); + if (this.encoding) headers.set("content-encoding", this.encoding); + const container = new URL(request.url).searchParams.get("restype") === "container"; + const error = + this.failFirstUpload && request.method === "PUT" && !container + ? { status: 404, code: "ContainerNotFound" } + : this.error; + if (error) { + this.failFirstUpload = false; + headers.set("x-ms-error-code", error.code); + headers.set("content-type", "application/xml"); + return { + request, + status: error.status, + headers, + bodyAsText: + request.method === "HEAD" ? "" : `${error.code}offline`, + }; + } + if (request.method === "PUT" && !container) { + const body = typeof request.body === "function" ? request.body() : request.body; + if (!Buffer.isBuffer(body)) throw new Error("Expected a buffered upload"); + this.bodies.push(body); + } + headers.set("content-length", String(this.content.length)); + return { + request, + status: request.method === "PUT" ? 201 : request.method === "DELETE" ? 202 : 200, + headers, + readableStreamBody: Readable.from([this.content]), + bodyAsText: "", + }; + } +} + +describe("BlobPayloadStore", () => { + let http: RecordingClient; + beforeEach(() => { + http = new RecordingClient(); + }); + const make = (http: RecordingClient, options: Partial[0]> = {}) => + new BlobPayloadStore({ + connectionString, + clientOptions: { httpClient: http, retryOptions: { maxTries: 1 } }, + ...options, + }); + + it.each([true, false])( + "writes credential-free v2 GUID tokens and ownership metadata (gzip=%s)", + async (compressionEnabled) => { + const store = make(http, { compressionEnabled }); + const token = await store.upload("\u00e9\ud83d\ude80"); + expect(token).toMatch(/^blob:v2:https:\/\/account\.invalid\/durabletask-payloads\/[a-f0-9]{32}$/); + const upload = http.requests.find((request) => request.method === "PUT" && !request.url.includes("restype="))!; + expect(upload.headers.get("x-ms-meta-managed_by")).toBe("dts"); + expect(upload.headers.get("x-ms-blob-content-encoding") ?? "").toBe(compressionEnabled ? "gzip" : ""); + expect((compressionEnabled ? gunzipSync(http.bodies[0]) : http.bodies[0]).toString("utf8")).toBe( + "\u00e9\ud83d\ude80", + ); + expect(http.requests[0].headers.get("x-ms-blob-public-access")).toBeUndefined(); + }, + ); + + it("single-flights container initialization and recovers one deleted-container generation", async () => { + const store = make(http); + await Promise.all(Array.from({ length: 12 }, () => store.upload("data"))); + expect(http.requests.filter((r) => r.url.includes("restype=container"))).toHaveLength(1); + http.failFirstUpload = true; + await store.upload("recovered"); + expect(http.requests.filter((r) => r.url.includes("restype=container"))).toHaveLength(2); + expect(http.bodies).toHaveLength(13); + }); + + it("cancels a waiting initializer promptly without canceling or replacing the leader", async () => { + let release!: () => void; + let entered!: () => void; + const blocked = new Promise((resolve) => { + release = resolve; + }); + const started = new Promise((resolve) => { + entered = resolve; + }); + const send = http.sendRequest.bind(http); + http.sendRequest = async (request) => { + if (request.url.includes("restype=container")) { + entered(); + await blocked; + } + return send(request); + }; + const store = make(http, { compressionEnabled: false }); + const leader = new AbortController(); + const waiter = new AbortController(); + const removed = jest.spyOn(waiter.signal, "removeEventListener"); + const first = store.upload("first", leader.signal); + await started; + const reason = new Error("Only cancel the waiter"); + let observed: unknown; + const second = store.upload("second", waiter.signal).catch((error) => { + observed = error; + }); + try { + waiter.abort(reason); + await new Promise((resolve) => setImmediate(resolve)); + expect(observed).toBe(reason); + expect(leader.signal.aborted).toBe(false); + expect(removed).toHaveBeenCalledWith("abort", expect.any(Function)); + release(); + await first; + await store.upload("third"); + expect(http.requests.filter((r) => r.url.includes("restype=container"))).toHaveLength(1); + expect(http.bodies.map((body) => body.toString())).toEqual(["first", "third"]); + } finally { + release(); + await Promise.allSettled([first, second]); + } + }); + + it("lets a surviving waiter initialize after the previous owner is canceled", async () => { + let entered!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + const send = http.sendRequest.bind(http); + let firstContainerRequest = true; + http.sendRequest = async (request) => { + if (request.url.includes("restype=container") && firstContainerRequest) { + firstContainerRequest = false; + if (!request.abortSignal) throw new Error("The owner must propagate cancellation."); + const signal = request.abortSignal; + await new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => + reject(Object.assign(new Error("Owner canceled"), { name: "AbortError" })), + ); + entered(); + }); + } + return send(request); + }; + const owner = new AbortController(); + const store = make(http, { compressionEnabled: false }); + const first = store.upload("owner", owner.signal); + const failedOwner = expect(first).rejects.toThrow(); + await started; + const survivor = store.upload("survivor"); + owner.abort(); + await failedOwner; + await expect(survivor).resolves.toMatch(/^blob:v2:/); + expect(http.bodies.map((body) => body.toString())).toEqual(["survivor"]); + }); + + it.each(["gzip", "GZip", ""])("reads UTF-8 storage representation with encoding %s", async (encoding) => { + http.content = encoding ? gzipSync(Buffer.from("\u00e9\ud83d\ude80")) : Buffer.from("\u00e9\ud83d\ude80"); + http.encoding = encoding; + expect(await make(http).download("blob:v1:durabletask-payloads:folder/a b%2Fc")).toBe("\u00e9\ud83d\ude80"); + expect(new URL(http.requests[0].url).pathname).toBe("/durabletask-payloads/folder/a%20b%252Fc"); + }); + + it.each(fixtures.vectors)("reads actual .NET 10 GZipStream fixture $id", async (fixture) => { + http.encoding = "gzip"; + http.content = Buffer.from(fixture.gzipBase64, "base64"); + const expected = fixture.seed.repeat(fixture.repeat) + fixture.suffix; + const actual = await make(http).download("blob:v2:https://account.invalid/durabletask-payloads/dotnet"); + expect(actual).toBe(expected); + expect(Buffer.byteLength(actual, "utf8")).toBe(fixture.utf8Bytes); + }); + + it.each([ + ["https://account.invalid", "https://account.invalid/old/folder/a%20b%252Fc"], + ["http://127.0.0.1:10000/accounta", "http://127.0.0.1:10000/accounta/old/blob"], + ])("preserves configured SharedKey pipeline across containers in %s", async (endpoint, target) => { + const store = make(http, { + connectionString: `DefaultEndpointsProtocol=http;AccountName=account;AccountKey=${accountKey};BlobEndpoint=${endpoint}`, + }); + expect(await store.download(`blob:v2:${target}`)).toBe("payload"); + expect(await store.delete(`blob:v2:${target}`)).toBe(PayloadDeleteOutcome.Deleted); + expect(http.requests.map((r) => r.method)).toEqual(["GET", "HEAD", "DELETE"]); + for (const request of http.requests) { + const url = new URL(request.url); + expect(url.origin + url.pathname).toBe(target); + expect(url.searchParams.get("timeout")).toBe("120"); + expect(request.headers.get("authorization")).toMatch(/^SharedKey /); + } + expect(http.requests[2].headers.get("if-match")).toBe('"owned-etag"'); + expect(http.requests[2].headers.get("x-ms-delete-snapshots")).toBe("include"); + }); + + it.each([ + "https://other.invalid/old/blob", + "http://account.invalid/old/blob", + "https://account.invalid:444/old/blob", + "http://127.0.0.1:10000/accountb/old/blob", + ])("never forwards configured account keys to %s", async (target) => { + const store = make(http); + await expect(store.download(`blob:v2:${target}`)).rejects.toBeInstanceOf(PayloadStorageException); + await expect(store.delete(`blob:v2:${target}`)).rejects.toBeInstanceOf(PayloadStorageException); + expect(http.requests).toHaveLength(0); + }); + + it("uses identity, not the configured account key, for cross-account reads", async () => { + const credential = { + getToken: async () => ({ token: "synthetic-access-token", expiresOnTimestamp: Date.now() + 3600000 }), + }; + const store = make(http, { credential }); + expect(await store.download("blob:v2:https://other.invalid/old/blob")).toBe("payload"); + expect(http.requests[0].headers.get("authorization")).toBe("Bearer synthetic-access-token"); + }); + + it("supports account URI and identity without a connection string", async () => { + const store = make(http, { + connectionString: undefined, + accountUri: "https://account.invalid", + credential: { + getToken: async () => ({ token: "synthetic-access-token", expiresOnTimestamp: Date.now() + 3600000 }), + }, + }); + expect(await store.download("blob:v2:https://account.invalid/durabletask-payloads/blob")).toBe("payload"); + expect(http.requests[0].headers.get("authorization")).toBe("Bearer synthetic-access-token"); + }); + + it("classifies the service endpoint independently of normalized blob-name escaping", async () => { + expect(await make(http).download("blob:v2:https://account.invalid/old/folder%2fblob")).toBe("payload"); + expect(http.requests[0].headers.get("authorization")).toMatch(/^SharedKey /); + expect(new URL(http.requests[0].url).pathname).toBe("/old/folder/blob"); + }); + + it("rejects another path-style account on the same host without sending SharedKey", async () => { + const store = make(http, { + connectionString: `DefaultEndpointsProtocol=http;AccountName=accounta;AccountKey=${accountKey};BlobEndpoint=http://127.0.0.1:10000/accounta`, + }); + await expect(store.download("blob:v2:http://127.0.0.1:10000/accountb/old/blob")).rejects.toBeInstanceOf( + PayloadStorageException, + ); + expect(http.requests).toHaveLength(0); + }); + + it("preserves a configured SAS across same-account containers without persisting it", async () => { + const store = make(http, { + connectionString: + "AccountName=account;SharedAccessSignature=sv=2025-01-05&sr=c&sp=rwd&sig=synthetic;BlobEndpoint=https://account.invalid", + }); + const token = await store.upload("payload"); + expect(token).not.toContain("sig="); + await store.download("blob:v2:https://account.invalid/old/blob"); + expect(new URL(http.requests.at(-1)!.url).searchParams.get("sig")).toBe("synthetic"); + }); + + it.each([ + [{}, PayloadDeleteOutcome.NotStoreOwned], + [{ managed_by: "customer" }, PayloadDeleteOutcome.NotStoreOwned], + [{ MANAGED_BY: "dts" }, PayloadDeleteOutcome.Deleted], + ])("checks ownership case-insensitively before conditional deletion (%#)", async (metadata, outcome) => { + http.metadata = metadata; + expect(await make(http).delete("blob:v2:https://account.invalid/durabletask-payloads/blob")).toBe(outcome); + expect(http.requests.some((r) => r.method === "DELETE")).toBe(outcome === PayloadDeleteOutcome.Deleted); + }); + + it.each(["BlobNotFound", "ContainerNotFound"])("confirms absence only for %s", async (code) => { + http.error = { status: 404, code }; + expect(await make(http).delete("blob:v2:https://account.invalid/durabletask-payloads/blob")).toBe( + PayloadDeleteOutcome.AlreadyAbsent, + ); + await expect( + make(http).download("blob:v2:https://account.invalid/durabletask-payloads/blob"), + ).rejects.toBeInstanceOf(PayloadStorageException); + }); + + it.each([ + [404, ""], + [412, "ConditionNotMet"], + [403, "AuthorizationPermissionMismatch"], + ])("propagates uncertain storage failures %s/%s", async (status, code) => { + http.error = { status, code }; + await expect(make(http).delete("blob:v2:https://account.invalid/durabletask-payloads/blob")).rejects.toMatchObject({ + statusCode: status, + }); + }); + + it.each([ + "blob:v1:bad", + "blob:v1:other:blob", + "blob:v2:invalid", + "blob:v2:https://account.invalid/container/", + "blob:v3:future", + ])("rejects malformed or mismatched tokens without storage I/O (%s)", async (token) => { + await expect(make(http).download(token)).rejects.toBeInstanceOf(TypeError); + expect(http.requests).toHaveLength(0); + }); +}); diff --git a/packages/durabletask-js-azureblob/test/dotnet-payload-fixtures.json b/packages/durabletask-js-azureblob/test/dotnet-payload-fixtures.json new file mode 100644 index 00000000..514e22aa --- /dev/null +++ b/packages/durabletask-js-azureblob/test/dotnet-payload-fixtures.json @@ -0,0 +1,91 @@ +{ + "referenceCommit": "91739687ff647699079cdadd627daa00562982b7", + "runtime": ".NET 10.0.13", + "compression": "System.IO.Compression.GZipStream / CompressionLevel.Optimal", + "vectors": [ + { + "id": "ascii-below-threshold", + "seed": "a", + "repeat": 262143, + "suffix": "", + "utf8Bytes": 262143, + "utf16CodeUnits": 262143, + "sha256": "0379854ae435b4c82999632f674694ce1f98103f645f44ba68c901a83a2b00fd", + "gzipBase64": "H4sIAAAAAAAACuzBMQEAAADCoKzrX8LbDkABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJwBAAD//wMAs7HRUv//AwA=", + "requiresExternalization": false, + "exceedsMaximum": false + }, + { + "id": "ascii-at-threshold", + "seed": "a", + "repeat": 262144, + "suffix": "", + "utf8Bytes": 262144, + "utf16CodeUnits": 262144, + "sha256": "dd3dde87623d9a6b354c68c943d189c89c63652d945e7bbdf0986cae91a49521", + "gzipBase64": "H4sIAAAAAAAACuzBMQEAAADCoKzrX8LbDkABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALwBAAD//wMAxI2NugAABAA=", + "requiresExternalization": true, + "exceedsMaximum": false + }, + { + "id": "ascii-above-threshold", + "seed": "a", + "repeat": 262145, + "suffix": "", + "utf8Bytes": 262145, + "utf16CodeUnits": 262145, + "sha256": "c592f4a6b099700b5050ce8bc67367f0c8f44810203124e86405c3e7b6f1a2ba", + "gzipBase64": "H4sIAAAAAAAACuzBMQEAAADCoKzrX8LfDEABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJwBAAD//wMAZzUEdAEABAA=", + "requiresExternalization": true, + "exceedsMaximum": false + }, + { + "id": "multibyte-at-threshold", + "seed": "é", + "repeat": 131072, + "suffix": "", + "utf8Bytes": 262144, + "utf16CodeUnits": 131072, + "sha256": "94914398e4fe14ac182b9e6080caa078bbde122682c352744929d55f7d038d10", + "gzipBase64": "H4sIAAAAAAAACuzQwRAAAAACMH+UOBLrncNuBmuKAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYOAGBgAA//8DAONphUEAAAQA", + "requiresExternalization": true, + "exceedsMaximum": false + }, + { + "id": "supplementary-at-threshold", + "seed": "🚀", + "repeat": 65536, + "suffix": "", + "utf8Bytes": 262144, + "utf16CodeUnits": 131072, + "sha256": "f802f2cdafbd3f25ef8ce8e48994ef484a1b860addef41067f94e621945eac6e", + "gzipBase64": "H4sIAAAAAAAACuzSMREAAAgDMevoYMFiJXRFRIY38JfcTuQBAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDwDBQAA//8DAIXOWJsAAAQA", + "requiresExternalization": true, + "exceedsMaximum": false + }, + { + "id": "ascii-at-maximum", + "seed": "a", + "repeat": 10485760, + "suffix": "", + "utf8Bytes": 10485760, + "utf16CodeUnits": 10485760, + "sha256": "b5eec3f68ef64d15e82dad91ff908582c5f081e61a62e22427af9bec2cd35f8d", + "gzipBase64": "H4sIAAAAAAAACuzBgQAAAACAINb9JRapCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgNmDAwEAAAAAIP/XRlBVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVhD44JAAAAEAZltX+JxfABBgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA3AQAA//8DALIc12sAAKAA", + "requiresExternalization": true, + "exceedsMaximum": false + }, + { + "id": "ascii-above-maximum", + "seed": "a", + "repeat": 10485761, + "suffix": "", + "utf8Bytes": 10485761, + "utf16CodeUnits": 10485761, + "sha256": "4ea73dbccbce283083f78555e86595e0b345c46ff188509412fee1c68914d0cb", + "gzipBase64": "H4sIAAAAAAAACuzBgQAAAACAINb9JRapCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgNmDAwEAAAAAIP/XRlBVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVhD44JAAAAEAZltX+JxfABBgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/AQAA//8DAP+7s80BAKAA", + "requiresExternalization": true, + "exceedsMaximum": true + } + ] +} diff --git a/packages/durabletask-js-azureblob/tsconfig.build.json b/packages/durabletask-js-azureblob/tsconfig.build.json new file mode 100644 index 00000000..b4fc9d03 --- /dev/null +++ b/packages/durabletask-js-azureblob/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": "./src", + "outDir": "./dist", + "baseUrl": ".", + "paths": { "@microsoft/durabletask-js": ["../durabletask-js/dist/index"] } + } +} diff --git a/packages/durabletask-js-azureblob/tsconfig.json b/packages/durabletask-js-azureblob/tsconfig.json new file mode 100644 index 00000000..8acab8b6 --- /dev/null +++ b/packages/durabletask-js-azureblob/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src"], + "exclude": ["node_modules", "dist", "test"] +} diff --git a/packages/durabletask-js-azuremanaged/CHANGELOG.md b/packages/durabletask-js-azuremanaged/CHANGELOG.md index 60244b3c..450932e7 100644 --- a/packages/durabletask-js-azuremanaged/CHANGELOG.md +++ b/packages/durabletask-js-azuremanaged/CHANGELOG.md @@ -2,6 +2,9 @@ ### New +- Add `.useExternalizedPayloads()` to client and worker builders, preserving payload settings + across connection-option replacement and enabling transparent payload storage and reserved + auto-purge tasks in the core SDK. Require core `>=0.4.1` for the new public payload types. - Support the `ResourceId` token audience URI in connection strings and an optional final `resourceId` argument in the client/worker factory functions, alongside existing builder/options setters. Trim surrounding whitespace and trailing slashes, remove one case-insensitive `/.default` suffix, diff --git a/packages/durabletask-js-azuremanaged/README.md b/packages/durabletask-js-azuremanaged/README.md index 08fe6b13..33c75915 100644 --- a/packages/durabletask-js-azuremanaged/README.md +++ b/packages/durabletask-js-azuremanaged/README.md @@ -53,6 +53,15 @@ const worker = createAzureManagedWorkerBuilder("https://myservice.durabletask.io await worker.start(); ``` +### Large payloads + +Both builders support `.useExternalizedPayloads({ store, thresholdBytes?, maxPayloadBytes? })`. +This release requires core `@microsoft/durabletask-js >=0.4.1`, including its payload types. +The opt-in setting survives `.connectionString(...)` replacement. Configure the same payload +storage on every participating client and worker; scheduler and storage credentials are separate. +See [`@microsoft/durabletask-js-azureblob`](../durabletask-js-azureblob/README.md) for .NET-compatible +blob storage, size defaults, authentication, and the separately enabled auto-purge lifecycle. + ### Token Audience and Azure Government The optional `resourceId` is a **token audience URI**, not an Azure Resource Manager resource path. @@ -60,11 +69,11 @@ Configure it with `.resourceId(value)` on either builder, `.setResourceId(value) the optional fourth argument to the explicit-parameter factory functions, or `ResourceId` in a connection string. Existing calls remain valid. -| Configuration | Selected resource ID | -| --- | --- | -| Explicit nonempty `resourceId` / `ResourceId` | The normalized explicit value | +| Configuration | Selected resource ID | +| ---------------------------------------------------------------------------------------------- | ------------------------------ | +| Explicit nonempty `resourceId` / `ResourceId` | The normalized explicit value | | Missing, `null`, or empty, and `REGION_NAME` starts with `usgov` or `usdod` (case-insensitive) | `https://durabletask.azure.us` | -| Otherwise | `https://durabletask.io` | +| Otherwise | `https://durabletask.io` | The default is captured when the connection options are created, including when a builder is constructed or its `.connectionString(...)` replaces the options. It stays fixed across token refreshes, worker reconnects, @@ -80,11 +89,11 @@ Explicit values have surrounding whitespace and trailing `/` characters trimmed, case-insensitive `/.default` suffix removed, followed by any remaining trailing `/` characters. The SDK requests `/.default` without changing the URI's casing: -| Input | Requested scope | -| --- | --- | -| `https://durabletask.azure.us/` | `https://durabletask.azure.us/.default` | -| `https://durabletask.azure.us//.DEFAULT//` | `https://durabletask.azure.us/.default` | -| `api://CustomAudience/resource/.DEFAULT/` | `api://CustomAudience/resource/.default` | +| Input | Requested scope | +| ------------------------------------------ | ---------------------------------------- | +| `https://durabletask.azure.us/` | `https://durabletask.azure.us/.default` | +| `https://durabletask.azure.us//.DEFAULT//` | `https://durabletask.azure.us/.default` | +| `api://CustomAudience/resource/.DEFAULT/` | `api://CustomAudience/resource/.default` | Whitespace-only values, `///`, `/.default`, and `/.DEFAULT///` throw a configuration `Error`, even with anonymous authentication. `ResourceId=` selects the default, but `ResourceId= ` is invalid. diff --git a/packages/durabletask-js-azuremanaged/package.json b/packages/durabletask-js-azuremanaged/package.json index 29f72a6e..151e9a49 100644 --- a/packages/durabletask-js-azuremanaged/package.json +++ b/packages/durabletask-js-azuremanaged/package.json @@ -53,7 +53,7 @@ }, "peerDependencies": { "@grpc/grpc-js": "^1.14.4", - "@microsoft/durabletask-js": ">=0.3.0" + "@microsoft/durabletask-js": ">=0.4.1" }, "devDependencies": { "@types/jest": "^29.5.1", diff --git a/packages/durabletask-js-azuremanaged/src/client-builder.ts b/packages/durabletask-js-azuremanaged/src/client-builder.ts index eab641be..1c848513 100644 --- a/packages/durabletask-js-azuremanaged/src/client-builder.ts +++ b/packages/durabletask-js-azuremanaged/src/client-builder.ts @@ -5,7 +5,7 @@ import { TokenCredential } from "@azure/identity"; import * as grpc from "@grpc/grpc-js"; import { DurableTaskAzureManagedClientOptions } from "./options"; import { ClientRetryOptions } from "./retry-policy"; -import { TaskHubGrpcClient, Logger, ConsoleLogger } from "@microsoft/durabletask-js"; +import { TaskHubGrpcClient, Logger, ConsoleLogger, PayloadStorageOptions } from "@microsoft/durabletask-js"; /** * Builder for creating DurableTaskClient instances that connect to Azure-managed Durable Task service. @@ -15,6 +15,7 @@ export class DurableTaskAzureManagedClientBuilder { private _options: DurableTaskAzureManagedClientOptions; private _grpcChannelOptions: grpc.ChannelOptions = {}; private _logger: Logger = new ConsoleLogger(); + private _payloads?: PayloadStorageOptions; /** * Creates a new instance of DurableTaskAzureManagedClientBuilder. @@ -142,6 +143,12 @@ export class DurableTaskAzureManagedClientBuilder { return this; } + /** Enables transparent externalized payloads using an independently configured payload store. */ + useExternalizedPayloads(options: PayloadStorageOptions): DurableTaskAzureManagedClientBuilder { + this._payloads = options; + return this; + } + /** * Builds and returns a configured TaskHubGrpcClient. * @@ -164,14 +171,15 @@ export class DurableTaskAzureManagedClientBuilder { }; // Use the core TaskHubGrpcClient with custom channel credentials and per-call metadata. - return new TaskHubGrpcClient( + return new TaskHubGrpcClient({ hostAddress, - combinedOptions, - true, - channelCredentials, + options: combinedOptions, + useTLS: true, + credentials: channelCredentials, metadataGenerator, - this._logger, - ); + logger: this._logger, + payloads: this._payloads, + }); } } diff --git a/packages/durabletask-js-azuremanaged/src/worker-builder.ts b/packages/durabletask-js-azuremanaged/src/worker-builder.ts index 8a0b1a76..05e61418 100644 --- a/packages/durabletask-js-azuremanaged/src/worker-builder.ts +++ b/packages/durabletask-js-azuremanaged/src/worker-builder.ts @@ -15,6 +15,7 @@ import { ConsoleLogger, VersioningOptions, WorkItemFilters, + PayloadStorageOptions, } from "@microsoft/durabletask-js"; /** @@ -33,6 +34,7 @@ export class DurableTaskAzureManagedWorkerBuilder { private _channelRecreateFailureThreshold?: number; private _versioning?: VersioningOptions; private _workItemFilters?: WorkItemFilters | "auto"; + private _payloads?: PayloadStorageOptions; /** * Creates a new instance of DurableTaskAzureManagedWorkerBuilder. @@ -305,6 +307,12 @@ export class DurableTaskAzureManagedWorkerBuilder { return this; } + /** Enables payload hydration, externalization and the reserved auto-purge tasks. */ + useExternalizedPayloads(options: PayloadStorageOptions): DurableTaskAzureManagedWorkerBuilder { + this._payloads = options; + return this; + } + /** * Builds and returns a configured TaskHubGrpcWorker. * @@ -339,6 +347,7 @@ export class DurableTaskAzureManagedWorkerBuilder { channelRecreateFailureThreshold: this._channelRecreateFailureThreshold, versioning: this._versioning, workItemFilters: this._workItemFilters, + payloads: this._payloads, // DTS natively supports long timers, matching the Python Azure-managed worker. maximumTimerIntervalMs: null, }); diff --git a/packages/durabletask-js-azuremanaged/test/unit/payload-builders.spec.ts b/packages/durabletask-js-azuremanaged/test/unit/payload-builders.spec.ts new file mode 100644 index 00000000..a9b191c0 --- /dev/null +++ b/packages/durabletask-js-azuremanaged/test/unit/payload-builders.spec.ts @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { PayloadStore, NoOpLogger } from "@microsoft/durabletask-js"; +import { DurableTaskAzureManagedClientBuilder, DurableTaskAzureManagedWorkerBuilder } from "../../src"; +import * as pb from "../../../durabletask-js/src/proto/orchestrator_service_pb"; + +class Store extends PayloadStore { + async upload(): Promise { + return "external:payload"; + } + async download(): Promise { + return '"resolved"'; + } + isKnownPayloadToken(value: string): boolean { + return value.startsWith("external:"); + } +} + +describe("Azure-managed payload configuration", () => { + it("preserves client payload settings when connection options are replaced", async () => { + const client = new DurableTaskAzureManagedClientBuilder() + .useExternalizedPayloads({ store: new Store(), thresholdBytes: 1 }) + .connectionString("Endpoint=http://localhost:8080;Authentication=None;TaskHub=payload-test") + .logger(new NoOpLogger()) + .build(); + expect(client["_payloads"]).toBeDefined(); + await client.stop(); + }); + it("preserves worker payload settings, business registrations, and internal wildcard filters", () => { + const worker = new DurableTaskAzureManagedWorkerBuilder() + .useExternalizedPayloads({ store: new Store() }) + .addNamedOrchestrator("business", () => undefined) + .connectionString("Endpoint=http://localhost:8080;Authentication=None;TaskHub=payload-test") + .useWorkItemFilters() + .build(); + const request = worker["_buildGetWorkItemsRequest"](); + expect(request.getCapabilitiesList()).toContain(pb.WorkerCapability.WORKER_CAPABILITY_LARGE_PAYLOADS); + expect( + request + .getWorkitemfilters()! + .getOrchestrationsList() + .map((f) => f.getName()), + ).toEqual(["business", "BlobPurgeJobOrchestrator"]); + }); +}); diff --git a/packages/durabletask-js/package.json b/packages/durabletask-js/package.json index 598a96c9..1b33d65c 100644 --- a/packages/durabletask-js/package.json +++ b/packages/durabletask-js/package.json @@ -1,6 +1,6 @@ { "name": "@microsoft/durabletask-js", - "version": "0.4.0", + "version": "0.4.1", "description": "A Durable Task Javascript SDK compatible with Dapr Workflow and its underlying Durable Task engine", "main": "./dist/index.js", "types": "./dist/index.d.ts", diff --git a/packages/durabletask-js/src/client/client.ts b/packages/durabletask-js/src/client/client.ts index 54d3bb7c..3850c58a 100644 --- a/packages/durabletask-js/src/client/client.ts +++ b/packages/durabletask-js/src/client/client.ts @@ -37,6 +37,13 @@ import { EntityInstanceId } from "../entities/entity-instance-id"; import { EntityMetadata, createEntityMetadata, createEntityMetadataWithoutState } from "../entities/entity-metadata"; import { EntityQuery, normalizeInstanceIdPrefix } from "../entities/entity-query"; import { SignalEntityOptions } from "../entities/signal-entity-options"; +import { PayloadStorageOptions } from "../payloads/payload-store"; +import { PayloadProcessor } from "../payloads/payload-processor"; +import { readPayloadHistory, PayloadHydrationError } from "../payloads/history-stream"; +import * as lp from "../proto/large_payload_purge_pb"; +import { LargePayloadPurgeService } from "../proto/large_payload_purge_grpc_pb"; +import { callPayloadPurge } from "../payloads/purge-transport"; +import { BlobPurgeConstants, validatePurgeBatchSize } from "../payloads/purge"; import { CleanEntityStorageRequest, CleanEntityStorageResult, @@ -59,6 +66,8 @@ export { MetadataGenerator } from "../utils/grpc-helper.util"; * Options for creating a TaskHubGrpcClient. */ export interface TaskHubGrpcClientOptions { + /** Optional externalized payload storage. Disabled by default. */ + payloads?: PayloadStorageOptions; /** The host address to connect to. Defaults to "localhost:4001". */ hostAddress?: string; /** gRPC channel options. */ @@ -84,6 +93,8 @@ export class TaskHubGrpcClient { private _metadataGenerator?: MetadataGenerator; private _logger: Logger; private _defaultVersion?: string; + private readonly _payloads?: PayloadProcessor; + private readonly _payloadCancellation = new AbortController(); /** * Creates a new TaskHubGrpcClient instance. @@ -137,6 +148,7 @@ export class TaskHubGrpcClient { resolvedMetadataGenerator = hostAddressOrOptions.metadataGenerator; resolvedLogger = hostAddressOrOptions.logger; resolvedDefaultVersion = hostAddressOrOptions.defaultVersion; + if (hostAddressOrOptions.payloads) this._payloads = new PayloadProcessor(hostAddressOrOptions.payloads); } else { // Deprecated positional parameters constructor resolvedHostAddress = hostAddressOrOptions; @@ -154,6 +166,7 @@ export class TaskHubGrpcClient { } async stop(): Promise { + this._payloadCancellation.abort(); await this._stub.close(); // Brief pause to allow gRPC cleanup - this is a known issue with grpc-node @@ -161,6 +174,95 @@ export class TaskHubGrpcClient { await new Promise((resolve) => setTimeout(resolve, 1000)); } + private async _call( + method: Parameters>[0], + request: TReq, + metadataGenerator?: MetadataGenerator, + signal?: AbortSignal, + ): Promise { + if (!this._payloads) return callWithMetadata(method, request, metadataGenerator, signal); + const payloadSignal = signal + ? AbortSignal.any([signal, this._payloadCancellation.signal]) + : this._payloadCancellation.signal; + await this._payloads.externalize(request, payloadSignal); + const response = await callWithMetadata(method, request, metadataGenerator, payloadSignal); + await this._payloads.resolve(response, payloadSignal); + return response; + } + + /** + * Enables or pauses DTS large-payload cleanup. Enabling starts/verifies the reserved singleton + * and enqueues its batch size (1..1000, default 500). Disabling only writes the backend setting. + * Steps are not transactional; errors and cancellation do not roll back an earlier successful step. + * A payload-enabled worker is required. Legacy v1 references are not deleted. + */ + async setLargePayloadAutoPurge(enabled: boolean, batchSize = 500, signal?: AbortSignal): Promise { + if (enabled) validatePurgeBatchSize(batchSize); + const cancellation = signal + ? AbortSignal.any([signal, this._payloadCancellation.signal]) + : this._payloadCancellation.signal; + await callPayloadPurge( + this._stub, + LargePayloadPurgeService.setLargePayloadAutoPurge, + new lp.SetLargePayloadAutoPurgeRequest().setEnabled(enabled), + this._metadataGenerator, + cancellation, + ); + if (!enabled) return; + const request = new pb.CreateInstanceRequest() + .setInstanceid(BlobPurgeConstants.instanceId) + .setName(BlobPurgeConstants.orchestrator) + .setVersion(new StringValue().setValue("")) + .setInput(new StringValue().setValue(JSON.stringify({ PurgeBatchSize: batchSize, PurgedCount: 0 }))) + .setOrchestrationidreusepolicy( + new pb.OrchestrationIdReusePolicy().setReplaceablestatusList([ + pb.OrchestrationStatus.ORCHESTRATION_STATUS_COMPLETED, + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + pb.OrchestrationStatus.ORCHESTRATION_STATUS_CANCELED, + pb.OrchestrationStatus.ORCHESTRATION_STATUS_TERMINATED, + ]), + ); + try { + const response = await this._call( + this._stub.startInstance.bind(this._stub), + request, + this._metadataGenerator, + cancellation, + ); + if (response.getInstanceid() !== BlobPurgeConstants.instanceId) { + throw new Error("The backend returned an unexpected auto-purge instance ID."); + } + } catch (error) { + if (!(error instanceof Error && "code" in error && error.code === grpc.status.ALREADY_EXISTS)) throw error; + } + const started = await this._call( + this._stub.waitForInstanceStart.bind(this._stub), + new pb.GetInstanceRequest().setInstanceid(BlobPurgeConstants.instanceId).setGetinputsandoutputs(false), + this._metadataGenerator, + cancellation, + ); + const state = started.getOrchestrationstate(); + if ( + !started.getExists() || + state?.getInstanceid() !== BlobPurgeConstants.instanceId || + state?.getName() !== BlobPurgeConstants.orchestrator + ) { + throw new Error("The reserved auto-purge instance ID belongs to a different orchestration."); + } + if (state.getOrchestrationstatus() !== pb.OrchestrationStatus.ORCHESTRATION_STATUS_RUNNING) { + throw new Error("Auto-purge did not start in Running status."); + } + await this._call( + this._stub.raiseEvent.bind(this._stub), + new pb.RaiseEventRequest() + .setInstanceid(BlobPurgeConstants.instanceId) + .setName(BlobPurgeConstants.configurationEvent) + .setInput(new StringValue().setValue(JSON.stringify(batchSize))), + this._metadataGenerator, + cancellation, + ); + } + /** * Schedules a new orchestrator using the DurableTask client. * @@ -256,7 +358,7 @@ export class TaskHubGrpcClient { ClientLogs.schedulingOrchestration(this._logger, req.getInstanceid(), name, inputSizeInBytes); try { - const res = await callWithMetadata( + const res = await this._call( this._stub.startInstance.bind(this._stub), req, this._metadataGenerator, @@ -301,7 +403,7 @@ export class TaskHubGrpcClient { req.setInstanceid(instanceId); req.setGetinputsandoutputs(fetchPayloads); - const res = await callWithMetadata( + const res = await this._call( this._stub.getInstance.bind(this._stub), req, this._metadataGenerator, @@ -343,7 +445,7 @@ export class TaskHubGrpcClient { signal, `Timed out waiting for orchestration '${instanceId}' to start after ${timeout}s`, (waitSignal) => - callWithMetadata(this._stub.waitForInstanceStart.bind(this._stub), req, this._metadataGenerator, waitSignal), + this._call(this._stub.waitForInstanceStart.bind(this._stub), req, this._metadataGenerator, waitSignal), ); return newOrchestrationState(req.getInstanceid(), res); @@ -415,7 +517,7 @@ export class TaskHubGrpcClient { // Metadata failures must not enter the remote-deadline retry path. const metadata = this._metadataGenerator ? await this._metadataGenerator() : new grpc.Metadata(); try { - return await callWithMetadata( + return await this._call( this._stub.waitForInstanceCompletion.bind(this._stub), req, async () => metadata, @@ -498,7 +600,7 @@ export class TaskHubGrpcClient { ClientLogs.raisingEvent(this._logger, instanceId, eventName); try { - await callWithMetadata( + await this._call( this._stub.raiseEvent.bind(this._stub), req, this._metadataGenerator, @@ -565,7 +667,7 @@ export class TaskHubGrpcClient { ClientLogs.terminatingInstance(this._logger, instanceId); - await callWithMetadata( + await this._call( this._stub.terminateInstance.bind(this._stub), req, this._metadataGenerator, @@ -591,7 +693,7 @@ export class TaskHubGrpcClient { ClientLogs.suspendingInstance(this._logger, instanceId); - await callWithMetadata( + await this._call( this._stub.suspendInstance.bind(this._stub), req, this._metadataGenerator, @@ -617,7 +719,7 @@ export class TaskHubGrpcClient { ClientLogs.resumingInstance(this._logger, instanceId); - await callWithMetadata( + await this._call( this._stub.resumeInstance.bind(this._stub), req, this._metadataGenerator, @@ -656,7 +758,7 @@ export class TaskHubGrpcClient { ClientLogs.rewindingInstance(this._logger, instanceId, reason); try { - await callWithMetadata( + await this._call( this._stub.rewindInstance.bind(this._stub), req, this._metadataGenerator, @@ -721,7 +823,7 @@ export class TaskHubGrpcClient { ClientLogs.restartingInstance(this._logger, instanceId, restartWithNewInstanceId); try { - const res = await callWithMetadata( + const res = await this._call( this._stub.restartInstance.bind(this._stub), req, this._metadataGenerator, @@ -772,10 +874,11 @@ export class TaskHubGrpcClient { const req = new pb.PurgeInstancesRequest(); req.setInstanceid(instanceId); req.setRecursive(options?.recursive ?? false); + if (this._payloads) req.setIsorchestration(!instanceId.startsWith("@")); ClientLogs.purgingInstanceMetadata(this._logger, instanceId); - res = await callWithMetadata( + res = await this._call( this._stub.purgeInstances.bind(this._stub), req, this._metadataGenerator, @@ -811,7 +914,7 @@ export class TaskHubGrpcClient { runtimeStatusList.map(String).join(", "), ); - const callPromise = callWithMetadata( + const callPromise = this._call( this._stub.purgeInstances.bind(this._stub), req, this._metadataGenerator, @@ -915,7 +1018,7 @@ export class TaskHubGrpcClient { req.setQuery(query); - const response = await callWithMetadata( + const response = await this._call( this._stub.queryInstances.bind(this._stub), req, this._metadataGenerator, @@ -1001,7 +1104,7 @@ export class TaskHubGrpcClient { req.setCompletedtimeto(timestamp); } - const response = await callWithMetadata( + const response = await this._call( this._stub.listInstanceIds.bind(this._stub), req, this._metadataGenerator, @@ -1049,6 +1152,26 @@ export class TaskHubGrpcClient { const metadata = this._metadataGenerator ? await this._metadataGenerator() : new grpc.Metadata(); const stream = this._stub.streamInstanceHistory(req, metadata); + if (this._payloads) { + try { + const events = await readPayloadHistory(stream, this._payloads, this._payloadCancellation.signal); + return events.map(convertProtoHistoryEvent).filter((event): event is HistoryEvent => event !== undefined); + } catch (error) { + if (error instanceof PayloadHydrationError) throw error.cause; + if (error instanceof Error && "code" in error) { + if (error.code === grpc.status.NOT_FOUND) return []; + if (error.code === grpc.status.CANCELLED) + throw new Error("The getOrchestrationHistory operation was canceled."); + if (error.code === grpc.status.INTERNAL) { + throw new Error( + `An error occurred while retrieving the history for orchestration with instanceId '${instanceId}'.`, + ); + } + } + throw error; + } + } + return new Promise((resolve, reject) => { const historyEvents: HistoryEvent[] = []; @@ -1147,7 +1270,7 @@ export class TaskHubGrpcClient { ClientLogs.signalingEntity(this._logger, id.toString(), operationName); try { - await callWithMetadata( + await this._call( this._stub.signalEntity.bind(this._stub), req, this._metadataGenerator, @@ -1183,7 +1306,7 @@ export class TaskHubGrpcClient { ClientLogs.gettingEntity(this._logger, id.toString()); - const res = await callWithMetadata( + const res = await this._call( this._stub.getEntity.bind(this._stub), req, this._metadataGenerator, @@ -1271,7 +1394,7 @@ export class TaskHubGrpcClient { req.setQuery(protoQuery); - const res = await callWithMetadata( + const res = await this._call( this._stub.queryEntities.bind(this._stub), req, this._metadataGenerator, @@ -1311,7 +1434,7 @@ export class TaskHubGrpcClient { protoReq.setContinuationtoken(token); } - const res = await callWithMetadata( + const res = await this._call( this._stub.cleanEntityStorage.bind(this._stub), protoReq, this._metadataGenerator, diff --git a/packages/durabletask-js/src/index.ts b/packages/durabletask-js/src/index.ts index 58e0275d..f319c996 100644 --- a/packages/durabletask-js/src/index.ts +++ b/packages/durabletask-js/src/index.ts @@ -2,6 +2,13 @@ // Licensed under the MIT License. // Client and Worker +export { + PayloadStore, + PayloadStorageOptions, + PayloadStorageException, + PayloadDeleteOutcome, +} from "./payloads/payload-store"; +export { LargePayloadPurgeDisposition, LargePayloadTombstone, LargePayloadPurgeResult } from "./payloads/purge"; export { TaskHubGrpcClient, TaskHubGrpcClientOptions, MetadataGenerator } from "./client/client"; export { TaskHubGrpcWorker, TaskHubGrpcWorkerOptions } from "./worker/task-hub-grpc-worker"; export { ConcurrencyOptions } from "./worker/concurrency-options"; diff --git a/packages/durabletask-js/src/payloads/history-stream.ts b/packages/durabletask-js/src/payloads/history-stream.ts new file mode 100644 index 00000000..81d09eaa --- /dev/null +++ b/packages/durabletask-js/src/payloads/history-stream.ts @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as grpc from "@grpc/grpc-js"; +import * as pb from "../proto/orchestrator_service_pb"; +import { PayloadProcessor } from "./payload-processor"; + +export class PayloadHydrationError extends Error { + constructor(cause: unknown) { + super("History payload hydration failed.", { cause }); + this.name = "PayloadHydrationError"; + } +} + +/** Serializes chunk hydration, applies stream backpressure, and drains I/O before settling. */ +export function readPayloadHistory( + stream: grpc.ClientReadableStream, + processor: PayloadProcessor, + signal?: AbortSignal, +): Promise { + return new Promise((resolve, reject) => { + const events: pb.HistoryEvent[] = []; + let pending = Promise.resolve(); + let finishing = false; + const finish = (error?: unknown) => { + if (finishing) return; + finishing = true; + stream.removeListener("data", onData); + stream.removeListener("end", onEnd); + stream.removeListener("error", onError); + stream.removeListener("close", onClose); + stream.on("error", () => {}); + signal?.removeEventListener("abort", onAbort); + if (error !== undefined) stream.cancel(); + stream.destroy(); + void pending.then(() => (error === undefined ? resolve(events) : reject(error)), reject); + }; + const onData = (chunk: pb.HistoryChunk) => { + stream.pause(); + pending = pending.then(async () => { + try { + await processor.resolve(chunk, signal); + } catch (error) { + throw new PayloadHydrationError(error); + } + for (const event of chunk.getEventsList()) events.push(event); + if (!finishing) stream.resume(); + }); + void pending.catch(finish); + }; + const onEnd = () => finish(); + const onError = (error: unknown) => finish(error); + const onClose = () => finish(new Error("History stream closed before all history was received.")); + const onAbort = () => finish(signal?.reason ?? new Error("History hydration was cancelled.")); + stream.on("data", onData); + stream.once("end", onEnd); + stream.once("error", onError); + stream.once("close", onClose); + signal?.addEventListener("abort", onAbort, { once: true }); + if (signal?.aborted) onAbort(); + }); +} diff --git a/packages/durabletask-js/src/payloads/payload-processor.ts b/packages/durabletask-js/src/payloads/payload-processor.ts new file mode 100644 index 00000000..235f6d3f --- /dev/null +++ b/packages/durabletask-js/src/payloads/payload-processor.ts @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { StringValue } from "google-protobuf/google/protobuf/wrappers_pb"; +import * as pb from "../proto/orchestrator_service_pb"; +import { newFailureDetails } from "../utils/pb-helper.util"; +import { PayloadStorageException, PayloadStorageOptions } from "./payload-store"; +import { runBounded } from "./run-bounded"; + +/** Explicit protobuf field mapping, aligned with the .NET AzureBlobPayloads SideCar interceptor. */ +export class PayloadProcessor { + private readonly _threshold: number; + private readonly _maximum: number; + + constructor(private readonly _options: PayloadStorageOptions) { + this._threshold = _options.thresholdBytes ?? 256 * 1024; + this._maximum = _options.maxPayloadBytes ?? 10 * 1024 * 1024; + if (!Number.isSafeInteger(this._threshold) || this._threshold < 0 || this._threshold > 1024 * 1024) { + throw new RangeError("thresholdBytes must be an integer between 0 and 1 MiB."); + } + if (!Number.isSafeInteger(this._maximum) || this._maximum <= 0) { + throw new RangeError("maxPayloadBytes must be a positive safe integer."); + } + if (!_options.store) throw new TypeError("A payload store is required."); + } + + async externalize(message: unknown, signal?: AbortSignal): Promise { + const fields: (StringValue | undefined)[] = []; + if ( + message instanceof pb.CreateInstanceRequest || + message instanceof pb.RaiseEventRequest || + message instanceof pb.SignalEntityRequest + ) { + fields.push(message.getInput()); + } else if (message instanceof pb.TerminateRequest) { + fields.push(message.getOutput()); + } else if ( + message instanceof pb.SuspendRequest || + message instanceof pb.ResumeRequest || + message instanceof pb.RewindInstanceRequest + ) { + fields.push(message.getReason()); + } else if (message instanceof pb.ActivityResponse) { + fields.push(message.getResult()); + } else if (message instanceof pb.OrchestratorResponse) { + fields.push(message.getCustomstatus()); + for (const action of message.getActionsList()) { + fields.push( + action.getScheduletask()?.getInput(), + action.getCreatesuborchestration()?.getInput(), + action.getCompleteorchestration()?.getResult(), + action.getCompleteorchestration()?.getDetails(), + action.getTerminateorchestration()?.getReason(), + action.getSendevent()?.getData(), + action.getSendentitymessage()?.getEntityoperationsignaled()?.getInput(), + action.getSendentitymessage()?.getEntityoperationcalled()?.getInput(), + ); + for (const event of action.getRewindorchestration()?.getNewhistoryList() ?? []) { + this._historyFields(event, fields); + } + } + } else if (message instanceof pb.EntityBatchResult) { + fields.push(message.getEntitystate()); + for (const result of message.getResultsList()) fields.push(result.getSuccess()?.getResult()); + for (const action of message.getActionsList()) { + fields.push(action.getSendsignal()?.getInput(), action.getStartneworchestration()?.getInput()); + } + } else if (message instanceof pb.EntityBatchRequest) { + fields.push(message.getEntitystate()); + for (const operation of message.getOperationsList()) fields.push(operation.getInput()); + } else if (message instanceof pb.EntityRequest) { + fields.push(message.getEntitystate()); + } + const operations: (() => Promise)[] = []; + for (const field of fields) { + const value = field?.getValue(); + if (!field || !value) continue; + const size = Buffer.byteLength(value, "utf8"); + if (size < this._threshold) continue; + operations.push(async () => { + if (size > this._maximum) { + throw new PayloadStorageException( + `Payload size ${size} bytes exceeds the configured maximum of ${this._maximum} bytes.`, + ); + } + field.setValue(await this._options.store.upload(value, signal)); + }); + } + try { + await runBounded(operations, signal); + } catch (error) { + if ( + !isPermanentStorageFailure(error) || + !(message instanceof pb.ActivityResponse || message instanceof pb.OrchestratorResponse) + ) + throw error; + const details = newFailureDetails(error); + details.setIsnonretriable(true); + if (message instanceof pb.ActivityResponse) { + message.clearResult(); + message.setFailuredetails(details); + } else { + message.clearCustomstatus(); + message.setIspartial(false); + message.clearChunkindex(); + message.setActionsList([ + new pb.OrchestratorAction().setCompleteorchestration( + new pb.CompleteOrchestrationAction() + .setOrchestrationstatus(pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED) + .setFailuredetails(details), + ), + ]); + } + } + } + + async resolve(message: unknown, signal?: AbortSignal): Promise { + const fields: (StringValue | undefined)[] = []; + this._responseFields(message, fields); + const operations: (() => Promise)[] = []; + for (const field of fields) { + const value = field?.getValue(); + if (field && value && this._options.store.isKnownPayloadToken(value)) { + operations.push(async () => { + field.setValue(await this._options.store.download(value, signal)); + }); + } + } + await runBounded(operations, signal); + } + + private _responseFields(message: unknown, fields: (StringValue | undefined)[]): void { + if (message instanceof pb.GetInstanceResponse) { + this._stateFields(message.getOrchestrationstate(), fields); + } else if (message instanceof pb.QueryInstancesResponse) { + for (const state of message.getOrchestrationstateList()) this._stateFields(state, fields); + } else if (message instanceof pb.HistoryChunk) { + for (const event of message.getEventsList()) this._historyFields(event, fields); + } else if (message instanceof pb.GetEntityResponse) { + fields.push(message.getEntity()?.getSerializedstate()); + } else if (message instanceof pb.QueryEntitiesResponse) { + for (const entity of message.getEntitiesList()) fields.push(entity.getSerializedstate()); + } else if (message instanceof pb.WorkItem) { + this._responseFields(message.getActivityrequest(), fields); + this._responseFields(message.getOrchestratorrequest(), fields); + this._responseFields(message.getEntityrequest(), fields); + this._responseFields(message.getEntityrequestv2(), fields); + } else if (message instanceof pb.ActivityRequest) { + fields.push(message.getInput()); + } else if (message instanceof pb.OrchestratorRequest) { + for (const event of [...message.getPasteventsList(), ...message.getNeweventsList()]) { + this._historyFields(event, fields); + } + } else if (message instanceof pb.EntityBatchRequest) { + fields.push(message.getEntitystate()); + for (const operation of message.getOperationsList()) fields.push(operation.getInput()); + } else if (message instanceof pb.EntityRequest) { + fields.push(message.getEntitystate()); + for (const event of message.getOperationrequestsList()) this._historyFields(event, fields); + } + } + + private _stateFields(state: pb.OrchestrationState | undefined, fields: (StringValue | undefined)[]): void { + if (state) fields.push(state.getInput(), state.getOutput(), state.getCustomstatus()); + } + + private _historyFields(event: pb.HistoryEvent, fields: (StringValue | undefined)[]): void { + fields.push( + event.getExecutionstarted()?.getInput(), + event.getExecutioncompleted()?.getResult(), + event.getExecutionterminated()?.getInput(), + event.getTaskscheduled()?.getInput(), + event.getTaskcompleted()?.getResult(), + event.getSuborchestrationinstancecreated()?.getInput(), + event.getSuborchestrationinstancecompleted()?.getResult(), + event.getEventsent()?.getInput(), + event.getEventraised()?.getInput(), + event.getGenericevent()?.getData(), + event.getContinueasnew()?.getInput(), + event.getExecutionsuspended()?.getInput(), + event.getExecutionresumed()?.getInput(), + event.getExecutionrewound()?.getReason(), + event.getEntityoperationsignaled()?.getInput(), + event.getEntityoperationcalled()?.getInput(), + event.getEntityoperationcompleted()?.getOutput(), + ); + this._stateFields(event.getHistorystate()?.getOrchestrationstate(), fields); + } +} + +export function isPermanentStorageFailure(error: unknown): error is Error { + if (error instanceof PayloadStorageException) return true; + if (!(error instanceof Error) || !("statusCode" in error)) return false; + const status = error.statusCode; + return typeof status === "number" && status >= 400 && status < 500 && status !== 408 && status !== 429; +} diff --git a/packages/durabletask-js/src/payloads/payload-store.ts b/packages/durabletask-js/src/payloads/payload-store.ts new file mode 100644 index 00000000..2eafae82 --- /dev/null +++ b/packages/durabletask-js/src/payloads/payload-store.ts @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +/** Confirmed results of a conditional, ownership-checked payload deletion. */ +export enum PayloadDeleteOutcome { + Unspecified = 0, + Deleted = 1, + AlreadyAbsent = 2, + NotStoreOwned = 3, +} + +/** Out-of-band storage for serialized UTF-8 payloads. Implementations must honor cancellation. */ +export abstract class PayloadStore { + abstract upload(payload: string, signal?: AbortSignal): Promise; + abstract download(token: string, signal?: AbortSignal): Promise; + abstract isKnownPayloadToken(value: string): boolean; + + /** Optional cleanup support. Never delete a blob without verifying ownership and its ETag. */ + async delete(_token: string, _signal?: AbortSignal): Promise { + throw new Error("This payload store does not support deletion."); + } +} + +/** A permanent payload storage or size-limit failure. */ +export class PayloadStorageException extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = "PayloadStorageException"; + } +} + +/** Opt-in payload externalization. Configure the same store on all clients and workers. */ +export interface PayloadStorageOptions { + store: PayloadStore; + /** Inclusive serialized UTF-8 size threshold. Defaults to 256 KiB; maximum 1 MiB. */ + thresholdBytes?: number; + /** Maximum serialized size of one externalized payload. Defaults to 10 MiB. */ + maxPayloadBytes?: number; +} diff --git a/packages/durabletask-js/src/payloads/purge-transport.ts b/packages/durabletask-js/src/payloads/purge-transport.ts new file mode 100644 index 00000000..62bfd7ee --- /dev/null +++ b/packages/durabletask-js/src/payloads/purge-transport.ts @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as grpc from "@grpc/grpc-js"; +import { callWithMetadata, MetadataGenerator } from "../utils/grpc-helper.util"; + +/** Reuses the effective sidecar's authenticated channel, including after worker channel recreation. */ +export async function callPayloadPurge( + client: grpc.Client, + definition: grpc.MethodDefinition, + request: TReq, + metadataGenerator?: MetadataGenerator, + signal?: AbortSignal, + deadline?: Date, +): Promise { + const response = await callWithMetadata( + (req, metadata, callback) => + client.makeUnaryRequest( + definition.path, + definition.requestSerialize, + definition.responseDeserialize, + req, + metadata, + { deadline }, + callback, + ), + request, + metadataGenerator, + signal, + ); + if (response === undefined) throw new Error("The large-payload purge RPC returned no response."); + return response; +} diff --git a/packages/durabletask-js/src/payloads/purge-worker.ts b/packages/durabletask-js/src/payloads/purge-worker.ts new file mode 100644 index 00000000..ecd2408d --- /dev/null +++ b/packages/durabletask-js/src/payloads/purge-worker.ts @@ -0,0 +1,191 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as grpc from "@grpc/grpc-js"; +import * as lp from "../proto/large_payload_purge_pb"; +import { LargePayloadPurgeService } from "../proto/large_payload_purge_grpc_pb"; +import { Registry } from "../worker/registry"; +import { WorkItemFilters } from "../worker/work-item-filters"; +import { Logger } from "../types/logger.type"; +import { MetadataGenerator } from "../utils/grpc-helper.util"; +import { PayloadStore, PayloadDeleteOutcome, PayloadStorageException } from "./payload-store"; +import { runBounded } from "./run-bounded"; +import { callPayloadPurge } from "./purge-transport"; +import { + BlobPurgeConstants, + blobPurgeJobOrchestrator, + BlobPurgeOutcome, + LargePayloadPurgeDisposition, + LargePayloadPurgeResult, + validatePurgeBatchSize, +} from "./purge"; + +const wireDispositions: Record = { + [LargePayloadPurgeDisposition.Unspecified]: + lp.LargePayloadPurgeDisposition.LARGE_PAYLOAD_PURGE_DISPOSITION_UNSPECIFIED, + [LargePayloadPurgeDisposition.Deleted]: lp.LargePayloadPurgeDisposition.LARGE_PAYLOAD_PURGE_DISPOSITION_DELETED, + [LargePayloadPurgeDisposition.Retry]: lp.LargePayloadPurgeDisposition.LARGE_PAYLOAD_PURGE_DISPOSITION_RETRY, + [LargePayloadPurgeDisposition.Quarantined]: + lp.LargePayloadPurgeDisposition.LARGE_PAYLOAD_PURGE_DISPOSITION_QUARANTINED, +}; + +export function registerPayloadPurge( + registry: Registry, + store: PayloadStore, + getClient: () => grpc.Client, + metadataGenerator: MetadataGenerator | undefined, + getSignal: () => AbortSignal | undefined, + logger: Logger, +): void { + registry.addNamedOrchestrator(BlobPurgeConstants.orchestrator, (ctx, input) => + blobPurgeJobOrchestrator(ctx, input, logger), + ); + registry.addNamedActivity(BlobPurgeConstants.fetch, async (_ctx, limit: number) => { + validatePurgeBatchSize(limit); + try { + const response = await callPayloadPurge( + getClient(), + LargePayloadPurgeService.getLargePayloadTombstones, + new lp.GetLargePayloadTombstonesRequest().setLimit(limit), + metadataGenerator, + getSignal(), + new Date(Date.now() + 60000), + ); + return response.getTombstonesList().map((t) => ({ + TombstoneToken: t.getTombstoneToken(), + PayloadToken: t.getPayloadToken(), + })); + } catch (error) { + if (grpcStatus(error) === grpc.status.FAILED_PRECONDITION) { + logger.info("Large payload purge fetch precondition not met; idling."); + return []; + } + if (grpcStatus(error) === grpc.status.UNIMPLEMENTED) throw unsupportedBackend(error); + throw error; + } + }); + registry.addNamedActivity(BlobPurgeConstants.delete, async (_ctx, tokens: string[]) => + deletePayloadChunk(store, tokens, logger, getSignal()), + ); + registry.addNamedActivity(BlobPurgeConstants.report, async (_ctx, results: LargePayloadPurgeResult[]) => { + if (!results?.length) return null; + const request = new lp.ReportLargePayloadPurgeResultsRequest().setResultsList( + results.map((r) => { + const disposition = wireDispositions[r.Disposition]; + if (disposition === undefined) throw new TypeError("Unknown large-payload purge disposition."); + return new lp.LargePayloadPurgeResult().setTombstoneToken(r.TombstoneToken).setDisposition(disposition); + }), + ); + try { + await callPayloadPurge( + getClient(), + LargePayloadPurgeService.reportLargePayloadPurgeResults, + request, + metadataGenerator, + getSignal(), + new Date(Date.now() + 60000), + ); + } catch (error) { + if (grpcStatus(error) === grpc.status.UNIMPLEMENTED) throw unsupportedBackend(error); + throw error; + } + return null; + }); +} + +export async function deletePayloadChunk( + store: PayloadStore, + tokens: string[], + logger: Logger, + signal?: AbortSignal, +): Promise { + const outcomes: BlobPurgeOutcome[] = new Array(tokens.length); + await runBounded( + tokens.map((token, index) => async () => { + outcomes[index] = { Disposition: await deletePayload(store, token, logger, signal) }; + }), + ); + return outcomes; +} + +async function deletePayload( + store: PayloadStore, + token: string, + logger: Logger, + signal?: AbortSignal, +): Promise { + const classify = (disposition: LargePayloadPurgeDisposition, reason: string) => { + logger.warn(`Large payload purge: ${reason}; disposition=${disposition}.`); + return disposition; + }; + if (token.startsWith("blob:v1:")) return classify(LargePayloadPurgeDisposition.Quarantined, "LegacyV1Token"); + if (!token.startsWith("blob:v2:")) return classify(LargePayloadPurgeDisposition.Retry, "UnsupportedTokenVersion"); + const timeout = new AbortController(); + const timer = setTimeout(() => timeout.abort(), 60000); + timer.unref(); + try { + const deleteSignal = signal ? AbortSignal.any([signal, timeout.signal]) : timeout.signal; + const outcome = await store.delete(token, deleteSignal); + switch (outcome) { + case PayloadDeleteOutcome.Deleted: + case PayloadDeleteOutcome.AlreadyAbsent: + return LargePayloadPurgeDisposition.Deleted; + case PayloadDeleteOutcome.NotStoreOwned: + logger.info("Large payload purge: NotStoreOwned; blob left untouched."); + return LargePayloadPurgeDisposition.Deleted; + default: + return classify(LargePayloadPurgeDisposition.Retry, "UnexpectedDeleteOutcome"); + } + } catch (error) { + if (error instanceof TypeError) return classify(LargePayloadPurgeDisposition.Quarantined, "MalformedToken"); + if (error instanceof PayloadStorageException) + return classify(LargePayloadPurgeDisposition.Retry, "StorageAccountUnreachable"); + const status = error !== null && typeof error === "object" && "statusCode" in error ? error.statusCode : undefined; + if (status === 400) return classify(LargePayloadPurgeDisposition.Quarantined, "InvalidStorageRequest"); + if (status === 401 || status === 403) + return classify(LargePayloadPurgeDisposition.Retry, "StorageAuthorizationFailed"); + return classify(LargePayloadPurgeDisposition.Retry, "UnconfirmedDelete"); + } finally { + clearTimeout(timer); + } +} + +export function isPayloadPurgeTask(name?: string): boolean { + return ( + name === BlobPurgeConstants.orchestrator || + name === BlobPurgeConstants.fetch || + name === BlobPurgeConstants.delete || + name === BlobPurgeConstants.report + ); +} + +export function withPayloadPurgeFilters(filters: WorkItemFilters): WorkItemFilters { + if (!(filters.orchestrations?.length || filters.activities?.length || filters.entities?.length)) return filters; + return { + ...filters, + orchestrations: [ + ...(filters.orchestrations ?? []).filter((f) => f.name !== BlobPurgeConstants.orchestrator), + { name: BlobPurgeConstants.orchestrator, versions: [] }, + ], + activities: [ + ...(filters.activities ?? []).filter((f) => !isPayloadPurgeTask(f.name)), + ...[BlobPurgeConstants.fetch, BlobPurgeConstants.delete, BlobPurgeConstants.report].map((name) => ({ + name, + versions: [], + })), + ], + }; +} + +function grpcStatus(error: unknown): unknown { + return error !== null && typeof error === "object" && "code" in error ? error.code : undefined; +} + +function unsupportedBackend(cause: unknown): Error { + const error = new Error( + "The backend does not implement large-payload auto-purge. Upgrade the backend or DTS emulator, then enable auto-purge again.", + { cause }, + ); + error.name = "System.NotImplementedException"; + return error; +} diff --git a/packages/durabletask-js/src/payloads/purge.ts b/packages/durabletask-js/src/payloads/purge.ts new file mode 100644 index 00000000..2afdd2e3 --- /dev/null +++ b/packages/durabletask-js/src/payloads/purge.ts @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { OrchestrationContext } from "../task/context/orchestration-context"; +import { Task } from "../task/task"; +import { whenAll, whenAny } from "../task"; +import { RetryPolicy } from "../task/retry/retry-policy"; +import { TaskFailureDetails } from "../task/failure-details"; +import { TaskFailedError } from "../task/exception/task-failed-error"; +import { Logger, ConsoleLogger } from "../types/logger.type"; + +export const BlobPurgeConstants = { + orchestrator: "BlobPurgeJobOrchestrator", + instanceId: "BlobPurgeJob-__dt_blob_payload_autopurge__", + fetch: "GetLargePayloadTombstonesActivity", + delete: "DeleteExternalBlobActivity", + report: "ReportLargePayloadPurgeResultsActivity", + configurationEvent: "SetBatchSize", +} as const; + +export enum LargePayloadPurgeDisposition { + Unspecified = 0, + Deleted = 1, + Retry = 2, + Quarantined = 3, +} + +// These activity/orchestration wire shapes intentionally retain .NET's default JSON property casing. +export interface LargePayloadTombstone { + TombstoneToken: string; + PayloadToken: string; +} +export interface LargePayloadPurgeResult { + TombstoneToken: string; + Disposition: LargePayloadPurgeDisposition; +} +export interface BlobPurgeOutcome { + Disposition: LargePayloadPurgeDisposition; +} +export interface BlobPurgeJobRunRequest { + PurgeBatchSize: number; + PurgedCount: number; +} + +export function validatePurgeBatchSize(batchSize: number): void { + if (!Number.isInteger(batchSize) || batchSize < 1 || batchSize > 1000) { + throw new RangeError("Purge batch size must be an integer between 1 and 1000."); + } +} + +export function isUnsupportedPurgeFailure(failure: TaskFailureDetails): boolean { + for (let current: TaskFailureDetails | undefined = failure; current; current = current.innerFailure) { + if (current.errorType === "System.NotImplementedException" || current.errorType === "NotImplementedException") + return true; + } + return false; +} + +const retry = new RetryPolicy({ + maxNumberOfAttempts: 3, + firstRetryIntervalInMilliseconds: 15000, + backoffCoefficient: 2, + maxRetryIntervalInMilliseconds: 60000, + handleFailure: (failure) => !isUnsupportedPurgeFailure(failure), +}); +const taskOptions = { retry, version: "" }; + +export async function* blobPurgeJobOrchestrator( + context: OrchestrationContext, + input: BlobPurgeJobRunRequest, + logger: Logger = new ConsoleLogger(), +): AsyncGenerator, void, any> { + validatePurgeBatchSize(input.PurgeBatchSize); + const replayLogger = context.createReplaySafeLogger(logger); + let batchSize = input.PurgeBatchSize; + let purgedCount = input.PurgedCount ?? 0; + let cycles = 0; + let configuration = context.waitForExternalEvent(BlobPurgeConstants.configurationEvent); + for (;;) { + // CAN cannot preserve a configuration already delivered to this waiter. + while (configuration.isComplete) { + batchSize = configuration.getResult(); + validatePurgeBatchSize(batchSize); + configuration = context.waitForExternalEvent(BlobPurgeConstants.configurationEvent); + } + if (cycles >= 5) { + context.continueAsNew({ PurgeBatchSize: batchSize, PurgedCount: purgedCount }, true); + return; + } + context.setCustomStatus({ Status: "Running", BatchSize: batchSize, PurgedCount: purgedCount }); + cycles++; + let wait = false; + try { + const tombstones: LargePayloadTombstone[] = yield context.callActivity( + BlobPurgeConstants.fetch, + batchSize, + taskOptions, + ); + if (!tombstones?.length) { + wait = true; + } else { + const results: LargePayloadPurgeResult[] = []; + for (let start = 0; start < tombstones.length; start += 50 * 4) { + const chunks: LargePayloadTombstone[][] = []; + const tasks: Task[] = []; + for (let index = start; index < Math.min(start + 50 * 4, tombstones.length); index += 50) { + const chunk = tombstones.slice(index, index + 50); + chunks.push(chunk); + tasks.push( + context.callActivity( + BlobPurgeConstants.delete, + chunk.map((t) => t.PayloadToken), + taskOptions, + ), + ); + } + const outcomes: BlobPurgeOutcome[][] = yield whenAll(tasks); + for (let index = 0; index < chunks.length; index++) { + const chunk = chunks[index]; + if (!outcomes[index] || outcomes[index].length !== chunk.length) { + throw new Error("The blob delete activity must return exactly one outcome per token."); + } + results.push( + ...chunk.map((tombstone, position) => ({ + TombstoneToken: tombstone.TombstoneToken, + Disposition: outcomes[index][position].Disposition, + })), + ); + } + } + yield context.callActivity(BlobPurgeConstants.report, results, taskOptions); + purgedCount += results.filter((r) => r.Disposition === LargePayloadPurgeDisposition.Deleted).length; + wait = results.every((r) => r.Disposition === LargePayloadPurgeDisposition.Retry); + } + } catch (error) { + if (error instanceof TaskFailedError && isUnsupportedPurgeFailure(error.details)) { + replayLogger.warn("Large payload auto-purge backend unsupported; waiting for explicit configuration."); + context.setCustomStatus({ Status: "BackendUnsupported", BatchSize: batchSize, PurgedCount: purgedCount }); + yield configuration; + } else { + replayLogger.error("Large payload auto-purge cycle failed; backing off before another attempt."); + wait = true; + } + } + if (wait && !configuration.isComplete) { + context.setCustomStatus({ Status: "Waiting", BatchSize: batchSize, PurgedCount: purgedCount }); + const timer = context.createTimer(new Date(context.currentUtcDateTime.getTime() + 60000)); + yield whenAny([timer, configuration]); + if (!timer.isComplete) timer.cancel(); + } + } +} diff --git a/packages/durabletask-js/src/payloads/run-bounded.ts b/packages/durabletask-js/src/payloads/run-bounded.ts new file mode 100644 index 00000000..f6646f61 --- /dev/null +++ b/packages/durabletask-js/src/payloads/run-bounded.ts @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +/** Stops dispatch on failure/cancellation, drains started I/O, then throws the first message-order failure. */ +export async function runBounded(operations: (() => Promise)[], signal?: AbortSignal): Promise { + const active = new Set>(); + const failures = new Map(); + let cancellationPreventedDispatch = false; + for (let ordinal = 0; ordinal < operations.length; ordinal++) { + if (active.size >= 8) await Promise.race(active); + if (failures.size) break; + if (signal?.aborted) { + cancellationPreventedDispatch = true; + break; + } + const operation = operations[ordinal]; + const tracked = (async () => { + try { + await operation(); + } catch (error) { + failures.set(ordinal, error); + } + })(); + active.add(tracked); + void tracked.then(() => active.delete(tracked)); + // Observe synchronously rejected operations before claiming the next dispatch. + await Promise.resolve(); + } + await Promise.all(active); + if (failures.size) throw failures.get(Math.min(...failures.keys())); + if (cancellationPreventedDispatch) signal?.throwIfAborted(); +} diff --git a/packages/durabletask-js/src/proto/large_payload_purge_grpc_pb.d.ts b/packages/durabletask-js/src/proto/large_payload_purge_grpc_pb.d.ts new file mode 100644 index 00000000..64917153 --- /dev/null +++ b/packages/durabletask-js/src/proto/large_payload_purge_grpc_pb.d.ts @@ -0,0 +1,75 @@ +// package: microsoft.durabletask.largepayloads +// file: large_payload_purge.proto + +/* tslint:disable */ +/* eslint-disable */ + +import * as grpc from "@grpc/grpc-js"; +import * as large_payload_purge_pb from "./large_payload_purge_pb"; + +interface ILargePayloadPurgeService extends grpc.ServiceDefinition { + setLargePayloadAutoPurge: ILargePayloadPurgeService_ISetLargePayloadAutoPurge; + getLargePayloadTombstones: ILargePayloadPurgeService_IGetLargePayloadTombstones; + reportLargePayloadPurgeResults: ILargePayloadPurgeService_IReportLargePayloadPurgeResults; +} + +interface ILargePayloadPurgeService_ISetLargePayloadAutoPurge extends grpc.MethodDefinition { + path: "/microsoft.durabletask.largepayloads.LargePayloadPurge/SetLargePayloadAutoPurge"; + requestStream: false; + responseStream: false; + requestSerialize: grpc.serialize; + requestDeserialize: grpc.deserialize; + responseSerialize: grpc.serialize; + responseDeserialize: grpc.deserialize; +} +interface ILargePayloadPurgeService_IGetLargePayloadTombstones extends grpc.MethodDefinition { + path: "/microsoft.durabletask.largepayloads.LargePayloadPurge/GetLargePayloadTombstones"; + requestStream: false; + responseStream: false; + requestSerialize: grpc.serialize; + requestDeserialize: grpc.deserialize; + responseSerialize: grpc.serialize; + responseDeserialize: grpc.deserialize; +} +interface ILargePayloadPurgeService_IReportLargePayloadPurgeResults extends grpc.MethodDefinition { + path: "/microsoft.durabletask.largepayloads.LargePayloadPurge/ReportLargePayloadPurgeResults"; + requestStream: false; + responseStream: false; + requestSerialize: grpc.serialize; + requestDeserialize: grpc.deserialize; + responseSerialize: grpc.serialize; + responseDeserialize: grpc.deserialize; +} + +export const LargePayloadPurgeService: ILargePayloadPurgeService; + +export interface ILargePayloadPurgeServer extends grpc.UntypedServiceImplementation { + setLargePayloadAutoPurge: grpc.handleUnaryCall; + getLargePayloadTombstones: grpc.handleUnaryCall; + reportLargePayloadPurgeResults: grpc.handleUnaryCall; +} + +export interface ILargePayloadPurgeClient { + setLargePayloadAutoPurge(request: large_payload_purge_pb.SetLargePayloadAutoPurgeRequest, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.SetLargePayloadAutoPurgeResponse) => void): grpc.ClientUnaryCall; + setLargePayloadAutoPurge(request: large_payload_purge_pb.SetLargePayloadAutoPurgeRequest, metadata: grpc.Metadata, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.SetLargePayloadAutoPurgeResponse) => void): grpc.ClientUnaryCall; + setLargePayloadAutoPurge(request: large_payload_purge_pb.SetLargePayloadAutoPurgeRequest, metadata: grpc.Metadata, options: Partial, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.SetLargePayloadAutoPurgeResponse) => void): grpc.ClientUnaryCall; + getLargePayloadTombstones(request: large_payload_purge_pb.GetLargePayloadTombstonesRequest, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.GetLargePayloadTombstonesResponse) => void): grpc.ClientUnaryCall; + getLargePayloadTombstones(request: large_payload_purge_pb.GetLargePayloadTombstonesRequest, metadata: grpc.Metadata, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.GetLargePayloadTombstonesResponse) => void): grpc.ClientUnaryCall; + getLargePayloadTombstones(request: large_payload_purge_pb.GetLargePayloadTombstonesRequest, metadata: grpc.Metadata, options: Partial, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.GetLargePayloadTombstonesResponse) => void): grpc.ClientUnaryCall; + reportLargePayloadPurgeResults(request: large_payload_purge_pb.ReportLargePayloadPurgeResultsRequest, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.ReportLargePayloadPurgeResultsResponse) => void): grpc.ClientUnaryCall; + reportLargePayloadPurgeResults(request: large_payload_purge_pb.ReportLargePayloadPurgeResultsRequest, metadata: grpc.Metadata, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.ReportLargePayloadPurgeResultsResponse) => void): grpc.ClientUnaryCall; + reportLargePayloadPurgeResults(request: large_payload_purge_pb.ReportLargePayloadPurgeResultsRequest, metadata: grpc.Metadata, options: Partial, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.ReportLargePayloadPurgeResultsResponse) => void): grpc.ClientUnaryCall; +} + +export class LargePayloadPurgeClient extends grpc.Client implements ILargePayloadPurgeClient { + constructor(address: string, credentials: grpc.ChannelCredentials, options?: Partial); + public setLargePayloadAutoPurge(request: large_payload_purge_pb.SetLargePayloadAutoPurgeRequest, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.SetLargePayloadAutoPurgeResponse) => void): grpc.ClientUnaryCall; + public setLargePayloadAutoPurge(request: large_payload_purge_pb.SetLargePayloadAutoPurgeRequest, metadata: grpc.Metadata, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.SetLargePayloadAutoPurgeResponse) => void): grpc.ClientUnaryCall; + public setLargePayloadAutoPurge(request: large_payload_purge_pb.SetLargePayloadAutoPurgeRequest, metadata: grpc.Metadata, options: Partial, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.SetLargePayloadAutoPurgeResponse) => void): grpc.ClientUnaryCall; + public getLargePayloadTombstones(request: large_payload_purge_pb.GetLargePayloadTombstonesRequest, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.GetLargePayloadTombstonesResponse) => void): grpc.ClientUnaryCall; + public getLargePayloadTombstones(request: large_payload_purge_pb.GetLargePayloadTombstonesRequest, metadata: grpc.Metadata, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.GetLargePayloadTombstonesResponse) => void): grpc.ClientUnaryCall; + public getLargePayloadTombstones(request: large_payload_purge_pb.GetLargePayloadTombstonesRequest, metadata: grpc.Metadata, options: Partial, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.GetLargePayloadTombstonesResponse) => void): grpc.ClientUnaryCall; + public reportLargePayloadPurgeResults(request: large_payload_purge_pb.ReportLargePayloadPurgeResultsRequest, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.ReportLargePayloadPurgeResultsResponse) => void): grpc.ClientUnaryCall; + public reportLargePayloadPurgeResults(request: large_payload_purge_pb.ReportLargePayloadPurgeResultsRequest, metadata: grpc.Metadata, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.ReportLargePayloadPurgeResultsResponse) => void): grpc.ClientUnaryCall; + public reportLargePayloadPurgeResults(request: large_payload_purge_pb.ReportLargePayloadPurgeResultsRequest, metadata: grpc.Metadata, options: Partial, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.ReportLargePayloadPurgeResultsResponse) => void): grpc.ClientUnaryCall; +} diff --git a/packages/durabletask-js/src/proto/large_payload_purge_grpc_pb.js b/packages/durabletask-js/src/proto/large_payload_purge_grpc_pb.js new file mode 100644 index 00000000..58a51048 --- /dev/null +++ b/packages/durabletask-js/src/proto/large_payload_purge_grpc_pb.js @@ -0,0 +1,118 @@ +// GENERATED CODE -- DO NOT EDIT! + +// Original file comments: +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// +// DTS-only contract from durabletask-dotnet 91739687ff647699079cdadd627daa00562982b7. +// +'use strict'; +var grpc = require('@grpc/grpc-js'); +var large_payload_purge_pb = require('./large_payload_purge_pb.js'); + +function serialize_microsoft_durabletask_largepayloads_GetLargePayloadTombstonesRequest(arg) { + if (!(arg instanceof large_payload_purge_pb.GetLargePayloadTombstonesRequest)) { + throw new Error('Expected argument of type microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest'); + } + return Buffer.from(arg.serializeBinary()); +} + +function deserialize_microsoft_durabletask_largepayloads_GetLargePayloadTombstonesRequest(buffer_arg) { + return large_payload_purge_pb.GetLargePayloadTombstonesRequest.deserializeBinary(new Uint8Array(buffer_arg)); +} + +function serialize_microsoft_durabletask_largepayloads_GetLargePayloadTombstonesResponse(arg) { + if (!(arg instanceof large_payload_purge_pb.GetLargePayloadTombstonesResponse)) { + throw new Error('Expected argument of type microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse'); + } + return Buffer.from(arg.serializeBinary()); +} + +function deserialize_microsoft_durabletask_largepayloads_GetLargePayloadTombstonesResponse(buffer_arg) { + return large_payload_purge_pb.GetLargePayloadTombstonesResponse.deserializeBinary(new Uint8Array(buffer_arg)); +} + +function serialize_microsoft_durabletask_largepayloads_ReportLargePayloadPurgeResultsRequest(arg) { + if (!(arg instanceof large_payload_purge_pb.ReportLargePayloadPurgeResultsRequest)) { + throw new Error('Expected argument of type microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest'); + } + return Buffer.from(arg.serializeBinary()); +} + +function deserialize_microsoft_durabletask_largepayloads_ReportLargePayloadPurgeResultsRequest(buffer_arg) { + return large_payload_purge_pb.ReportLargePayloadPurgeResultsRequest.deserializeBinary(new Uint8Array(buffer_arg)); +} + +function serialize_microsoft_durabletask_largepayloads_ReportLargePayloadPurgeResultsResponse(arg) { + if (!(arg instanceof large_payload_purge_pb.ReportLargePayloadPurgeResultsResponse)) { + throw new Error('Expected argument of type microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse'); + } + return Buffer.from(arg.serializeBinary()); +} + +function deserialize_microsoft_durabletask_largepayloads_ReportLargePayloadPurgeResultsResponse(buffer_arg) { + return large_payload_purge_pb.ReportLargePayloadPurgeResultsResponse.deserializeBinary(new Uint8Array(buffer_arg)); +} + +function serialize_microsoft_durabletask_largepayloads_SetLargePayloadAutoPurgeRequest(arg) { + if (!(arg instanceof large_payload_purge_pb.SetLargePayloadAutoPurgeRequest)) { + throw new Error('Expected argument of type microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest'); + } + return Buffer.from(arg.serializeBinary()); +} + +function deserialize_microsoft_durabletask_largepayloads_SetLargePayloadAutoPurgeRequest(buffer_arg) { + return large_payload_purge_pb.SetLargePayloadAutoPurgeRequest.deserializeBinary(new Uint8Array(buffer_arg)); +} + +function serialize_microsoft_durabletask_largepayloads_SetLargePayloadAutoPurgeResponse(arg) { + if (!(arg instanceof large_payload_purge_pb.SetLargePayloadAutoPurgeResponse)) { + throw new Error('Expected argument of type microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse'); + } + return Buffer.from(arg.serializeBinary()); +} + +function deserialize_microsoft_durabletask_largepayloads_SetLargePayloadAutoPurgeResponse(buffer_arg) { + return large_payload_purge_pb.SetLargePayloadAutoPurgeResponse.deserializeBinary(new Uint8Array(buffer_arg)); +} + + +// The backend owns the tombstone ledger, task-hub setting, and retry scheduling. +// The credentialed worker performs ownership-checked blob deletes and reports outcomes. +var LargePayloadPurgeService = exports.LargePayloadPurgeService = { + setLargePayloadAutoPurge: { + path: '/microsoft.durabletask.largepayloads.LargePayloadPurge/SetLargePayloadAutoPurge', + requestStream: false, + responseStream: false, + requestType: large_payload_purge_pb.SetLargePayloadAutoPurgeRequest, + responseType: large_payload_purge_pb.SetLargePayloadAutoPurgeResponse, + requestSerialize: serialize_microsoft_durabletask_largepayloads_SetLargePayloadAutoPurgeRequest, + requestDeserialize: deserialize_microsoft_durabletask_largepayloads_SetLargePayloadAutoPurgeRequest, + responseSerialize: serialize_microsoft_durabletask_largepayloads_SetLargePayloadAutoPurgeResponse, + responseDeserialize: deserialize_microsoft_durabletask_largepayloads_SetLargePayloadAutoPurgeResponse, + }, + getLargePayloadTombstones: { + path: '/microsoft.durabletask.largepayloads.LargePayloadPurge/GetLargePayloadTombstones', + requestStream: false, + responseStream: false, + requestType: large_payload_purge_pb.GetLargePayloadTombstonesRequest, + responseType: large_payload_purge_pb.GetLargePayloadTombstonesResponse, + requestSerialize: serialize_microsoft_durabletask_largepayloads_GetLargePayloadTombstonesRequest, + requestDeserialize: deserialize_microsoft_durabletask_largepayloads_GetLargePayloadTombstonesRequest, + responseSerialize: serialize_microsoft_durabletask_largepayloads_GetLargePayloadTombstonesResponse, + responseDeserialize: deserialize_microsoft_durabletask_largepayloads_GetLargePayloadTombstonesResponse, + }, + reportLargePayloadPurgeResults: { + path: '/microsoft.durabletask.largepayloads.LargePayloadPurge/ReportLargePayloadPurgeResults', + requestStream: false, + responseStream: false, + requestType: large_payload_purge_pb.ReportLargePayloadPurgeResultsRequest, + responseType: large_payload_purge_pb.ReportLargePayloadPurgeResultsResponse, + requestSerialize: serialize_microsoft_durabletask_largepayloads_ReportLargePayloadPurgeResultsRequest, + requestDeserialize: deserialize_microsoft_durabletask_largepayloads_ReportLargePayloadPurgeResultsRequest, + responseSerialize: serialize_microsoft_durabletask_largepayloads_ReportLargePayloadPurgeResultsResponse, + responseDeserialize: deserialize_microsoft_durabletask_largepayloads_ReportLargePayloadPurgeResultsResponse, + }, +}; + +exports.LargePayloadPurgeClient = grpc.makeGenericClientConstructor(LargePayloadPurgeService, 'LargePayloadPurge'); diff --git a/packages/durabletask-js/src/proto/large_payload_purge_pb.d.ts b/packages/durabletask-js/src/proto/large_payload_purge_pb.d.ts new file mode 100644 index 00000000..de303194 --- /dev/null +++ b/packages/durabletask-js/src/proto/large_payload_purge_pb.d.ts @@ -0,0 +1,178 @@ +// package: microsoft.durabletask.largepayloads +// file: large_payload_purge.proto + +/* tslint:disable */ +/* eslint-disable */ + +import * as jspb from "google-protobuf"; + +export class SetLargePayloadAutoPurgeRequest extends jspb.Message { + getEnabled(): boolean; + setEnabled(value: boolean): SetLargePayloadAutoPurgeRequest; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): SetLargePayloadAutoPurgeRequest.AsObject; + static toObject(includeInstance: boolean, msg: SetLargePayloadAutoPurgeRequest): SetLargePayloadAutoPurgeRequest.AsObject; + static extensions: {[key: number]: jspb.ExtensionFieldInfo}; + static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo}; + static serializeBinaryToWriter(message: SetLargePayloadAutoPurgeRequest, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): SetLargePayloadAutoPurgeRequest; + static deserializeBinaryFromReader(message: SetLargePayloadAutoPurgeRequest, reader: jspb.BinaryReader): SetLargePayloadAutoPurgeRequest; +} + +export namespace SetLargePayloadAutoPurgeRequest { + export type AsObject = { + enabled: boolean, + } +} + +export class SetLargePayloadAutoPurgeResponse extends jspb.Message { + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): SetLargePayloadAutoPurgeResponse.AsObject; + static toObject(includeInstance: boolean, msg: SetLargePayloadAutoPurgeResponse): SetLargePayloadAutoPurgeResponse.AsObject; + static extensions: {[key: number]: jspb.ExtensionFieldInfo}; + static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo}; + static serializeBinaryToWriter(message: SetLargePayloadAutoPurgeResponse, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): SetLargePayloadAutoPurgeResponse; + static deserializeBinaryFromReader(message: SetLargePayloadAutoPurgeResponse, reader: jspb.BinaryReader): SetLargePayloadAutoPurgeResponse; +} + +export namespace SetLargePayloadAutoPurgeResponse { + export type AsObject = { + } +} + +export class LargePayloadTombstone extends jspb.Message { + getTombstoneToken(): string; + setTombstoneToken(value: string): LargePayloadTombstone; + getPayloadToken(): string; + setPayloadToken(value: string): LargePayloadTombstone; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): LargePayloadTombstone.AsObject; + static toObject(includeInstance: boolean, msg: LargePayloadTombstone): LargePayloadTombstone.AsObject; + static extensions: {[key: number]: jspb.ExtensionFieldInfo}; + static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo}; + static serializeBinaryToWriter(message: LargePayloadTombstone, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): LargePayloadTombstone; + static deserializeBinaryFromReader(message: LargePayloadTombstone, reader: jspb.BinaryReader): LargePayloadTombstone; +} + +export namespace LargePayloadTombstone { + export type AsObject = { + tombstoneToken: string, + payloadToken: string, + } +} + +export class LargePayloadPurgeResult extends jspb.Message { + getTombstoneToken(): string; + setTombstoneToken(value: string): LargePayloadPurgeResult; + getDisposition(): LargePayloadPurgeDisposition; + setDisposition(value: LargePayloadPurgeDisposition): LargePayloadPurgeResult; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): LargePayloadPurgeResult.AsObject; + static toObject(includeInstance: boolean, msg: LargePayloadPurgeResult): LargePayloadPurgeResult.AsObject; + static extensions: {[key: number]: jspb.ExtensionFieldInfo}; + static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo}; + static serializeBinaryToWriter(message: LargePayloadPurgeResult, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): LargePayloadPurgeResult; + static deserializeBinaryFromReader(message: LargePayloadPurgeResult, reader: jspb.BinaryReader): LargePayloadPurgeResult; +} + +export namespace LargePayloadPurgeResult { + export type AsObject = { + tombstoneToken: string, + disposition: LargePayloadPurgeDisposition, + } +} + +export class GetLargePayloadTombstonesRequest extends jspb.Message { + getLimit(): number; + setLimit(value: number): GetLargePayloadTombstonesRequest; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetLargePayloadTombstonesRequest.AsObject; + static toObject(includeInstance: boolean, msg: GetLargePayloadTombstonesRequest): GetLargePayloadTombstonesRequest.AsObject; + static extensions: {[key: number]: jspb.ExtensionFieldInfo}; + static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo}; + static serializeBinaryToWriter(message: GetLargePayloadTombstonesRequest, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetLargePayloadTombstonesRequest; + static deserializeBinaryFromReader(message: GetLargePayloadTombstonesRequest, reader: jspb.BinaryReader): GetLargePayloadTombstonesRequest; +} + +export namespace GetLargePayloadTombstonesRequest { + export type AsObject = { + limit: number, + } +} + +export class GetLargePayloadTombstonesResponse extends jspb.Message { + clearTombstonesList(): void; + getTombstonesList(): Array; + setTombstonesList(value: Array): GetLargePayloadTombstonesResponse; + addTombstones(value?: LargePayloadTombstone, index?: number): LargePayloadTombstone; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetLargePayloadTombstonesResponse.AsObject; + static toObject(includeInstance: boolean, msg: GetLargePayloadTombstonesResponse): GetLargePayloadTombstonesResponse.AsObject; + static extensions: {[key: number]: jspb.ExtensionFieldInfo}; + static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo}; + static serializeBinaryToWriter(message: GetLargePayloadTombstonesResponse, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetLargePayloadTombstonesResponse; + static deserializeBinaryFromReader(message: GetLargePayloadTombstonesResponse, reader: jspb.BinaryReader): GetLargePayloadTombstonesResponse; +} + +export namespace GetLargePayloadTombstonesResponse { + export type AsObject = { + tombstonesList: Array, + } +} + +export class ReportLargePayloadPurgeResultsRequest extends jspb.Message { + clearResultsList(): void; + getResultsList(): Array; + setResultsList(value: Array): ReportLargePayloadPurgeResultsRequest; + addResults(value?: LargePayloadPurgeResult, index?: number): LargePayloadPurgeResult; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): ReportLargePayloadPurgeResultsRequest.AsObject; + static toObject(includeInstance: boolean, msg: ReportLargePayloadPurgeResultsRequest): ReportLargePayloadPurgeResultsRequest.AsObject; + static extensions: {[key: number]: jspb.ExtensionFieldInfo}; + static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo}; + static serializeBinaryToWriter(message: ReportLargePayloadPurgeResultsRequest, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): ReportLargePayloadPurgeResultsRequest; + static deserializeBinaryFromReader(message: ReportLargePayloadPurgeResultsRequest, reader: jspb.BinaryReader): ReportLargePayloadPurgeResultsRequest; +} + +export namespace ReportLargePayloadPurgeResultsRequest { + export type AsObject = { + resultsList: Array, + } +} + +export class ReportLargePayloadPurgeResultsResponse extends jspb.Message { + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): ReportLargePayloadPurgeResultsResponse.AsObject; + static toObject(includeInstance: boolean, msg: ReportLargePayloadPurgeResultsResponse): ReportLargePayloadPurgeResultsResponse.AsObject; + static extensions: {[key: number]: jspb.ExtensionFieldInfo}; + static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo}; + static serializeBinaryToWriter(message: ReportLargePayloadPurgeResultsResponse, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): ReportLargePayloadPurgeResultsResponse; + static deserializeBinaryFromReader(message: ReportLargePayloadPurgeResultsResponse, reader: jspb.BinaryReader): ReportLargePayloadPurgeResultsResponse; +} + +export namespace ReportLargePayloadPurgeResultsResponse { + export type AsObject = { + } +} + +export enum LargePayloadPurgeDisposition { + LARGE_PAYLOAD_PURGE_DISPOSITION_UNSPECIFIED = 0, + LARGE_PAYLOAD_PURGE_DISPOSITION_DELETED = 1, + LARGE_PAYLOAD_PURGE_DISPOSITION_RETRY = 2, + LARGE_PAYLOAD_PURGE_DISPOSITION_QUARANTINED = 3, +} diff --git a/packages/durabletask-js/src/proto/large_payload_purge_pb.js b/packages/durabletask-js/src/proto/large_payload_purge_pb.js new file mode 100644 index 00000000..327f6692 --- /dev/null +++ b/packages/durabletask-js/src/proto/large_payload_purge_pb.js @@ -0,0 +1,1313 @@ +// source: large_payload_purge.proto +/** + * @fileoverview + * @enhanceable + * @suppress {missingRequire} reports error on implicit type usages. + * @suppress {messageConventions} JS Compiler reports an error if a variable or + * field starts with 'MSG_' and isn't a translatable message. + * @public + */ +// GENERATED CODE -- DO NOT EDIT! +/* eslint-disable */ +// @ts-nocheck + +var jspb = require('google-protobuf'); +var goog = jspb; +var global = (function() { + if (this) { return this; } + if (typeof window !== 'undefined') { return window; } + if (typeof global !== 'undefined') { return global; } + if (typeof self !== 'undefined') { return self; } + return Function('return this')(); +}.call(null)); + +goog.exportSymbol('proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest', null, global); +goog.exportSymbol('proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse', null, global); +goog.exportSymbol('proto.microsoft.durabletask.largepayloads.LargePayloadPurgeDisposition', null, global); +goog.exportSymbol('proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult', null, global); +goog.exportSymbol('proto.microsoft.durabletask.largepayloads.LargePayloadTombstone', null, global); +goog.exportSymbol('proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest', null, global); +goog.exportSymbol('proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse', null, global); +goog.exportSymbol('proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest', null, global); +goog.exportSymbol('proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse', null, global); +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.displayName = 'proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.displayName = 'proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.microsoft.durabletask.largepayloads.LargePayloadTombstone, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.displayName = 'proto.microsoft.durabletask.largepayloads.LargePayloadTombstone'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.displayName = 'proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.displayName = 'proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.repeatedFields_, null); +}; +goog.inherits(proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.displayName = 'proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.repeatedFields_, null); +}; +goog.inherits(proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.displayName = 'proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.displayName = 'proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse'; +} + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.prototype.toObject = function(opt_includeInstance) { + return proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.toObject = function(includeInstance, msg) { + var f, obj = { + enabled: jspb.Message.getBooleanFieldWithDefault(msg, 1, false) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest} + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest; + return proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest} + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setEnabled(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getEnabled(); + if (f) { + writer.writeBool( + 1, + f + ); + } +}; + + +/** + * optional bool enabled = 1; + * @return {boolean} + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.prototype.getEnabled = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 1, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest} returns this + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.prototype.setEnabled = function(value) { + return jspb.Message.setProto3BooleanField(this, 1, value); +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.prototype.toObject = function(opt_includeInstance) { + return proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.toObject = function(includeInstance, msg) { + var f, obj = { + + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse} + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse; + return proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse} + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.serializeBinaryToWriter = function(message, writer) { + var f = undefined; +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.prototype.toObject = function(opt_includeInstance) { + return proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.microsoft.durabletask.largepayloads.LargePayloadTombstone} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.toObject = function(includeInstance, msg) { + var f, obj = { + tombstoneToken: jspb.Message.getFieldWithDefault(msg, 1, ""), + payloadToken: jspb.Message.getFieldWithDefault(msg, 2, "") + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadTombstone} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.microsoft.durabletask.largepayloads.LargePayloadTombstone; + return proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.microsoft.durabletask.largepayloads.LargePayloadTombstone} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadTombstone} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {string} */ (reader.readString()); + msg.setTombstoneToken(value); + break; + case 2: + var value = /** @type {string} */ (reader.readString()); + msg.setPayloadToken(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.microsoft.durabletask.largepayloads.LargePayloadTombstone} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getTombstoneToken(); + if (f.length > 0) { + writer.writeString( + 1, + f + ); + } + f = message.getPayloadToken(); + if (f.length > 0) { + writer.writeString( + 2, + f + ); + } +}; + + +/** + * optional string tombstone_token = 1; + * @return {string} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.prototype.getTombstoneToken = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +}; + + +/** + * @param {string} value + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadTombstone} returns this + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.prototype.setTombstoneToken = function(value) { + return jspb.Message.setProto3StringField(this, 1, value); +}; + + +/** + * optional string payload_token = 2; + * @return {string} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.prototype.getPayloadToken = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +}; + + +/** + * @param {string} value + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadTombstone} returns this + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.prototype.setPayloadToken = function(value) { + return jspb.Message.setProto3StringField(this, 2, value); +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.prototype.toObject = function(opt_includeInstance) { + return proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.toObject = function(includeInstance, msg) { + var f, obj = { + tombstoneToken: jspb.Message.getFieldWithDefault(msg, 1, ""), + disposition: jspb.Message.getFieldWithDefault(msg, 2, 0) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult; + return proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {string} */ (reader.readString()); + msg.setTombstoneToken(value); + break; + case 2: + var value = /** @type {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeDisposition} */ (reader.readEnum()); + msg.setDisposition(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getTombstoneToken(); + if (f.length > 0) { + writer.writeString( + 1, + f + ); + } + f = message.getDisposition(); + if (f !== 0.0) { + writer.writeEnum( + 2, + f + ); + } +}; + + +/** + * optional string tombstone_token = 1; + * @return {string} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.prototype.getTombstoneToken = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +}; + + +/** + * @param {string} value + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult} returns this + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.prototype.setTombstoneToken = function(value) { + return jspb.Message.setProto3StringField(this, 1, value); +}; + + +/** + * optional LargePayloadPurgeDisposition disposition = 2; + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeDisposition} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.prototype.getDisposition = function() { + return /** @type {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeDisposition} */ (jspb.Message.getFieldWithDefault(this, 2, 0)); +}; + + +/** + * @param {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeDisposition} value + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult} returns this + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.prototype.setDisposition = function(value) { + return jspb.Message.setProto3EnumField(this, 2, value); +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.prototype.toObject = function(opt_includeInstance) { + return proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.toObject = function(includeInstance, msg) { + var f, obj = { + limit: jspb.Message.getFieldWithDefault(msg, 1, 0) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest; + return proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {number} */ (reader.readInt32()); + msg.setLimit(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getLimit(); + if (f !== 0) { + writer.writeInt32( + 1, + f + ); + } +}; + + +/** + * optional int32 limit = 1; + * @return {number} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.prototype.getLimit = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); +}; + + +/** + * @param {number} value + * @return {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest} returns this + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.prototype.setLimit = function(value) { + return jspb.Message.setProto3IntField(this, 1, value); +}; + + + +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.repeatedFields_ = [1]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.prototype.toObject = function(opt_includeInstance) { + return proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.toObject = function(includeInstance, msg) { + var f, obj = { + tombstonesList: jspb.Message.toObjectList(msg.getTombstonesList(), + proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.toObject, includeInstance) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse; + return proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new proto.microsoft.durabletask.largepayloads.LargePayloadTombstone; + reader.readMessage(value,proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.deserializeBinaryFromReader); + msg.addTombstones(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getTombstonesList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 1, + f, + proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.serializeBinaryToWriter + ); + } +}; + + +/** + * repeated LargePayloadTombstone tombstones = 1; + * @return {!Array} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.prototype.getTombstonesList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, proto.microsoft.durabletask.largepayloads.LargePayloadTombstone, 1)); +}; + + +/** + * @param {!Array} value + * @return {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse} returns this +*/ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.prototype.setTombstonesList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); +}; + + +/** + * @param {!proto.microsoft.durabletask.largepayloads.LargePayloadTombstone=} opt_value + * @param {number=} opt_index + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadTombstone} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.prototype.addTombstones = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.microsoft.durabletask.largepayloads.LargePayloadTombstone, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse} returns this + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.prototype.clearTombstonesList = function() { + return this.setTombstonesList([]); +}; + + + +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.repeatedFields_ = [1]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.toObject = function(includeInstance, msg) { + var f, obj = { + resultsList: jspb.Message.toObjectList(msg.getResultsList(), + proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.toObject, includeInstance) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest; + return proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult; + reader.readMessage(value,proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.deserializeBinaryFromReader); + msg.addResults(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getResultsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 1, + f, + proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.serializeBinaryToWriter + ); + } +}; + + +/** + * repeated LargePayloadPurgeResult results = 1; + * @return {!Array} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.prototype.getResultsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult, 1)); +}; + + +/** + * @param {!Array} value + * @return {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest} returns this +*/ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.prototype.setResultsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); +}; + + +/** + * @param {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult=} opt_value + * @param {number=} opt_index + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.prototype.addResults = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest} returns this + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.prototype.clearResultsList = function() { + return this.setResultsList([]); +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.toObject = function(includeInstance, msg) { + var f, obj = { + + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse; + return proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.serializeBinaryToWriter = function(message, writer) { + var f = undefined; +}; + + +/** + * @enum {number} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeDisposition = { + LARGE_PAYLOAD_PURGE_DISPOSITION_UNSPECIFIED: 0, + LARGE_PAYLOAD_PURGE_DISPOSITION_DELETED: 1, + LARGE_PAYLOAD_PURGE_DISPOSITION_RETRY: 2, + LARGE_PAYLOAD_PURGE_DISPOSITION_QUARANTINED: 3 +}; + +goog.object.extend(exports, proto.microsoft.durabletask.largepayloads); diff --git a/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts b/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts index 962b6fec..c5a124df 100644 --- a/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts +++ b/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts @@ -26,6 +26,10 @@ import { ExponentialBackoff, sleep, withTimeout } from "../utils/backoff.util"; import { VersioningOptions, VersionMatchStrategy, VersionFailureStrategy } from "./versioning-options"; import { WorkItemFilters, generateWorkItemFiltersFromRegistry, toGrpcWorkItemFilters } from "./work-item-filters"; import { compareVersions } from "../utils/versioning.util"; +import { PayloadStorageOptions } from "../payloads/payload-store"; +import { PayloadProcessor } from "../payloads/payload-processor"; +import { readPayloadHistory, PayloadHydrationError } from "../payloads/history-stream"; +import { registerPayloadPurge, withPayloadPurgeFilters, isPayloadPurgeTask } from "../payloads/purge-worker"; import * as WorkerLogs from "./logs"; import { ConcurrencyOptions, ResolvedConcurrencyOptions, resolveConcurrencyOptions } from "./concurrency-options"; import { @@ -61,6 +65,8 @@ type WorkItemStreamResult = { * Options for creating a TaskHubGrpcWorker. */ export interface TaskHubGrpcWorkerOptions { + /** Optional externalized payload storage. Disabled by default. */ + payloads?: PayloadStorageOptions; /** * Maximum backend timer segment in milliseconds. Defaults to three days, like Python. * null, zero, or negative values disable segmentation. Must be finite; positive @@ -139,6 +145,7 @@ export class TaskHubGrpcWorker { private _workerLoopPromise: Promise | null; private _deferredStubCloseTimers: Map>; private readonly _maximumTimerIntervalMs: number | null; + private readonly _payloads?: PayloadProcessor; /** * Creates a new TaskHubGrpcWorker instance. @@ -207,6 +214,7 @@ export class TaskHubGrpcWorker { resolvedWorkItemFilters = hostAddressOrOptions.workItemFilters; resolvedConcurrency = hostAddressOrOptions.concurrency; resolvedMaximumTimerIntervalMs = hostAddressOrOptions.maximumTimerIntervalMs; + if (hostAddressOrOptions.payloads) this._payloads = new PayloadProcessor(hostAddressOrOptions.payloads); } else { // Deprecated positional parameters constructor resolvedHostAddress = hostAddressOrOptions; @@ -259,6 +267,19 @@ export class TaskHubGrpcWorker { this._abortController = null; this._workerLoopPromise = null; this._deferredStubCloseTimers = new Map(); + if (typeof hostAddressOrOptions === "object" && hostAddressOrOptions?.payloads) { + registerPayloadPurge( + this._registry, + hostAddressOrOptions.payloads.store, + () => { + if (!this._stub) throw new Error("The payload purge transport is not connected."); + return this._stub; + }, + this._metadataGenerator, + () => this._abortController?.signal, + this._logger, + ); + } } /** The configured maximum backend timer interval in milliseconds. */ @@ -857,6 +878,7 @@ export class TaskHubGrpcWorker { private _buildGetWorkItemsRequest(): pb.GetWorkItemsRequest { const request = new pb.GetWorkItemsRequest(); request.setCapabilitiesList([pb.WorkerCapability.WORKER_CAPABILITY_HISTORY_STREAMING]); + if (this._payloads) request.addCapabilities(pb.WorkerCapability.WORKER_CAPABILITY_LARGE_PAYLOADS); request.setMaxconcurrentactivityworkitems( Math.min(this._concurrency.maximumConcurrentActivityWorkItems, MAX_PROTOCOL_CONCURRENCY), ); @@ -872,7 +894,7 @@ export class TaskHubGrpcWorker { this._workItemFilters === "auto" ? generateWorkItemFiltersFromRegistry(this._registry, this._versioning) : this._workItemFilters; - request.setWorkitemfilters(toGrpcWorkItemFilters(filters)); + request.setWorkitemfilters(toGrpcWorkItemFilters(this._payloads ? withPayloadPurgeFilters(filters) : filters)); } return request; @@ -881,13 +903,17 @@ export class TaskHubGrpcWorker { /** * Result of version compatibility check. */ - private _checkVersionCompatibility(orchestrationVersion?: string): { + private _checkVersionCompatibility( + orchestrationVersion?: string, + taskName?: string, + ): { compatible: boolean; shouldFail: boolean; orchestrationVersion?: string; errorType?: string; errorMessage?: string; } { + if (this._payloads && isPayloadPurgeTask(taskName)) return { compatible: true, shouldFail: false }; // If no versioning options configured or match strategy is None, always compatible if ( !this._versioning || @@ -974,6 +1000,7 @@ export class TaskHubGrpcWorker { request: TReq, signal?: AbortSignal, ): Promise { + if (this._payloads) await this._payloads.externalize(request, signal); const backoff = new ExponentialBackoff({ initialDelayMs: 200, maxDelayMs: 15000, @@ -1059,6 +1086,13 @@ export class TaskHubGrpcWorker { if (settled) return; signal?.throwIfAborted(); stream = stub.streamInstanceHistory(request, metadata); + if (this._payloads) { + readPayloadHistory(stream, this._payloads, signal).then((hydrated) => { + for (const event of hydrated) events.push(event); + finish(); + }, onError); + return; + } stream.on("data", onData); stream.once("end", onEnd); stream.once("error", onError); @@ -1079,6 +1113,8 @@ export class TaskHubGrpcWorker { ): Promise { const instanceId = req.getInstanceid(); + if (this._payloads) await this._payloads.resolve(req, retrySignal); + if (!instanceId) { throw new Error(`Could not execute the orchestrator as the instanceId was not provided (${instanceId})`); } @@ -1096,6 +1132,7 @@ export class TaskHubGrpcWorker { req.setPasteventsList(pastEvents); } catch (e: unknown) { if (retrySignal?.aborted) return; + if (e instanceof PayloadHydrationError) throw e.cause; const error = e instanceof Error ? e : new Error(String(e)); WorkerLogs.executionError(this._logger, instanceId, error); const res = new pb.OrchestratorResponse(); @@ -1120,7 +1157,11 @@ export class TaskHubGrpcWorker { } // Check version compatibility if versioning is enabled - const versionCheckResult = this._checkVersionCompatibility(this._getOrchestrationVersion(req)); + const taskName = [...req.getPasteventsList(), ...req.getNeweventsList()] + .find((event) => event.hasExecutionstarted()) + ?.getExecutionstarted() + ?.getName(); + const versionCheckResult = this._checkVersionCompatibility(this._getOrchestrationVersion(req), taskName); if (!versionCheckResult.compatible) { if (versionCheckResult.shouldFail) { // Fail the orchestration with version mismatch error @@ -1327,20 +1368,16 @@ export class TaskHubGrpcWorker { stub: stubs.TaskHubSidecarServiceClient, retrySignal?: AbortSignal, ): Promise { + if (this._payloads) await this._payloads.resolve(req, retrySignal); const instanceId = req.getOrchestrationinstance()?.getInstanceid(); if (!instanceId) { throw new Error("Activity request does not contain an orchestration instance id"); } - const versionCheck = this._checkVersionCompatibility(req.getVersion()?.getValue()); + const versionCheck = this._checkVersionCompatibility(req.getVersion()?.getValue(), req.getName()); if (!versionCheck.compatible && !versionCheck.shouldFail) { - WorkerLogs.versionMismatchAbandon( - this._logger, - instanceId, - versionCheck.errorType!, - versionCheck.errorMessage!, - ); + WorkerLogs.versionMismatchAbandon(this._logger, instanceId, versionCheck.errorType!, versionCheck.errorMessage!); try { const abandon = new pb.AbandonActivityTaskRequest().setCompletiontoken(completionToken); await this._deliverResponse(stub.abandonTaskActivityWorkItem.bind(stub), abandon, retrySignal); @@ -1444,6 +1481,7 @@ export class TaskHubGrpcWorker { operationInfos?: pb.OperationInfo[], retrySignal?: AbortSignal, ): Promise { + if (this._payloads) await this._payloads.resolve(req, retrySignal); const instanceIdString = req.getInstanceid(); if (!instanceIdString) { @@ -1546,6 +1584,7 @@ export class TaskHubGrpcWorker { stub: stubs.TaskHubSidecarServiceClient, retrySignal?: AbortSignal, ): Promise { + if (this._payloads) await this._payloads.resolve(req, retrySignal); // Convert EntityRequest (V2) to EntityBatchRequest (V1) format const batchRequest = new pb.EntityBatchRequest(); batchRequest.setInstanceid(req.getInstanceid()); diff --git a/packages/durabletask-js/test/client-purge.spec.ts b/packages/durabletask-js/test/client-purge.spec.ts index e5a284ca..adf479a3 100644 --- a/packages/durabletask-js/test/client-purge.spec.ts +++ b/packages/durabletask-js/test/client-purge.spec.ts @@ -11,6 +11,7 @@ import { PurgeInstanceCriteria } from "../src/orchestration/orchestration-purge- import { PurgeResult } from "../src/orchestration/orchestration-purge-result"; import * as pb from "../src/proto/orchestrator_service_pb"; import { NoOpLogger } from "../src/types/logger.type"; +import { PayloadStore } from "../src/payloads/payload-store"; type Callback = (error: grpc.ServiceError | null, response?: pb.PurgeInstancesResponse) => void; @@ -137,4 +138,31 @@ describe("TaskHubGrpcClient purge", () => { expect(cancel).not.toHaveBeenCalled(); callback(null, new pb.PurgeInstancesResponse()); }); + + it.each(["instance-1", "@counter@key"])( + "routes payload-enabled purge %s to its backend instance kind", + async (id) => { + client["_stub"].close(); + const store = new (class extends PayloadStore { + async upload(): Promise { + return ""; + } + async download(): Promise { + return ""; + } + isKnownPayloadToken(): boolean { + return false; + } + })(); + client = new TaskHubGrpcClient({ payloads: { store }, logger: new NoOpLogger() }); + Object.defineProperty(client["_stub"], "purgeInstances", { value: rpc }); + const result = client.purgeOrchestration(id); + await new Promise((resolve) => setImmediate(resolve)); + expect(rpc).toHaveBeenCalledTimes(1); + const request = rpc.mock.calls[0][0]; + callback(null, new pb.PurgeInstancesResponse()); + await result; + expect(request.getIsorchestration()).toBe(!id.startsWith("@")); + }, + ); }); diff --git a/packages/durabletask-js/test/payload-purge.spec.ts b/packages/durabletask-js/test/payload-purge.spec.ts new file mode 100644 index 00000000..18a0095a --- /dev/null +++ b/packages/durabletask-js/test/payload-purge.spec.ts @@ -0,0 +1,416 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as grpc from "@grpc/grpc-js"; +import { + TaskHubGrpcClient, + TaskHubGrpcWorker, + NoOpLogger, + PayloadStore, + PayloadDeleteOutcome, + ActivityContext, + VersionMatchStrategy, + WorkItemFilters, + TaskFailedError, +} from "../src"; +import * as pb from "../src/proto/orchestrator_service_pb"; +import * as stubs from "../src/proto/orchestrator_service_grpc_pb"; +import * as lp from "../src/proto/large_payload_purge_pb"; +import { LargePayloadPurgeService, ILargePayloadPurgeServer } from "../src/proto/large_payload_purge_grpc_pb"; +import { + blobPurgeJobOrchestrator, + BlobPurgeConstants, + BlobPurgeOutcome, + LargePayloadPurgeDisposition, +} from "../src/payloads/purge"; +import { deletePayloadChunk } from "../src/payloads/purge-worker"; +import { RuntimeOrchestrationContext } from "../src/worker/runtime-orchestration-context"; +import { RetryPolicy } from "../src/task/retry/retry-policy"; + +class Store extends PayloadStore { + async upload(): Promise { + return "blob:v2:https://account.invalid/payloads/blob"; + } + async download(): Promise { + return "null"; + } + isKnownPayloadToken(value: string): boolean { + return value.startsWith("blob:"); + } + async delete(_token: string): Promise { + return PayloadDeleteOutcome.Deleted; + } +} +const logger = new NoOpLogger(); + +describe("large payload auto-purge lifecycle transport", () => { + let server: grpc.Server; + let client: TaskHubGrpcClient; + let calls: string[]; + let start: pb.CreateInstanceRequest | undefined; + let event: pb.RaiseEventRequest | undefined; + let startId: string; + let stateName: string; + let status: pb.OrchestrationStatus; + let collision: boolean; + let hostAddress: string; + let rpcError: grpc.status | undefined; + + beforeEach(async () => { + calls = []; + start = undefined; + event = undefined; + startId = BlobPurgeConstants.instanceId; + stateName = BlobPurgeConstants.orchestrator; + status = pb.OrchestrationStatus.ORCHESTRATION_STATUS_RUNNING; + collision = false; + rpcError = undefined; + server = new grpc.Server(); + const sidecar = { + startInstance: (call, callback) => { + calls.push("start"); + expect(call.metadata.get("taskhub")).toEqual(["payload-test"]); + start = call.request; + if (collision) callback({ code: grpc.status.ALREADY_EXISTS, message: "exists" }); + else callback(null, new pb.CreateInstanceResponse().setInstanceid(startId)); + }, + waitForInstanceStart: (call, callback) => { + calls.push("wait"); + expect(call.metadata.get("taskhub")).toEqual(["payload-test"]); + callback( + null, + new pb.GetInstanceResponse() + .setExists(true) + .setOrchestrationstate( + new pb.OrchestrationState() + .setInstanceid(BlobPurgeConstants.instanceId) + .setName(stateName) + .setOrchestrationstatus(status), + ), + ); + }, + raiseEvent: (call, callback) => { + calls.push("event"); + event = call.request; + expect(call.metadata.get("taskhub")).toEqual(["payload-test"]); + callback(null, new pb.RaiseEventResponse()); + }, + } satisfies Pick; + server.addService(stubs.TaskHubSidecarServiceService, sidecar); + const purge = { + setLargePayloadAutoPurge: (call, callback) => { + calls.push(`setting:${call.request.getEnabled()}`); + expect(call.metadata.get("taskhub")).toEqual(["payload-test"]); + callback(null, new lp.SetLargePayloadAutoPurgeResponse()); + }, + getLargePayloadTombstones: (call, callback) => { + calls.push("fetch"); + expect(call.metadata.get("taskhub")).toEqual(["payload-test"]); + expect(Number(call.getDeadline()) - Date.now()).toBeLessThanOrEqual(61000); + if (rpcError !== undefined) callback({ code: rpcError, message: "backend" }); + else + callback( + null, + new lp.GetLargePayloadTombstonesResponse().setTombstonesList([ + new lp.LargePayloadTombstone() + .setTombstoneToken("opaque:\u00e9:do-not-parse") + .setPayloadToken("blob:v2:https://a.invalid/c/b"), + ]), + ); + }, + reportLargePayloadPurgeResults: (call, callback) => { + calls.push("report"); + expect(call.metadata.get("taskhub")).toEqual(["payload-test"]); + expect(Number(call.getDeadline()) - Date.now()).toBeLessThanOrEqual(61000); + expect(call.request.getResultsList()[0].getTombstoneToken()).toBe("opaque:\u00e9:do-not-parse"); + callback(null, new lp.ReportLargePayloadPurgeResultsResponse()); + }, + } satisfies ILargePayloadPurgeServer; + server.addService(LargePayloadPurgeService, purge); + const port = await new Promise((resolve, reject) => + server.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (error, port) => + error ? reject(error) : resolve(port), + ), + ); + hostAddress = `127.0.0.1:${port}`; + client = new TaskHubGrpcClient({ + hostAddress, + defaultVersion: "business", + metadataGenerator: async () => { + const metadata = new grpc.Metadata(); + metadata.set("taskhub", "payload-test"); + return metadata; + }, + }); + }); + afterEach(async () => { + await client.stop(); + server.forceShutdown(); + }); + + it("writes setting first, schedules unversioned singleton, verifies start and sends configuration", async () => { + await client.setLargePayloadAutoPurge(true, 500); + expect(calls).toEqual(["setting:true", "start", "wait", "event"]); + expect(start!.getInstanceid()).toBe(BlobPurgeConstants.instanceId); + expect(start!.getName()).toBe(BlobPurgeConstants.orchestrator); + expect(start!.getVersion()?.getValue() ?? "").toBe(""); + expect(JSON.parse(start!.getInput()!.getValue())).toEqual({ PurgeBatchSize: 500, PurgedCount: 0 }); + expect(start!.getOrchestrationidreusepolicy()!.getReplaceablestatusList()).toEqual([ + pb.OrchestrationStatus.ORCHESTRATION_STATUS_COMPLETED, + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + pb.OrchestrationStatus.ORCHESTRATION_STATUS_CANCELED, + pb.OrchestrationStatus.ORCHESTRATION_STATUS_TERMINATED, + ]); + expect(event!.getName()).toBe("SetBatchSize"); + expect(event!.getInput()!.getValue()).toBe("500"); + }); + + it("disables without validating batch, starting, waiting, or signaling", async () => { + await client.setLargePayloadAutoPurge(false, NaN); + expect(calls).toEqual(["setting:false"]); + }); + + it.each([0, 1001, NaN, 1.5])("rejects invalid enabling batch %s before mutating the backend", async (batch) => { + await expect(client.setLargePayloadAutoPurge(true, batch)).rejects.toThrow(RangeError); + expect(calls).toEqual([]); + }); + + it("rejects an unexpected backend start ID without waiting or signaling", async () => { + startId = "different"; + await expect(client.setLargePayloadAutoPurge(true)).rejects.toThrow("unexpected"); + expect(calls).toEqual(["setting:true", "start"]); + }); + + it("accepts a deduplicated start only after verifying identity and Running status", async () => { + collision = true; + await client.setLargePayloadAutoPurge(true); + expect(calls).toEqual(["setting:true", "start", "wait", "event"]); + }); + + it.each(["wrong-name", "suspended"])("does not configure a %s reserved-ID collision", async (kind) => { + collision = true; + if (kind === "wrong-name") stateName = "business"; + else status = pb.OrchestrationStatus.ORCHESTRATION_STATUS_SUSPENDED; + await expect(client.setLargePayloadAutoPurge(true)).rejects.toThrow(); + expect(calls).toEqual(["setting:true", "start", "wait"]); + }); + + it("uses the current authenticated sidecar transport for reserved fetch/report activities", async () => { + const worker = new TaskHubGrpcWorker({ + payloads: { store: new Store() }, + metadataGenerator: client["_metadataGenerator"], + logger, + }); + worker["_stub"] = client["_stub"]; + const fetch = worker["_registry"].getActivity(BlobPurgeConstants.fetch)!; + const report = worker["_registry"].getActivity(BlobPurgeConstants.report)!; + const tombstones = await fetch(new ActivityContext("job", 0), 10); + expect(tombstones).toEqual([ + { TombstoneToken: "opaque:\u00e9:do-not-parse", PayloadToken: "blob:v2:https://a.invalid/c/b" }, + ]); + await report(new ActivityContext("job", 1), [{ TombstoneToken: tombstones[0].TombstoneToken, Disposition: 1 }]); + expect(calls).toEqual(["fetch", "report"]); + rpcError = grpc.status.FAILED_PRECONDITION; + expect(await fetch(new ActivityContext("job", 2), 10)).toEqual([]); + rpcError = grpc.status.UNIMPLEMENTED; + await expect(fetch(new ActivityContext("job", 3), 10)).rejects.toMatchObject({ + name: "System.NotImplementedException", + }); + }); +}); + +describe("payload purge worker registrations", () => { + it.each([{}, { orchestrations: [], activities: [], entities: [] }])( + "leaves all-empty filters unfiltered", + (filters) => { + const worker = new TaskHubGrpcWorker({ payloads: { store: new Store() }, workItemFilters: filters }); + expect(worker["_buildGetWorkItemsRequest"]().getWorkitemfilters()!.toObject()).toEqual({ + orchestrationsList: [], + activitiesList: [], + entitiesList: [], + }); + }, + ); + it("augments entity-only filters, bypasses business versions, and does not mutate caller lists", () => { + const filters: WorkItemFilters = { entities: [{ name: "counter" }] }; + const worker = new TaskHubGrpcWorker({ + payloads: { store: new Store() }, + workItemFilters: filters, + versioning: { version: "business", matchStrategy: VersionMatchStrategy.Strict }, + }); + const wire = worker["_buildGetWorkItemsRequest"]().getWorkitemfilters()!; + expect(wire.getOrchestrationsList().map((f) => [f.getName(), f.getVersionsList()])).toEqual([ + [BlobPurgeConstants.orchestrator, []], + ]); + expect(wire.getActivitiesList()).toHaveLength(3); + expect(filters).toEqual({ entities: [{ name: "counter" }] }); + expect(worker["_checkVersionCompatibility"]("", BlobPurgeConstants.orchestrator).compatible).toBe(true); + expect(worker["_checkVersionCompatibility"]("", "business").compatible).toBe(false); + }); +}); + +describe("purge delete classification", () => { + it("bounds deletes at eight and preserves positional dispositions without logging tokens", async () => { + const store = new Store(); + let active = 0; + let max = 0; + store.delete = async (token) => { + max = Math.max(max, ++active); + await new Promise((resolve) => setImmediate(resolve)); + active--; + if (token.endsWith("error")) throw Object.assign(new Error(token), { statusCode: 403 }); + return PayloadDeleteOutcome.Deleted; + }; + const tokens = Array.from({ length: 50 }, (_, i) => `blob:v2:https://a.invalid/c/${i === 3 ? "error" : i}`); + const warn = jest.fn(); + const outcomes = await deletePayloadChunk(store, tokens, { + ...logger, + warn, + error: jest.fn(), + info: jest.fn(), + debug: jest.fn(), + }); + expect(max).toBe(8); + expect(outcomes.map((o) => o.Disposition)).toEqual(tokens.map((_, i) => (i === 3 ? 2 : 1))); + expect(JSON.stringify(warn.mock.calls)).not.toContain("a.invalid"); + }); + it.each([ + ["blob:v1:c:b", 3], + ["blob:v3:future", 2], + ])("does not delete %s", async (token, disposition) => { + const store = new Store(); + store.delete = jest.fn(); + expect(await deletePayloadChunk(store, [token], logger)).toEqual([{ Disposition: disposition }]); + expect(store.delete).not.toHaveBeenCalled(); + }); + it.each([ + [new TypeError("bad token"), 3], + [Object.assign(new Error("bad"), { statusCode: 400 }), 3], + [Object.assign(new Error("bad"), { statusCode: 401 }), 2], + [Object.assign(new Error("bad"), { statusCode: 412 }), 2], + [new Error("network"), 2], + ])("classifies storage errors without losing tombstones (%#)", async (error, disposition) => { + const store = new Store(); + store.delete = async () => { + throw error; + }; + expect(await deletePayloadChunk(store, ["blob:v2:https://a.invalid/c/b"], logger)).toEqual([ + { Disposition: disposition }, + ]); + }); + it.each([PayloadDeleteOutcome.Deleted, PayloadDeleteOutcome.AlreadyAbsent, PayloadDeleteOutcome.NotStoreOwned])( + "maps confirmed delete outcome %s to terminal success", + async (outcome) => { + const store = new Store(); + store.delete = async () => outcome; + expect(await deletePayloadChunk(store, ["blob:v2:https://a.invalid/c/b"], logger)).toEqual([{ Disposition: 1 }]); + }, + ); +}); + +describe("replay-safe purge runner", () => { + it("chunks 50 tokens, schedules four chunks per wave, and echoes opaque tombstones unchanged", async () => { + const context = new RuntimeOrchestrationContext("job"); + const run = blobPurgeJobOrchestrator(context, { PurgeBatchSize: 500, PurgedCount: 2 }); + await run.next(); + const tombstones = Array.from({ length: 251 }, (_, i) => ({ + TombstoneToken: `opaque:${i}`, + PayloadToken: `blob:v2:https://a/c/${i}`, + })); + await run.next(tombstones); + let actions = Object.values(context._pendingActions); + expect(actions.filter((a) => a.getScheduletask()?.getName() === BlobPurgeConstants.delete)).toHaveLength(4); + await run.next(Array.from({ length: 4 }, () => Array.from({ length: 50 }, () => ({ Disposition: 1 })))); + actions = Object.values(context._pendingActions); + expect(actions.filter((a) => a.getScheduletask()?.getName() === BlobPurgeConstants.delete)).toHaveLength(6); + await run.next([Array.from({ length: 50 }, () => ({ Disposition: 1 })), [{ Disposition: 1 }]]); + const report = Object.values(context._pendingActions).find( + (a) => a.getScheduletask()?.getName() === BlobPurgeConstants.report, + )!; + expect(JSON.parse(report.getScheduletask()!.getInput()!.getValue())).toEqual( + tombstones.map((t) => ({ TombstoneToken: t.TombstoneToken, Disposition: 1 })), + ); + await run.return(); + }); + + it("drains delivered and buffered configuration before continue-as-new on cycle five", async () => { + const context = new RuntimeOrchestrationContext("job"); + const run = blobPurgeJobOrchestrator(context, { PurgeBatchSize: 500, PurgedCount: 3 }); + await run.next(); + for (let cycle = 0; cycle < 5; cycle++) { + await run.next([{ TombstoneToken: "opaque", PayloadToken: "blob:v2:https://a/c/b" }]); + await run.next([[{ Disposition: 1 }]]); + if (cycle === 4) { + context._pendingEvents.setbatchsize[0].complete(60); + context._receivedEvents.setbatchsize = [70, 80]; + } + await run.next(null); + } + expect(context._newInput).toEqual({ PurgeBatchSize: 80, PurgedCount: 8 }); + expect(context._saveEvents).toBe(true); + }); + + it("waits for explicit configuration on unsupported backends without scheduling a retry timer", async () => { + const context = new RuntimeOrchestrationContext("job"); + const call = jest.spyOn(context, "callActivity"); + const run = blobPurgeJobOrchestrator(context, { PurgeBatchSize: 500, PurgedCount: 0 }); + await run.next(); + const policy = call.mock.calls[0][2]!.retry; + expect(policy).toBeInstanceOf(RetryPolicy); + expect( + (policy as RetryPolicy).shouldRetry({ errorType: "System.NotImplementedException", message: "unsupported" }), + ).toBe(false); + const failure = new TaskFailedError( + "unsupported", + new pb.TaskFailureDetails().setErrortype("System.NotImplementedException"), + ); + const waiting = await run.throw(failure); + expect(waiting.value).toBe(context._pendingEvents.setbatchsize[0]); + expect(Object.values(context._pendingActions).some((a) => a.hasCreatetimer())).toBe(false); + expect(JSON.parse(context.getCustomStatus()!).Status).toBe("BackendUnsupported"); + context._pendingEvents.setbatchsize[0].complete(100); + await run.next(100); + expect(call).toHaveBeenCalledTimes(2); + await run.return(); + }); + + it("backs off all-Retry results and cancels its idle timer when configuration arrives", async () => { + const context = new RuntimeOrchestrationContext("job"); + const timer = jest.spyOn(context, "createTimer"); + const run = blobPurgeJobOrchestrator(context, { PurgeBatchSize: 10, PurgedCount: 0 }); + await run.next(); + await run.next([{ TombstoneToken: "opaque", PayloadToken: "blob:v2:https://a/c/b" }]); + await run.next([[{ Disposition: LargePayloadPurgeDisposition.Retry } satisfies BlobPurgeOutcome]]); + await run.next(null); + expect(timer).toHaveBeenCalledTimes(1); + expect((timer.mock.calls[0][0] as Date).getTime() - context.currentUtcDateTime.getTime()).toBe(60000); + context._pendingEvents.setbatchsize[0].complete(20); + await run.next(context._pendingEvents.setbatchsize[0]); + expect(timer.mock.results[0].value.isCanceled).toBe(true); + await run.return(); + }); + + it("waits for configuration before rolling over an unsupported fifth cycle", async () => { + const context = new RuntimeOrchestrationContext("job"); + const run = blobPurgeJobOrchestrator(context, { PurgeBatchSize: 500, PurgedCount: 0 }, logger); + await run.next(); + for (let cycle = 0; cycle < 4; cycle++) { + await run.next([{ TombstoneToken: "opaque", PayloadToken: "blob:v2:https://a/c/b" }]); + await run.next([[{ Disposition: 1 }]]); + await run.next(null); + } + const failure = new TaskFailedError( + "unsupported", + new pb.TaskFailureDetails().setErrortype("System.NotImplementedException"), + ); + const waiting = await run.throw(failure); + expect(waiting.done).toBe(false); + expect(context._newInput).toBeUndefined(); + expect(Object.values(context._pendingActions).some((action) => action.hasCreatetimer())).toBe(false); + context._pendingEvents.setbatchsize[0].complete(60); + context._receivedEvents.setbatchsize = [70]; + expect((await run.next(60)).done).toBe(true); + expect(context._newInput).toEqual({ PurgeBatchSize: 70, PurgedCount: 4 }); + expect(context._saveEvents).toBe(true); + }); +}); diff --git a/packages/durabletask-js/test/payload-transport.spec.ts b/packages/durabletask-js/test/payload-transport.spec.ts new file mode 100644 index 00000000..a1bc197b --- /dev/null +++ b/packages/durabletask-js/test/payload-transport.spec.ts @@ -0,0 +1,233 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as grpc from "@grpc/grpc-js"; +import { Empty } from "google-protobuf/google/protobuf/empty_pb"; +import { StringValue } from "google-protobuf/google/protobuf/wrappers_pb"; +import { TaskHubGrpcClient, TaskHubGrpcWorker, PayloadStore, NoOpLogger } from "../src"; +import * as pb from "../src/proto/orchestrator_service_pb"; +import * as stubs from "../src/proto/orchestrator_service_grpc_pb"; +import * as pbh from "../src/utils/pb-helper.util"; + +const text = (value: string) => new StringValue().setValue(value); +class MemoryStore extends PayloadStore { + readonly values = new Map(); + async upload(value: string): Promise { + const token = `payload:${this.values.size}`; + this.values.set(token, value); + return token; + } + async download(token: string): Promise { + await new Promise((resolve) => setTimeout(resolve, 5)); + if (!this.values.has(token)) throw new Error("Missing blob"); + return this.values.get(token)!; + } + isKnownPayloadToken(value: string): boolean { + return value.startsWith("payload:"); + } +} + +async function waitFor(predicate: () => boolean): Promise { + const deadline = Date.now() + 3000; + while (!predicate()) { + if (Date.now() > deadline) throw new Error("Timed out"); + await new Promise((resolve) => setTimeout(resolve, 5)); + } +} + +describe("transparent payload transport over gRPC", () => { + let server: grpc.Server; + let client: TaskHubGrpcClient; + let worker: TaskHubGrpcWorker; + let store: MemoryStore; + let hostAddress: string; + let input: string; + let subscription: grpc.ServerWritableStream | undefined; + let activityResponse: pb.ActivityResponse | undefined; + let activityCalls: number; + let requestedInput: string | undefined; + let orchestrationResponses: pb.OrchestratorResponse[]; + let historyEvents: pb.HistoryEvent[]; + + beforeEach(async () => { + store = new MemoryStore(); + input = await store.upload(JSON.stringify({ message: "hydrated" })); + subscription = undefined; + activityResponse = undefined; + activityCalls = 0; + requestedInput = undefined; + orchestrationResponses = []; + historyEvents = [pbh.newExecutionStartedEvent("flow", "instance", input)]; + server = new grpc.Server(); + const state = () => + new pb.OrchestrationState() + .setInstanceid("instance") + .setName("workflow") + .setInput(text(input)) + .setOutput(text(input)) + .setCustomstatus(text(input)); + const service = { + hello: (_call, callback) => callback(null, new Empty()), + startInstance: (call, callback) => { + requestedInput = call.request.getInput()?.getValue(); + callback(null, new pb.CreateInstanceResponse().setInstanceid(call.request.getInstanceid())); + }, + getInstance: (_call, callback) => + callback(null, new pb.GetInstanceResponse().setExists(true).setOrchestrationstate(state())), + waitForInstanceStart: (_call, callback) => + callback(null, new pb.GetInstanceResponse().setExists(true).setOrchestrationstate(state())), + queryInstances: (_call, callback) => + callback(null, new pb.QueryInstancesResponse().setOrchestrationstateList([state()])), + streamInstanceHistory: (call) => { + call.write(new pb.HistoryChunk().setEventsList(historyEvents)); + call.end(); + }, + getWorkItems: (call) => { + subscription = call; + call.on("cancelled", () => call.end()); + }, + completeActivityTask: (call, callback) => { + activityResponse = call.request; + callback(null, new pb.CompleteTaskResponse()); + }, + completeOrchestratorTask: (call, callback) => { + orchestrationResponses.push(call.request); + callback(null, new pb.CompleteTaskResponse()); + }, + } satisfies Pick< + stubs.ITaskHubSidecarServiceServer, + | "hello" + | "startInstance" + | "getInstance" + | "waitForInstanceStart" + | "queryInstances" + | "streamInstanceHistory" + | "getWorkItems" + | "completeActivityTask" + | "completeOrchestratorTask" + >; + server.addService(stubs.TaskHubSidecarServiceService, service); + const port = await new Promise((resolve, reject) => + server.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (error, port) => + error ? reject(error) : resolve(port), + ), + ); + hostAddress = `127.0.0.1:${port}`; + const payloads = { store, thresholdBytes: 1 }; + client = new TaskHubGrpcClient({ hostAddress, payloads, logger: new NoOpLogger() }); + worker = new TaskHubGrpcWorker({ hostAddress, payloads, logger: new NoOpLogger() }); + worker.addNamedActivity("echo", async (_context, value) => { + activityCalls++; + return value; + }); + }); + + afterEach(async () => { + if (worker["_isRunning"]) await worker.stop(); + await client.stop(); + server.forceShutdown(); + }); + + it("externalizes before sending and hydrates state, wait, query and streamed history before conversion", async () => { + await client.scheduleNewOrchestration("workflow", { message: "request" }); + expect(requestedInput).toMatch(/^payload:/); + expect(await store.download(requestedInput!)).toBe('{"message":"request"}'); + expect((await client.getOrchestrationState("instance"))!.serializedInput).toBe('{"message":"hydrated"}'); + expect((await client.waitForOrchestrationStart("instance"))!.serializedInput).toBe('{"message":"hydrated"}'); + const states = []; + for await (const item of client.getAllInstances({ fetchInputsAndOutputs: true })) states.push(item); + expect(states[0].serializedInput).toBe('{"message":"hydrated"}'); + const history = await client.getOrchestrationHistory("instance"); + expect(history[0]).toMatchObject({ input: '{"message":"hydrated"}' }); + }); + + it("advertises support, awaits input hydration and externalizes worker output", async () => { + await worker.start(); + await waitFor(() => !!subscription); + expect(subscription!.request.getCapabilitiesList()).toContain(pb.WorkerCapability.WORKER_CAPABILITY_LARGE_PAYLOADS); + subscription!.write( + new pb.WorkItem() + .setCompletiontoken("completion") + .setActivityrequest( + new pb.ActivityRequest() + .setName("echo") + .setTaskid(1) + .setInput(text(input)) + .setOrchestrationinstance(new pb.OrchestrationInstance().setInstanceid("instance")), + ), + ); + await waitFor(() => !!activityResponse); + expect(activityCalls).toBe(1); + expect(activityResponse!.getCompletiontoken()).toBe("completion"); + expect(await store.download(activityResponse!.getResult()!.getValue())).toBe('{"message":"hydrated"}'); + }); + + it("propagates missing streamed payloads rather than returning tokens or succeeding early at end", async () => { + store.values.clear(); + await expect(client.getOrchestrationHistory("instance")).rejects.toThrow("Missing blob"); + }); + + it("hydrates streamed orchestration history before user execution and externalizes its result", async () => { + let observed: unknown; + worker.addNamedOrchestrator("flow", async (_ctx, value) => { + observed = value; + return value; + }); + await worker.start(); + await waitFor(() => !!subscription); + subscription!.write( + new pb.WorkItem() + .setCompletiontoken("streamed") + .setOrchestratorrequest( + new pb.OrchestratorRequest() + .setInstanceid("instance") + .setRequireshistorystreaming(true) + .setNeweventsList([pbh.newOrchestratorStartedEvent()]), + ), + ); + await waitFor(() => orchestrationResponses.length > 0); + expect(observed).toEqual({ message: "hydrated" }); + const result = orchestrationResponses[0].getActionsList()[0].getCompleteorchestration()!.getResult()!.getValue(); + expect(await store.download(result)).toBe('{"message":"hydrated"}'); + }); + + it("leaves a streamed payload download failure retryable instead of failing the orchestration", async () => { + const errors = jest.fn(); + worker = new TaskHubGrpcWorker({ + hostAddress, + payloads: { store }, + logger: { + error: errors, + warn: () => {}, + info: () => {}, + debug: () => {}, + }, + }); + const flow = jest.fn(async () => "should not run"); + worker.addNamedOrchestrator("flow", flow); + store.values.clear(); + await worker.start(); + await waitFor(() => !!subscription); + subscription!.write( + new pb.WorkItem() + .setCompletiontoken("retryable") + .setOrchestratorrequest( + new pb.OrchestratorRequest() + .setInstanceid("instance") + .setRequireshistorystreaming(true) + .setNeweventsList([pbh.newOrchestratorStartedEvent()]), + ), + ); + await waitFor(() => errors.mock.calls.length > 0); + await waitFor(() => worker["_pendingWorkItems"].size === 0); + expect(flow).not.toHaveBeenCalled(); + expect(orchestrationResponses).toEqual([]); + }); + + it("collects large history chunks without the JavaScript variadic argument limit", async () => { + historyEvents = Array.from({ length: 150000 }, () => + new pb.HistoryEvent().setGenericevent(new pb.GenericEvent().setData(text("x"))), + ); + expect(await client.getOrchestrationHistory("instance")).toHaveLength(historyEvents.length); + }, 20000); +}); diff --git a/packages/durabletask-js/test/payloads.spec.ts b/packages/durabletask-js/test/payloads.spec.ts new file mode 100644 index 00000000..96b8580c --- /dev/null +++ b/packages/durabletask-js/test/payloads.spec.ts @@ -0,0 +1,334 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { StringValue, Int32Value } from "google-protobuf/google/protobuf/wrappers_pb"; +import * as pb from "../src/proto/orchestrator_service_pb"; +import { PayloadProcessor } from "../src/payloads/payload-processor"; +import { PayloadStore, PayloadStorageException } from "../src/payloads/payload-store"; +import { runBounded } from "../src/payloads/run-bounded"; + +class MemoryPayloadStore extends PayloadStore { + readonly values = new Map(); + uploads = 0; + async upload(value: string): Promise { + const token = `test:${++this.uploads}`; + this.values.set(token, value); + return token; + } + async download(token: string): Promise { + const value = this.values.get(token); + if (value === undefined) throw new Error("Missing payload"); + return value; + } + isKnownPayloadToken(value: string): boolean { + return value.startsWith("test:"); + } +} + +const text = (value: string) => new StringValue().setValue(value); + +describe("externalized payloads", () => { + let store: MemoryPayloadStore; + let processor: PayloadProcessor; + beforeEach(() => { + store = new MemoryPayloadStore(); + processor = new PayloadProcessor({ store }); + }); + + it.each([ + ["a".repeat(262143), false], + ["a".repeat(262144), true], + ["a".repeat(262145), true], + ["\u00e9".repeat(131072), true], + ["\ud83d\ude00".repeat(65536), true], + ["", false], + ])("uses the inclusive UTF-8 threshold (%#)", async (value, externalized) => { + const request = new pb.CreateInstanceRequest().setInput(text(value)); + await processor.externalize(request); + expect(store.uploads).toBe(externalized ? 1 : 0); + if (externalized) { + const response = new pb.GetInstanceResponse().setOrchestrationstate( + new pb.OrchestrationState().setInput(request.getInput()), + ); + await processor.resolve(response); + expect(response.getOrchestrationstate()!.getInput()!.getValue()).toBe(value); + } else { + expect(request.getInput()!.getValue()).toBe(value); + } + }); + + it("keeps absent payloads absent", async () => { + const request = new pb.CreateInstanceRequest(); + await processor.externalize(request); + expect(request.hasInput()).toBe(false); + expect(store.uploads).toBe(0); + }); + + it("maps large entity batches without the JavaScript variadic argument limit", async () => { + const request = new pb.EntityBatchRequest().setOperationsList( + Array.from({ length: 150000 }, () => new pb.OperationRequest().setInput(text("x"))), + ); + await expect(processor.resolve(request)).resolves.toBeUndefined(); + await expect(processor.externalize(request)).resolves.toBeUndefined(); + expect(store.uploads).toBe(0); + }); + + it("accepts exactly 10 MiB and rejects larger serialized payloads before storage", async () => { + const value = "x".repeat(10 * 1024 * 1024); + await processor.externalize(new pb.CreateInstanceRequest().setInput(text(value))); + await expect( + processor.externalize(new pb.CreateInstanceRequest().setInput(text(value + "x"))), + ).rejects.toBeInstanceOf(PayloadStorageException); + expect(store.uploads).toBe(1); + }); + + it.each([1048577, NaN, Infinity, -1, 0.5])("rejects invalid thresholds (%s)", (thresholdBytes) => { + expect(() => new PayloadProcessor({ store, thresholdBytes })).toThrow(RangeError); + }); + + it.each([ + () => new pb.CreateInstanceRequest().setInput(text("large")), + () => new pb.RaiseEventRequest().setInput(text("large")), + () => new pb.TerminateRequest().setOutput(text("large")), + () => new pb.SuspendRequest().setReason(text("large")), + () => new pb.ResumeRequest().setReason(text("large")), + () => new pb.RewindInstanceRequest().setReason(text("large")), + () => new pb.SignalEntityRequest().setInput(text("large")), + () => new pb.ActivityResponse().setResult(text("large")), + () => new pb.EntityRequest().setEntitystate(text("large")), + () => new pb.EntityBatchRequest().setEntitystate(text("large")), + ])("externalizes every client and worker scalar request (%#)", async (makeRequest) => { + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + await processor.externalize(makeRequest()); + expect(store.uploads).toBe(1); + }); + + it("externalizes all orchestration actions, custom status and rewritten history", async () => { + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + const response = new pb.OrchestratorResponse() + .setCustomstatus(text("status")) + .setActionsList([ + new pb.OrchestratorAction().setScheduletask(new pb.ScheduleTaskAction().setInput(text("activity"))), + new pb.OrchestratorAction().setCreatesuborchestration( + new pb.CreateSubOrchestrationAction().setInput(text("child")), + ), + new pb.OrchestratorAction().setCompleteorchestration( + new pb.CompleteOrchestrationAction().setResult(text("result")).setDetails(text("details")), + ), + new pb.OrchestratorAction().setTerminateorchestration( + new pb.TerminateOrchestrationAction().setReason(text("reason")), + ), + new pb.OrchestratorAction().setSendevent(new pb.SendEventAction().setData(text("event"))), + new pb.OrchestratorAction().setSendentitymessage( + new pb.SendEntityMessageAction().setEntityoperationcalled( + new pb.EntityOperationCalledEvent().setInput(text("entity")), + ), + ), + new pb.OrchestratorAction().setRewindorchestration( + new pb.RewindOrchestrationAction().setNewhistoryList([ + new pb.HistoryEvent().setExecutionstarted(new pb.ExecutionStartedEvent().setInput(text("rewritten"))), + ]), + ), + ]); + await processor.externalize(response); + expect([...store.values.values()]).toEqual([ + "status", + "activity", + "child", + "result", + "details", + "reason", + "event", + "entity", + "rewritten", + ]); + }); + + it("preserves entity result oneofs and externalizes state, outputs and outgoing operations", async () => { + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + const success = new pb.OperationResult().setSuccess(new pb.OperationResultSuccess().setResult(text("result"))); + const failure = new pb.OperationResult().setFailure(new pb.OperationResultFailure()); + const result = new pb.EntityBatchResult() + .setEntitystate(text("state")) + .setResultsList([success, failure]) + .setActionsList([ + new pb.OperationAction().setSendsignal(new pb.SendSignalAction().setInput(text("signal"))), + new pb.OperationAction().setStartneworchestration(new pb.StartNewOrchestrationAction().setInput(text("start"))), + ]); + await processor.externalize(result); + expect(store.uploads).toBe(4); + expect(success.hasSuccess()).toBe(true); + expect(failure.hasFailure()).toBe(true); + }); + + it("hydrates nested history state, activity and entity V1/V2 work items", async () => { + const token = await store.upload("resolved"); + const history = new pb.HistoryEvent().setHistorystate( + new pb.HistoryStateEvent().setOrchestrationstate( + new pb.OrchestrationState().setInput(text(token)).setOutput(text(token)).setCustomstatus(text(token)), + ), + ); + const messages = [ + new pb.WorkItem().setActivityrequest(new pb.ActivityRequest().setInput(text(token))), + new pb.WorkItem().setOrchestratorrequest(new pb.OrchestratorRequest().setPasteventsList([history])), + new pb.WorkItem().setEntityrequest( + new pb.EntityBatchRequest() + .setEntitystate(text(token)) + .setOperationsList([new pb.OperationRequest().setInput(text(token))]), + ), + new pb.WorkItem().setEntityrequestv2( + new pb.EntityRequest() + .setEntitystate(text(token)) + .setOperationrequestsList([ + new pb.HistoryEvent().setEntityoperationcalled(new pb.EntityOperationCalledEvent().setInput(text(token))), + ]), + ), + new pb.GetEntityResponse().setEntity(new pb.EntityMetadata().setSerializedstate(text(token))), + new pb.QueryEntitiesResponse().setEntitiesList([new pb.EntityMetadata().setSerializedstate(text(token))]), + new pb.QueryInstancesResponse().setOrchestrationstateList([new pb.OrchestrationState().setInput(text(token))]), + new pb.HistoryChunk().setEventsList([ + new pb.HistoryEvent().setExecutionrewound(new pb.ExecutionRewoundEvent().setReason(text(token))), + ]), + ]; + for (const message of messages) { + await processor.resolve(message); + expect(JSON.stringify(message.toObject())).not.toContain(token); + expect(JSON.stringify(message.toObject())).toContain("resolved"); + } + }); + + it.each([ + (value: StringValue) => new pb.HistoryEvent().setExecutionstarted(new pb.ExecutionStartedEvent().setInput(value)), + (value: StringValue) => + new pb.HistoryEvent().setExecutioncompleted(new pb.ExecutionCompletedEvent().setResult(value)), + (value: StringValue) => + new pb.HistoryEvent().setExecutionterminated(new pb.ExecutionTerminatedEvent().setInput(value)), + (value: StringValue) => new pb.HistoryEvent().setTaskscheduled(new pb.TaskScheduledEvent().setInput(value)), + (value: StringValue) => new pb.HistoryEvent().setTaskcompleted(new pb.TaskCompletedEvent().setResult(value)), + (value: StringValue) => + new pb.HistoryEvent().setSuborchestrationinstancecreated( + new pb.SubOrchestrationInstanceCreatedEvent().setInput(value), + ), + (value: StringValue) => + new pb.HistoryEvent().setSuborchestrationinstancecompleted( + new pb.SubOrchestrationInstanceCompletedEvent().setResult(value), + ), + (value: StringValue) => new pb.HistoryEvent().setEventsent(new pb.EventSentEvent().setInput(value)), + (value: StringValue) => new pb.HistoryEvent().setEventraised(new pb.EventRaisedEvent().setInput(value)), + (value: StringValue) => new pb.HistoryEvent().setGenericevent(new pb.GenericEvent().setData(value)), + (value: StringValue) => new pb.HistoryEvent().setContinueasnew(new pb.ContinueAsNewEvent().setInput(value)), + (value: StringValue) => + new pb.HistoryEvent().setExecutionsuspended(new pb.ExecutionSuspendedEvent().setInput(value)), + (value: StringValue) => new pb.HistoryEvent().setExecutionresumed(new pb.ExecutionResumedEvent().setInput(value)), + (value: StringValue) => new pb.HistoryEvent().setExecutionrewound(new pb.ExecutionRewoundEvent().setReason(value)), + (value: StringValue) => + new pb.HistoryEvent().setEntityoperationsignaled(new pb.EntityOperationSignaledEvent().setInput(value)), + (value: StringValue) => + new pb.HistoryEvent().setEntityoperationcalled(new pb.EntityOperationCalledEvent().setInput(value)), + (value: StringValue) => + new pb.HistoryEvent().setEntityoperationcompleted(new pb.EntityOperationCompletedEvent().setOutput(value)), + ])("hydrates and rewrites every payload-bearing history oneof (%#)", async (makeEvent) => { + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + const event = makeEvent(text(await store.upload("payload"))); + const originalCase = event.getEventtypeCase(); + await processor.resolve(new pb.HistoryChunk().setEventsList([event])); + expect(JSON.stringify(event.toObject())).toContain("payload"); + const response = new pb.OrchestratorResponse().setActionsList([ + new pb.OrchestratorAction().setRewindorchestration(new pb.RewindOrchestrationAction().setNewhistoryList([event])), + ]); + await processor.externalize(response); + expect([...store.values.values()]).toEqual(["payload", "payload"]); + expect(event.getEventtypeCase()).toBe(originalCase); + }); + + it("converts permanent activity upload failures without sending the result", async () => { + store.upload = async () => { + throw Object.assign(new Error("denied"), { statusCode: 403 }); + }; + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + const response = new pb.ActivityResponse().setResult(text("large")).setCompletiontoken("completion"); + await processor.externalize(response); + expect(response.hasResult()).toBe(false); + expect(response.getFailuredetails()!.getIsnonretriable()).toBe(true); + expect(response.getCompletiontoken()).toBe("completion"); + }); + + it("clears partial orchestration responses before a non-retriable failure", async () => { + processor = new PayloadProcessor({ store, thresholdBytes: 1, maxPayloadBytes: 2 }); + const response = new pb.OrchestratorResponse() + .setCustomstatus(text("large")) + .setIspartial(true) + .setChunkindex(new Int32Value().setValue(2)) + .setActionsList([new pb.OrchestratorAction().setScheduletask(new pb.ScheduleTaskAction().setInput(text("x")))]); + await processor.externalize(response); + expect(response.hasCustomstatus()).toBe(false); + expect(response.getIspartial()).toBe(false); + expect(response.hasChunkindex()).toBe(false); + expect(response.getActionsList()).toHaveLength(1); + const complete = response.getActionsList()[0].getCompleteorchestration()!; + expect(complete.getOrchestrationstatus()).toBe(pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED); + expect(complete.getFailuredetails()!.getIsnonretriable()).toBe(true); + }); + + it.each([408, 429, 500, 503])("propagates transient upload failure %s", async (statusCode) => { + const error = Object.assign(new Error("transient"), { statusCode }); + store.upload = async () => { + throw error; + }; + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + await expect(processor.externalize(new pb.ActivityResponse().setResult(text("large")))).rejects.toBe(error); + }); +}); + +describe("bounded payload I/O", () => { + it("caps concurrency at eight", async () => { + let active = 0; + let maximum = 0; + await runBounded( + Array.from({ length: 40 }, () => async () => { + maximum = Math.max(maximum, ++active); + await new Promise((resolve) => setImmediate(resolve)); + active--; + }), + ); + expect(maximum).toBe(8); + expect(active).toBe(0); + }); + + it("stops dispatch, drains every started operation and throws the lowest-ordinal failure", async () => { + const first = new Error("first in message order"); + const later = new Error("first to finish"); + const started: number[] = []; + const completed: number[] = []; + await expect( + runBounded( + Array.from({ length: 20 }, (_, index) => async () => { + started.push(index); + if (index === 1) throw later; + await new Promise((resolve) => setImmediate(resolve)); + completed.push(index); + if (index === 0) throw first; + }), + ), + ).rejects.toBe(first); + expect(started).toHaveLength(2); + expect(completed).toEqual([0]); + }); + + it("drains on cancellation and does not observe cancellation for no-op messages", async () => { + const abort = new AbortController(); + let completed = 0; + await expect( + runBounded( + Array.from({ length: 20 }, () => async () => { + abort.abort(); + await new Promise((resolve) => setImmediate(resolve)); + completed++; + }), + abort.signal, + ), + ).rejects.toBe(abort.signal.reason); + expect(completed).toBe(1); + await expect(runBounded([], abort.signal)).resolves.toBeUndefined(); + }); +}); diff --git a/scripts/test-payload-package-compatibility.js b/scripts/test-payload-package-compatibility.js new file mode 100644 index 00000000..1b52aced --- /dev/null +++ b/scripts/test-payload-package-compatibility.js @@ -0,0 +1,161 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { pathToFileURL } = require("node:url"); +const { spawnSync } = require("node:child_process"); +const ts = require("typescript"); + +const root = path.resolve(__dirname, ".."); +const [oldCoreArgument, ...extraArguments] = process.argv.slice(2); +if (!oldCoreArgument || extraArguments.length) { + throw new Error("Usage: npm run test:payload-packages -- "); +} +const oldCore = path.resolve(oldCoreArgument); +assert.ok(fs.statSync(oldCore).isFile(), "The published core archive must be a file."); +const npmCli = process.env.npm_execpath; +assert.ok(npmCli, "Run this check through npm run test:payload-packages."); +const temporary = fs.mkdtempSync(path.join(os.tmpdir(), "durabletask-payload-packages-")); + +function npm(args, cwd) { + const result = spawnSync(process.execPath, [npmCli, ...args], { + cwd, + encoding: "utf8", + timeout: 120000, + maxBuffer: 2 * 1024 * 1024, + }); + if (result.error) throw result.error; + return result; +} + +function pack(name) { + const result = npm( + ["pack", "--json", "--ignore-scripts", "--workspace", name, "--pack-destination", temporary], + root, + ); + assert.equal(result.status, 0, result.stderr || result.stdout); + const [metadata] = JSON.parse(result.stdout); + return path.join(temporary, metadata.filename); +} + +function install(name, dependencies, devDependencies = {}) { + const directory = path.join(temporary, name); + fs.mkdirSync(directory); + fs.writeFileSync( + path.join(directory, "package.json"), + JSON.stringify({ + name, + version: "1.0.0", + private: true, + dependencies: Object.fromEntries( + Object.entries(dependencies).map(([dependency, archive]) => [dependency, pathToFileURL(archive).href]), + ), + devDependencies, + }), + ); + const result = npm( + ["install", "--json", "--ignore-scripts", "--strict-peer-deps", "--no-audit", "--no-fund"], + directory, + ); + return { ...result, directory }; +} + +try { + const coreName = "@microsoft/durabletask-js"; + const blobName = "@microsoft/durabletask-js-azureblob"; + const managedName = "@microsoft/durabletask-js-azuremanaged"; + const core = pack(coreName); + const blob = pack(blobName); + const managed = pack(managedName); + for (const [index, name, archive] of [ + [0, blobName, blob], + [1, managedName, managed], + ]) { + const result = install(`reject-published-core-${index}`, { [coreName]: oldCore, [name]: archive }); + assert.notEqual(result.status, 0, `${name} accepted published core 0.4.0, which lacks PayloadStore.`); + const output = `${result.stdout}\n${result.stderr}`; + assert.match(output, /ERESOLVE/, `Expected a peer conflict, not an unrelated install failure:\n${output}`); + assert.match(output, /@microsoft\/durabletask-js@0\.4\.0/, output); + console.log(`${name}: npm rejected published core 0.4.0 with ERESOLVE.`); + } + const result = install( + "accept-payload-capable-core", + { [coreName]: core, [blobName]: blob, [managedName]: managed }, + { + "@types/node": require("@types/node/package.json").version, + "@types/google-protobuf": require("@types/google-protobuf/package.json").version, + "@opentelemetry/api": require(path.join(root, "package.json")).devDependencies["@opentelemetry/api"], + "@azure/storage-blob": require("@azure/storage-blob/package.json").version, + }, + ); + assert.equal(result.status, 0, result.stderr || result.stdout); + const consumer = path.join(result.directory, "consumer.ts"); + fs.writeFileSync( + consumer, + ` + import { PayloadStorageException, PayloadStore } from "@microsoft/durabletask-js"; + import { BlobPayloadStore } from "@microsoft/durabletask-js-azureblob"; + import { DurableTaskAzureManagedClientBuilder } from "@microsoft/durabletask-js-azuremanaged"; + const store: PayloadStore = new BlobPayloadStore({ connectionString: "UseDevelopmentStorage=true" }); + new DurableTaskAzureManagedClientBuilder().useExternalizedPayloads({ store }); + new PayloadStorageException("failure", { cause: new Error("cause") }); + `, + ); + const program = ts.createProgram([consumer], { + target: ts.ScriptTarget.ES2020, + lib: ["lib.es2020.d.ts"], + module: ts.ModuleKind.CommonJS, + moduleResolution: ts.ModuleResolutionKind.Node10, + typeRoots: [path.join(result.directory, "node_modules", "@types")], + types: ["node"], + strict: true, + skipLibCheck: false, + esModuleInterop: true, + noEmit: true, + }); + assert.ok( + !program + .getTypeChecker() + .getSymbolsInScope(program.getSourceFile(consumer), ts.SymbolFlags.Type) + .some((symbol) => symbol.name === "ErrorOptions"), + "The ES2020 check must not inherit the global ES2022 ErrorOptions type.", + ); + const diagnostics = ts.getPreEmitDiagnostics(program); + assert.equal( + diagnostics.length, + 0, + ts.formatDiagnosticsWithColorAndContext(diagnostics, { + getCanonicalFileName: (name) => name, + getCurrentDirectory: () => result.directory, + getNewLine: () => "\n", + }), + ); + console.log("Packed public declarations type-check against ES2020 with skipLibCheck=false."); + const runtime = spawnSync( + process.execPath, + [ + "-e", + ` + const assert = require("node:assert/strict"); + const core = require("@microsoft/durabletask-js"); + const { BlobPayloadStore } = require("@microsoft/durabletask-js-azureblob"); + const { DurableTaskAzureManagedClientBuilder } = require("@microsoft/durabletask-js-azuremanaged"); + assert.equal(typeof core.PayloadStore, "function"); + const cause = new Error("cause"); + assert.equal(new core.PayloadStorageException("failure", { cause }).cause, cause); + assert.equal(typeof core.TaskHubGrpcClient.prototype.setLargePayloadAutoPurge, "function"); + assert.ok(new BlobPayloadStore({ connectionString: "UseDevelopmentStorage=true" }) instanceof core.PayloadStore); + assert.equal(typeof DurableTaskAzureManagedClientBuilder.prototype.useExternalizedPayloads, "function"); + `, + ], + { cwd: result.directory, encoding: "utf8", timeout: 30000 }, + ); + if (runtime.error) throw runtime.error; + assert.equal(runtime.status, 0, runtime.stderr || runtime.stdout); + console.log("Packed core and extensions install successfully; public APIs load without storage or DTS I/O."); +} finally { + fs.rmSync(temporary, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }); +} diff --git a/tools/generate-grpc-javascript.sh b/tools/generate-grpc-javascript.sh index 4f7979c3..fb488839 100644 --- a/tools/generate-grpc-javascript.sh +++ b/tools/generate-grpc-javascript.sh @@ -27,7 +27,11 @@ generateGrpc() { --js_out="import_style=commonjs,binary:$PATH_PROTO_OUT" \ --ts_out="grpc_js:$PATH_PROTO_OUT" \ --grpc_out="grpc_js:$PATH_PROTO_OUT" \ - ${PROTO_FILE_PATH_ABSOLUTE} + ${PROTO_FILE_PATH_ABSOLUTE} || return $? + + # The TS plugin emits trailing spaces on class declarations. + node -e 'const fs = require("fs"); const file = process.argv[1]; fs.writeFileSync(file, fs.readFileSync(file, "utf8").replace(/[ \t]+$/gm, ""));' \ + "${PATH_PROTO_OUT}/${PROTO_FILE_NAME_NO_PROTO}_pb.d.ts" || return $? echo "- Generated '${PATH_PROTO_OUT}/${PROTO_FILE_NAME_NO_PROTO}_pb.js'" echo "- Generated '${PATH_PROTO_OUT}/${PROTO_FILE_NAME_NO_PROTO}_pb.d.ts'" @@ -39,7 +43,7 @@ fail_trap() { result=$? if [ $result != 0 ]; then - echo "Failed to generate gRPC interface and proto buf: $ret_val" + echo "Failed to generate gRPC interface and proto buf: $result" fi exit $result @@ -79,4 +83,5 @@ fi echo "==============================================" echo "Generating Javascript code in $PATH_PROTO_OUT" -generateGrpc "$PATH_PROTO_SRC/orchestrator_service.proto" \ No newline at end of file +generateGrpc "$PATH_PROTO_SRC/orchestrator_service.proto" || exit $? +generateGrpc "$PATH_PROTO_SRC/large_payload_purge.proto" || exit $? \ No newline at end of file diff --git a/tsconfig.base.json b/tsconfig.base.json index 1dfb5658..e366f1d3 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1,20 +1,21 @@ -{ - "compilerOptions": { - "target": "ES2022", - "lib": ["ES2022"], - "module": "commonjs", - "declaration": true, - "strict": true, - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "skipLibCheck": true, - "resolveJsonModule": true, - "moduleResolution": "node", - "baseUrl": ".", - "paths": { - "@microsoft/durabletask-js": ["packages/durabletask-js/src/index.ts"], - "@microsoft/durabletask-js-azuremanaged": ["packages/durabletask-js-azuremanaged/src/index.ts"], - "@microsoft/durabletask-js-export-history": ["packages/durabletask-js-export-history/src/index.ts"] - } - } -} +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022"], + "module": "commonjs", + "declaration": true, + "strict": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true, + "resolveJsonModule": true, + "moduleResolution": "node", + "baseUrl": ".", + "paths": { + "@microsoft/durabletask-js": ["packages/durabletask-js/src/index.ts"], + "@microsoft/durabletask-js-azuremanaged": ["packages/durabletask-js-azuremanaged/src/index.ts"], + "@microsoft/durabletask-js-azureblob": ["packages/durabletask-js-azureblob/src/index.ts"], + "@microsoft/durabletask-js-export-history": ["packages/durabletask-js-export-history/src/index.ts"] + } + } +}