diff --git a/.github/workflows/prepare-release.yaml b/.github/workflows/prepare-release.yaml index 2df2802f..f18964db 100644 --- a/.github/workflows/prepare-release.yaml +++ b/.github/workflows/prepare-release.yaml @@ -4,7 +4,7 @@ # 1. Bumps the selected package's version in its own package.json # 2. Generates a changelog from git history since the package's last release tag # 3. Updates the package's changelog file and creates a release branch + package-scoped tag -# Packages are versioned and tagged independently (tag prefixes: v, azuremanaged-v, durable-functions-v). +# Packages are versioned and tagged independently (tag prefixes: v, azureblob-v, azuremanaged-v, durable-functions-v). # After the workflow completes, manually open a PR from the release branch to main, then publish # from the package directory using the npm command shown in the run summary (mind the dist-tag). @@ -19,6 +19,7 @@ on: type: choice options: - durabletask-js + - durabletask-js-azureblob - durabletask-js-azuremanaged - azure-functions-durable version: @@ -60,6 +61,12 @@ jobs: TAG_PREFIX="azuremanaged-v" CHANGELOG_PATH="packages/durabletask-js-azuremanaged/CHANGELOG.md" ;; + durabletask-js-azureblob) + PKG_DIR="packages/durabletask-js-azureblob" + NPM_NAME="@microsoft/durabletask-js-azureblob" + TAG_PREFIX="azureblob-v" + CHANGELOG_PATH="packages/durabletask-js-azureblob/CHANGELOG.md" + ;; azure-functions-durable) PKG_DIR="packages/azure-functions-durable" NPM_NAME="durable-functions" @@ -67,7 +74,7 @@ jobs: CHANGELOG_PATH="packages/azure-functions-durable/CHANGELOG.md" ;; *) - echo "::error::Unknown package '$PACKAGE'. Expected one of: durabletask-js, durabletask-js-azuremanaged, azure-functions-durable." + echo "::error::Unknown package '$PACKAGE'. Expected one of: durabletask-js, durabletask-js-azureblob, durabletask-js-azuremanaged, azure-functions-durable." exit 1 ;; esac diff --git a/CHANGELOG.md b/CHANGELOG.md index c4f417fa..672ecf24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,16 +2,12 @@ ### New -- Add reserved cleanup worker tasks with bounded deletion chunks, retry/quarantine handling, - configuration-preserving continue-as-new, authenticated current-channel RPCs and filter/version isolation. - This completes the cleanup runtime; aggregate release/consumer validation follows in phase 6. -- Add the DTS payload-purge protocol and `setLargePayloadAutoPurge()` client lifecycle, - including setting-first enable/disable and reserved-instance identity verification. - A compatible .NET worker or the later cleanup-worker phase is required; do not publish partial phases. -- Add payload storage contracts and bounded protobuf payload processing, with inclusive UTF-8 - thresholds, explicit storage/size failures, and non-retriable worker upload-failure conversion. - This is phase 1 of the external-payload feature; transport, providers, and cleanup follow separately. - Reserve core `0.4.1` for the new APIs; do not publish partial feature phases. +- Add opt-in `payloads` storage to gRPC clients/workers, with transparent UTF-8-size-based + externalization and awaited query/history/work-item hydration across orchestration, activity, + and entity V1/V2 payloads. Add the optional `@microsoft/durabletask-js-azureblob` storage package. +- Add .NET-aligned `setLargePayloadAutoPurge()`, reserved replay-safe cleanup tasks, and DTS + lifecycle transport. Storage configuration alone does not enable deletion; cleanup checks + blob ownership and ETags, preserves uncertain outcomes, and leaves legacy v1 blobs untouched. - Expose readonly `OrchestrationContext.sourceInstanceId` from the existing protobuf request properties on initial execution and replay, independently of parent identity. Strings are preserved verbatim; null or missing metadata returns `undefined`. @@ -53,8 +49,10 @@ ### Fixes -- Determine single-instance purge routing from the instance ID regardless of payload configuration, - matching .NET management clients: ordinary IDs are orchestrations and `@name@key` IDs are entities. +- Determine single-instance purge routing from ordinary/entity IDs regardless of payload storage, + matching .NET management clients without changing purge timeout/completion semantics. +- Keep `PayloadStorageException` cause options structural so its public declarations remain + usable by ES2020 consumers without the global ES2022 `ErrorOptions` type. - Apply payload externalization and UTF-8 size limits to native continue-as-new carryover history. Event metadata, ordering, and hydrated payload values are preserved. - Validate every eligible payload's UTF-8 size before starting uploads, preventing storage writes diff --git a/README.md b/README.md index 9cd5cd75..2b758f38 100644 --- a/README.md +++ b/README.md @@ -23,37 +23,22 @@ const entityResponseBytes = await worker.processEntityBatchRequest(entityBatchRe `TaskHubGrpcClient` already exposes orchestration start/query/event/terminate/suspend/resume/purge APIs and entity signal/read/query/clean APIs through its existing `hostAddress` and `metadataGenerator` options. Host integrations that need task-hub routing metadata should provide it through `metadataGenerator`, keeping host-specific metadata policy outside the core client. Azure-managed scheduler connection strings remain in `@microsoft/durabletask-js-azuremanaged`. -## External payload transport - -Core clients and workers accept opt-in `payloads: { store }` options, where `store` implements -the exported `PayloadStore` contract. The same store must be configured on every participant. -The default threshold is 256 KiB of serialized UTF-8 bytes (inclusive), with a 10 MiB maximum. -Requests are externalized before sending; queries, history chunks and work items are hydrated -before conversion or execution. Empty payloads remain inline, and disabling the option preserves -the existing transport behavior. - -Azure-managed builders expose `.useExternalizedPayloads({ store })` and require core `>=0.4.1` -for the public payload types. A custom/in-memory store or the optional Azure Blob provider can -implement storage independently of the transport. Complete the six-phase feature stack before release. - -### Payload cleanup client protocol - -`client.setLargePayloadAutoPurge(enabled, batchSize?, signal?)` manages the authenticated DTS -task-hub cleanup setting. Enabling writes the setting first, starts/deduplicates and verifies -the reserved Running orchestration, then signals its batch size. Disabling only writes the -backend setting. These steps are not transactional; earlier successful steps are not rolled back. - -This client operation does not require blob credentials. It requires a backend implementing -the DTS `LargePayloadPurge` service and a compatible worker. Payload-enabled JavaScript workers -now register the reserved cleanup orchestrator and activities automatically, preserve their -wildcard filters and bypass business version policy without changing customer filters. -Deletion verifies ownership and ETags, batches 50 tokens per activity and limits concurrency; -the runner backs off empty/retry-only batches and preserves configuration through continue-as-new. -Do not publish the feature until all six phases are complete. - -Single-instance purges now identify the instance kind even on management-only clients without -payload storage: ordinary IDs set `isOrchestration=true`, while `@name@key` entity IDs set it false. -The existing timeout and completion-status behavior is unchanged. +## Large payloads + +Clients and workers can opt into transparent external payload storage with the `payloads` +option. The optional [`@microsoft/durabletask-js-azureblob`](./packages/durabletask-js-azureblob) +package implements .NET-compatible Azure Blob tokens and UTF-8/gzip storage. The default +externalization threshold is 256 KiB (inclusive); the maximum payload size is 10 MiB. +Core users do not acquire Azure Storage dependencies; payload transport remains opt-in. + +Configure all participating clients and workers before storing externalized payloads. +Azure-managed builders expose `.useExternalizedPayloads({ store })`. Cleanup is separately +opt-in through `client.setLargePayloadAutoPurge(true)` and requires a supporting DTS backend. +Single-instance purges now route ordinary/entity IDs correctly even without payload storage. +See the package's [configuration, interoperability, and cleanup contract](./packages/durabletask-js-azureblob/README.md). + +This feature is reviewed as a six-phase dependency stack. Merge all phases before preparing +a release; publish the completed core `0.4.1` before the dependent packages. ## Long durable timers diff --git a/doc/release_process.md b/doc/release_process.md index 4bccad11..5f3985b8 100644 --- a/doc/release_process.md +++ b/doc/release_process.md @@ -1,12 +1,13 @@ # Release Process -This document describes how to cut a new release of the three npm packages published from this monorepo: `@microsoft/durabletask-js` (core), `@microsoft/durabletask-js-azuremanaged`, and `durable-functions` (the Azure Functions v4 compatibility provider, whose source lives in `packages/azure-functions-durable`). +This document describes how to cut a new release of the core SDK, its Azure Blob and Azure-managed extensions, and `durable-functions` (the Azure Functions v4 compatibility provider, whose source lives in `packages/azure-functions-durable`). ## Overview | Package (npm name) | Directory | Changelog | npm | |---|---|---|---| | `@microsoft/durabletask-js` (core) | `packages/durabletask-js` | `CHANGELOG.md` | [npmjs](https://www.npmjs.com/package/@microsoft/durabletask-js) | +| `@microsoft/durabletask-js-azureblob` | `packages/durabletask-js-azureblob` | `packages/durabletask-js-azureblob/CHANGELOG.md` | [npmjs](https://www.npmjs.com/package/@microsoft/durabletask-js-azureblob) | | `@microsoft/durabletask-js-azuremanaged` | `packages/durabletask-js-azuremanaged` | `packages/durabletask-js-azuremanaged/CHANGELOG.md` | [npmjs](https://www.npmjs.com/package/@microsoft/durabletask-js-azuremanaged) | | `durable-functions` | `packages/azure-functions-durable` | `packages/azure-functions-durable/CHANGELOG.md` | [npmjs](https://www.npmjs.com/package/durable-functions) | @@ -16,10 +17,12 @@ This is an **npm workspaces** monorepo. The official build (`eng/ci/official-bui **Each package is versioned, changelogged, and tagged independently.** A release covers exactly one package; its version number, its changelog, and its git tag advance on their own and do **not** have to match the other packages. -There is one hard ordering constraint between packages: +Core must be published before a dependent package requiring that version: -- **`@microsoft/durabletask-js` (core) must be published before `durable-functions`.** The compat package declares an **exact** dependency on core (currently `"@microsoft/durabletask-js": "0.4.0"`). If `durable-functions` is published while that exact core version is not yet on npm, the published compat package is **uninstallable**. -- **`@microsoft/durabletask-js-azuremanaged` is unordered relative to `durable-functions`.** It depends on core only through a peer **floor** (`"@microsoft/durabletask-js": ">=0.3.0"`), which is already satisfied by the published core, so it can be released before or after the other packages. +- **`@microsoft/durabletask-js` (core) must be published before `durable-functions`.** The compat package declares an **exact** dependency on core (currently `"@microsoft/durabletask-js": "0.4.1"`). If `durable-functions` is published while that exact core version is not yet on npm, the published compat package is **uninstallable**. +- **Azure Blob and Azure-managed now require core `>=0.4.1`.** Core `0.4.1` is the first source version containing `PayloadStore` and the payload transport APIs; the already-published `0.4.0` cannot satisfy this floor. Publish core before either extension. The extensions remain unordered relative to each other and `durable-functions`, and their own version numbers remain independent. + +Merge the complete six-phase large-payload review stack before releasing any of these changes. Partial phases are review boundaries, not releases. For the first payload-capable core release, explicitly select `version: 0.4.1` in **Prepare Release**. The source version is already reserved as `0.4.1`; leaving the version input empty would auto-increment it. No package is published by the feature PRs. ## Versioning Scheme @@ -50,7 +53,7 @@ The changelog step lists commits since the released package's **last package-sco 1. Go to **Actions** → **Prepare Release** in GitHub 2. Click **Run workflow** -3. **Select the `package` to release** (required) — one of `durabletask-js`, `durabletask-js-azuremanaged`, or `azure-functions-durable`. Each run releases exactly one package. +3. **Select the `package` to release** (required) — one of `durabletask-js`, `durabletask-js-azureblob`, `durabletask-js-azuremanaged`, or `azure-functions-durable`. Each run releases exactly one package. 4. Optionally specify a version (e.g., `0.2.0-beta.1` for core, or `4.0.0-beta.1` for `durable-functions`). Leave empty to auto-increment the selected package's current version (bumps the prerelease number for a prerelease, otherwise the patch). 5. Click **Run workflow** @@ -61,8 +64,8 @@ For the **one package you selected** (and only that package): 1. **Determines the next version**: uses the `version` input, or auto-increments the selected package's current version 2. **Generates a changelog**: lists commits since that package's last release tag, scoped to the package's directory (`git log ..HEAD -- `), so only commits that touched that package are included 3. **Bumps the version**: updates `version` in that package's own `package.json` -4. **Updates that package's changelog**: core writes `CHANGELOG.md`, Azure Managed writes `packages/durabletask-js-azuremanaged/CHANGELOG.md`, and `durable-functions` writes `packages/azure-functions-durable/CHANGELOG.md` -5. **Creates a release branch and a package-scoped tag**: tag `` and branch `release/`, where the prefix is `v` (core), `azuremanaged-v`, or `durable-functions-v` +4. **Updates that package's changelog**: core writes `CHANGELOG.md`; Azure Blob, Azure Managed, and `durable-functions` write their own package's `CHANGELOG.md` +5. **Creates a release branch and a package-scoped tag**: tag `` and branch `release/`, where the prefix is `v` (core), `azureblob-v`, `azuremanaged-v`, or `durable-functions-v` 6. **For `durable-functions` only**: verifies the exact-pinned `@microsoft/durabletask-js` version is already published on public npm, and fails the run if it is not (guards the uninstallable-dependency case) ### After the Workflow Completes @@ -98,8 +101,9 @@ Trigger the official build pipeline to produce signed `.tgz` artifacts. The offi 1. Click **Run pipeline** on `main` (or locate the `main` run that already contains the release commit) 2. Wait for it to complete -3. A single `main` build packs **all three** packages into one `drop` artifact; which single package publishes is decided later by the `package` parameter in the release pipeline (Step 3). Verify `drop` contains the correctly versioned `.tgz` files: +3. A single `main` build packs **all four** packages into one `drop` artifact; which single package publishes is decided later by the `package` parameter in the release pipeline (Step 3). Verify `drop` contains the correctly versioned `.tgz` files: - `buildoutputs/durabletask-js/microsoft-durabletask-js-X.Y.Z.tgz` + - `buildoutputs/durabletask-js-azureblob/microsoft-durabletask-js-azureblob-X.Y.Z.tgz` - `buildoutputs/durabletask-js-azuremanaged/microsoft-durabletask-js-azuremanaged-X.Y.Z.tgz` - `buildoutputs/azure-functions-durable/durable-functions-X.Y.Z.tgz` @@ -110,6 +114,7 @@ Never publish from the release branch — it exists only to carry the version/ch Trigger the release pipeline to publish one signed package to npm via ESRP. **This is the sanctioned publish path.** `eng/ci/release.yml` consumes the `durabletask-js.official` build artifact from **`main`** (its pipeline resource is pinned to `branch: main`), so its source is the `main` official build selected in Step 2 — not the release branch. The pipeline's `package` runtime parameter controls which one of these release stages is inserted into the compiled 1ES plan. It defaults to `durabletask-js`, so verify or change it before queueing: a wrong or unchanged default compiles the wrong package stage. ESRP approval remains the final safety gate before publishing. - `durabletask-js` inserts `release_durabletask_js` for core `@microsoft/durabletask-js`. +- `durabletask-js-azureblob` inserts `release_durabletask_js_azureblob` for `@microsoft/durabletask-js-azureblob`. - `durabletask-js-azuremanaged` inserts `release_durabletask_js_azuremanaged` for `@microsoft/durabletask-js-azuremanaged`. - `azure-functions-durable` inserts `release_durable_functions` for `durable-functions`. @@ -122,7 +127,7 @@ Trigger the release pipeline to publish one signed package to npm via ESRP. **Th This compile-time selection eliminates the failure demonstrated by run **294746**: de-selecting the core stage at queue time did not expose its result to the Azure Managed dependency condition as the expected literal `Skipped`, so the selected package's stage was skipped too. The selected package now has no cross-stage dependency because the other package stages do not exist in that run's compiled plan. -**Two runs for core + compat.** The compat **Prepare Release** run guards on core already being published to **public npm** — it runs `npm view @microsoft/durabletask-js@ --registry https://registry.npmjs.org/` and fails if that exact version is absent (`durable-functions` exact-pins core). Publish and verify core first; only then prepare `durable-functions` and publish it in a later release-pipeline run. (`azuremanaged` depends on core only through a peer floor that is already satisfied and has no ordering constraint.) +**Separate runs for core and dependents.** The compat **Prepare Release** run guards on core already being published to **public npm** — it runs `npm view @microsoft/durabletask-js@ --registry https://registry.npmjs.org/` and fails if that exact version is absent (`durable-functions` exact-pins core). Publish and verify core first; only then publish the payload extensions and prepare/publish `durable-functions` in later release-pipeline runs. > **Open question (B11) — hard blocker for ESRP prereleases:** it is not yet confirmed whether the ESRP release task can set the npm **dist-tag** (e.g. publish a prerelease under `preview` rather than moving `latest`). Publishing a prerelease such as `durable-functions@4.0.0-beta.1` through ESRP is **blocked** until the ESRP / 1ES pipeline owners confirm how to apply the `preview` tag — do **not** assume ESRP applies a dist-tag. The `--tag` guidance in *Quick Reference: npm Dist Tags* applies to a manual `npm publish`. @@ -130,6 +135,7 @@ This compile-time selection eliminates the failure demonstrated by run **294746* ```bash npm view @microsoft/durabletask-js versions +npm view @microsoft/durabletask-js-azureblob versions npm view @microsoft/durabletask-js-azuremanaged versions npm view durable-functions versions @@ -160,13 +166,14 @@ Bump the `"version"` field in **only the package you are releasing**: ```bash # core: packages/durabletask-js/package.json → "version": "X.Y.Z" +# azureblob: packages/durabletask-js-azureblob/package.json → "version": "X.Y.Z" # azuremanaged: packages/durabletask-js-azuremanaged/package.json → "version": "X.Y.Z" # compat: packages/azure-functions-durable/package.json → "version": "X.Y.Z" ``` Then update the affected cross-package dependency **for the package you are releasing**: -- **`@microsoft/durabletask-js-azuremanaged`** declares a peer **floor** on core. Only raise it if this release actually requires a newer core — it does not have to equal the release version: +- **`@microsoft/durabletask-js-azureblob` and `@microsoft/durabletask-js-azuremanaged`** declare a peer **floor** on core. Raise it when the package requires a newer core API — it does not have to equal the extension's own release version. The payload API floor is `>=0.4.1`; never lower it to the incompatible published `0.4.0`: ```jsonc "peerDependencies": { @@ -189,6 +196,7 @@ Move items from the `## Upcoming` section of the package's changelog into a new | Package | Changelog | |---|---| | `@microsoft/durabletask-js` | `CHANGELOG.md` | +| `@microsoft/durabletask-js-azureblob` | `packages/durabletask-js-azureblob/CHANGELOG.md` | | `@microsoft/durabletask-js-azuremanaged` | `packages/durabletask-js-azuremanaged/CHANGELOG.md` | | `durable-functions` | `packages/azure-functions-durable/CHANGELOG.md` | diff --git a/eng/ci/release.yml b/eng/ci/release.yml index 206d4c49..9270a82c 100644 --- a/eng/ci/release.yml +++ b/eng/ci/release.yml @@ -8,6 +8,7 @@ parameters: default: durabletask-js values: - durabletask-js + - durabletask-js-azureblob - durabletask-js-azuremanaged - azure-functions-durable @@ -101,6 +102,41 @@ extends: mainpublisher: 'durabletask-java' domaintenantid: '33e01921-4d64-4f8c-a055-5bdaffd5e33d' + - ${{ if eq(parameters.package, 'durabletask-js-azureblob') }}: + - stage: release_durabletask_js_azureblob + displayName: 'Release @microsoft/durabletask-js-azureblob' + jobs: + - job: durabletask_js_azureblob + displayName: 'Release @microsoft/durabletask-js-azureblob' + templateContext: + type: releaseJob + isProduction: true + inputs: + - input: pipelineArtifact + pipeline: DurableTaskJSBuildPipeline + artifactName: drop + targetPath: $(System.DefaultWorkingDirectory)/drop + + steps: + - task: SFP.release-tasks.custom-build-release-task.EsrpRelease@9 + displayName: 'ESRP Release @microsoft/durabletask-js-azureblob' + inputs: + connectedservicename: 'dtfx-internal-esrp-prod' + usemanagedidentity: true + keyvaultname: 'durable-esrp-akv' + signcertname: 'dts-esrp-cert' + clientid: '0b3ed1a4-0727-4a50-b82a-02c2bd9dec89' + intent: 'PackageDistribution' + contenttype: 'npm' + contentsource: 'Folder' + folderlocation: '$(System.DefaultWorkingDirectory)/drop/buildoutputs/durabletask-js-azureblob' + waitforreleasecompletion: true + owners: 'wangbill@microsoft.com' + approvers: 'kaibocai@microsoft.com' + serviceendpointurl: 'https://api.esrp.microsoft.com' + mainpublisher: 'durabletask-java' + domaintenantid: '33e01921-4d64-4f8c-a055-5bdaffd5e33d' + - ${{ if eq(parameters.package, 'azure-functions-durable') }}: - stage: release_durable_functions displayName: 'Release durable-functions' diff --git a/eng/templates/build.yml b/eng/templates/build.yml index 06c1ab26..a6110554 100644 --- a/eng/templates/build.yml +++ b/eng/templates/build.yml @@ -48,6 +48,12 @@ jobs: npm pack displayName: "pack @microsoft/durabletask-js" + # Pack @microsoft/durabletask-js-azureblob + - script: | + cd packages/durabletask-js-azureblob + npm pack + displayName: "pack @microsoft/durabletask-js-azureblob" + # Pack @microsoft/durabletask-js-azuremanaged - script: | cd packages/durabletask-js-azuremanaged @@ -73,6 +79,12 @@ jobs: SourceFolder: $(System.DefaultWorkingDirectory)/packages/durabletask-js-azuremanaged Contents: "*.tgz" TargetFolder: $(Build.ArtifactStagingDirectory)/buildoutputs/durabletask-js-azuremanaged + - task: CopyFiles@2 + displayName: "Copy durabletask-js-azureblob to staging" + inputs: + SourceFolder: $(System.DefaultWorkingDirectory)/packages/durabletask-js-azureblob + Contents: "*.tgz" + TargetFolder: $(Build.ArtifactStagingDirectory)/buildoutputs/durabletask-js-azureblob - task: CopyFiles@2 displayName: "Copy azure-functions-durable to staging" inputs: diff --git a/examples/azure-managed/README.md b/examples/azure-managed/README.md index 28cfedc8..db2456e7 100644 --- a/examples/azure-managed/README.md +++ b/examples/azure-managed/README.md @@ -11,6 +11,7 @@ Runnable samples demonstrating every major feature of the Durable Task JavaScrip | [human-interaction](human-interaction/) | Event-driven workflows | External events, timers, `whenAny` race, `sendEvent`, custom status | Yes | | [lifecycle-management](lifecycle-management/) | Orchestration control | Terminate (recursive), suspend/resume, restart, continue-as-new, purge, tags | Yes | | [query-and-history](query-and-history/) | Monitoring & debugging | `getAllInstances`, pagination, `listInstanceIds`, `getOrchestrationHistory`, typed events | Yes | +| [large-payloads](large-payloads/) | External payloads | Transparent UTF-8/gzip blob storage and large activity/orchestration inputs/results | DTS and Blob/Azurite | | [versioning](versioning/) | Safe deployments | Version match strategies, failure strategies, `ctx.version`, `ctx.compareVersionTo()` | Yes | | [unit-testing](unit-testing/) | Testing without infra | `InMemoryOrchestrationBackend`, `TestOrchestrationClient`, `TestOrchestrationWorker`, `ReplaySafeLogger` | **No** | | [basics](basics/) | Azure-managed basics | Connection strings, `DefaultAzureCredential`, `createAzureManagedClient` | Yes | diff --git a/examples/azure-managed/large-payloads/README.md b/examples/azure-managed/large-payloads/README.md new file mode 100644 index 00000000..2a24df36 --- /dev/null +++ b/examples/azure-managed/large-payloads/README.md @@ -0,0 +1,33 @@ +# Large payloads + +This sample round-trips a 300 KiB input through an orchestration and an activity, +verifying that the client receives the original output rather than a blob token. +The standard 256 KiB threshold is measured on serialized UTF-8 payloads. + +Build the SDK, configure a DTS endpoint and a blob endpoint, then run from the repository root: + +```sh +npm install +npm run build +npm run example -- ./examples/azure-managed/large-payloads/index.ts +``` + +Set these in `examples/azure-managed/.env` or your environment: + +```text +DURABLE_TASK_SCHEDULER_CONNECTION_STRING=Endpoint=http://localhost:8080;Authentication=None;TaskHub=default +AZURE_STORAGE_CONNECTION_STRING=UseDevelopmentStorage=true +``` + +For local execution, start DTS using the parent directory's Docker Compose file and run +Azurite separately on its default blob port 10000. The sample can instead use existing +Azure resources; no account or role is provisioned. Connection strings must never be committed. +For identity-based storage authentication, see the +[payload package documentation](../../../packages/durabletask-js-azureblob/README.md). + +The sample leaves its instance intact and does **not** modify the task hub's auto-purge setting. +To opt into cleanup separately, configure a supporting DTS backend and payload-enabled workers, +call `client.setLargePayloadAutoPurge(true)`, then purge terminal instances through the usual +client API. Keep a payload-enabled worker running to process tombstones. Disabling with +`client.setLargePayloadAutoPurge(false)` pauses backend fetches without terminating the runner. +Legacy v1 blobs, soft-deleted data, and retained blob versions are not immediately reclaimed. diff --git a/examples/azure-managed/large-payloads/index.ts b/examples/azure-managed/large-payloads/index.ts new file mode 100644 index 00000000..04e576d5 --- /dev/null +++ b/examples/azure-managed/large-payloads/index.ts @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as dotenv from "dotenv"; +import * as path from "path"; +import { ActivityContext, OrchestrationContext } from "@microsoft/durabletask-js"; +import { BlobPayloadStore } from "@microsoft/durabletask-js-azureblob"; +import { + DurableTaskAzureManagedClientBuilder, + createAzureManagedWorkerBuilder, +} from "@microsoft/durabletask-js-azuremanaged"; + +dotenv.config({ path: path.join(__dirname, "..", ".env") }); + +async function echo(_context: ActivityContext, input: string): Promise { + return input; +} + +async function* largePayload(context: OrchestrationContext, input: string): AsyncGenerator { + context.setCustomStatus({ phase: "echo", inputBytes: input.length }); + return yield context.callActivity(echo, input); +} + +async function main(): Promise { + const scheduler = process.env.DURABLE_TASK_SCHEDULER_CONNECTION_STRING; + const storage = process.env.AZURE_STORAGE_CONNECTION_STRING; + if (!scheduler || !storage) { + throw new Error("Set DURABLE_TASK_SCHEDULER_CONNECTION_STRING and AZURE_STORAGE_CONNECTION_STRING."); + } + const store = new BlobPayloadStore({ connectionString: storage }); + const payloads = { store }; + const worker = createAzureManagedWorkerBuilder(scheduler) + .useExternalizedPayloads(payloads) + .addOrchestrator(largePayload) + .addActivity(echo) + .build(); + const client = new DurableTaskAzureManagedClientBuilder() + .connectionString(scheduler) + .useExternalizedPayloads(payloads) + .build(); + let started = false; + try { + await worker.start(); + started = true; + const input = "x".repeat(300 * 1024); + const id = await client.scheduleNewOrchestration(largePayload, input); + const state = await client.waitForOrchestrationCompletion(id, true, 60); + if (!state) throw new Error("The orchestration was not found."); + state.raiseIfFailed(); + if (state.serializedOutput !== JSON.stringify(input)) + throw new Error("The hydrated output did not match the input."); + console.log(`Round-tripped ${Buffer.byteLength(input, "utf8")} bytes through orchestration and activity payloads.`); + console.log(`Instance ${id} retained. Auto-purge is a separate, task-hub-wide opt-in; see the sample README.`); + } finally { + if (started) await worker.stop(); + await client.stop(); + } +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/examples/azure-managed/large-payloads/sample.json b/examples/azure-managed/large-payloads/sample.json new file mode 100644 index 00000000..a5490f47 --- /dev/null +++ b/examples/azure-managed/large-payloads/sample.json @@ -0,0 +1,7 @@ +{ + "name": "large-payloads", + "description": "Transparent orchestration/activity payload externalization and .NET-compatible Azure Blob storage", + "requiresEmulator": true, + "skipCi": true, + "skipCiReason": "Requires an independently configured Azure Blob/Azurite endpoint in addition to DTS." +} diff --git a/package.json b/package.json index 28d397c7..194ba76f 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "build:azuremanaged": "npm run build -w @microsoft/durabletask-js-azuremanaged", "test": "npm run test --workspaces", "test:unit": "npm run test:unit --workspaces --if-present", + "test:payload-packages": "node scripts/test-payload-package-compatibility.js", "test:e2e": "./scripts/test-e2e.sh", "test:e2e:internal": "jest tests/e2e --runInBand --detectOpenHandles", "test:e2e:one": "jest tests/e2e --runInBand --detectOpenHandles --testNamePattern", diff --git a/packages/durabletask-js-azureblob/CHANGELOG.md b/packages/durabletask-js-azureblob/CHANGELOG.md index d9b00525..5706c52b 100644 --- a/packages/durabletask-js-azureblob/CHANGELOG.md +++ b/packages/durabletask-js-azureblob/CHANGELOG.md @@ -6,6 +6,5 @@ - Add opt-in Azure Blob payload storage, interoperable with durabletask-dotnet `91739687ff647699079cdadd627daa00562982b7`: UTF-8/gzip blobs, credential-free v2 tokens, legacy v1 reads, safe same-service and identity-only cross-account routing. -- Support explicit ownership-checked, ETag-conditional deletion, including snapshots and - confirmed-absence handling. Scheduler-driven cleanup follows in later phases; do not release - partial feature phases. +- Support ownership-checked, ETag-conditional deletion for DTS automatic payload cleanup, + including snapshots and confirmed-absence handling. diff --git a/packages/durabletask-js-azureblob/README.md b/packages/durabletask-js-azureblob/README.md index 3f7c0f83..caae47d1 100644 --- a/packages/durabletask-js-azureblob/README.md +++ b/packages/durabletask-js-azureblob/README.md @@ -2,13 +2,9 @@ `@microsoft/durabletask-js-azureblob` adds opt-in external payload storage to the Durable Task JavaScript SDK. Core users do not acquire Azure Storage dependencies. -Its tokens, UTF-8/gzip representation, and ownership metadata +Its tokens, UTF-8/gzip representation, ownership metadata, and auto-purge protocol match [durabletask-dotnet at `91739687`](https://github.com/microsoft/durabletask-dotnet/tree/91739687ff647699079cdadd627daa00562982b7/src/Extensions/AzureBlobPayloads). -This is phase 3 of the six-PR feature stack. The provider works with the preceding SDK -transport integration; scheduler-driven automatic cleanup is implemented in later phases. -Do not publish a partial feature phase. - ## Configure clients and workers Requires `@microsoft/durabletask-js >=0.4.1`, the first core source version with @@ -45,7 +41,7 @@ const store = new BlobPayloadStore({ ``` The identity needs permission to create the private container and read/write its blobs, -and delete permission when invoking the store's deletion API. The SDK does not provision accounts or roles. +and delete permission when auto-purge is enabled. The SDK does not provision accounts or roles. Azure-managed scheduler builders expose `.useExternalizedPayloads(payloads)` on both `DurableTaskAzureManagedClientBuilder` and `DurableTaskAzureManagedWorkerBuilder`. Scheduler credentials and blob credentials are independent. @@ -65,8 +61,9 @@ This feature targets the Durable Task Scheduler; it does not reconfigure a Durab | `compressionEnabled` | `true` | Gzip payload bytes; record `Content-Encoding: gzip`. | Sizes refer to serialized values, including JSON quotes/escapes, not JavaScript string length -or compressed size. Absent and empty payloads remain inline. With the feature disabled, -existing transport and payload behavior are unchanged. +or compressed size. Absent and empty payloads remain inline. Payload transport is opt-in. +Single-instance purge routing is corrected independently of payload configuration: ordinary +IDs identify orchestrations, and `@name@key` IDs identify entities, including management-only clients. The SDK transparently externalizes client start/event/entity-signal inputs and management reasons/outputs, activity results, orchestration actions and custom status, continue-as-new @@ -100,16 +97,54 @@ or recover existing orphan blobs. Custom stores can extend `PayloadStore` from the core package. They must honor `AbortSignal`, recognize only their own tokens, and return confirmed `PayloadDeleteOutcome` values for cleanup. -Deletion is optional. The provider's explicit deletion API has the safety contract below. +Deletion is optional; a store without deletion support leaves cleanup work retryable. + +## Automatic cleanup + +```typescript +await worker.start(); +await client.setLargePayloadAutoPurge(true); // default batch size 500; range 1..1000 +await client.setLargePayloadAutoPurge(true, 250); // update through SetBatchSize +await client.setLargePayloadAutoPurge(false); // pause backend fetches; do not terminate the runner +``` -## Explicit deletion safety +This requires a DTS backend implementing `microsoft.durabletask.largepayloads.LargePayloadPurge`. +Merely configuring storage does **not** enable deletion. The last explicit setting wins; +coordinate calls from multiple clients. + +Enabling writes the backend setting first, schedules the unversioned singleton +`BlobPurgeJobOrchestrator` at `BlobPurgeJob-__dt_blob_payload_autopurge__`, verifies its identity +and Running status, then enqueues `SetBatchSize`. Live instances are deduplicated, not replaced. +Do not use these names/IDs for business tasks. The steps are not transactional: earlier successful +steps remain applied if a later step fails or is canceled. Completion acknowledges event enqueue, +not application of the configuration. The optional third argument is an `AbortSignal`. +Disabling ignores batch size and only writes the backend setting; already-fetched deletes may finish. + +Payload-enabled workers automatically register the runner and its three activities, include +them with wildcard versions in nonempty filters, and bypass business version policy for those +reserved tasks. All-empty/unfiltered subscriptions remain unfiltered. Purge RPCs reuse the +worker's current authenticated transport across reconnects. + +The runner fetches due tombstones, deletes chunks of 50 with at most four chunk activities +concurrent, and reports one positional outcome per attempted tombstone, preserving correlation +tokens unchanged. Each chunk limits deletion to eight concurrent blobs with a 60-second ceiling +per cooperative store call. Fetch/report RPCs have 60-second deadlines. Activities make up to three +attempts with 15/30-second retry delays, excluding unsupported backends. Empty/disabled fetches and +all-Retry results idle on a one-minute durable timer, interruptible by configuration. The runner +continues as new after five cycles, retaining configuration events, batch size, and diagnostic count. +Unsupported fetch/report RPCs wait for explicit configuration instead of polling indefinitely. Deletion checks the case-insensitive ownership marker, then uses that read's ETag and includes snapshots. Unowned blobs are never deleted. Only confirmed `BlobNotFound`/`ContainerNotFound` -responses count as already absent; uncertain failures propagate. Configuring the provider alone -does not invoke deletion. Later cleanup phases add the authenticated backend lifecycle and runner. +responses count as already absent. Malformed v2 tokens, legacy v1 tokens, and invalid storage +requests are quarantined; authorization, uncertain errors, ETag conflicts, unsupported stores, +and unknown token versions remain retryable. The backend owns retry scheduling and quarantined +evidence. Logs contain bounded classifications, not payload tokens or raw storage errors. + +Only tracked v2 references are eligible for automatic cleanup; v1 blobs remain in storage. `Deleted` means storage accepted deletion or confirmed a terminal outcome, **not** immediate physical reclamation. Soft-delete retention and blob versions can continue to consume storage. +The progress count can include duplicate/already-absent/unowned outcomes. ## Local verification @@ -118,3 +153,20 @@ gzip vectors generated by .NET 10.0.13 using the pinned implementation's write/f Run `npm run test:unit -w @microsoft/durabletask-js-azureblob`. The optional `test:azurite` suite requires a local-only `DURABLETASK_BLOB_TEST_CONNECTION_STRING` and never provisions cloud resources. + +After `npm run build`, run the packed-consumer regression from the repository root: + +```sh +npm pack @microsoft/durabletask-js@0.4.0 --ignore-scripts --pack-destination +npm run test:payload-packages -- /microsoft-durabletask-js-0.4.0.tgz +``` + +This checks actual npm peer resolution against the published archive: both payload-enabled +extensions must reject core `0.4.0` with `ERESOLVE`. It then installs the new packed core and +extensions in an isolated consumer and loads their public APIs without contacting storage or DTS. +It also type-checks the packed public declarations with `lib: ES2020`, strict checking, and +`skipLibCheck: false`, without inheriting the ambient ES2022 `ErrorOptions` type. +All temporary consumer installations are removed afterward. + +Merge all six review phases before preparing any package release. The complete core `0.4.1` +must be published before the dependent extensions; no partial feature phase is a release. diff --git a/packages/durabletask-js-azuremanaged/CHANGELOG.md b/packages/durabletask-js-azuremanaged/CHANGELOG.md index a57bd6cb..450932e7 100644 --- a/packages/durabletask-js-azuremanaged/CHANGELOG.md +++ b/packages/durabletask-js-azuremanaged/CHANGELOG.md @@ -2,9 +2,9 @@ ### New -- Add `.useExternalizedPayloads()` to client and worker builders, preserving settings when - connection options change. The new public option types require core `>=0.4.1`. - Cleanup registration is deferred to the later worker phase; do not release partial feature phases. +- Add `.useExternalizedPayloads()` to client and worker builders, preserving payload settings + across connection-option replacement and enabling transparent payload storage and reserved + auto-purge tasks in the core SDK. Require core `>=0.4.1` for the new public payload types. - Support the `ResourceId` token audience URI in connection strings and an optional final `resourceId` argument in the client/worker factory functions, alongside existing builder/options setters. Trim surrounding whitespace and trailing slashes, remove one case-insensitive `/.default` suffix, diff --git a/packages/durabletask-js-azuremanaged/README.md b/packages/durabletask-js-azuremanaged/README.md index 08fe6b13..996cc6b9 100644 --- a/packages/durabletask-js-azuremanaged/README.md +++ b/packages/durabletask-js-azuremanaged/README.md @@ -53,6 +53,15 @@ const worker = createAzureManagedWorkerBuilder("https://myservice.durabletask.io await worker.start(); ``` +### Large payloads + +Both builders support `.useExternalizedPayloads({ store, thresholdBytes?, maxPayloadBytes? })`. +This release requires core `@microsoft/durabletask-js >=0.4.1`, including its payload types. +The opt-in setting survives `.connectionString(...)` replacement. Configure the same payload +storage on every participating client and worker; scheduler and storage credentials are separate. +See [`@microsoft/durabletask-js-azureblob`](../durabletask-js-azureblob/README.md) for .NET-compatible +blob storage, size defaults, authentication, and the separately enabled auto-purge lifecycle. + ### Token Audience and Azure Government The optional `resourceId` is a **token audience URI**, not an Azure Resource Manager resource path. diff --git a/scripts/test-payload-package-compatibility.js b/scripts/test-payload-package-compatibility.js new file mode 100644 index 00000000..1b52aced --- /dev/null +++ b/scripts/test-payload-package-compatibility.js @@ -0,0 +1,161 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { pathToFileURL } = require("node:url"); +const { spawnSync } = require("node:child_process"); +const ts = require("typescript"); + +const root = path.resolve(__dirname, ".."); +const [oldCoreArgument, ...extraArguments] = process.argv.slice(2); +if (!oldCoreArgument || extraArguments.length) { + throw new Error("Usage: npm run test:payload-packages -- "); +} +const oldCore = path.resolve(oldCoreArgument); +assert.ok(fs.statSync(oldCore).isFile(), "The published core archive must be a file."); +const npmCli = process.env.npm_execpath; +assert.ok(npmCli, "Run this check through npm run test:payload-packages."); +const temporary = fs.mkdtempSync(path.join(os.tmpdir(), "durabletask-payload-packages-")); + +function npm(args, cwd) { + const result = spawnSync(process.execPath, [npmCli, ...args], { + cwd, + encoding: "utf8", + timeout: 120000, + maxBuffer: 2 * 1024 * 1024, + }); + if (result.error) throw result.error; + return result; +} + +function pack(name) { + const result = npm( + ["pack", "--json", "--ignore-scripts", "--workspace", name, "--pack-destination", temporary], + root, + ); + assert.equal(result.status, 0, result.stderr || result.stdout); + const [metadata] = JSON.parse(result.stdout); + return path.join(temporary, metadata.filename); +} + +function install(name, dependencies, devDependencies = {}) { + const directory = path.join(temporary, name); + fs.mkdirSync(directory); + fs.writeFileSync( + path.join(directory, "package.json"), + JSON.stringify({ + name, + version: "1.0.0", + private: true, + dependencies: Object.fromEntries( + Object.entries(dependencies).map(([dependency, archive]) => [dependency, pathToFileURL(archive).href]), + ), + devDependencies, + }), + ); + const result = npm( + ["install", "--json", "--ignore-scripts", "--strict-peer-deps", "--no-audit", "--no-fund"], + directory, + ); + return { ...result, directory }; +} + +try { + const coreName = "@microsoft/durabletask-js"; + const blobName = "@microsoft/durabletask-js-azureblob"; + const managedName = "@microsoft/durabletask-js-azuremanaged"; + const core = pack(coreName); + const blob = pack(blobName); + const managed = pack(managedName); + for (const [index, name, archive] of [ + [0, blobName, blob], + [1, managedName, managed], + ]) { + const result = install(`reject-published-core-${index}`, { [coreName]: oldCore, [name]: archive }); + assert.notEqual(result.status, 0, `${name} accepted published core 0.4.0, which lacks PayloadStore.`); + const output = `${result.stdout}\n${result.stderr}`; + assert.match(output, /ERESOLVE/, `Expected a peer conflict, not an unrelated install failure:\n${output}`); + assert.match(output, /@microsoft\/durabletask-js@0\.4\.0/, output); + console.log(`${name}: npm rejected published core 0.4.0 with ERESOLVE.`); + } + const result = install( + "accept-payload-capable-core", + { [coreName]: core, [blobName]: blob, [managedName]: managed }, + { + "@types/node": require("@types/node/package.json").version, + "@types/google-protobuf": require("@types/google-protobuf/package.json").version, + "@opentelemetry/api": require(path.join(root, "package.json")).devDependencies["@opentelemetry/api"], + "@azure/storage-blob": require("@azure/storage-blob/package.json").version, + }, + ); + assert.equal(result.status, 0, result.stderr || result.stdout); + const consumer = path.join(result.directory, "consumer.ts"); + fs.writeFileSync( + consumer, + ` + import { PayloadStorageException, PayloadStore } from "@microsoft/durabletask-js"; + import { BlobPayloadStore } from "@microsoft/durabletask-js-azureblob"; + import { DurableTaskAzureManagedClientBuilder } from "@microsoft/durabletask-js-azuremanaged"; + const store: PayloadStore = new BlobPayloadStore({ connectionString: "UseDevelopmentStorage=true" }); + new DurableTaskAzureManagedClientBuilder().useExternalizedPayloads({ store }); + new PayloadStorageException("failure", { cause: new Error("cause") }); + `, + ); + const program = ts.createProgram([consumer], { + target: ts.ScriptTarget.ES2020, + lib: ["lib.es2020.d.ts"], + module: ts.ModuleKind.CommonJS, + moduleResolution: ts.ModuleResolutionKind.Node10, + typeRoots: [path.join(result.directory, "node_modules", "@types")], + types: ["node"], + strict: true, + skipLibCheck: false, + esModuleInterop: true, + noEmit: true, + }); + assert.ok( + !program + .getTypeChecker() + .getSymbolsInScope(program.getSourceFile(consumer), ts.SymbolFlags.Type) + .some((symbol) => symbol.name === "ErrorOptions"), + "The ES2020 check must not inherit the global ES2022 ErrorOptions type.", + ); + const diagnostics = ts.getPreEmitDiagnostics(program); + assert.equal( + diagnostics.length, + 0, + ts.formatDiagnosticsWithColorAndContext(diagnostics, { + getCanonicalFileName: (name) => name, + getCurrentDirectory: () => result.directory, + getNewLine: () => "\n", + }), + ); + console.log("Packed public declarations type-check against ES2020 with skipLibCheck=false."); + const runtime = spawnSync( + process.execPath, + [ + "-e", + ` + const assert = require("node:assert/strict"); + const core = require("@microsoft/durabletask-js"); + const { BlobPayloadStore } = require("@microsoft/durabletask-js-azureblob"); + const { DurableTaskAzureManagedClientBuilder } = require("@microsoft/durabletask-js-azuremanaged"); + assert.equal(typeof core.PayloadStore, "function"); + const cause = new Error("cause"); + assert.equal(new core.PayloadStorageException("failure", { cause }).cause, cause); + assert.equal(typeof core.TaskHubGrpcClient.prototype.setLargePayloadAutoPurge, "function"); + assert.ok(new BlobPayloadStore({ connectionString: "UseDevelopmentStorage=true" }) instanceof core.PayloadStore); + assert.equal(typeof DurableTaskAzureManagedClientBuilder.prototype.useExternalizedPayloads, "function"); + `, + ], + { cwd: result.directory, encoding: "utf8", timeout: 30000 }, + ); + if (runtime.error) throw runtime.error; + assert.equal(runtime.status, 0, runtime.stderr || runtime.stdout); + console.log("Packed core and extensions install successfully; public APIs load without storage or DTS I/O."); +} finally { + fs.rmSync(temporary, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }); +}