diff --git a/CHANGELOG.md b/CHANGELOG.md index fe3fb6e3..262e62d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,9 @@ ### Fixes +- Preserve received structured failure `properties` in task errors, retry callbacks, client + state/history, entity failures, and testing. Forward them with rethrown task failure chains. + Empty wire maps remain `undefined`; arbitrary JavaScript `Error` fields are not collected. - Preserve nested `innerFailure` details in task errors, retry handlers and predicates, client state/history, and in-memory testing. Uncaught or rethrown task failures retain their received chain beneath the existing `TaskFailedError` wrapper. Retry decisions diff --git a/README.md b/README.md index 943a6575..b2fd2032 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,7 @@ Retries do not guarantee connection recovery, acceptance of expired tokens, or e ### Inspecting nested failures Task errors expose an optional, read-only `innerFailure` chain, with `errorType`, `message`, -and optional `stackTrace` at each level. For an activity failure +and optional `stackTrace` and `properties` at each level. For an activity failure `OrderFailed -> PaymentFailed -> ConnectionTimeout`, orchestrator code can inspect: ```typescript @@ -202,7 +202,12 @@ try { if (error instanceof TaskFailedError) { const paymentFailure = error.details.innerFailure; const rootFailure = paymentFailure?.innerFailure; - ctx.setCustomStatus({ rootErrorType: rootFailure?.errorType }); + // If the backend supplied a structured "code" property, narrow its unknown value. + const code = paymentFailure?.properties?.["code"]; + ctx.setCustomStatus({ + rootErrorType: rootFailure?.errorType, + paymentCode: typeof code === "string" ? code : undefined, + }); } throw error; } @@ -216,13 +221,25 @@ task details under `state.failureDetails.innerFailure`. Missing inner failures remain `undefined`; existing two- and three-argument `FailureDetails` constructors still work, with an optional fourth argument for the inner -details. Remote failures are not reconstructed as JavaScript `Error.cause` objects. +details and fifth argument for properties. Remote failures are not reconstructed as +JavaScript `Error.cause` objects. The existing writer still limits ordinary `Error.cause` chains to ten inner levels; reading or forwarding already-received failure details does not add a truncation limit. If user code mutates received details into a cycle, forwarding preserves each unique failure and ends the chain with `errorType: "CircularFailureDetails"` and `message: "A circular innerFailure reference was detected."` rather than looping. +`properties` is a `Readonly>` containing values already received +from the backend: strings, numbers, booleans, `null`, arrays, and objects. Empty or absent +wire maps become `undefined`. Wire strings stay strings, including .NET `dt:`/`dto:` +date encodings; this is not CLR type reconstruction or a custom data converter. +The same properties reach retry callbacks, client state/history, entity failure details, +and test adapters. Ordinary JavaScript `Error` fields are **not** automatically collected. +Forwarding user-mutated properties rejects unsupported values (such as `Date`, `bigint`, +or `undefined`) and circular objects instead of silently dropping data. The current +protobuf dependency cannot retain the reserved `__proto__` map key on receipt; adding +that key before forwarding also raises a serialization error. + ### Reusing orchestration instance IDs Set the top-level `dedupeStatuses` start option when an instance ID may be reused. The list diff --git a/packages/azure-functions-durable/CHANGELOG.md b/packages/azure-functions-durable/CHANGELOG.md index 68018694..4caac949 100644 --- a/packages/azure-functions-durable/CHANGELOG.md +++ b/packages/azure-functions-durable/CHANGELOG.md @@ -16,6 +16,8 @@ ### Fixes +- Preserve received failure `properties` in `durable-functions/testing` results, including + nested failures. This does not collect arbitrary JavaScript `Error` fields. - Forward the optional `reason` from the classic `suspend()` / `resume()` aliases to the core client instead of ignoring it, preserving literal strings including empty strings. - Preserve nested task `innerFailure` details in `durable-functions/testing` results. diff --git a/packages/azure-functions-durable/src/testing/index.ts b/packages/azure-functions-durable/src/testing/index.ts index 10777df1..21df1b6e 100644 --- a/packages/azure-functions-durable/src/testing/index.ts +++ b/packages/azure-functions-durable/src/testing/index.ts @@ -98,6 +98,7 @@ export async function runOrchestrator( message: state.failureDetails.message, stackTrace: state.failureDetails.stackTrace, innerFailure: state.failureDetails.innerFailure, + properties: state.failureDetails.properties, } : undefined, }; diff --git a/packages/azure-functions-durable/test/unit/testing.spec.ts b/packages/azure-functions-durable/test/unit/testing.spec.ts index 202089fb..8694077b 100644 --- a/packages/azure-functions-durable/test/unit/testing.spec.ts +++ b/packages/azure-functions-durable/test/unit/testing.spec.ts @@ -4,6 +4,9 @@ import { InvocationContext } from "@azure/functions"; import { InMemoryOrchestrationBackend, + FailureDetails, + OrchestrationState, + OrchestrationStatus, TestOrchestrationClient, TestOrchestrationWorker, TOrchestrator, @@ -53,6 +56,40 @@ describe("durable-functions/testing", () => { }); describe("runOrchestrator", () => { + it("preserves received root and nested properties in its failure projection", async () => { + const inner = new FailureDetails("inner", "InnerError", undefined, undefined, { retryable: false }); + const failure = new FailureDetails("remote", "RemoteError", undefined, inner, { code: "RemoteCode" }); + const state = new OrchestrationState( + "remote", + "orchestrator", + OrchestrationStatus.FAILED, + new Date(0), + new Date(0), + undefined, + undefined, + undefined, + failure, + ); + // Inject received client state to isolate the adapter's public projection. + const wait = jest + .spyOn(TestOrchestrationClient.prototype, "waitForOrchestrationCompletion") + .mockResolvedValue(state); + try { + const orchestrator: TOrchestrator = async function* (ctx) { + yield ctx.createTimer(0); + }; + const result = await runOrchestrator(orchestrator); + expect(result.failure).toMatchObject({ + errorType: "RemoteError", + message: "remote", + properties: { code: "RemoteCode" }, + innerFailure: { errorType: "InnerError", message: "inner", properties: { retryable: false } }, + }); + } finally { + wait.mockRestore(); + } + }); + it("inherits the core three-day timer default, like the Functions worker", async () => { const complete = jest.spyOn(InMemoryOrchestrationBackend.prototype, "completeOrchestration"); const day = 24 * 60 * 60 * 1000; diff --git a/packages/durabletask-js/src/entities/entity-operation-failed-exception.ts b/packages/durabletask-js/src/entities/entity-operation-failed-exception.ts index 46b3dea5..00210a03 100644 --- a/packages/durabletask-js/src/entities/entity-operation-failed-exception.ts +++ b/packages/durabletask-js/src/entities/entity-operation-failed-exception.ts @@ -3,6 +3,7 @@ import { EntityInstanceId } from "./entity-instance-id"; import * as pb from "../proto/orchestrator_service_pb"; +import { convertFailureProperties } from "../utils/failure-details.util"; /** * Details about a task failure. @@ -31,6 +32,8 @@ export interface TaskFailureDetails { * Details about an inner failure, if any. */ readonly innerFailure?: TaskFailureDetails; + /** Structured backend values, if any; empty wire maps are undefined. Not collected from Error fields. */ + readonly properties?: Readonly>; } /** @@ -49,6 +52,7 @@ export function createTaskFailureDetails(proto: pb.TaskFailureDetails | undefine errorMessage: proto.getErrormessage(), stackTrace: proto.getStacktrace()?.getValue(), innerFailure: createTaskFailureDetails(proto.getInnerfailure()), + properties: convertFailureProperties(proto), }; } diff --git a/packages/durabletask-js/src/task/failure-details.ts b/packages/durabletask-js/src/task/failure-details.ts index 83ee7031..72c761e1 100644 --- a/packages/durabletask-js/src/task/failure-details.ts +++ b/packages/durabletask-js/src/task/failure-details.ts @@ -15,6 +15,12 @@ export interface TaskFailureDetails { readonly stackTrace?: string; /** Details of the underlying failure, if supplied by the task or backend. */ readonly innerFailure?: TaskFailureDetails; + /** + * Structured values received from the backend, not collected from JavaScript Error fields. + * Empty wire maps are undefined. Values are protobuf Value data (strings, numbers, booleans, + * null, arrays, or objects); narrow unknown values before using them. + */ + readonly properties?: Readonly>; } export class FailureDetails implements TaskFailureDetails { @@ -22,12 +28,20 @@ export class FailureDetails implements TaskFailureDetails { private _errorType: string; private _stackTrace: string | undefined; private _innerFailure: TaskFailureDetails | undefined; + private _properties: Readonly> | undefined; - constructor(message: string, errorType: string, stackTrace?: string, innerFailure?: TaskFailureDetails) { + constructor( + message: string, + errorType: string, + stackTrace?: string, + innerFailure?: TaskFailureDetails, + properties?: Readonly>, + ) { this._message = message; this._errorType = errorType; this._stackTrace = stackTrace; this._innerFailure = innerFailure; + this._properties = properties; } get message(): string { @@ -45,4 +59,8 @@ export class FailureDetails implements TaskFailureDetails { get innerFailure(): TaskFailureDetails | undefined { return this._innerFailure; } + + get properties(): Readonly> | undefined { + return this._properties; + } } diff --git a/packages/durabletask-js/src/task/retry-handler-task.ts b/packages/durabletask-js/src/task/retry-handler-task.ts index 2caa1408..5e678c3a 100644 --- a/packages/durabletask-js/src/task/retry-handler-task.ts +++ b/packages/durabletask-js/src/task/retry-handler-task.ts @@ -80,6 +80,7 @@ export class RetryHandlerTask extends RetryTaskBase { message: details.message, stackTrace: details.stackTrace, innerFailure: details.innerFailure, + properties: details.properties, }; const totalRetryTimeMs = currentTime.getTime() - this.startTime.getTime(); diff --git a/packages/durabletask-js/src/task/retryable-task.ts b/packages/durabletask-js/src/task/retryable-task.ts index 00028249..eb464022 100644 --- a/packages/durabletask-js/src/task/retryable-task.ts +++ b/packages/durabletask-js/src/task/retryable-task.ts @@ -73,6 +73,7 @@ export class RetryableTask extends RetryTaskBase { message: details.message, stackTrace: details.stackTrace, innerFailure: details.innerFailure, + properties: details.properties, }; if (!this._retryPolicy.shouldRetry(failureDetails)) { diff --git a/packages/durabletask-js/src/utils/failure-details.util.ts b/packages/durabletask-js/src/utils/failure-details.util.ts index 7d2ce7f1..f7de0be1 100644 --- a/packages/durabletask-js/src/utils/failure-details.util.ts +++ b/packages/durabletask-js/src/utils/failure-details.util.ts @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +import { JavaScriptValue, Value } from "google-protobuf/google/protobuf/struct_pb"; import * as pb from "../proto/orchestrator_service_pb"; import { FailureDetails } from "../task/failure-details"; @@ -16,5 +17,50 @@ export function convertFailureDetails(details: pb.TaskFailureDetails | undefined details.getErrortype(), details.getStacktrace()?.getValue(), convertFailureDetails(details.getInnerfailure()), + convertFailureProperties(details), ); } + +export function convertFailureProperties( + details: pb.TaskFailureDetails, +): Readonly> | undefined { + const properties = details.getPropertiesMap(); + return properties.getLength() === 0 + ? undefined + : Object.fromEntries(Array.from(properties.entries(), ([key, value]) => [key, value.toJavaScript()])); +} + +export function setFailureProperties( + details: pb.TaskFailureDetails, + properties: Readonly> | undefined, +): void { + if (properties === undefined) return; + for (const [key, value] of Object.entries(properties)) { + validatePropertyValue(value, key); + details.getPropertiesMap().set(key, Value.fromJavaScript(value)); + } +} + +function validatePropertyValue( + value: unknown, + key: string, + ancestors = new Set(), +): asserts value is JavaScriptValue { + // google-protobuf maps silently drop this key instead of preserving it as data. + if (key === "__proto__") throw new TypeError("google-protobuf cannot serialize a '__proto__' failure property."); + if (value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") return; + if ( + typeof value !== "object" || + (!Array.isArray(value) && + Object.getPrototypeOf(value) !== Object.prototype && + Object.getPrototypeOf(value) !== null) + ) { + throw new TypeError("Failure properties must contain only protobuf Value-compatible data."); + } + if (ancestors.has(value)) throw new TypeError("Cannot serialize circular failure properties."); + ancestors.add(value); + for (const [childKey, child] of Object.entries(value)) { + validatePropertyValue(child, childKey, ancestors); + } + ancestors.delete(value); +} diff --git a/packages/durabletask-js/src/utils/pb-helper.util.ts b/packages/durabletask-js/src/utils/pb-helper.util.ts index 8993601b..197d959d 100644 --- a/packages/durabletask-js/src/utils/pb-helper.util.ts +++ b/packages/durabletask-js/src/utils/pb-helper.util.ts @@ -8,6 +8,7 @@ import { ActivityNotRegisteredError } from "../worker/exception/activity-not-reg import { OrchestratorNotRegisteredError } from "../worker/exception/orchestrator-not-registered-error"; import { TaskFailedError } from "../task/exception/task-failed-error"; import { TaskFailureDetails } from "../task/failure-details"; +import { setFailureProperties } from "./failure-details.util"; export function newOrchestratorStartedEvent(timestamp?: Date | null): pb.HistoryEvent { const ts = new Timestamp(); @@ -258,6 +259,7 @@ function buildFailureDetails(e: unknown, depth: number): pb.TaskFailureDetails { if (details.stackTrace !== undefined) { inner.setStacktrace(new StringValue().setValue(details.stackTrace)); } + setFailureProperties(inner, details.properties); current.setInnerfailure(inner); current = inner; } diff --git a/packages/durabletask-js/test/entity-operation-failed-exception.spec.ts b/packages/durabletask-js/test/entity-operation-failed-exception.spec.ts index c36ea06f..448a038f 100644 --- a/packages/durabletask-js/test/entity-operation-failed-exception.spec.ts +++ b/packages/durabletask-js/test/entity-operation-failed-exception.spec.ts @@ -10,6 +10,7 @@ import { import { TaskFailedError } from "../src/task/exception/task-failed-error"; import * as pb from "../src/proto/orchestrator_service_pb"; import { StringValue } from "google-protobuf/google/protobuf/wrappers_pb"; +import { Value } from "google-protobuf/google/protobuf/struct_pb"; describe("EntityOperationFailedException", () => { describe("constructor", () => { @@ -155,11 +156,13 @@ describe("createTaskFailureDetails", () => { const innerProto = new pb.TaskFailureDetails(); innerProto.setErrortype("InnerError"); innerProto.setErrormessage("Inner message"); + innerProto.getPropertiesMap().set("retryable", Value.fromJavaScript(false)); const proto = new pb.TaskFailureDetails(); proto.setErrortype("OuterError"); proto.setErrormessage("Outer message"); proto.setInnerfailure(innerProto); + proto.getPropertiesMap().set("code", Value.fromJavaScript("Conflict")); // Act const result = createTaskFailureDetails(proto); @@ -169,6 +172,8 @@ describe("createTaskFailureDetails", () => { expect(result!.innerFailure).toBeDefined(); expect(result!.innerFailure!.errorType).toBe("InnerError"); expect(result!.innerFailure!.errorMessage).toBe("Inner message"); + expect(Reflect.get(result!, "properties")).toEqual({ code: "Conflict" }); + expect(Reflect.get(result!.innerFailure!, "properties")).toEqual({ retryable: false }); }); it("should handle missing optional fields", () => { @@ -185,5 +190,6 @@ describe("createTaskFailureDetails", () => { expect(result).toBeDefined(); expect(result!.stackTrace).toBeUndefined(); expect(result!.innerFailure).toBeUndefined(); + expect(Reflect.get(result!, "properties")).toBeUndefined(); }); }); diff --git a/packages/durabletask-js/test/failure-details-grpc.spec.ts b/packages/durabletask-js/test/failure-details-grpc.spec.ts index 16088f1e..0a7b06c0 100644 --- a/packages/durabletask-js/test/failure-details-grpc.spec.ts +++ b/packages/durabletask-js/test/failure-details-grpc.spec.ts @@ -3,10 +3,145 @@ import * as grpc from "@grpc/grpc-js"; import { Empty } from "google-protobuf/google/protobuf/empty_pb"; -import { NoOpLogger, TaskHubGrpcClient, TaskHubGrpcWorker } from "../src"; +import { Value } from "google-protobuf/google/protobuf/struct_pb"; +import { NoOpLogger, RetryPolicy, TaskFailedError, TaskHubGrpcClient, TaskHubGrpcWorker } from "../src"; +import type { TaskFailureDetails } from "../src"; import * as pb from "../src/proto/orchestrator_service_pb"; import * as stubs from "../src/proto/orchestrator_service_grpc_pb"; import { withTimeout } from "../src/utils/backoff.util"; +import { + newExecutionStartedEvent, + newOrchestratorStartedEvent, + newTaskFailedEvent, + newTaskScheduledEvent, +} from "../src/utils/pb-helper.util"; + +it.each(["uncaught", "handler", "policy"] as const)( + "forwards received wire properties through %s orchestration failure and all client projections", + async (mode) => { + // Simulate a foreign worker's failure; no .NET worker or cloud backend runs in this fixture. + const incoming = new pb.TaskFailureDetails().setErrortype("PaymentFailed").setErrormessage("Declined"); + incoming.getPropertiesMap().set("code", Value.fromJavaScript("CardDeclined")); + incoming + .getPropertiesMap() + .set("context", Value.fromJavaScript({ attempts: 0, retryable: false, tags: ["", null] })); + const inner = new pb.TaskFailureDetails().setErrortype("GatewayError").setErrormessage("Rejected"); + inner.getPropertiesMap().set("status", Value.fromJavaScript(402)); + incoming.setInnerfailure(inner); + const expected = { + errorType: "PaymentFailed", + message: "Declined", + properties: { code: "CardDeclined", context: { attempts: 0, retryable: false, tags: ["", null] } }, + innerFailure: { errorType: "GatewayError", message: "Rejected", properties: { status: 402 } }, + }; + const server = new grpc.Server(); + const state = new pb.OrchestrationState().setInstanceid("order").setName("order"); + const response = new pb.GetInstanceResponse().setExists(true).setOrchestrationstate(state); + const get: grpc.handleUnaryCall = (_call, callback) => + callback(null, response); + let completed!: () => void; + const completion = new Promise((resolve) => (completed = resolve)); + const failedEvent = newTaskFailedEvent(1, new Error("unused"), incoming); + server.addService(stubs.TaskHubSidecarServiceService, { + hello: (_call, callback) => callback(null, new Empty()), + getWorkItems: (call) => { + call.once("cancelled", () => call.end()); + call.write( + new pb.WorkItem().setCompletiontoken("order").setOrchestratorrequest( + new pb.OrchestratorRequest() + .setInstanceid("order") + .setPasteventsList([ + newOrchestratorStartedEvent(), + newExecutionStartedEvent("order", "order"), + newTaskScheduledEvent(1, "pay"), + ]) + .setNeweventsList([newOrchestratorStartedEvent(), failedEvent]), + ), + ); + }, + completeOrchestratorTask: (call, callback) => { + const action = call.request + .getActionsList() + .find((item) => item.hasCompleteorchestration()) + ?.getCompleteorchestration(); + state.setFailuredetails(action?.getFailuredetails()); + if (action) state.setOrchestrationstatus(action.getOrchestrationstatus()); + callback(null, new pb.CompleteTaskResponse()); + completed(); + }, + getInstance: get, + waitForInstanceStart: get, + waitForInstanceCompletion: get, + queryInstances: (_call, callback) => + callback(null, new pb.QueryInstancesResponse().setOrchestrationstateList([state])), + streamInstanceHistory: (call) => { + call.write(new pb.HistoryChunk().setEventsList([failedEvent])); + call.end(); + }, + } satisfies Partial); + const port = await new Promise((resolve, reject) => { + server.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (error, boundPort) => + error ? reject(error) : resolve(boundPort), + ); + }); + const options = { hostAddress: `127.0.0.1:${port}`, logger: new NoOpLogger() }; + const worker = new TaskHubGrpcWorker({ ...options, shutdownTimeoutMs: 500 }); + const client = new TaskHubGrpcClient(options); + const inspected: TaskFailureDetails[] = []; + const caught: TaskFailureDetails[] = []; + const inspect = (failure: TaskFailureDetails) => { + inspected.push(failure); + return false; + }; + worker.addNamedOrchestrator("order", async function* (ctx) { + const retry = + mode === "policy" + ? new RetryPolicy({ maxNumberOfAttempts: 3, firstRetryIntervalInMilliseconds: 1, handleFailure: inspect }) + : mode === "handler" + ? (context: { lastFailure: TaskFailureDetails }) => inspect(context.lastFailure) + : undefined; + if (mode === "uncaught") { + yield ctx.callActivity("pay"); + } else { + try { + yield ctx.callActivity("pay", undefined, { retry }); + } catch (error) { + if (!(error instanceof TaskFailedError)) throw error; + caught.push(error.details); + throw error; + } + } + }); + await worker.start(); + try { + await withTimeout(completion, 5000); + if (mode !== "uncaught") { + expect(inspected).toHaveLength(1); + expect(inspected[0]).toMatchObject(expected); + expect(caught).toHaveLength(1); + expect(caught[0]).toMatchObject(expected); + } + const terminal = { errorType: "TaskFailedError", innerFailure: expected }; + expect(state.getOrchestrationstatus()).toBe(pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED); + for (const method of [ + "getOrchestrationState", + "waitForOrchestrationStart", + "waitForOrchestrationCompletion", + ] as const) { + expect((await client[method]("order", false))?.failureDetails).toMatchObject(terminal); + } + const states = []; + for await (const instance of client.getAllInstances()) states.push(instance); + expect(states).toHaveLength(1); + expect(states[0].failureDetails).toMatchObject(terminal); + expect(await client.getOrchestrationHistory("order")).toMatchObject([{ failureDetails: expected }]); + } finally { + await client.stop(); + await worker.stop(); + server.forceShutdown(); + } + }, +); it("preserves serialized worker failures through gRPC client get, waits, query and history", async () => { const server = new grpc.Server(); diff --git a/packages/durabletask-js/test/failure-details.spec.ts b/packages/durabletask-js/test/failure-details.spec.ts index 276a8f89..69395ab7 100644 --- a/packages/durabletask-js/test/failure-details.spec.ts +++ b/packages/durabletask-js/test/failure-details.spec.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. import { StringValue } from "google-protobuf/google/protobuf/wrappers_pb"; +import { ListValue, Struct, Value } from "google-protobuf/google/protobuf/struct_pb"; import { FailureDetails, InMemoryOrchestrationBackend, @@ -17,6 +18,25 @@ import * as pb from "../src/proto/orchestrator_service_pb"; import { convertProtoHistoryEvent } from "../src/utils/history-event-converter"; import { newFailureDetails } from "../src/utils/pb-helper.util"; +const wireProperties = { + code: "PaymentDeclined", + count: 0, + retryable: false, + reason: "", + optional: null, + context: { values: [1.5, true, null, "", { currency: "USD" }] }, + timestamp: "dt:2026-09-28T00:00:00Z", +}; + +function receivedFailure(): pb.TaskFailureDetails { + const failure = newFailureDetails(errorChain()); + for (const [key, value] of Object.entries(wireProperties)) { + failure.getPropertiesMap().set(key, Value.fromJavaScript(value)); + } + failure.getInnerfailure()!.getPropertiesMap().set("status", Value.fromJavaScript(402)); + return pb.TaskFailureDetails.deserializeBinary(failure.serializeBinary()); +} + function errorChain(): Error { const inner = new Error("Connection timed out"); inner.name = "ConnectionTimeout"; @@ -58,7 +78,69 @@ const expectedChain = { }, }; +const expectedReceivedChain = { + ...expectedChain, + properties: wireProperties, + innerFailure: { ...expectedChain.innerFailure, properties: { status: 402 } }, +}; + describe("Public failure details", () => { + it("receives and forwards structured wire properties at each failure level", () => { + const proto = receivedFailure(); + const error = new TaskFailedError("Activity task failed", proto); + expect(error.details.properties).toEqual(wireProperties); + expect(error.details.innerFailure?.properties).toEqual({ status: 402 }); + expect(newFailureDetails(error).getInnerfailure()?.serializeBinary()).toEqual(proto.serializeBinary()); + }); + + it("keeps delivered property keys as data at the root and inside lists and structs", () => { + const fields = new Struct(); + fields.getFieldsMap().set("constructor", Value.fromJavaScript("also data")); + fields.getFieldsMap().set("toString", Value.fromJavaScript({ marker: "data" })); + const value = new Value().setStructValue(fields); + const proto = new pb.TaskFailureDetails(); + proto.getPropertiesMap().set("nested", value); + proto.getPropertiesMap().set("constructor", Value.fromJavaScript(false)); + proto.getPropertiesMap().set("list", new Value().setListValue(new ListValue().setValuesList([value]))); + const decoded = pb.TaskFailureDetails.deserializeBinary(proto.serializeBinary()); + const error = new TaskFailedError("failed", decoded); + const nested = { constructor: "also data", toString: { marker: "data" } }; + const expected = { nested, constructor: false, list: [nested] }; + expect(Reflect.get(error.details, "properties")).toEqual(expected); + expect(Object.prototype).not.toHaveProperty("marker"); + expect(newFailureDetails(error).getInnerfailure()?.serializeBinary()).toEqual(decoded.serializeBinary()); + }); + + it.each([undefined, 1n, () => 1, new Date(0), { nested: undefined }, Object.fromEntries([["__proto__", "data"]])])( + "rejects unsupported values added to received properties (%p)", + (value) => { + const error = new TaskFailedError("failed", receivedFailure()); + Object.defineProperty(error.details, "properties", { value: { invalid: value } }); + expect(() => newFailureDetails(error)).toThrow(); + }, + ); + + it("rejects a cyclic property graph rather than returning a truncated success", () => { + const cyclic: Record = {}; + cyclic.self = cyclic; + const error = new TaskFailedError("failed", receivedFailure()); + Object.defineProperty(error.details, "properties", { value: cyclic }); + expect(() => newFailureDetails(error)).toThrow(); + }); + + it("forwards deeply nested property values without an arbitrary depth limit", () => { + let value: unknown = { leaf: false }; + for (let i = 0; i < 15; i++) value = { child: value }; + const error = new TaskFailedError("failed", receivedFailure()); + Object.defineProperty(error.details, "properties", { value: { deep: value } }); + expect(newFailureDetails(error).getInnerfailure()?.getPropertiesMap().get("deep")?.toJavaScript()).toEqual(value); + }); + + it("does not collect arbitrary properties from ordinary JavaScript errors", () => { + const error = Object.assign(new Error("local"), { properties: wireProperties, code: "local" }); + expect(newFailureDetails(error).getPropertiesMap().getLength()).toBe(0); + }); + it("preserves a three-level wire chain in TaskFailedError without changing its message or cause", () => { const error = new TaskFailedError("Activity task failed", newFailureDetails(errorChain())); expect(error.details).toMatchObject(expectedChain); @@ -73,6 +155,7 @@ describe("Public failure details", () => { errorType: "Error", stackTrace: "stack", innerFailure: undefined, + properties: undefined, }); expect(new FailureDetails("message", "Error").stackTrace).toBeUndefined(); const proto = new pb.TaskFailureDetails(); @@ -81,11 +164,16 @@ describe("Public failure details", () => { errorType: "", stackTrace: undefined, innerFailure: undefined, + properties: undefined, }); proto.setStacktrace(new StringValue().setValue("")); expect(new TaskFailedError("failed", proto).details.stackTrace).toBe(""); const inner = new FailureDetails("inner", "InnerError"); expect(new FailureDetails("outer", "OuterError", undefined, inner).innerFailure).toBe(inner); + const properties = { code: "RemoteCode" }; + const details = new FailureDetails("outer", "OuterError", undefined, inner, properties); + expect(details.properties).toBe(properties); + expect(Reflect.set(details, "properties", {})).toBe(false); }); it("does not apply the outbound Error.cause depth limit to an incoming wire chain", () => { @@ -114,7 +202,7 @@ describe("Public failure details", () => { }); it.each([1, 3])("marks a %i-node cycle after preserving each unique failure and the task wrapper", (length) => { - const proto = newFailureDetails(errorChain()); + const proto = receivedFailure(); const error = new TaskFailedError("Activity task failed", proto); let tail: TaskFailureDetails = error.details; for (let i = 1; i < length; i++) tail = tail.innerFailure!; @@ -130,6 +218,7 @@ describe("Public failure details", () => { expect(actual?.getErrortype()).toBe(expected?.getErrortype()); expect(actual?.getErrormessage()).toBe(expected?.getErrormessage()); expect(actual?.getStacktrace()?.getValue()).toBe(expected?.getStacktrace()?.getValue()); + expect(actual?.getPropertiesMap().toArray()).toEqual(expected?.getPropertiesMap().toArray()); actual = actual?.getInnerfailure(); expected = expected?.getInnerfailure(); } @@ -151,12 +240,12 @@ describe("Public failure details", () => { }); it("preserves the chain in orchestration state and raiseIfFailed", () => { - const proto = new pb.OrchestrationState().setFailuredetails(newFailureDetails(errorChain())); + const proto = new pb.OrchestrationState().setFailuredetails(receivedFailure()); const state = newOrchestrationState( "order", new pb.GetInstanceResponse().setExists(true).setOrchestrationstate(proto), ); - expect(state?.failureDetails).toMatchObject(expectedChain); + expect(state?.failureDetails).toMatchObject(expectedReceivedChain); let thrown: unknown; try { state?.raiseIfFailed(); @@ -166,14 +255,14 @@ describe("Public failure details", () => { expect(thrown).toMatchObject({ name: "OrchestrationFailedError", message: "Orchestration 'order' failed: Order failed", - failureDetails: expectedChain, + failureDetails: expectedReceivedChain, }); }); it.each(["execution", "activity", "subOrchestration", "entity"] as const)( "preserves the chain in %s history events", (kind) => { - const details = newFailureDetails(errorChain()); + const details = receivedFailure(); const event = new pb.HistoryEvent(); if (kind === "execution") event.setExecutioncompleted(new pb.ExecutionCompletedEvent().setFailuredetails(details)); @@ -185,12 +274,14 @@ describe("Public failure details", () => { } if (kind === "entity") event.setEntityoperationfailed(new pb.EntityOperationFailedEvent().setFailuredetails(details)); - expect(convertProtoHistoryEvent(event)).toMatchObject({ failureDetails: expectedChain }); + expect(convertProtoHistoryEvent(event)).toMatchObject({ + failureDetails: expectedReceivedChain, + }); }, ); }); -describe.each(["activity", "subOrchestration"] as const)("In-memory %s failure chains", (kind) => { +describe.each(["activity", "subOrchestration"] as const)("In-memory %s received failure chains", (kind) => { it.each(["handler", "policy"] as const)( "reaches %s retry inspection, catch, history and client state", async (mode) => { @@ -212,8 +303,10 @@ describe.each(["activity", "subOrchestration"] as const)("In-memory %s failure c }) : (context: { lastFailure: TaskFailureDetails }) => inspectFailure(context.lastFailure); const fail = () => { - throw errorChain(); + // Simulate an error already received from a foreign worker, not local Error field collection. + throw new TaskFailedError("Foreign task failed", receivedFailure()); }; + const expected = { errorType: "TaskFailedError", innerFailure: expectedReceivedChain }; worker.addNamedActivity("fail", fail); worker.addNamedOrchestrator("fail", fail); worker.addNamedOrchestrator("order", async function* (ctx) { @@ -231,24 +324,23 @@ describe.each(["activity", "subOrchestration"] as const)("In-memory %s failure c const id = await client.scheduleNewOrchestration("order"); const state = await client.waitForOrchestrationCompletion(id, true, 5); expect(inspected).toHaveLength(1); - expect(inspected[0]).toMatchObject(expectedChain); - expect(inspected[0].innerFailure?.innerFailure?.errorType).toBe("ConnectionTimeout"); + expect(inspected[0]).toMatchObject(expected); expect(caught).toHaveLength(1); - expect(caught[0]).toMatchObject(expectedChain); + expect(caught[0]).toMatchObject(expected); expect(state?.runtimeStatus).toBe(OrchestrationStatus.FAILED); const terminalFailure = { errorType: "TaskFailedError", - innerFailure: expectedChain, + innerFailure: expected, }; expect(state?.failureDetails).toMatchObject(terminalFailure); expect((await client.getOrchestrationState(id, false))?.failureDetails).toMatchObject(terminalFailure); const history = backend.getInstance(id)!.history.map(convertProtoHistoryEvent); const failedEvent = history.find( - (event) => event && "failureDetails" in event && event.failureDetails?.errorType === "OrderFailed", + (event) => event && "failureDetails" in event && event.failureDetails?.errorType === "TaskFailedError", ); - expect(failedEvent).toMatchObject({ failureDetails: expectedChain }); + expect(failedEvent).toMatchObject({ failureDetails: expected }); if (kind === "subOrchestration") { - expect((await client.getOrchestrationState("child"))?.failureDetails).toMatchObject(expectedChain); + expect((await client.getOrchestrationState("child"))?.failureDetails).toMatchObject(expected); } } finally { await worker.stop();