Add Cloud Run OpenTelemetry worker sample

Directory.Packages.props

@@ -24,6 +24,7 @@
     <PackageVersion Include="Temporalio.Extensions.Aws.Lambda" Version="1.19.0" />
     <PackageVersion Include="Temporalio.Extensions.Aws.Lambda.OpenTelemetry" Version="1.19.0" />
     <PackageVersion Include="Temporalio.Extensions.DiagnosticSource" Version="1.19.0" />
+    <PackageVersion Include="Temporalio.Extensions.Gcp.CloudRun.OpenTelemetry" Version="1.19.0" />
     <PackageVersion Include="Temporalio.Extensions.Hosting" Version="1.19.0" />
     <PackageVersion Include="Temporalio.Extensions.OpenTelemetry" Version="1.19.0" />
     <PackageVersion Include="TemporalCommunity.Aspire.Hosting" Version="0.1.0" />

README.md

@@ -25,6 +25,7 @@ Prerequisites:
 * [EagerWorkflowStart](src/EagerWorkflowStart) - Demonstrates usage of Eager Workflow Start to reduce latency for workflows that start with a local activity.
 * [Encryption](src/Encryption) - End-to-end encryption with Temporal payload codecs.
 * [EnvConfig](src/EnvConfig) - Load client configuration from TOML files with programmatic overrides
+* [Gcp/CloudRun/OpenTelemetry](src/Gcp/CloudRun/OpenTelemetry) - Run a Temporal Worker on a Google Cloud Run worker pool, exporting OpenTelemetry metrics and traces to a collector sidecar.
 * [LambdaWorker](src/LambdaWorker) - Run a Temporal Worker inside an AWS Lambda function.
 * [Mutex](src/Mutex) - How to implement a mutex as a workflow. Demonstrates how to avoid race conditions or parallel mutually exclusive operations on the same resource.
 * [NexusCancellation](src/NexusCancellation) - Demonstrates how to cancel a running Nexus operation from a caller workflow.

TemporalioSamples.sln

@@ -7,6 +7,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{1A647B41-53D
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TemporalioSamples.ActivityWorker", "src\ActivityWorker\TemporalioSamples.ActivityWorker.csproj", "{7AECC7C6-9A21-4B8A-84D9-AFC4F5840CAF}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TemporalioSamples.Gcp.CloudRun.OpenTelemetry", "src\Gcp\CloudRun\OpenTelemetry\TemporalioSamples.Gcp.CloudRun.OpenTelemetry.csproj", "{E0F934F9-10A7-41A0-A85E-EE6D0E267367}"
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TemporalioSamples.Tests", "tests\TemporalioSamples.Tests.csproj", "{3FA7E5DF-03B7-4586-A980-85C155B376C5}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AspNet", "AspNet", "{E431D279-E02B-4670-B934-3DB9F15D8CCC}"
@@ -167,6 +169,18 @@ Global
 		{7AECC7C6-9A21-4B8A-84D9-AFC4F5840CAF}.Release|x64.Build.0 = Release|Any CPU
 		{7AECC7C6-9A21-4B8A-84D9-AFC4F5840CAF}.Release|x86.ActiveCfg = Release|Any CPU
 		{7AECC7C6-9A21-4B8A-84D9-AFC4F5840CAF}.Release|x86.Build.0 = Release|Any CPU
+		{E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Debug|x64.Build.0 = Debug|Any CPU
+		{E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Debug|x86.Build.0 = Debug|Any CPU
+		{E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Release|x64.ActiveCfg = Release|Any CPU
+		{E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Release|x64.Build.0 = Release|Any CPU
+		{E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Release|x86.ActiveCfg = Release|Any CPU
+		{E0F934F9-10A7-41A0-A85E-EE6D0E267367}.Release|x86.Build.0 = Release|Any CPU
 		{3FA7E5DF-03B7-4586-A980-85C155B376C5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{3FA7E5DF-03B7-4586-A980-85C155B376C5}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{3FA7E5DF-03B7-4586-A980-85C155B376C5}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -821,6 +835,7 @@ Global
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{7AECC7C6-9A21-4B8A-84D9-AFC4F5840CAF} = {1A647B41-53D0-4638-AE5A-6630BAAE45FC}
+		{E0F934F9-10A7-41A0-A85E-EE6D0E267367} = {1A647B41-53D0-4638-AE5A-6630BAAE45FC}
 		{E431D279-E02B-4670-B934-3DB9F15D8CCC} = {1A647B41-53D0-4638-AE5A-6630BAAE45FC}
 		{31EC2647-6A5A-42D1-B7B5-02804B340726} = {E431D279-E02B-4670-B934-3DB9F15D8CCC}
 		{AFFA4143-DC28-4FBE-A33B-D6414F541EA4} = {E431D279-E02B-4670-B934-3DB9F15D8CCC}

nuget.config

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Restores the unreleased Temporalio.Extensions.Gcp.CloudRun.OpenTelemetry from a committed local feed; remove once it ships on nuget.org. -->
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+    <add key="local" value="./src/Gcp/CloudRun/OpenTelemetry/local-packages" />
+  </packageSources>
+  <packageSourceMapping>
+    <packageSource key="nuget.org">
+      <package pattern="*" />
+    </packageSource>
+    <packageSource key="local">
+      <package pattern="Temporalio.Extensions.Gcp.CloudRun.OpenTelemetry" />
+    </packageSource>
+  </packageSourceMapping>
+</configuration>

src/Gcp/CloudRun/OpenTelemetry/CloudRunWorkerSample.cs

@@ -0,0 +1,12 @@
+namespace TemporalioSamples.Gcp.CloudRun.OpenTelemetry;
+
+using Temporalio.Worker;
+
+// Shared so the entrypoint and tests register the same workflow and activities.
+public static class CloudRunWorkerSample
+{
+    public static TemporalWorkerOptions ConfigureOptions(TemporalWorkerOptions options) =>
+        options.
+            AddWorkflow<GreetingWorkflow>().
+            AddActivity(GreetingActivities.SayHello);
+}

src/Gcp/CloudRun/OpenTelemetry/Dockerfile

@@ -0,0 +1,14 @@
+# syntax=docker/dockerfile:1
+# Build from the repo root: docker build -f src/Gcp/CloudRun/OpenTelemetry/Dockerfile -t <image> .
+FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
+WORKDIR /src
+COPY global.json Directory.Build.props Directory.Packages.props .editorconfig nuget.config ./
+COPY src/Gcp/CloudRun/OpenTelemetry/ ./src/Gcp/CloudRun/OpenTelemetry/
+RUN dotnet publish src/Gcp/CloudRun/OpenTelemetry/TemporalioSamples.Gcp.CloudRun.OpenTelemetry.csproj -c Release -o /app
+
+FROM mcr.microsoft.com/dotnet/runtime:8.0
+RUN useradd --create-home --uid 10001 worker
+WORKDIR /app
+COPY --from=build --chown=worker:worker /app ./
+USER 10001
+ENTRYPOINT ["dotnet", "TemporalioSamples.Gcp.CloudRun.OpenTelemetry.dll"]

src/Gcp/CloudRun/OpenTelemetry/GreetingActivities.cs

@@ -0,0 +1,14 @@
+namespace TemporalioSamples.Gcp.CloudRun.OpenTelemetry;
+
+using Microsoft.Extensions.Logging;
+using Temporalio.Activities;
+
+public static class GreetingActivities
+{
+    [Activity]
+    public static string SayHello(string name)
+    {
+        ActivityExecutionContext.Current.Logger.LogInformation("SayHello activity: {Name}", name);
+        return $"Hello, {name}!";
+    }
+}

src/Gcp/CloudRun/OpenTelemetry/GreetingWorkflow.workflow.cs

@@ -0,0 +1,19 @@
+namespace TemporalioSamples.Gcp.CloudRun.OpenTelemetry;
+
+using Microsoft.Extensions.Logging;
+using Temporalio.Workflows;
+
+[Workflow]
+public class GreetingWorkflow
+{
+    [WorkflowRun]
+    public async Task<string> RunAsync(string name)
+    {
+        Workflow.Logger.LogInformation("GreetingWorkflow started: {Name}", name);
+        var result = await Workflow.ExecuteActivityAsync(
+            () => GreetingActivities.SayHello(name),
+            new() { StartToCloseTimeout = TimeSpan.FromSeconds(10) });
+        Workflow.Logger.LogInformation("GreetingWorkflow completed: {Result}", result);
+        return result;
+    }
+}

src/Gcp/CloudRun/OpenTelemetry/Program.cs

@@ -0,0 +1,67 @@
+using System.Runtime.InteropServices;
+using Microsoft.Extensions.Logging;
+using Temporalio.Client;
+using Temporalio.Common.EnvConfig;
+using Temporalio.Extensions.Gcp.CloudRun.OpenTelemetry;
+using Temporalio.Worker;
+using TemporalioSamples.Gcp.CloudRun.OpenTelemetry;
+
+// Connect from environment config (TEMPORAL_ADDRESS, TEMPORAL_NAMESPACE, TEMPORAL_API_KEY, ...).
+var connectOptions = ClientEnvConfig.LoadClientConnectOptions();
+connectOptions.TargetHost ??= "localhost:7233";
+connectOptions.LoggerFactory = LoggerFactory.Create(builder =>
+    builder.
+        AddSimpleConsole(options => options.TimestampFormat = "[HH:mm:ss] ").
+        SetMinimumLevel(LogLevel.Information));
+
+var taskQueue = Environment.GetEnvironmentVariable("TEMPORAL_TASK_QUEUE") ?? "cloud-run-worker";
+
+// --starter runs a single workflow, e.g. to kick off work against the same server the worker polls.
+if (args.Contains("--starter"))
+{
+    using var starterTelemetry = connectOptions.ApplyGoogleCloudRunOpenTelemetryDefaults();
+    var starterClient = await TemporalClient.ConnectAsync(connectOptions);
+    var greeting = await starterClient.ExecuteWorkflowAsync(
+        (GreetingWorkflow wf) => wf.RunAsync("Temporal"),
+        new($"cloud-run-worker-{Guid.NewGuid():N}", taskQueue));
+    Console.WriteLine("Workflow result: {0}", greeting);
+    await starterTelemetry.FlushAsync(TimeSpan.FromSeconds(2));
+    return;
+}
+
+// Adds the tracing interceptor and a runtime exporting Core metrics + traces over OTLP to the
+// collector sidecar; the returned handle owns the tracer provider.
+using var telemetry = connectOptions.ApplyGoogleCloudRunOpenTelemetryDefaults();
+
+var client = await TemporalClient.ConnectAsync(connectOptions);
+
+using var cts = new CancellationTokenSource();
+Console.CancelKeyPress += (_, eventArgs) =>
+{
+    eventArgs.Cancel = true;
+    cts.Cancel();
+};
+
+// Cloud Run sends SIGTERM ~10s before SIGKILL.
+using var sigterm = PosixSignalRegistration.Create(PosixSignal.SIGTERM, _ => cts.Cancel());
+
+using var worker = new TemporalWorker(
+    client, CloudRunWorkerSample.ConfigureOptions(new(taskQueue)));
+
+Console.WriteLine(
+    "Worker running: taskQueue={0} address={1} namespace={2}",
+    taskQueue,
+    connectOptions.TargetHost,
+    connectOptions.Namespace ?? "default");
+try
+{
+    await worker.ExecuteAsync(cts.Token);
+}
+catch (OperationCanceledException)
+{
+    Console.WriteLine("Worker shutting down");
+}
+
+// Flush traces within the shutdown grace window (Core metrics export periodically, no explicit flush).
+await telemetry.FlushAsync(TimeSpan.FromSeconds(2));
+Console.WriteLine("Worker stopped");

src/Gcp/CloudRun/OpenTelemetry/README.md

@@ -0,0 +1,52 @@
+# Cloud Run OpenTelemetry Worker
+
+Run a Temporal Worker on a
+[Google Cloud Run worker pool](https://cloud.google.com/run/docs/deploy-worker-pools) that exports
+Core SDK metrics and traces to a
+[Google-Built OpenTelemetry Collector](https://cloud.google.com/stackdriver/docs/instrumentation/opentelemetry-collector-cloud-run)
+sidecar (metrics to Managed Service for Prometheus, traces to Cloud Trace) via the
+`Temporalio.Extensions.Gcp.CloudRun.OpenTelemetry` extension.
+
+`Program.cs` calls `ApplyGoogleCloudRunOpenTelemetryDefaults()`, which adds the tracing interceptor
+and an OTLP exporter aimed at the sidecar, then runs a greeting workflow and activity until SIGTERM.
+
+> The extension is not on nuget.org yet, so the sample restores it from the committed
+> `local-packages/` feed (see `nuget.config`) until it ships.
+
+## Prerequisites
+
+- A Temporal server the worker pool can reach (`TEMPORAL_ADDRESS` / `TEMPORAL_NAMESPACE`)
+- A Google Cloud project with the Cloud Run and Artifact Registry APIs enabled
+- [`gcloud`](https://cloud.google.com/sdk/docs/install), authenticated with the project set
+- The [Temporal CLI](https://docs.temporal.io/cli) and .NET 8
+
+## Deploy
+
+Set the placeholders used by `worker-pool.yaml`, then build the image, store the collector config as
+a secret, and deploy. Run from the repo root:
+
+```bash
+export REGION=us-central1 WORKER_POOL=temporal-otel-worker INSTANCE_COUNT=1
+export SERVICE_ACCOUNT_EMAIL=<sa>@<project>.iam.gserviceaccount.com
+export WORKER_IMAGE=$REGION-docker.pkg.dev/$(gcloud config get-value project)/samples/cloud-run-otel
+export TEMPORAL_ADDRESS=<host:7233> TEMPORAL_NAMESPACE=<namespace> TEMPORAL_TASK_QUEUE=cloud-run-worker
+export COLLECTOR_CONFIG_SECRET=otel-collector-config COLLECTOR_CONFIG_SECRET_VERSION=latest
+
+docker build -f src/Gcp/CloudRun/OpenTelemetry/Dockerfile -t "$WORKER_IMAGE" . && docker push "$WORKER_IMAGE"
+gcloud secrets create "$COLLECTOR_CONFIG_SECRET" --data-file=src/Gcp/CloudRun/OpenTelemetry/collector-config.yaml
+
+envsubst < src/Gcp/CloudRun/OpenTelemetry/worker-pool.yaml > /tmp/worker-pool.yaml
+gcloud run worker-pools replace /tmp/worker-pool.yaml --region "$REGION"
+```
+
+The service account needs the `monitoring.metricWriter`, `cloudtrace.agent`, and
+`secretmanager.secretAccessor` roles.
+
+## Run a workflow
+
+```bash
+temporal workflow execute --task-queue cloud-run-worker --type GreetingWorkflow --input '"Temporal"'
+```
+
+Metrics appear in Metrics Explorer and traces in Trace Explorer. Delete the pool with
+`gcloud run worker-pools delete "$WORKER_POOL" --region "$REGION"`.

src/Gcp/CloudRun/OpenTelemetry/TemporalioSamples.Gcp.CloudRun.OpenTelemetry.csproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Temporalio.Extensions.Gcp.CloudRun.OpenTelemetry" />
+  </ItemGroup>
+
+</Project>

src/Gcp/CloudRun/OpenTelemetry/collector-config.yaml

@@ -0,0 +1,82 @@
+# Google-Built OpenTelemetry Collector config for the sidecar: metrics -> Managed Service for Prometheus, traces -> Cloud Trace. Auth via the worker-pool service account (ADC).
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: localhost:4317
+
+processors:
+  # Batch traces for throughput. Do NOT add a batch processor to the cumulative-metrics pipeline: a
+  # shutdown flush could be batched with a recent periodic export of the same series and rejected as
+  # a duplicate time series.
+  batch/traces:
+    send_batch_max_size: 200
+    send_batch_size: 200
+    timeout: 5s
+  memory_limiter:
+    check_interval: 1s
+    limit_percentage: 65
+    spike_limit_percentage: 20
+  resourcedetection:
+    detectors: [gcp]
+    timeout: 10s
+  # Rename Temporal datapoint labels that collide with the target labels Managed Service for Prometheus injects (e.g. `namespace`).
+  transform/collision:
+    metric_statements:
+      - context: datapoint
+        statements:
+          - set(attributes["exported_location"], attributes["location"])
+          - delete_key(attributes, "location")
+          - set(attributes["exported_cluster"], attributes["cluster"])
+          - delete_key(attributes, "cluster")
+          - set(attributes["exported_namespace"], attributes["namespace"])
+          - delete_key(attributes, "namespace")
+          - set(attributes["exported_job"], attributes["job"])
+          - delete_key(attributes, "job")
+          - set(attributes["exported_instance"], attributes["instance"])
+          - delete_key(attributes, "instance")
+          - set(attributes["exported_project_id"], attributes["project_id"])
+          - delete_key(attributes, "project_id")
+  # The Telemetry API expects the Google Cloud project in gcp.project_id.
+  transform/set_project_id:
+    error_mode: ignore
+    trace_statements:
+      - set(resource.attributes["gcp.project_id"], resource.attributes["gcp.project.id"]) where resource.attributes["gcp.project.id"] != nil
+      - set(resource.attributes["gcp.project_id"], resource.attributes["cloud.account.id"]) where resource.attributes["gcp.project_id"] == nil and resource.attributes["cloud.account.id"] != nil
+
+exporters:
+  googlemanagedprometheus:
+  otlp:
+    endpoint: telemetry.googleapis.com:443
+    compression: none
+    balancer_name: pick_first
+    auth:
+      authenticator: googleclientauth
+
+extensions:
+  health_check:
+    endpoint: 0.0.0.0:13133
+  googleclientauth:
+
+service:
+  extensions:
+    - health_check
+    - googleclientauth
+  pipelines:
+    metrics:
+      receivers: [otlp]
+      processors: [memory_limiter, resourcedetection, transform/collision]
+      exporters: [googlemanagedprometheus]
+    traces:
+      receivers: [otlp]
+      processors: [memory_limiter, resourcedetection, transform/set_project_id, batch/traces]
+      exporters: [otlp]
+  telemetry:
+    metrics:
+      readers:
+        - periodic:
+            exporter:
+              otlp:
+                protocol: grpc
+                endpoint: http://localhost:4317
+                insecure: true