From 62b32a9dfef32b976d18c8caf183673cb109f63a Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:25:25 +0000 Subject: [PATCH 1/2] feat(apidocs): Publish chunk upload and assemble endpoints as experimental Mark chunk upload (GET options, POST chunks), DIF assemble, and artifact bundle assemble as PUBLIC_EXPERIMENTAL with full request and response schemas, so generated clients such as `@sentry/api` can call them and pick up the `x-sentry-idempotent` markers from #128930. The assemble endpoints share their jsonschema between validation and the docs. The DIF body is a map keyed by file checksum; OpenAPI 3.0 has no `patternProperties`, so its schema uses `additionalProperties`, and the request body check now validates field descriptions under a map's `additionalProperties`. Refs #128927 Co-Authored-By: BYK --- src/sentry/api/endpoints/chunk.py | 107 ++++++++++++---- src/sentry/api/endpoints/debug_files.py | 117 +++++++++++++----- .../organization_artifactbundle_assemble.py | 109 ++++++++++++---- src/sentry/apidocs/hooks.py | 3 + tests/apidocs/test_hooks.py | 42 +++++++ .../sentry/api/endpoints/test_chunk_upload.py | 2 +- 6 files changed, 305 insertions(+), 75 deletions(-) diff --git a/src/sentry/api/endpoints/chunk.py b/src/sentry/api/endpoints/chunk.py index 92281ab1df86..f1962f9e8d8a 100644 --- a/src/sentry/api/endpoints/chunk.py +++ b/src/sentry/api/endpoints/chunk.py @@ -2,11 +2,13 @@ import re from gzip import BadGzipFile, GzipFile from io import BytesIO +from typing import TypedDict import zstandard from django.conf import settings from django.urls import reverse -from rest_framework import status +from drf_spectacular.utils import extend_schema +from rest_framework import serializers, status from rest_framework.request import Request from rest_framework.response import Response @@ -16,6 +18,15 @@ from sentry.api.base import cell_silo_endpoint from sentry.api.bases.organization import OrganizationEndpoint, OrganizationReleasePermission from sentry.api.utils import generate_locality_url +from sentry.apidocs.constants import ( + RESPONSE_BAD_REQUEST, + RESPONSE_FORBIDDEN, + RESPONSE_NOT_FOUND, + RESPONSE_SUCCESS, + RESPONSE_UNAUTHORIZED, +) +from sentry.apidocs.parameters import GlobalParams +from sentry.apidocs.utils import inline_sentry_response_serializer from sentry.models.files.fileblob import FileBlob from sentry.models.files.utils import MAX_FILE_SIZE from sentry.models.organization import Organization @@ -126,11 +137,40 @@ def has_object_permission(self, request: Request, view, organization) -> bool: return super().has_object_permission(request, view, organization) +class ChunkUploadOptionsResponse(TypedDict): + url: str + chunkSize: int + chunksPerRequest: int + maxFileSize: int + maxRequestSize: int + concurrency: int + hashAlgorithm: str + compression: list[str] + accept: list[str] + + +# Only describes the multipart body in the OpenAPI schema; the endpoint reads +# ``request.FILES`` directly. +class ChunkUploadSerializer(serializers.Serializer): + file = serializers.ListField( + child=serializers.FileField(), + help_text=( + "The chunks to upload. Name each part after the SHA-1 checksum of its uncompressed " + "content. Repeat the field for each chunk, up to `chunksPerRequest` per request." + ), + ) + + +class _ChunkUploadErrorResponse(TypedDict): + error: str + + +@extend_schema(tags=["Organizations"]) @cell_silo_endpoint class ChunkUploadEndpoint(OrganizationEndpoint): publish_status = { - "GET": ApiPublishStatus.PRIVATE, - "POST": ApiPublishStatus.PRIVATE, + "GET": ApiPublishStatus.PUBLIC_EXPERIMENTAL, + "POST": ApiPublishStatus.PUBLIC_EXPERIMENTAL, } # Blobs are unique per checksum, so resending a chunk leaves the same single blob. idempotent_methods = frozenset({"POST"}) @@ -143,11 +183,25 @@ class ChunkUploadEndpoint(OrganizationEndpoint): permission_classes = (ChunkUploadPermission,) rate_limits = RateLimitConfig(group="CLI") - def get(self, request: Request, organization: Organization) -> Response: + @extend_schema( + operation_id="getOrganizationChunkUploadOptions", + summary="Retrieve Chunk Upload Options", + parameters=[GlobalParams.ORG_ID_OR_SLUG], + responses={ + 200: inline_sentry_response_serializer( + "ChunkUploadOptionsResponse", ChunkUploadOptionsResponse + ), + 401: RESPONSE_UNAUTHORIZED, + 403: RESPONSE_FORBIDDEN, + 404: RESPONSE_NOT_FOUND, + }, + ) + def get( + self, request: Request, organization: Organization + ) -> Response[ChunkUploadOptionsResponse]: """ - Return chunk upload parameters - `````````````````````````````` - :auth: required + Return the URL and limits to use when uploading file chunks, such as the chunk size, + the number of chunks per request, and the supported compression. """ endpoint = options.get("system.upload-url-prefix") relative_url = reverse("sentry-api-0-chunk-upload", args=[organization.slug]) @@ -193,7 +247,7 @@ def get(self, request: Request, organization: Organization) -> Response: if organization.id in options.get("chunk-upload.no-compression") else list(CHUNK_UPLOAD_COMPRESSION) ) - accept = CHUNK_UPLOAD_ACCEPT + accept = list(CHUNK_UPLOAD_ACCEPT) # Sentry CLI versions ≤2.39.1 require "chunkSize" to be a power of two, and will error otherwise, # with no way for the user to work around the error. This restriction has been removed from @@ -214,23 +268,30 @@ def get(self, request: Request, organization: Organization) -> Response: } ) - def post(self, request: Request, organization) -> Response: + @extend_schema( + operation_id="uploadOrganizationChunks", + summary="Upload File Chunks", + parameters=[GlobalParams.ORG_ID_OR_SLUG], + request={"multipart/form-data": ChunkUploadSerializer}, + responses={ + 200: RESPONSE_SUCCESS, + 400: RESPONSE_BAD_REQUEST, + 401: RESPONSE_UNAUTHORIZED, + 403: RESPONSE_FORBIDDEN, + 404: RESPONSE_NOT_FOUND, + }, + ) + def post( + self, request: Request, organization + ) -> Response[None] | Response[_ChunkUploadErrorResponse]: """ - Upload chunks and store them as FileBlobs - ````````````````````````````````````````` - Requests to this endpoint should use the region-specific domain - eg. `us.sentry.io` or `de.sentry.io` - - Chunks may be compressed with ``gzip`` or ``zstd``. Codec is selected - via the ``Content-Encoding`` request header; the legacy ``file_gzip`` - multipart field is still accepted but cannot be combined with the - header. - - :pparam file file: The filename should be sha1 hash of the content. - Also not you can add up to MAX_CHUNKS_PER_REQUEST files - in this request. + Upload file chunks so they can be assembled into debug information files or artifact + bundles. Send requests to the URL from the chunk upload options, which uses the + region-specific domain (for example `us.sentry.io` or `de.sentry.io`). - :auth: required + Chunks may be compressed with one of the codecs listed in the options. Select it with + the `Content-Encoding` request header. Resending a chunk is safe: chunks are stored + by checksum, so an identical upload changes nothing. """ # Create a unique instance so our logger can be decoupled from the request # and used in threads. diff --git a/src/sentry/api/endpoints/debug_files.py b/src/sentry/api/endpoints/debug_files.py index aac427d32140..a173a69ae996 100644 --- a/src/sentry/api/endpoints/debug_files.py +++ b/src/sentry/api/endpoints/debug_files.py @@ -36,7 +36,12 @@ from sentry.api.serializers import serialize from sentry.api.serializers.models.debug_file import DebugFileSerializerResponse from sentry.api.utils import to_valid_int_id -from sentry.apidocs.constants import RESPONSE_FORBIDDEN, RESPONSE_NOT_FOUND, RESPONSE_UNAUTHORIZED +from sentry.apidocs.constants import ( + RESPONSE_BAD_REQUEST, + RESPONSE_FORBIDDEN, + RESPONSE_NOT_FOUND, + RESPONSE_UNAUTHORIZED, +) from sentry.apidocs.examples.dsym_examples import DebugFileExamples from sentry.apidocs.parameters import CursorQueryParam, GlobalParams from sentry.apidocs.utils import inline_sentry_response_serializer @@ -592,29 +597,61 @@ class AssembleRequestFile(TypedDict): """Mapping from file checksums to the corresponding assemble request payload.""" +_SHA1_PATTERN = "^[0-9a-f]{40}$" + +# One file in the assemble request. Shared by validation and the OpenAPI schema. +_ASSEMBLE_REQUEST_FILE_SCHEMA: dict[str, object] = { + "type": "object", + "required": ["name", "chunks"], + "properties": { + "name": { + "type": "string", + "description": "The file name. A ProGuard mapping without `debug_id` can carry it in the name, as in `/proguard/mapping-.txt`.", + }, + "debug_id": { + "type": "string", + "pattern": "^[A-Fa-f0-9-]+$", + "description": "The debug ID to store the file under, overriding the one read from the file.", + }, + "chunks": { + "type": "array", + "items": {"type": "string", "pattern": _SHA1_PATTERN}, + "description": "The SHA-1 checksums of the file's chunks, in order. Upload missing chunks with the chunk upload endpoint. Send an empty list to poll the file's state.", + }, + }, + "additionalProperties": True, +} + +_ASSEMBLE_REQUEST_SCHEMA: dict[str, object] = { + "type": "object", + "patternProperties": {_SHA1_PATTERN: _ASSEMBLE_REQUEST_FILE_SCHEMA}, + "additionalProperties": False, +} + +# OpenAPI 3.0 has no patternProperties, so the docs describe the checksum keys in prose. +_ASSEMBLE_REQUEST_OPENAPI_SCHEMA: dict[str, object] = { + "type": "object", + "description": "The files to assemble, keyed by the SHA-1 checksum of each whole file.", + "additionalProperties": _ASSEMBLE_REQUEST_FILE_SCHEMA, +} + + +class DifAssembleFileResponse(TypedDict): + # One of the ChunkFileState values: ok, not_found, created, assembling, or error. + state: str + missingChunks: list[str] + detail: NotRequired[str | None] + dif: NotRequired[DebugFileSerializerResponse] + + +class _AssembleErrorResponse(TypedDict): + error: str + + def parse_assemble_request_payload(body: bytes) -> AssembleRequestPayload: """Parse and validate the DIF assemble request body.""" - schema: dict[str, object] = { - "type": "object", - "patternProperties": { - "^[0-9a-f]{40}$": { - "type": "object", - "required": ["name", "chunks"], - "properties": { - "name": {"type": "string"}, - "debug_id": {"type": "string", "pattern": "^[A-Fa-f0-9-]+$"}, - "chunks": { - "type": "array", - "items": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, - }, - }, - "additionalProperties": True, - } - }, - "additionalProperties": False, - } payload_obj: object = orjson.loads(body) - jsonschema.validate(payload_obj, schema) + jsonschema.validate(payload_obj, _ASSEMBLE_REQUEST_SCHEMA) return cast(AssembleRequestPayload, payload_obj) @@ -629,12 +666,14 @@ def get_file_info(file: AssembleRequestFile) -> tuple[str, str | None, list[str] return name, debug_id, chunks -def batch_assemble(project: Project, files: AssembleRequestPayload): +def batch_assemble( + project: Project, files: AssembleRequestPayload +) -> dict[str, DifAssembleFileResponse]: """ Performs assembling in a batch fashion, issuing queries that span multiple files. """ files_to_check = files.copy() - file_response = {} + file_response: dict[str, DifAssembleFileResponse] = {} # 1. Exclude all files that already have an assemble status. for checksum, file in list(files_to_check.items()): @@ -903,7 +942,7 @@ def _clone_proguard_debug_file_for_reupload( debug_file: ProjectDebugFile, requested_debug_id: str, is_proguard_clone_source: bool, -) -> dict[str, object]: +) -> DifAssembleFileResponse: """Clone a ProGuard debug file row for a reupload and return a batch response. ``is_proguard_clone_source`` must reflect the caller's annotation-based @@ -979,23 +1018,43 @@ def _clone_proguard_debug_file_for_reupload( } +@extend_schema(tags=["Projects"]) @cell_silo_endpoint class DifAssembleEndpoint(ProjectEndpoint): owner = ApiOwner.OWNERS_INGEST publish_status = { - "POST": ApiPublishStatus.PRIVATE, + "POST": ApiPublishStatus.PUBLIC_EXPERIMENTAL, } # Assemble state is cached per checksum, so a resend reports it instead of # starting a second job. idempotent_methods = frozenset({"POST"}) permission_classes = (ProjectReleasePermission,) - def post(self, request: Request, project: Project) -> Response: + @extend_schema( + operation_id="assembleProjectDebugFiles", + summary="Assemble Debug Information Files", + parameters=[GlobalParams.ORG_ID_OR_SLUG, GlobalParams.PROJECT_ID_OR_SLUG], + request={"application/json": _ASSEMBLE_REQUEST_OPENAPI_SCHEMA}, + responses={ + 200: inline_sentry_response_serializer( + "AssembleProjectDebugFilesResponse", dict[str, DifAssembleFileResponse] + ), + 400: RESPONSE_BAD_REQUEST, + 401: RESPONSE_UNAUTHORIZED, + 403: RESPONSE_FORBIDDEN, + 404: RESPONSE_NOT_FOUND, + }, + ) + def post( + self, request: Request, project: Project + ) -> Response[dict[str, DifAssembleFileResponse]] | Response[_AssembleErrorResponse]: """ - Assemble one or multiple chunks (FileBlob) into debug files - ```````````````````````````````````````````````````````````` + Assemble previously uploaded chunks into debug information files. - :auth: required + The response reports each file's `state`, keyed by the same checksums. If chunks are + missing, upload them and send the request again. Repeat the request to poll until the + state is `ok` or `error`. Resending an identical request is safe: a file that is + already assembling or assembled reports its state instead of starting a new job. """ try: files = parse_assemble_request_payload(request.body) diff --git a/src/sentry/api/endpoints/organization_artifactbundle_assemble.py b/src/sentry/api/endpoints/organization_artifactbundle_assemble.py index d67459f8d735..ee55231a47da 100644 --- a/src/sentry/api/endpoints/organization_artifactbundle_assemble.py +++ b/src/sentry/api/endpoints/organization_artifactbundle_assemble.py @@ -1,6 +1,9 @@ +from typing import NotRequired, TypedDict + import jsonschema import orjson from django.db.models import Q +from drf_spectacular.utils import extend_schema from rest_framework.request import Request from rest_framework.response import Response from sentry_sdk import traces @@ -10,6 +13,14 @@ from sentry.api.base import cell_silo_endpoint from sentry.api.bases.organization import OrganizationReleasesBaseEndpoint from sentry.api.exceptions import ResourceDoesNotExist +from sentry.apidocs.constants import ( + RESPONSE_BAD_REQUEST, + RESPONSE_FORBIDDEN, + RESPONSE_NOT_FOUND, + RESPONSE_UNAUTHORIZED, +) +from sentry.apidocs.parameters import GlobalParams +from sentry.apidocs.utils import inline_sentry_response_serializer from sentry.constants import ObjectStatus from sentry.debug_files.upload import find_missing_chunks from sentry.models.orgauthtoken import is_org_auth_token_auth, update_org_auth_token_last_used @@ -21,41 +32,95 @@ set_assemble_status, ) - +_SHA1_PATTERN = "^[0-9a-f]{40}$" + +# Shared by request validation and the OpenAPI schema. +_ASSEMBLE_REQUEST_SCHEMA: dict[str, object] = { + "type": "object", + "properties": { + # The version pattern has been extracted from the url definition of OrganizationReleaseAssembleEndpoint. + "version": { + "type": "string", + "pattern": "^[^/]+$", + "description": "The release to associate the bundle with.", + }, + "dist": { + "type": "string", + "description": "The distribution to associate the bundle with. Requires `version`.", + }, + "projects": { + "type": "array", + "items": {"type": "string"}, + "description": "The IDs or slugs of the projects the bundle belongs to.", + }, + "checksum": { + "type": "string", + "pattern": _SHA1_PATTERN, + "description": "The SHA-1 checksum of the whole bundle.", + }, + "chunks": { + "type": "array", + "items": {"type": "string", "pattern": _SHA1_PATTERN}, + "description": "The SHA-1 checksums of the bundle's chunks, in order. Upload missing chunks with the chunk upload endpoint. Send an empty list to poll the bundle's state.", + }, + }, + "required": ["checksum", "chunks", "projects"], + "additionalProperties": False, +} + + +class ArtifactBundleAssembleResponse(TypedDict): + # One of the ChunkFileState values: ok, not_found, created, assembling, or error. + state: str + missingChunks: list[str] + detail: NotRequired[str | None] + + +class _AssembleErrorResponse(TypedDict): + error: str + + +@extend_schema(tags=["Releases"]) @cell_silo_endpoint class OrganizationArtifactBundleAssembleEndpoint(OrganizationReleasesBaseEndpoint): owner = ApiOwner.OWNERS_INGEST publish_status = { - "POST": ApiPublishStatus.PRIVATE, + "POST": ApiPublishStatus.PUBLIC_EXPERIMENTAL, } # Assemble state is cached per checksum, so a resend reports it instead of # starting a second job. idempotent_methods = frozenset({"POST"}) - def post(self, request: Request, organization) -> Response: + @extend_schema( + operation_id="assembleOrganizationArtifactBundle", + summary="Assemble an Artifact Bundle", + parameters=[GlobalParams.ORG_ID_OR_SLUG], + request={"application/json": _ASSEMBLE_REQUEST_SCHEMA}, + responses={ + 200: inline_sentry_response_serializer( + "ArtifactBundleAssembleResponse", ArtifactBundleAssembleResponse + ), + 400: RESPONSE_BAD_REQUEST, + 401: RESPONSE_UNAUTHORIZED, + 403: RESPONSE_FORBIDDEN, + 404: RESPONSE_NOT_FOUND, + }, + ) + def post( + self, request: Request, organization + ) -> Response[ArtifactBundleAssembleResponse] | Response[_AssembleErrorResponse]: """ - Assembles an artifact bundle and stores the debug ids in the database. + Assemble previously uploaded chunks into an artifact bundle, such as source maps, and + store its debug IDs. + + The response reports the bundle's `state`. If chunks are missing, upload them and send + the request again. Repeat the request to poll until the state is `ok` or `error`. + Resending an identical request is safe: a bundle that is already assembling or + assembled reports its state instead of starting a new job. """ with traces.start_span( name="artifact_bundle.assemble", attributes={"sentry.op": "artifact_bundle.assemble"} ): - schema = { - "type": "object", - "properties": { - # The version pattern has been extracted from the url definition of OrganizationReleaseAssembleEndpoint. - "version": {"type": "string", "pattern": "^[^/]+$"}, - "dist": {"type": "string"}, - "projects": {"type": "array", "items": {"type": "string"}}, - "checksum": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, - "chunks": { - "type": "array", - "items": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, - }, - }, - "required": ["checksum", "chunks", "projects"], - "additionalProperties": False, - } - error_messages = { "version": 'The version field cannot be empty and cannot contain any "/" characters.', "dist": "The dist field must be a string.", @@ -66,7 +131,7 @@ def post(self, request: Request, organization) -> Response: try: data = orjson.loads(request.body) - jsonschema.validate(data, schema) + jsonschema.validate(data, _ASSEMBLE_REQUEST_SCHEMA) except jsonschema.ValidationError as e: error_message = e.message # Get the field from the path if available diff --git a/src/sentry/apidocs/hooks.py b/src/sentry/apidocs/hooks.py index d413163ac294..f5755a9d7296 100644 --- a/src/sentry/apidocs/hooks.py +++ b/src/sentry/apidocs/hooks.py @@ -284,6 +284,9 @@ def _validate_request_body( # Dereference schema if needed and raise error on schema component collisions schema = dereference_schema(schema, schema_components) + if "properties" not in schema and isinstance(schema.get("additionalProperties"), dict): + # A map body, such as files keyed by checksum: its values carry the named fields. + schema = dereference_schema(schema["additionalProperties"], schema_components) for body_param, param_data in schema["properties"].items(): # Ensure body parameters have a description. Our API docs don't diff --git a/tests/apidocs/test_hooks.py b/tests/apidocs/test_hooks.py index b9e5f2b1eb2e..0e801be9be58 100644 --- a/tests/apidocs/test_hooks.py +++ b/tests/apidocs/test_hooks.py @@ -303,6 +303,48 @@ def test_unique_summaries_pass(self) -> None: custom_postprocessing_hook(result, None) +class MapRequestBodyTest(TestCase): + """A body keyed by arbitrary strings documents the fields of its values.""" + + def _result(self, value_properties: dict[str, Any]) -> dict[str, Any]: + return { + "components": {"schemas": {}}, + "paths": { + "/api/0/assemble/": { + "post": { + "tags": ["Events"], + "description": "An endpoint", + "operationId": "assemble", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": value_properties, + }, + } + } + } + }, + } + } + }, + } + + def test_undescribed_value_field_raises(self) -> None: + with pytest.raises(SentryApiBuildError, match="'name' is missing a description"): + custom_postprocessing_hook(self._result({"name": {"type": "string"}}), None) + + def test_described_value_fields_pass(self) -> None: + # Should not raise. + custom_postprocessing_hook( + self._result({"name": {"type": "string", "description": "The file name."}}), None + ) + + class FixIssueRoutesTest(TestCase): def test_issue_route_fixes(self) -> None: BEFORE = { diff --git a/tests/sentry/api/endpoints/test_chunk_upload.py b/tests/sentry/api/endpoints/test_chunk_upload.py index d350b6e482ae..d9756572d535 100644 --- a/tests/sentry/api/endpoints/test_chunk_upload.py +++ b/tests/sentry/api/endpoints/test_chunk_upload.py @@ -61,7 +61,7 @@ def test_chunk_parameters(self) -> None: assert response.data["concurrency"] == MAX_CONCURRENCY assert response.data["hashAlgorithm"] == HASH_ALGORITHM assert response.data["url"] == generate_locality_url() + self.url - assert response.data["accept"] == CHUNK_UPLOAD_ACCEPT + assert response.data["accept"] == list(CHUNK_UPLOAD_ACCEPT) assert response.data["compression"] == list(CHUNK_UPLOAD_COMPRESSION) assert "gzip" in response.data["compression"] assert "zstd" in response.data["compression"] From 5ddb64b773c6687332632fdfcce6140237e3bf10 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:37:08 +0000 Subject: [PATCH 2/2] test(seer): Add artifact bundle assemble payload to agent token matrix The public mutation matrix now covers the artifact bundle assemble endpoint, which rejects an empty body. Send a status poll (no chunks) so the session control succeeds without starting an assemble task. --- .../sentry/seer/endpoints/test_organization_agent_token.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/sentry/seer/endpoints/test_organization_agent_token.py b/tests/sentry/seer/endpoints/test_organization_agent_token.py index f4350200d333..0aae0aea9e6e 100644 --- a/tests/sentry/seer/endpoints/test_organization_agent_token.py +++ b/tests/sentry/seer/endpoints/test_organization_agent_token.py @@ -2006,6 +2006,12 @@ def _mutation_payload(self, endpoint: PublicMutationEndpoint) -> dict[str, Any]: ("OrganizationDashboardDetailsEndpoint", "PUT"): { "title": "Updated permission matrix dashboard" }, + # No chunks makes this a status poll, so no assemble task is started. + ("OrganizationArtifactBundleAssembleEndpoint", "POST"): { + "checksum": "a" * 40, + "chunks": [], + "projects": [self.project.slug], + }, ("OrganizationDashboardsEndpoint", "POST"): { "title": "Permission matrix dashboard mutation" },