feat(apidocs): Publish chunk upload and assemble endpoints as experimental

src/sentry/api/endpoints/chunk.py

@@ -2,11 +2,13 @@
 import re
 from gzip import BadGzipFile, GzipFile
 from io import BytesIO
+from typing import TypedDict
 
 import zstandard
 from django.conf import settings
 from django.urls import reverse
-from rest_framework import status
+from drf_spectacular.utils import extend_schema
+from rest_framework import serializers, status
 from rest_framework.request import Request
 from rest_framework.response import Response
 
@@ -16,6 +18,15 @@
 from sentry.api.base import cell_silo_endpoint
 from sentry.api.bases.organization import OrganizationEndpoint, OrganizationReleasePermission
 from sentry.api.utils import generate_locality_url
+from sentry.apidocs.constants import (
+    RESPONSE_BAD_REQUEST,
+    RESPONSE_FORBIDDEN,
+    RESPONSE_NOT_FOUND,
+    RESPONSE_SUCCESS,
+    RESPONSE_UNAUTHORIZED,
+)
+from sentry.apidocs.parameters import GlobalParams
+from sentry.apidocs.utils import inline_sentry_response_serializer
 from sentry.models.files.fileblob import FileBlob
 from sentry.models.files.utils import MAX_FILE_SIZE
 from sentry.models.organization import Organization
@@ -126,11 +137,40 @@ def has_object_permission(self, request: Request, view, organization) -> bool:
         return super().has_object_permission(request, view, organization)
 
 
+class ChunkUploadOptionsResponse(TypedDict):
+    url: str
+    chunkSize: int
+    chunksPerRequest: int
+    maxFileSize: int
+    maxRequestSize: int
+    concurrency: int
+    hashAlgorithm: str
+    compression: list[str]
+    accept: list[str]
+
+
+# Only describes the multipart body in the OpenAPI schema; the endpoint reads
+# ``request.FILES`` directly.
+class ChunkUploadSerializer(serializers.Serializer):
+    file = serializers.ListField(
+        child=serializers.FileField(),
+        help_text=(
+            "The chunks to upload. Name each part after the SHA-1 checksum of its uncompressed "
+            "content. Repeat the field for each chunk, up to `chunksPerRequest` per request."
+        ),
+    )
+
+
+class _ChunkUploadErrorResponse(TypedDict):
+    error: str
+
+
+@extend_schema(tags=["Organizations"])
 @cell_silo_endpoint
 class ChunkUploadEndpoint(OrganizationEndpoint):
     publish_status = {
-        "GET": ApiPublishStatus.PRIVATE,
-        "POST": ApiPublishStatus.PRIVATE,
+        "GET": ApiPublishStatus.PUBLIC_EXPERIMENTAL,
+        "POST": ApiPublishStatus.PUBLIC_EXPERIMENTAL,
     }
     # Blobs are unique per checksum, so resending a chunk leaves the same single blob.
     idempotent_methods = frozenset({"POST"})
@@ -143,11 +183,25 @@ class ChunkUploadEndpoint(OrganizationEndpoint):
     permission_classes = (ChunkUploadPermission,)
     rate_limits = RateLimitConfig(group="CLI")
 
-    def get(self, request: Request, organization: Organization) -> Response:
+    @extend_schema(
+        operation_id="getOrganizationChunkUploadOptions",
+        summary="Retrieve Chunk Upload Options",
+        parameters=[GlobalParams.ORG_ID_OR_SLUG],
+        responses={
+            200: inline_sentry_response_serializer(
+                "ChunkUploadOptionsResponse", ChunkUploadOptionsResponse
+            ),
+            401: RESPONSE_UNAUTHORIZED,
+            403: RESPONSE_FORBIDDEN,
+            404: RESPONSE_NOT_FOUND,
+        },
+    )
+    def get(
+        self, request: Request, organization: Organization
+    ) -> Response[ChunkUploadOptionsResponse]:
         """
-        Return chunk upload parameters
-        ``````````````````````````````
-        :auth: required
+        Return the URL and limits to use when uploading file chunks, such as the chunk size,
+        the number of chunks per request, and the supported compression.
         """
         endpoint = options.get("system.upload-url-prefix")
         relative_url = reverse("sentry-api-0-chunk-upload", args=[organization.slug])
@@ -193,7 +247,7 @@ def get(self, request: Request, organization: Organization) -> Response:
             if organization.id in options.get("chunk-upload.no-compression")
             else list(CHUNK_UPLOAD_COMPRESSION)
         )
-        accept = CHUNK_UPLOAD_ACCEPT
+        accept = list(CHUNK_UPLOAD_ACCEPT)
 
         # Sentry CLI versions ≤2.39.1 require "chunkSize" to be a power of two, and will error otherwise,
         # with no way for the user to work around the error. This restriction has been removed from
@@ -214,23 +268,30 @@ def get(self, request: Request, organization: Organization) -> Response:
             }
         )
 
-    def post(self, request: Request, organization) -> Response:
+    @extend_schema(
+        operation_id="uploadOrganizationChunks",
+        summary="Upload File Chunks",
+        parameters=[GlobalParams.ORG_ID_OR_SLUG],
+        request={"multipart/form-data": ChunkUploadSerializer},
+        responses={
+            200: RESPONSE_SUCCESS,
+            400: RESPONSE_BAD_REQUEST,
+            401: RESPONSE_UNAUTHORIZED,
+            403: RESPONSE_FORBIDDEN,
+            404: RESPONSE_NOT_FOUND,
+        },
+    )
+    def post(
+        self, request: Request, organization
+    ) -> Response[None] | Response[_ChunkUploadErrorResponse]:
         """
-        Upload chunks and store them as FileBlobs
-        `````````````````````````````````````````
-        Requests to this endpoint should use the region-specific domain
-        eg. `us.sentry.io` or `de.sentry.io`
-
-        Chunks may be compressed with ``gzip`` or ``zstd``. Codec is selected
-        via the ``Content-Encoding`` request header; the legacy ``file_gzip``
-        multipart field is still accepted but cannot be combined with the
-        header.
-
-        :pparam file file: The filename should be sha1 hash of the content.
-                            Also not you can add up to MAX_CHUNKS_PER_REQUEST files
-                            in this request.
+        Upload file chunks so they can be assembled into debug information files or artifact
+        bundles. Send requests to the URL from the chunk upload options, which uses the
+        region-specific domain (for example `us.sentry.io` or `de.sentry.io`).
 
-        :auth: required
+        Chunks may be compressed with one of the codecs listed in the options. Select it with
+        the `Content-Encoding` request header. Resending a chunk is safe: chunks are stored
+        by checksum, so an identical upload changes nothing.
         """
         # Create a unique instance so our logger can be decoupled from the request
         # and used in threads.

src/sentry/api/endpoints/debug_files.py

@@ -36,7 +36,12 @@
 from sentry.api.serializers import serialize
 from sentry.api.serializers.models.debug_file import DebugFileSerializerResponse
 from sentry.api.utils import to_valid_int_id
-from sentry.apidocs.constants import RESPONSE_FORBIDDEN, RESPONSE_NOT_FOUND, RESPONSE_UNAUTHORIZED
+from sentry.apidocs.constants import (
+    RESPONSE_BAD_REQUEST,
+    RESPONSE_FORBIDDEN,
+    RESPONSE_NOT_FOUND,
+    RESPONSE_UNAUTHORIZED,
+)
 from sentry.apidocs.examples.dsym_examples import DebugFileExamples
 from sentry.apidocs.parameters import CursorQueryParam, GlobalParams
 from sentry.apidocs.utils import inline_sentry_response_serializer
@@ -592,29 +597,61 @@ class AssembleRequestFile(TypedDict):
 """Mapping from file checksums to the corresponding assemble request payload."""
 
 
+_SHA1_PATTERN = "^[0-9a-f]{40}$"
+
+# One file in the assemble request. Shared by validation and the OpenAPI schema.
+_ASSEMBLE_REQUEST_FILE_SCHEMA: dict[str, object] = {
+    "type": "object",
+    "required": ["name", "chunks"],
+    "properties": {
+        "name": {
+            "type": "string",
+            "description": "The file name. A ProGuard mapping without `debug_id` can carry it in the name, as in `/proguard/mapping-<uuid>.txt`.",
+        },
+        "debug_id": {
+            "type": "string",
+            "pattern": "^[A-Fa-f0-9-]+$",
+            "description": "The debug ID to store the file under, overriding the one read from the file.",
+        },
+        "chunks": {
+            "type": "array",
+            "items": {"type": "string", "pattern": _SHA1_PATTERN},
+            "description": "The SHA-1 checksums of the file's chunks, in order. Upload missing chunks with the chunk upload endpoint. Send an empty list to poll the file's state.",
+        },
+    },
+    "additionalProperties": True,
+}
+
+_ASSEMBLE_REQUEST_SCHEMA: dict[str, object] = {
+    "type": "object",
+    "patternProperties": {_SHA1_PATTERN: _ASSEMBLE_REQUEST_FILE_SCHEMA},
+    "additionalProperties": False,
+}
+
+# OpenAPI 3.0 has no patternProperties, so the docs describe the checksum keys in prose.
+_ASSEMBLE_REQUEST_OPENAPI_SCHEMA: dict[str, object] = {
+    "type": "object",
+    "description": "The files to assemble, keyed by the SHA-1 checksum of each whole file.",
+    "additionalProperties": _ASSEMBLE_REQUEST_FILE_SCHEMA,
+}
+
+
+class DifAssembleFileResponse(TypedDict):
+    # One of the ChunkFileState values: ok, not_found, created, assembling, or error.
+    state: str
+    missingChunks: list[str]
+    detail: NotRequired[str | None]
+    dif: NotRequired[DebugFileSerializerResponse]
+
+
+class _AssembleErrorResponse(TypedDict):
+    error: str
+
+
 def parse_assemble_request_payload(body: bytes) -> AssembleRequestPayload:
     """Parse and validate the DIF assemble request body."""
-    schema: dict[str, object] = {
-        "type": "object",
-        "patternProperties": {
-            "^[0-9a-f]{40}$": {
-                "type": "object",
-                "required": ["name", "chunks"],
-                "properties": {
-                    "name": {"type": "string"},
-                    "debug_id": {"type": "string", "pattern": "^[A-Fa-f0-9-]+$"},
-                    "chunks": {
-                        "type": "array",
-                        "items": {"type": "string", "pattern": "^[0-9a-f]{40}$"},
-                    },
-                },
-                "additionalProperties": True,
-            }
-        },
-        "additionalProperties": False,
-    }
     payload_obj: object = orjson.loads(body)
-    jsonschema.validate(payload_obj, schema)
+    jsonschema.validate(payload_obj, _ASSEMBLE_REQUEST_SCHEMA)
     return cast(AssembleRequestPayload, payload_obj)
 
 
@@ -629,12 +666,14 @@ def get_file_info(file: AssembleRequestFile) -> tuple[str, str | None, list[str]
     return name, debug_id, chunks
 
 
-def batch_assemble(project: Project, files: AssembleRequestPayload):
+def batch_assemble(
+    project: Project, files: AssembleRequestPayload
+) -> dict[str, DifAssembleFileResponse]:
     """
     Performs assembling in a batch fashion, issuing queries that span multiple files.
     """
     files_to_check = files.copy()
-    file_response = {}
+    file_response: dict[str, DifAssembleFileResponse] = {}
 
     # 1. Exclude all files that already have an assemble status.
     for checksum, file in list(files_to_check.items()):
@@ -903,7 +942,7 @@ def _clone_proguard_debug_file_for_reupload(
     debug_file: ProjectDebugFile,
     requested_debug_id: str,
     is_proguard_clone_source: bool,
-) -> dict[str, object]:
+) -> DifAssembleFileResponse:
     """Clone a ProGuard debug file row for a reupload and return a batch response.
 
     ``is_proguard_clone_source`` must reflect the caller's annotation-based
@@ -979,23 +1018,43 @@ def _clone_proguard_debug_file_for_reupload(
     }
 
 
+@extend_schema(tags=["Projects"])
 @cell_silo_endpoint
 class DifAssembleEndpoint(ProjectEndpoint):
     owner = ApiOwner.OWNERS_INGEST
     publish_status = {
-        "POST": ApiPublishStatus.PRIVATE,
+        "POST": ApiPublishStatus.PUBLIC_EXPERIMENTAL,
     }
     # Assemble state is cached per checksum, so a resend reports it instead of
     # starting a second job.
     idempotent_methods = frozenset({"POST"})
     permission_classes = (ProjectReleasePermission,)
 
-    def post(self, request: Request, project: Project) -> Response:
+    @extend_schema(
+        operation_id="assembleProjectDebugFiles",
+        summary="Assemble Debug Information Files",
+        parameters=[GlobalParams.ORG_ID_OR_SLUG, GlobalParams.PROJECT_ID_OR_SLUG],
+        request={"application/json": _ASSEMBLE_REQUEST_OPENAPI_SCHEMA},
+        responses={
+            200: inline_sentry_response_serializer(
+                "AssembleProjectDebugFilesResponse", dict[str, DifAssembleFileResponse]
+            ),
+            400: RESPONSE_BAD_REQUEST,
+            401: RESPONSE_UNAUTHORIZED,
+            403: RESPONSE_FORBIDDEN,
+            404: RESPONSE_NOT_FOUND,
+        },
+    )
+    def post(
+        self, request: Request, project: Project
+    ) -> Response[dict[str, DifAssembleFileResponse]] | Response[_AssembleErrorResponse]:
         """
-        Assemble one or multiple chunks (FileBlob) into debug files
-        ````````````````````````````````````````````````````````````
+        Assemble previously uploaded chunks into debug information files.
 
-        :auth: required
+        The response reports each file's `state`, keyed by the same checksums. If chunks are
+        missing, upload them and send the request again. Repeat the request to poll until the
+        state is `ok` or `error`. Resending an identical request is safe: a file that is
+        already assembling or assembled reports its state instead of starting a new job.
         """
         try:
             files = parse_assemble_request_payload(request.body)