From 6d27a4a41265586da395f449242ed00e4a072cce Mon Sep 17 00:00:00 2001 From: Dr Alexander Mikhalev Date: Wed, 14 Oct 2026 08:44:55 +0100 Subject: [PATCH] feat(release): add reviewed private-source LSP artifact handoff Refs #3488 --- .github/scripts/release/lsp_handoff.py | 343 +++++ .../scripts/release/lsp_private_producer.py | 171 +++ .../scripts/release/publish_lsp_release.py | 406 ++++++ .github/workflows/lsp-producer.yml | 223 ++++ .github/workflows/lsp-publish.yml | 44 + .../workflows/release-manifest-contract.yml | 19 +- .../schemas/lsp-lane-receipt.schema.json | 1178 +++++++++++++++++ .../schemas/lsp-release-handoff.schema.json | 90 ++ .../schemas/lsp-source-request.schema.json | 82 ++ docs/plans/lsp-provenance-handoff.md | 35 + tests/lsp_handoff_test.py | 196 +++ tests/lsp_private_producer_test.py | 85 ++ tests/workflow_lsp_handoff_test.py | 100 ++ ...workflow_release_recovery_contract_test.py | 5 +- 14 files changed, 2975 insertions(+), 2 deletions(-) create mode 100644 .github/scripts/release/lsp_handoff.py create mode 100644 .github/scripts/release/lsp_private_producer.py create mode 100644 .github/scripts/release/publish_lsp_release.py create mode 100644 .github/workflows/lsp-producer.yml create mode 100644 .github/workflows/lsp-publish.yml create mode 100644 docs/contracts/schemas/lsp-lane-receipt.schema.json create mode 100644 docs/contracts/schemas/lsp-release-handoff.schema.json create mode 100644 docs/contracts/schemas/lsp-source-request.schema.json create mode 100644 docs/plans/lsp-provenance-handoff.md create mode 100644 tests/lsp_handoff_test.py create mode 100644 tests/lsp_private_producer_test.py create mode 100644 tests/workflow_lsp_handoff_test.py diff --git a/.github/scripts/release/lsp_handoff.py b/.github/scripts/release/lsp_handoff.py new file mode 100644 index 000000000..8f17cf418 --- /dev/null +++ b/.github/scripts/release/lsp_handoff.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 +"""Explicit private-source provenance; never relax the same-source publisher.""" +from __future__ import annotations + +import argparse +import hashlib +import hmac +import importlib.util +import json +import os +from pathlib import Path +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import zipfile + +_spec = importlib.util.spec_from_file_location('lsp_publisher', Path(__file__).with_name('publish_lsp_release.py')) +pub = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(pub) +REPO = 'terraphim/terraphim-ai' +PRIVATE_REPO = 'https://git.terraphim.cloud/' + REPO +WORKFLOW = '.github/workflows/lsp-producer.yml' +PUBLISH_WORKFLOW = '.github/workflows/lsp-publish.yml' +RECIPE = '.github/scripts/release/build_lsp_lane.py' +SOURCE_CHECKS = ('native-ci / build (push)', 'native-clippy / lint (push)') +LANES = ('qualify', 'x86_64-unknown-linux-gnu', 'x86_64-unknown-linux-musl', 'aarch64-unknown-linux-musl', 'macos') +LANE_ASSETS = {name: (('terraphim-lsp-' + name,) if name not in ('qualify', 'macos') else ()) for name in LANES} +LANE_ASSETS['macos'] = tuple(n for n in pub.RAW_ASSETS if n.endswith('apple-darwin')) +COMMAND_NAMES = { + 'qualify': ('recipe-tests', 'install-toolchain', 'fmt', 'clippy', 'test'), + 'x86_64-unknown-linux-gnu': ('install-toolchain', 'fetch', 'build', 'architecture', 'version'), + 'x86_64-unknown-linux-musl': ('install-toolchain', 'install-cross', 'fetch', 'build', 'architecture', 'version'), + 'aarch64-unknown-linux-musl': ('install-toolchain', 'install-cross', 'fetch', 'build', 'architecture'), + 'macos': ('install-toolchain', 'fetch', 'build-arm64', 'build-x86_64', 'lipo', 'architecture-arm64', + 'architecture-x86_64', 'architecture-universal', 'version-native', 'version-universal'), +} +HEX64 = re.compile(r'[0-9a-f]{64}') +MAX_BYTES = 6 * 128 * 1024 * 1024 + 1024 * 1024 + + +def require(condition, message='LSP_PROVENANCE_REJECTED'): + if not condition: + raise ValueError(message) + + +def mapping(value): + require(isinstance(value, dict), 'LSP_METADATA_SHAPE_REJECTED') + return value + + +def keys(value, expected): + require(isinstance(value, dict) and set(value) == set(expected), 'LSP_UNSAFE_OR_MISSING_FIELDS') + + +def hex_value(value, length=64): + require(isinstance(value, str) and re.fullmatch(r'[0-9a-f]{' + str(length) + r'}', value)) + + +def integer(value): + require(type(value) is int and value > 0) + + +def canonical_digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest() + + +def validate_request(value): + keys(value, ('schema', 'component', 'version', 'tag', 'toolchain', 'source', 'recipe')) + require(type(value['schema']) is int and value['schema'] == 1 and value['component'] == 'terraphim-lsp') + require(isinstance(value['version'], str) and re.fullmatch(r'[0-9]+\.[0-9]+\.[0-9]+', value['version'])) + require(isinstance(value['tag'], str)); pub.validate_tag(value['tag'], value['version']) + require(isinstance(value['toolchain'], str) and re.fullmatch(r'[0-9]+\.[0-9]+(?:\.[0-9]+)?', value['toolchain'])) + source = value['source']; keys(source, ('repository', 'commit', 'tag', 'requiredChecks')) + require(source['repository'] == PRIVATE_REPO and source['tag'] == value['tag']) + hex_value(source['commit'], 40) + require(source['requiredChecks'] == list(SOURCE_CHECKS)) + keys(value['recipe'], ('path', 'sha256')); require(value['recipe']['path'] == RECIPE) + hex_value(value['recipe']['sha256']) + return value + + +def validate_producer(value): + keys(value, ('repository', 'workflow', 'controlCommit', 'runId', 'runAttempt', 'event')) + require(value['repository'] == REPO and value['workflow'] == WORKFLOW and value['event'] == 'workflow_dispatch') + hex_value(value['controlCommit'], 40); integer(value['runId']); integer(value['runAttempt']) + return value + + +def validate_assets(records, names): + require(isinstance(records, list) and len(records) == len(names)) + for item, name in zip(records, names): + keys(item, ('name', 'sha256', 'bytes')) + require(item['name'] == name); hex_value(item['sha256']); integer(item['bytes']) + require(item['bytes'] <= 128 * 1024 * 1024) + + +def validate_lane(value, request): + validate_request(request) + keys(value, ('schema', 'lane', 'status', 'sourceCommit', 'sourceTag', 'version', 'toolchain', + 'recipeSha256', 'signing', 'commands', 'assets')) + name = value['lane']; require(isinstance(name, str) and name in LANES) + require(type(value['schema']) is int and value['schema'] == 1 and value['status'] == 'PASS') + require(value['sourceCommit'] == request['source']['commit'] and value['sourceTag'] == request['tag'] + and value['version'] == request['version'] and value['toolchain'] == request['toolchain'] + and value['recipeSha256'] == request['recipe']['sha256']) + # This producer has no broad signing credential. Never infer signing from an + # executable's ad-hoc signature or a previous release. + require(value['signing'] == ('unsigned' if name == 'macos' else 'not-applicable')) + require(isinstance(value['commands'], list) and len(value['commands']) == len(COMMAND_NAMES[name])) + for command, expected in zip(value['commands'], COMMAND_NAMES[name]): + keys(command, ('name', 'exitCode', 'logSha256')) + require(command['name'] == expected and type(command['exitCode']) is int and command['exitCode'] == 0) + hex_value(command['logSha256']) + validate_assets(value['assets'], LANE_ASSETS[name]) + return value + + +def artifact_name(tag, attempt=None): + require(isinstance(tag, str) and pub.TAG.fullmatch(tag)) + if attempt is not None: integer(attempt) + return 'lsp-frozen-' + tag + (f'-attempt{attempt}' if attempt is not None else '') + + +def assemble_public_bundle(folders, request, producer, output): + validate_request(request); validate_producer(producer) + require(not output.exists() and not output.is_symlink()) + require(len(folders) == len(LANES)) + receipts = [] + for folder, name in zip(folders, LANES): + pub.asset_paths(folder, (*LANE_ASSETS[name], 'lane.json')) + envelope = pub.read_json(folder / 'lane.json') + keys(envelope, ('schema', 'producer', 'receipt')) + require(type(envelope['schema']) is int and envelope['schema'] == 1 and envelope['producer'] == producer) + receipt = validate_lane(envelope['receipt'], request) + require(receipt['lane'] == name) + require(receipt['assets'] == pub.asset_records([folder / n for n in LANE_ASSETS[name]])) + receipts.append(receipt) + output.mkdir(parents=True) + for folder, name in zip(folders, LANES): + for asset in LANE_ASSETS[name]: shutil.copyfile(folder / asset, output / asset) + (output / 'checksums.txt').write_text(''.join(f'{pub.digest(output / n)} {n}\n' for n in pub.RAW_ASSETS), encoding='ascii') + manifest = {'schema': 1, 'component': 'terraphim-lsp', 'provenanceMode': 'private-source-handoff-v1', + 'requestSha256': canonical_digest(request), 'privateSource': request['source'], + 'version': request['version'], 'toolchain': request['toolchain'], 'recipe': request['recipe'], + 'producer': producer, 'lanes': receipts, 'universalMacosSigning': 'unsigned', + 'assets': pub.asset_records([output / n for n in (*pub.RAW_ASSETS, 'checksums.txt')])} + pub.write_new(output / pub.MANIFEST, manifest) + verify_bundle(output, request, producer) + return manifest + + +def verify_bundle(folder, request, producer): + validate_request(request); validate_producer(producer) + paths = pub.asset_paths(folder, pub.RELEASE_FILES); pub.verify_sums(folder) + manifest = pub.read_json(folder / pub.MANIFEST) + keys(manifest, ('schema', 'component', 'provenanceMode', 'requestSha256', 'privateSource', 'version', + 'toolchain', 'recipe', 'producer', 'lanes', 'universalMacosSigning', 'assets')) + require(type(manifest['schema']) is int and manifest['schema'] == 1 and manifest['component'] == 'terraphim-lsp' + and manifest['provenanceMode'] == 'private-source-handoff-v1') + require(manifest['requestSha256'] == canonical_digest(request) and manifest['privateSource'] == request['source'] + and manifest['version'] == request['version'] and manifest['toolchain'] == request['toolchain'] + and manifest['recipe'] == request['recipe'] and manifest['producer'] == producer + and manifest['universalMacosSigning'] == 'unsigned') + require(isinstance(manifest['lanes'], list) and len(manifest['lanes']) == len(LANES)) + for lane, name in zip(manifest['lanes'], LANES): + validate_lane(lane, request); require(lane['lane'] == name) + require(lane['assets'] == pub.asset_records([folder / n for n in LANE_ASSETS[name]])) + require(manifest['assets'] == pub.asset_records(paths[:-1])) + return manifest, paths + + +def validate_handoff(value): + keys(value, ('schema', 'tag', 'requestSha256', 'manifestSha256', 'producer', 'artifact')) + require(type(value['schema']) is int and value['schema'] == 1) + artifact_name(value['tag']); hex_value(value['requestSha256']); hex_value(value['manifestSha256']) + validate_producer(value['producer']) + artifact = value['artifact']; keys(artifact, ('id', 'name', 'digest')); integer(artifact['id']) + require(artifact['name'] == artifact_name(value['tag'], value['producer']['runAttempt']) and isinstance(artifact['digest'], str) + and artifact['digest'].startswith('sha256:')) + hex_value(artifact['digest'].removeprefix('sha256:')) + return value + + +def verify_producer(handoff, run, artifact): + validate_handoff(handoff); producer = handoff['producer']; wanted = handoff['artifact'] + mapping(run); mapping(artifact) + repository = mapping(run.get('repository')); head_repository = mapping(run.get('head_repository')) + artifact_run = mapping(artifact.get('workflow_run')) + require(run.get('id') == producer['runId'] and run.get('run_attempt') == producer['runAttempt'] + and repository.get('full_name') == REPO and head_repository.get('full_name') == REPO + and run.get('path') == WORKFLOW and run.get('head_sha') == producer['controlCommit'] + and run.get('head_branch') == 'main' and run.get('event') == 'workflow_dispatch' + and run.get('status') == 'completed' and run.get('conclusion') == 'success', 'LSP_PRODUCER_PROOF_REJECTED') + require(all(artifact.get(k) == wanted[k] for k in ('id', 'name', 'digest')) + and artifact.get('expired') is False and type(artifact.get('size_in_bytes')) is int + and 0 < artifact['size_in_bytes'] <= MAX_BYTES and artifact_run.get('id') == producer['runId'] + and artifact_run.get('head_sha') == producer['controlCommit'] + and artifact_run.get('head_branch') == 'main', 'LSP_ARTIFACT_PROOF_REJECTED') + + +def extract_bundle(archive, destination, max_bytes=MAX_BYTES): + require(archive.is_file() and not archive.is_symlink() and archive.stat().st_size <= MAX_BYTES) + require(not destination.exists() and not destination.is_symlink()) + with zipfile.ZipFile(archive) as source: + entries = source.infolist() + require(len(entries) == len(pub.RELEASE_FILES) and {e.filename for e in entries} == set(pub.RELEASE_FILES)) + require(sum(e.file_size for e in entries) <= max_bytes) + for entry in entries: + kind = stat.S_IFMT(entry.external_attr >> 16) + require(not entry.is_dir() and kind in (0, stat.S_IFREG) and not (entry.flag_bits & 1) + and 0 < entry.file_size <= 128 * 1024 * 1024) + destination.mkdir(parents=True) + for entry in entries: + with source.open(entry) as src, (destination / entry.filename).open('xb') as dest: + shutil.copyfileobj(src, dest) + pub.asset_paths(destination, pub.RELEASE_FILES) + + +def public_control(root, commit, tag=None): + hex_value(commit, 40) + require(pub.git(root, 'remote', 'get-url', 'origin').removesuffix('.git') == 'https://github.com/' + REPO) + require(pub.git(root, 'rev-parse', 'HEAD') == commit and not pub.git(root, 'status', '--porcelain', '--untracked-files=no')) + subprocess.run(['git', 'merge-base', '--is-ancestor', commit, 'origin/main'], cwd=root, check=True, capture_output=True, timeout=30) + if tag: + artifact_name(tag) + try: actual = subprocess.check_output(['git', 'rev-parse', '--verify', f'refs/tags/{tag}^{{commit}}'], cwd=root, text=True, stderr=subprocess.PIPE, timeout=30).strip() + except subprocess.SubprocessError as error: raise ValueError('LSP_CONTROL_TAG_MISSING') from error + require(actual == commit) + + +def verify_handoff_bundle(handoff, request, folder): + validate_handoff(handoff); validate_request(request) + require(handoff['tag'] == request['tag'] and handoff['requestSha256'] == canonical_digest(request) + and hmac.compare_digest(handoff['manifestSha256'], pub.digest(folder / pub.MANIFEST))) + return verify_bundle(folder, request, handoff['producer']) + + +def validate_environment(metadata, policies, name): + mapping(metadata); mapping(policies) + expected = ('main', 'branch') if name == 'lsp-producer' else ('terraphim_lsp-v*', 'tag') + require(name in ('lsp-producer', 'lsp-releases') and metadata.get('name') == name + and metadata.get('deployment_branch_policy') == {'protected_branches': False, 'custom_branch_policies': True}) + entries = policies.get('branch_policies') + require(isinstance(entries, list) and len(entries) == 1 + and (mapping(entries[0]).get('name'), mapping(entries[0]).get('type')) == expected, + 'LSP_ENVIRONMENT_PROTECTION_REQUIRED') + + +def verify_environment(name): + prefix = f'repos/{REPO}/environments/{name}' + validate_environment(pub.api(prefix), pub.api(prefix + '/deployment-branch-policies'), name) + + +def producer_context(): + require(os.environ.get('GITHUB_REPOSITORY') == REPO and os.environ.get('GITHUB_EVENT_NAME') == 'workflow_dispatch' + and os.environ.get('GITHUB_REF') == 'refs/heads/main' + and os.environ.get('GITHUB_WORKFLOW_REF') == f'{REPO}/{WORKFLOW}@refs/heads/main' + and os.environ.get('GITHUB_WORKFLOW_SHA') == os.environ.get('GITHUB_SHA')) + return validate_producer({'repository': REPO, 'workflow': WORKFLOW, 'controlCommit': os.environ['GITHUB_SHA'], + 'runId': int(os.environ['GITHUB_RUN_ID']), 'runAttempt': int(os.environ['GITHUB_RUN_ATTEMPT']), + 'event': 'workflow_dispatch'}) + + +def request_path(root, tag): + artifact_name(tag) + path = root / 'release/lsp/requests' / (tag + '.json') + require(path.is_file() and not path.is_symlink()) + return path + + +def publish_handoff(root, tag, commit, receipt): + public_control(root, commit, tag) + require(os.environ.get('GITHUB_REPOSITORY') == REPO and os.environ.get('GITHUB_EVENT_NAME') == 'push' + and os.environ.get('GITHUB_REF') == 'refs/tags/' + tag + and os.environ.get('GITHUB_SHA') == commit and os.environ.get('GITHUB_WORKFLOW_SHA') == commit + and os.environ.get('GITHUB_WORKFLOW_REF') == f'{REPO}/{PUBLISH_WORKFLOW}@refs/tags/{tag}') + verify_environment('lsp-releases') + handoff_path = root / 'release/lsp/handoffs' / (tag + '.json') + require(handoff_path.is_file() and not handoff_path.is_symlink()) + handoff = validate_handoff(pub.read_json(handoff_path)); request = validate_request(pub.read_json(request_path(root, tag))) + producer = handoff['producer']; artifact = handoff['artifact'] + # Read the original run AND current attempt: a later rerun cannot silently + # replace an already reviewed producer attempt. + run = pub.api(f'repos/{REPO}/actions/runs/{producer["runId"]}') + attempt = pub.api(f'repos/{REPO}/actions/runs/{producer["runId"]}/attempts/{producer["runAttempt"]}') + metadata = pub.api(f'repos/{REPO}/actions/artifacts/{artifact["id"]}') + verify_producer(handoff, run, metadata); verify_producer(handoff, attempt, metadata) + with tempfile.TemporaryDirectory(prefix='lsp-handoff-') as temp: + folder = Path(temp); archive = folder / 'artifact.zip' + with archive.open('xb') as output: + subprocess.run(['gh', 'api', '--hostname', 'github.com', f'repos/{REPO}/actions/artifacts/{artifact["id"]}/zip'], + stdout=output, stderr=subprocess.PIPE, check=True, timeout=600) + require(hmac.compare_digest('sha256:' + pub.digest(archive), artifact['digest']), 'LSP_ARCHIVE_DIGEST_REJECTED') + assets = folder / 'assets'; extract_bundle(archive, assets) + _, files = verify_handoff_bundle(handoff, request, assets) + notes = (f'Private-source LSP prerelease {tag}.\n\n' + f'Build source: `{request["source"]["commit"]}` on `{PRIVATE_REPO}`.\n' + f'Public release control: `{commit}`.\n' + f'Producer: https://github.com/{REPO}/actions/runs/{producer["runId"]} ' + f'(attempt {producer["runAttempt"]}).\n' + 'Six frozen binaries, checksums and provenance manifest. macOS binaries are unsigned.\n') + result = pub.publish(REPO, tag, files, commit, notes) + # The shared transport's source means tag target. Do not mislabel it as + # the private build source in this explicit mode. + result['publicControlCommit'] = result.pop('sourceCommit') + result['privateSource'] = request['source']; result['producer'] = producer + result['provenanceMode'] = 'private-source-handoff-v1' + pub.write_new(receipt, result) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('command', choices=('check-request', 'assemble', 'publish')) + parser.add_argument('--root', type=Path, default=Path(__file__).resolve().parents[3]) + parser.add_argument('--tag', required=True); parser.add_argument('--lanes', type=Path) + parser.add_argument('--output', type=Path); parser.add_argument('--receipt', type=Path) + args = parser.parse_args() + try: + require(args.receipt and not args.receipt.exists() and not args.receipt.is_symlink()) + request = validate_request(pub.read_json(request_path(args.root, args.tag))) + if args.command == 'publish': + publish_handoff(args.root, args.tag, os.environ['GITHUB_SHA'], args.receipt) + else: + producer = producer_context(); public_control(args.root, producer['controlCommit']) + if args.command == 'check-request': + verify_environment('lsp-producer') + require(pub.api(f'repos/{REPO}/git/ref/heads/main')['object']['sha'] == producer['controlCommit']) + if args.command == 'assemble': + assemble_public_bundle([args.lanes / name for name in LANES], request, producer, args.output) + pub.write_new(args.receipt, {'schema': 1, 'status': 'PASS', 'requestSha256': canonical_digest(request), 'producer': producer}) + return 0 + except (ValueError, KeyError, TypeError, OSError, subprocess.SubprocessError, zipfile.BadZipFile): + # Never copy raw exception text or private stderr into public Actions logs. + if args.receipt and not args.receipt.exists() and not args.receipt.is_symlink(): + pub.write_new(args.receipt, {'schema': 1, 'status': 'FAIL', 'errorCode': 'LSP_HANDOFF_REJECTED'}) + print('LSP_HANDOFF_REJECTED', file=sys.stderr) + return 1 + + +if __name__ == '__main__': sys.exit(main()) diff --git a/.github/scripts/release/lsp_private_producer.py b/.github/scripts/release/lsp_private_producer.py new file mode 100644 index 000000000..a866ff050 --- /dev/null +++ b/.github/scripts/release/lsp_private_producer.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Protected private checkout boundary. Never upload source or raw diagnostics.""" +from __future__ import annotations +import argparse +import json +import hmac +import os +import re +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import urllib.request +import urllib.error +from urllib.parse import urlsplit + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import lsp_handoff as h + +API = 'https://git.terraphim.cloud/api/v1' +PRIVATE_ENV = ('PATH', 'HOME', 'RUSTUP_HOME', 'TMPDIR', 'TMP', 'TEMP', 'SDKROOT', 'DEVELOPER_DIR', + 'GITEA_LSP_SOURCE_TOKEN', 'CARGO_REGISTRIES_TERRAPHIM_TOKEN') + + +def credentials(environment): + token = environment.get('GITEA_LSP_SOURCE_TOKEN', '') + registry = environment.get('CARGO_REGISTRIES_TERRAPHIM_TOKEN', '') + h.require(bool(token) and '\n' not in token and '\r' not in token, 'LSP_SOURCE_CREDENTIAL_MISSING') + h.require(registry.startswith('Bearer ') and len(registry) > 7 and '\n' not in registry and '\r' not in registry, + 'LSP_REGISTRY_CREDENTIAL_MISSING') + # No fallback to broad GITEA_TOKEN, OP_SERVICE_ACCOUNT_TOKEN or runner auth. + return {k: environment[k] for k in PRIVATE_ENV if k in environment} + + +def askpass_response(prompt, token): + match = re.fullmatch(r"(Username|Password) for '([^']+)': ?", prompt) + h.require(match is not None, 'LSP_CREDENTIAL_PROMPT_REJECTED') + url = urlsplit(match[2]) + h.require(url.scheme == 'https' and url.hostname == 'git.terraphim.cloud' + and url.netloc in ('git.terraphim.cloud', 'token@git.terraphim.cloud') + and url.path in ('/terraphim/terraphim-ai.git', '/terraphim/terraphim-editor.git') + and not url.query and not url.fragment, 'LSP_CREDENTIAL_PROMPT_REJECTED') + return 'token' if match[1] == 'Username' else token + + +def private_api(path, token): + request = urllib.request.Request(API + path, headers={'Authorization': 'token ' + token, 'Accept': 'application/json'}) + # Disallow credential-bearing redirects, including a changed API hostname. + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): return None + with urllib.request.build_opener(NoRedirect).open(request, timeout=60) as response: + data = response.read(2 * 1024 * 1024 + 1) + h.require(len(data) <= 2 * 1024 * 1024) + return json.loads(data, object_pairs_hook=h.pub.unique_object) + + +def verify_source_metadata(request, repository, tag, status, editor): + h.validate_request(request) + for value in (repository, tag, status, editor): h.mapping(value) + commit = h.mapping(tag.get('commit')) + h.require(repository.get('full_name') == h.REPO and repository.get('default_branch') == 'main' + and repository.get('private') is True and repository.get('archived') is False) + h.require(editor.get('full_name') == 'terraphim/terraphim-editor' and editor.get('archived') is False) + h.require(tag.get('name') == request['tag'] and commit.get('sha') == request['source']['commit']) + h.require(status.get('sha') == request['source']['commit'] and status.get('state') == 'success') + records = status.get('statuses') + h.require(isinstance(records, list)) + for record in records: h.mapping(record) + for context in h.SOURCE_CHECKS: + found = [v for v in records if v.get('context') == context] + h.require(found and all(v.get('status') == 'success' for v in found) + and any(isinstance(v.get('target_url'), str) and v['target_url'].startswith('/' + h.REPO + '/actions/runs/') for v in found)) + return {'repository': h.PRIVATE_REPO, 'commit': request['source']['commit'], 'tag': request['tag'], + 'requiredChecks': list(h.SOURCE_CHECKS), 'metadataSha256': h.canonical_digest({'tag': tag, 'status': status})} + + +def preflight(request, token): + prefix = '/repos/' + h.REPO + return verify_source_metadata(request, private_api(prefix, token), private_api(prefix + '/tags/' + request['tag'], token), + private_api(prefix + '/commits/' + request['source']['commit'] + '/status', token), + private_api('/repos/terraphim/terraphim-editor', token)) + + +def run_private(argv, cwd, env, log): + with log.open('xb') as output: + status = subprocess.run(argv, cwd=cwd, env=env, stdout=output, stderr=subprocess.STDOUT, timeout=14400, check=False) + h.require(status.returncode == 0, 'LSP_PRIVATE_STEP_FAILED') + + +def protected_directory(home): + # RUNNER_TEMP is erased by Actions after a job. Keep protected diagnostics + # outside it so a failed build remains inspectable by the runner owner. + root = home / '.local/state/terraphim-lsp-private' + for path in (home, home/'.local', home/'.local/state', root): + h.require(not path.is_symlink(), 'LSP_PRIVATE_PATH_REJECTED') + root.mkdir(parents=True, exist_ok=True, mode=0o700) + h.require(root.is_dir() and root.stat().st_uid == os.getuid() and root.stat().st_mode & 0o077 == 0, + 'LSP_PRIVATE_PATH_REJECTED') + return Path(tempfile.mkdtemp(prefix='producer-', dir=root)) + + +def produce(root, request, lane, output): + producer = h.producer_context(); h.public_control(root, producer['controlCommit']) + h.require(h.pub.api(f'repos/{h.REPO}/git/ref/heads/main')['object']['sha'] == producer['controlCommit']) + h.validate_request(request); h.require(lane in h.LANES and not output.exists() and not output.is_symlink()) + env = credentials(os.environ) + proof = preflight(request, env['GITEA_LSP_SOURCE_TOKEN']) + private = protected_directory(Path(env['HOME'])) + h.pub.write_new(private/'source-proof.json', proof) + h.pub.write_new(private/'producer.json', producer) + source = private/'source'; askpass = private/'askpass.py' + askpass.write_text('#!' + sys.executable + '\nimport os,sys\n' + + 'sys.path.insert(0, ' + repr(str(Path(__file__).resolve().parent)) + ')\n' + + 'from lsp_private_producer import askpass_response\n' + + 'try: print(askpass_response(sys.argv[1], os.environ["GITEA_LSP_SOURCE_TOKEN"]))\n' + + 'except (ValueError,KeyError,IndexError): raise SystemExit(1)\n') + askpass.chmod(0o700) + env.update(GIT_ASKPASS=str(askpass), GIT_TERMINAL_PROMPT='0', GIT_CONFIG_NOSYSTEM='1', + GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_COUNT='2', GIT_CONFIG_KEY_0='credential.helper', GIT_CONFIG_VALUE_0='', + GIT_CONFIG_KEY_1='credential.useHttpPath', GIT_CONFIG_VALUE_1='true') + try: + run_private(['git', 'clone', '--no-checkout', '--single-branch', '--branch', 'main', h.PRIVATE_REPO + '.git', str(source)], + private, env, private/'clone.log') + run_private(['git', 'fetch', 'origin', f'refs/tags/{request["tag"]}:refs/tags/{request["tag"]}'], source, env, private/'tag.log') + run_private(['git', 'checkout', '--detach', request['source']['commit']], source, env, private/'checkout.log') + h.require(h.pub.toolchain(source) == request['toolchain'] and h.pub.version(source) == request['version']) + recipe = source / h.RECIPE + h.require(recipe.is_file() and not recipe.is_symlink() and hmac.compare_digest(h.pub.digest(recipe), request['recipe']['sha256'])) + # Existing exact-source guard is re-used on the private checkout. + run_private([sys.executable, str(source/'.github/scripts/release/publish_lsp_release.py'), 'source', '--root', str(source), + '--tag', request['tag'], '--source', request['source']['commit'], '--require-tag'], source, env, private/'source-guard.log') + staged = private/'safe-output' + run_private([sys.executable, str(recipe), '--root', str(source), '--lane', lane, '--source', request['source']['commit'], + '--tag', request['tag'], '--output', str(staged), '--private-work', str(private/'work')], source, env, private/'recipe.log') + h.pub.asset_paths(staged, (*h.LANE_ASSETS[lane], 'lane.json')) + receipt = h.validate_lane(h.pub.read_json(staged/'lane.json'), request) + h.require(receipt['lane'] == lane and receipt['assets'] == h.pub.asset_records([staged/n for n in h.LANE_ASSETS[lane]])) + # Recheck movable private tags and CI metadata after the long build. + preflight(request, env['GITEA_LSP_SOURCE_TOKEN']) + output.mkdir(parents=True) + for name in h.LANE_ASSETS[lane]: shutil.copyfile(staged/name, output/name) + h.pub.write_new(output/'lane.json', {'schema': 1, 'producer': producer, 'receipt': receipt}) + finally: + # No credential value is ever written to disk. The helper also goes away; + # source, exact receipts and raw diagnostics stay owner-only for review. + askpass.unlink(missing_ok=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--root', type=Path, default=Path(__file__).resolve().parents[3]) + parser.add_argument('--tag', required=True); parser.add_argument('--lane', choices=h.LANES, required=True) + parser.add_argument('--output', type=Path, required=True); parser.add_argument('--receipt', type=Path, required=True) + args = parser.parse_args() + try: + h.require(not args.receipt.exists() and not args.receipt.is_symlink()) + request = h.validate_request(h.pub.read_json(h.request_path(args.root, args.tag))) + produce(args.root, request, args.lane, args.output) + h.pub.write_new(args.receipt, {'schema': 1, 'status': 'PASS', 'lane': args.lane}) + return 0 + except (ValueError, KeyError, TypeError, OSError, subprocess.SubprocessError) as error: + allowed = {'LSP_SOURCE_CREDENTIAL_MISSING', 'LSP_REGISTRY_CREDENTIAL_MISSING', 'LSP_PRIVATE_PATH_REJECTED'} + code = str(error) if str(error) in allowed else 'LSP_PRIVATE_PRODUCER_REJECTED' + if not args.receipt.exists() and not args.receipt.is_symlink(): + h.pub.write_new(args.receipt, {'schema': 1, 'status': 'FAIL', 'errorCode': code, 'lane': args.lane}) + print(code, file=sys.stderr) + return 1 + + +if __name__ == '__main__': sys.exit(main()) diff --git a/.github/scripts/release/publish_lsp_release.py b/.github/scripts/release/publish_lsp_release.py new file mode 100644 index 000000000..084bb5e4b --- /dev/null +++ b/.github/scripts/release/publish_lsp_release.py @@ -0,0 +1,406 @@ +#!/usr/bin/env python3 +"""Qualify and publish immutable LSP bytes; never move tags or replace assets. + +The resumable draft/readback policy follows the existing VS Code publisher. +Source provenance is the actual checkout, including its repository. This helper +cannot bridge a private source commit into a public tag: that integration must +be reviewed separately, and a missing or different public tag fails closed. +""" +from __future__ import annotations + +import argparse +import hashlib +import hmac +import json +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import time +import tomllib +from urllib.parse import urlparse + +ROOT = Path(__file__).resolve().parents[3] +SHA = re.compile(r"[0-9a-f]{40}") +TAG = re.compile(r"terraphim_lsp-v([0-9]+\.[0-9]+\.[0-9]+)(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?") +REPOSITORY = "terraphim/terraphim-ai" +SOURCE_REPOSITORIES = {f"https://{host}/{REPOSITORY}" for host in ("github.com", "git.terraphim.cloud")} +RAW_ASSETS = tuple(sorted("terraphim-lsp-" + target for target in ( + "aarch64-apple-darwin", "aarch64-unknown-linux-musl", "universal-apple-darwin", + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl", +))) +MANIFEST = "lsp-release-manifest.json" +RELEASE_FILES = (*RAW_ASSETS, "checksums.txt", MANIFEST) +COMMANDS = ( + ("cargo", "fmt", "-p", "terraphim_lsp_core", "-p", "terraphim_lsp", "--", "--check"), + ("cargo", "clippy", "--locked", "-p", "terraphim_lsp_core", "-p", "terraphim_lsp", + "--all-targets", "--all-features", "--", "-D", "warnings"), + ("cargo", "test", "--locked", "-p", "terraphim_lsp_core", "-p", "terraphim_lsp", "--all-features"), +) + + +def run(*args, cwd=None): + return subprocess.check_output(list(args), cwd=cwd, text=True, timeout=300).strip() + + +def git(root, *args): + return run("git", *args, cwd=root) + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"Duplicate JSON key: {key}") + result[key] = value + return result + + +def read_json(path): + return json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=unique_object) + + +def write_new(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("x", encoding="utf-8") as output: + output.write(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def version(root): + crate = tomllib.loads((root / "crates/terraphim_lsp/Cargo.toml").read_text())["package"]["version"] + if crate == {"workspace": True}: + crate = tomllib.loads((root / "Cargo.toml").read_text())["workspace"]["package"]["version"] + return crate + + +def validate_tag(tag, crate_version): + match = TAG.fullmatch(tag) + if not match or match[1] != crate_version: + raise ValueError("Exact LSP tag and crate version must match") + + +def source_repository(root): + url = git(root, "remote", "get-url", "origin").removesuffix(".git").rstrip("/") + if url.startswith("git@"): + host, path = url[4:].split(":", 1) + url = f"https://{host}/{path}" + if url not in SOURCE_REPOSITORIES: + raise ValueError("Source origin is not an approved Terraphim source repository") + return url + + +def source_identity(root, tag, source, require_tag=True): + validate_tag(tag, version(root)) + if not SHA.fullmatch(source) or git(root, "rev-parse", "HEAD") != source: + raise ValueError("Checkout does not match exact source commit") + if git(root, "status", "--porcelain", "--untracked-files=no"): + raise ValueError("Tracked source is dirty") + # Fail closed if reviewed main history was not fetched or the source is not + # merged there. A matching version string alone is never source approval. + subprocess.run(["git", "merge-base", "--is-ancestor", source, "origin/main"], + cwd=root, check=True, timeout=30) + local = subprocess.run(["git", "rev-parse", "--verify", f"refs/tags/{tag}^{{commit}}"], + cwd=root, text=True, capture_output=True, timeout=30) + if local.returncode == 0: + if local.stdout.strip() != source: + raise ValueError("Local release tag differs from exact source") + elif require_tag: + raise ValueError("Publishing requires the existing exact source tag") + return {"repository": source_repository(root), "commit": source, + "tag": tag, "trackedTreeDirty": False} + + +def toolchain(root): + return tomllib.loads((root / "rust-toolchain.toml").read_text())["toolchain"]["channel"] + + +def qualification_commands(root): + return [["rustup", "run", toolchain(root), *command] for command in COMMANDS] + + +def qualify(root, tag, source, output, require_tag): + if output.exists(): + raise ValueError("Choose a new qualification directory") + identity = source_identity(root, tag, source, require_tag) + output.mkdir(parents=True) + result = {"schema": 1, "status": "FAIL", "source": identity, "commands": []} + try: + for index, command in enumerate(qualification_commands(root)): + path = output / f"command-{index}.log" + with path.open("xb") as log: + # Execute the real crate tests, including save/export/stdio tests. + status = subprocess.run(command, cwd=root, stdout=log, stderr=subprocess.STDOUT, + timeout=1800, check=False) + result["commands"].append({"argv": command, "exitCode": status.returncode, + "log": path.name, "sha256": digest(path)}) + if status.returncode: + raise ValueError(f"Qualification command {index} failed; inspect {path.name}") + source_identity(root, tag, source, require_tag) + result["status"] = "PASS" + finally: + write_new(output / "qualification.json", result) + + +def verify_qualification(root, folder, identity): + report = read_json(folder / "qualification.json") + commands = report.get("commands", []) + expected = qualification_commands(root) + if (type(report.get("schema")) is not int or report.get("schema") != 1 or report.get("status") != "PASS" + or report.get("source") != identity or len(commands) != len(expected)): + raise ValueError("Qualification does not identify this successful exact source") + for index, (record, argv) in enumerate(zip(commands, expected)): + log = folder / f"command-{index}.log" + if (record.get("argv") != argv or type(record.get("exitCode")) is not int or record.get("exitCode") != 0 + or record.get("log") != log.name or not log.is_file() or log.is_symlink() + or not isinstance(record.get("sha256"), str) + or not hmac.compare_digest(record["sha256"], digest(log))): + raise ValueError("Qualification command or saved log differs") + return report + + +def asset_paths(folder, names): + if folder.is_symlink() or not folder.is_dir(): + raise ValueError("Assets must be a regular directory") + if set(p.name for p in folder.iterdir()) != set(names): + raise ValueError("Unexpected or missing release assets") + paths = [folder / name for name in names] + if any(p.is_symlink() or not p.is_file() or p.stat().st_size == 0 for p in paths): + raise ValueError("Release assets must be nonempty regular files") + return paths + + +def asset_records(paths): + return [{"name": p.name, "sha256": digest(p), "bytes": p.stat().st_size} for p in paths] + + +def verify_sums(folder): + expected = "".join(f"{digest(folder / name)} {name}\n" for name in RAW_ASSETS) + if (folder / "checksums.txt").read_text(encoding="ascii") != expected: + raise ValueError("Checksums differ from the exact six raw binaries") + + +def assemble(root, folder, qualification, tag, source, signing, require_tag): + identity = source_identity(root, tag, source, require_tag) + report = verify_qualification(root, qualification, identity) + paths = asset_paths(folder, (*RAW_ASSETS, "checksums.txt")) + verify_sums(folder) + if signing not in ("signed", "unsigned"): + raise ValueError("Unknown universal macOS signing state") + result = {"schema": 1, "component": "terraphim-lsp", "source": identity, + "version": version(root), "toolchain": toolchain(root), + "qualification": report, "universalMacosSigning": signing, + "assets": asset_records(paths)} + write_new(folder / MANIFEST, result) + return result + + +def verify_bundle(root, folder, qualification, tag, source): + identity = source_identity(root, tag, source) + report = verify_qualification(root, qualification, identity) + paths = asset_paths(folder, RELEASE_FILES) + manifest = read_json(folder / MANIFEST) + verify_sums(folder) + if (manifest.get("schema") != 1 or manifest.get("component") != "terraphim-lsp" + or manifest.get("source") != identity or manifest.get("version") != version(root) + or manifest.get("toolchain") != toolchain(root) or manifest.get("qualification") != report + or manifest.get("universalMacosSigning") not in ("signed", "unsigned") + or manifest.get("assets") != asset_records(paths[:-1])): + raise ValueError("Manifest differs from qualified source or frozen assets") + return manifest, paths + + +def gh(*args): + return run("gh", *args) + + +def api(path): + return json.loads(gh("api", "--hostname", "github.com", path), object_pairs_hook=unique_object) + + +def verify_remote_tag(repo, tag, source): + value = api(f"repos/{repo}/git/ref/tags/{tag}")["object"] + for _ in range(5): + if value["type"] == "commit" and value["sha"] == source: + return + if value["type"] != "tag" or not SHA.fullmatch(value["sha"]): + break + value = api(f"repos/{repo}/git/tags/{value['sha']}")["object"] + raise ValueError("Public tag does not identify the qualified source; no source bridge is configured") + + +def select_release(pages, tag): + matches = [item for page in pages for item in page if item.get("tag_name") == tag] + if len(matches) > 1: + raise ValueError("Ambiguous release tag") + return matches[0] if matches else None + + +def release(repo, tag): + # Listing includes private drafts, unlike the anonymous exact-tag endpoint. + pages = json.loads(gh("api", "--hostname", "github.com", "--paginate", "--slurp", + f"repos/{repo}/releases?per_page=100")) + return select_release(pages, tag) + + +def ownership(files): + manifest = next(p for p in files if p.name == MANIFEST) + return f"" + + +def resume_plan(current, tag, files): + body = current.get("body") + if not isinstance(body, str): + raise ValueError("Existing release has no ownership marker") + if (current.get("tag_name") != tag or type(current.get("prerelease")) is not bool or not current["prerelease"] + or type(current.get("draft")) is not bool + or ownership(files) not in body.replace("\r", "").splitlines()): + raise ValueError("Existing release does not belong to this exact candidate manifest") + assets = current["assets"] + names = [item["name"] for item in assets] + wanted = {p.name: p for p in files} + if len(names) != len(set(names)) or not set(names).issubset(wanted): + raise ValueError("Unexpected or duplicate release assets") + for item in assets: + path = wanted[item["name"]] + if (item.get("state") != "uploaded" or item.get("size") != path.stat().st_size + or item.get("digest") not in (None, "sha256:" + digest(path))): + raise ValueError("Existing asset differs or is incomplete; refusing replacement") + missing = [p for p in files if p.name not in names] + if missing and not current["draft"]: + raise ValueError("Published releases are immutable; missing assets require a new version") + return missing + + +def verify_bytes(expected, downloaded): + if downloaded.is_symlink() or not downloaded.is_file() or expected.read_bytes() != downloaded.read_bytes(): + raise ValueError(f"Downloaded {expected.name} differs; refusing replacement") + + +def anonymous_download(url, destination): + parsed = urlparse(url) + if (parsed.scheme != "https" or parsed.hostname != "github.com" or parsed.username + or parsed.password or parsed.query or parsed.fragment or destination.exists() + or destination.is_symlink()): + raise ValueError("Expected a new anonymous HTTPS GitHub download") + # No curl config, credentials, cookies or netrc; redirect protocols stay HTTPS. + status = subprocess.run([ + "curl", "--disable", "--fail", "--silent", "--show-error", "--location", + "--proto", "=https", "--proto-redir", "=https", "--max-redirs", "5", + "--connect-timeout", "15", "--max-time", "300", "--max-filesize", str(128 * 1024 * 1024), + "--retry", "2", "--output", str(destination), "--write-out", "%{http_code}", url, + ], text=True, capture_output=True, timeout=1000) + if status.returncode or status.stdout != "200": + raise ValueError(f"Anonymous readback failed: HTTP {status.stdout}; curl {status.returncode}") + + +def recovered_release(repo, tag): + # Draft creation/listing is eventually consistent. Bounded reads recover the + # original draft; never retry create blindly or replace an existing release. + for attempt in range(5): + current = release(repo, tag) + if current is not None: + return current + if attempt < 4: + time.sleep(2) + raise ValueError("Created draft is not yet readable; rerun the same qualified artifact") + + +def publish(repo, tag, files, source, notes): + if repo != REPOSITORY or not TAG.fullmatch(tag) or not SHA.fullmatch(source): + raise ValueError("Unexpected public release repository, tag or source") + metadata = api(f"repos/{repo}") + if type(metadata.get("private")) is not bool or metadata["private"] or metadata.get("full_name") != repo: + raise ValueError("Exact public repository is required") + verify_remote_tag(repo, tag, source) + current = release(repo, tag) + with tempfile.TemporaryDirectory(prefix="lsp-publication-") as folder: + temp = Path(folder) + if current is None: + note_file = temp / "notes.md" + note_file.write_text(ownership(files) + "\n\n" + notes, encoding="utf-8") + gh("release", "create", tag, "--repo", f"github.com/{repo}", "--verify-tag", + "--draft", "--prerelease", "--latest=false", "--title", tag, "--notes-file", str(note_file)) + current = recovered_release(repo, tag) + missing = resume_plan(current, tag, files) + # Verify all existing bytes before any missing-asset write. Matching + # metadata alone is insufficient, even when GitHub supplies a digest. + for path in files: + if path not in missing: + gh("release", "download", tag, "--repo", f"github.com/{repo}", + "--pattern", path.name, "--dir", str(temp)) + verify_bytes(path, temp / path.name) + for path in missing: + gh("release", "upload", tag, str(path), "--repo", f"github.com/{repo}") + gh("release", "download", tag, "--repo", f"github.com/{repo}", + "--pattern", path.name, "--dir", str(temp)) + verify_bytes(path, temp / path.name) + current = recovered_release(repo, tag) + if resume_plan(current, tag, files): + raise ValueError("Draft asset set is incomplete") + verify_remote_tag(repo, tag, source) + if current["draft"]: + gh("release", "edit", tag, "--repo", f"github.com/{repo}", "--verify-tag", "--draft=false") + published = recovered_release(repo, tag) + if published["draft"] or resume_plan(published, tag, files): + raise ValueError("Release is not public with the exact asset set") + verify_remote_tag(repo, tag, source) + for path in files: + destination = temp / ("public-" + path.name) + anonymous_download(f"https://github.com/{repo}/releases/download/{tag}/{path.name}", destination) + verify_bytes(path, destination) + return {"schema": 1, "status": "PUBLISHED_PUBLIC_VERIFIED", "repository": repo, + "tag": tag, "sourceCommit": source, "releaseId": published["id"], "assets": asset_records(files)} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("source", "qualify", "assemble", "publish")) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--tag", required=True) + parser.add_argument("--source", required=True) + parser.add_argument("--require-tag", action="store_true") + parser.add_argument("--assets", type=Path) + parser.add_argument("--qualification", type=Path) + parser.add_argument("--signing", choices=("signed", "unsigned")) + parser.add_argument("--repo", default=REPOSITORY) + parser.add_argument("--notes", type=Path) + parser.add_argument("--receipt", type=Path) + args = parser.parse_args() + result = {"schema": 1, "status": "FAIL"} + try: + if args.command == "source": + source_identity(args.root, args.tag, args.source, args.require_tag) + elif args.command == "qualify": + qualify(args.root, args.tag, args.source, args.qualification, args.require_tag) + elif args.command == "assemble": + assemble(args.root, args.assets, args.qualification, args.tag, args.source, + args.signing, args.require_tag) + else: + if not args.receipt or args.receipt.exists() or args.receipt.is_symlink(): + raise ValueError("Choose a new publication receipt path") + verify_bundle(args.root, args.assets, args.qualification, args.tag, args.source) + # Upload only a revalidated snapshot; later workspace edits cannot + # replace bytes between qualification and upload. + with tempfile.TemporaryDirectory(prefix="lsp-frozen-") as folder: + frozen = Path(folder) / "assets" + shutil.copytree(args.assets, frozen, symlinks=True) + _, paths = verify_bundle(args.root, frozen, args.qualification, args.tag, args.source) + result = publish(args.repo, args.tag, paths, args.source, args.notes.read_text()) + write_new(args.receipt, result) + return 0 + except (ValueError, KeyError, TypeError, OSError, subprocess.SubprocessError) as error: + result["error"] = str(error) + if args.command == "publish" and args.receipt and not args.receipt.exists() and not args.receipt.is_symlink(): + write_new(args.receipt, result) + print(f"publish_lsp_release: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/lsp-producer.yml b/.github/workflows/lsp-producer.yml new file mode 100644 index 000000000..731285a07 --- /dev/null +++ b/.github/workflows/lsp-producer.yml @@ -0,0 +1,223 @@ +name: Build private-source LSP artifact + +on: + workflow_dispatch: + inputs: + tag: + description: 'Reviewed release/lsp/requests/.json on main' + required: true + type: string + +permissions: + contents: read + +concurrency: + group: lsp-producer-${{ inputs.tag }} + cancel-in-progress: false + +jobs: + resolve: + permissions: + contents: read + actions: read + if: github.repository == 'terraphim/terraphim-ai' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.11' + - name: Validate reviewed main request + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ inputs.tag }} + run: python3 .github/scripts/release/lsp_handoff.py check-request --tag "$TAG" --receipt resolve-receipt.json + - name: Preserve safe resolution receipt + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: lsp-resolve-${{ github.run_attempt }} + path: resolve-receipt.json + if-no-files-found: error + + qualify: + needs: resolve + environment: lsp-producer + runs-on: + group: lsp-private + labels: [self-hosted, Linux, X64] + timeout-minutes: 180 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.11' + - name: Qualify private exact source without exposing raw logs + env: + GH_TOKEN: ${{ github.token }} + GITEA_LSP_SOURCE_TOKEN: ${{ secrets.GITEA_LSP_SOURCE_TOKEN }} + CARGO_REGISTRIES_TERRAPHIM_TOKEN: ${{ secrets.CARGO_REGISTRIES_TERRAPHIM_TOKEN }} + TAG: ${{ inputs.tag }} + run: python3 .github/scripts/release/lsp_private_producer.py --tag "$TAG" --lane qualify --output public-lane --receipt producer-receipt.json + - name: Upload only validated safe qualification receipt + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: lsp-lane-${{ github.run_attempt }}-qualify + path: public-lane/* + if-no-files-found: error + - name: Preserve fixed-code producer outcome + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: lsp-outcome-${{ github.run_attempt }}-qualify + path: producer-receipt.json + if-no-files-found: error + + linux: + needs: [resolve, qualify] + environment: lsp-producer + runs-on: + group: lsp-private + labels: [self-hosted, Linux, X64] + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: + target: [x86_64-unknown-linux-gnu, x86_64-unknown-linux-musl, aarch64-unknown-linux-musl] + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.11' + - name: Build isolated private Linux lane + env: + GH_TOKEN: ${{ github.token }} + GITEA_LSP_SOURCE_TOKEN: ${{ secrets.GITEA_LSP_SOURCE_TOKEN }} + CARGO_REGISTRIES_TERRAPHIM_TOKEN: ${{ secrets.CARGO_REGISTRIES_TERRAPHIM_TOKEN }} + TAG: ${{ inputs.tag }} + LANE: ${{ matrix.target }} + run: python3 .github/scripts/release/lsp_private_producer.py --tag "$TAG" --lane "$LANE" --output public-lane --receipt producer-receipt.json + - name: Upload only validated safe lane + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: lsp-lane-${{ github.run_attempt }}-${{ matrix.target }} + path: public-lane/* + if-no-files-found: error + - name: Preserve fixed-code producer outcome + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: lsp-outcome-${{ github.run_attempt }}-${{ matrix.target }} + path: producer-receipt.json + if-no-files-found: error + + macos: + needs: [resolve, qualify] + environment: lsp-producer + runs-on: + group: lsp-private + labels: [self-hosted, macOS] + timeout-minutes: 240 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.11' + - name: Build isolated private macOS lane + env: + GH_TOKEN: ${{ github.token }} + GITEA_LSP_SOURCE_TOKEN: ${{ secrets.GITEA_LSP_SOURCE_TOKEN }} + CARGO_REGISTRIES_TERRAPHIM_TOKEN: ${{ secrets.CARGO_REGISTRIES_TERRAPHIM_TOKEN }} + TAG: ${{ inputs.tag }} + run: python3 .github/scripts/release/lsp_private_producer.py --tag "$TAG" --lane macos --output public-lane --receipt producer-receipt.json + - name: Upload only validated safe lane + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: lsp-lane-${{ github.run_attempt }}-macos + path: public-lane/* + if-no-files-found: error + - name: Preserve fixed-code producer outcome + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: lsp-outcome-${{ github.run_attempt }}-macos + path: producer-receipt.json + if-no-files-found: error + + assemble: + needs: [resolve, qualify, linux, macos] + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.11' + - name: Download only this run attempt's lane artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + pattern: lsp-lane-${{ github.run_attempt }}-* + path: lanes + - name: Assemble frozen public-safe bytes + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ inputs.tag }} + ATTEMPT: ${{ github.run_attempt }} + run: | + python3 - <<'PY' + import os + from pathlib import Path + root = Path('lanes') + for path in root.iterdir(): + path.rename(root / path.name.removeprefix('lsp-lane-' + os.environ['ATTEMPT'] + '-')) + PY + python3 .github/scripts/release/lsp_handoff.py assemble --tag "$TAG" --lanes lanes --output frozen --receipt assembly-receipt.json + - name: Upload immutable frozen release artifact + id: frozen + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: lsp-frozen-${{ inputs.tag }}-attempt${{ github.run_attempt }} + path: frozen/* + if-no-files-found: error + retention-days: 90 + - name: Record handoff artifact coordinates + env: + ARTIFACT_ID: ${{ steps.frozen.outputs.artifact-id }} + ARTIFACT_DIGEST: ${{ steps.frozen.outputs.artifact-digest }} + run: | + python3 - <<'PY' + import json, os + from pathlib import Path + Path('artifact-coordinate.json').write_text(json.dumps({'id': int(os.environ['ARTIFACT_ID']), 'digest': 'sha256:' + os.environ['ARTIFACT_DIGEST']}) + '\n') + PY + - name: Preserve safe assembly and artifact coordinates + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: lsp-assembly-${{ github.run_attempt }} + path: | + assembly-receipt.json + artifact-coordinate.json + if-no-files-found: error diff --git a/.github/workflows/lsp-publish.yml b/.github/workflows/lsp-publish.yml new file mode 100644 index 000000000..5a489217b --- /dev/null +++ b/.github/workflows/lsp-publish.yml @@ -0,0 +1,44 @@ +name: Publish reviewed LSP handoff + +on: + push: + tags: + - 'terraphim_lsp-v*' + +permissions: + contents: read + +concurrency: + group: lsp-publish-${{ github.ref_name }} + cancel-in-progress: false + +jobs: + publish: + if: github.repository == 'terraphim/terraphim-ai' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/terraphim_lsp-v') + environment: lsp-releases + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write + actions: read + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.11' + - name: Verify provenance and publish immutable frozen bytes + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.ref_name }} + run: python3 .github/scripts/release/lsp_handoff.py publish --tag "$TAG" --receipt publication-receipt.json + - name: Preserve publication or fixed-code failure receipt + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: lsp-publication-${{ github.run_id }}-${{ github.run_attempt }} + path: publication-receipt.json + if-no-files-found: error diff --git a/.github/workflows/release-manifest-contract.yml b/.github/workflows/release-manifest-contract.yml index a9129b6bb..c5a3a01ea 100644 --- a/.github/workflows/release-manifest-contract.yml +++ b/.github/workflows/release-manifest-contract.yml @@ -14,6 +14,13 @@ on: - ".github/workflows/release-sign.yml" - ".github/workflows/continue-integration.yml" - ".github/workflows/vscode-release.yml" + - ".github/workflows/lsp-producer.yml" + - ".github/workflows/lsp-publish.yml" + - "docs/contracts/schemas/lsp-*.schema.json" + - "release/lsp/**" + - "tests/lsp_handoff_test.py" + - "tests/lsp_private_producer_test.py" + - "tests/workflow_lsp_handoff_test.py" - "integrations/continue/verify_source.py" - "integrations/continue/integration.json" - "vs-code-terraphim-it/package.json" @@ -53,6 +60,13 @@ on: - ".github/workflows/release-sign.yml" - ".github/workflows/continue-integration.yml" - ".github/workflows/vscode-release.yml" + - ".github/workflows/lsp-producer.yml" + - ".github/workflows/lsp-publish.yml" + - "docs/contracts/schemas/lsp-*.schema.json" + - "release/lsp/**" + - "tests/lsp_handoff_test.py" + - "tests/lsp_private_producer_test.py" + - "tests/workflow_lsp_handoff_test.py" - "integrations/continue/verify_source.py" - "integrations/continue/integration.json" - "vs-code-terraphim-it/package.json" @@ -184,7 +198,7 @@ jobs: - name: actionlint (static workflow analysis) run: | set -euo pipefail - actionlint .github/workflows/release-coordinator.yml .github/workflows/release-sign.yml .github/workflows/release-comprehensive.yml .github/workflows/publish-npm.yml .github/workflows/publish-pypi.yml .github/workflows/publish-bun.yml .github/workflows/publish-tauri.yml .github/workflows/release-manifest-contract.yml + actionlint .github/workflows/lsp-producer.yml .github/workflows/lsp-publish.yml .github/workflows/release-coordinator.yml .github/workflows/release-sign.yml .github/workflows/release-comprehensive.yml .github/workflows/publish-npm.yml .github/workflows/publish-pypi.yml .github/workflows/publish-bun.yml .github/workflows/publish-tauri.yml .github/workflows/release-manifest-contract.yml - name: Parse workflow YAML run: | @@ -202,6 +216,9 @@ jobs: - name: Run workflow static contract tests run: python -m unittest -v tests.workflow_release_coordinator_contract_test + - name: Private-source LSP handoff and workflow contracts + run: python -m unittest -v tests.lsp_handoff_test tests.lsp_private_producer_test tests.workflow_lsp_handoff_test + - name: Run exhaustive release-writer and recovery contract tests run: python -m unittest -v tests.workflow_release_recovery_contract_test diff --git a/docs/contracts/schemas/lsp-lane-receipt.schema.json b/docs/contracts/schemas/lsp-lane-receipt.schema.json new file mode 100644 index 000000000..885f2d95d --- /dev/null +++ b/docs/contracts/schemas/lsp-lane-receipt.schema.json @@ -0,0 +1,1178 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://terraphim.ai/schemas/lsp-lane-receipt.schema.json", + "title": "LSP lane-receipt", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "producer", + "receipt" + ], + "properties": { + "schema": { + "const": 1 + }, + "producer": { + "type": "object", + "additionalProperties": false, + "required": [ + "repository", + "workflow", + "controlCommit", + "runId", + "runAttempt", + "event" + ], + "properties": { + "repository": { + "const": "terraphim/terraphim-ai" + }, + "workflow": { + "const": ".github/workflows/lsp-producer.yml" + }, + "controlCommit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "runId": { + "type": "integer", + "minimum": 1 + }, + "runAttempt": { + "type": "integer", + "minimum": 1 + }, + "event": { + "const": "workflow_dispatch" + } + } + }, + "receipt": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "lane", + "status", + "sourceCommit", + "sourceTag", + "version", + "toolchain", + "recipeSha256", + "signing", + "commands", + "assets" + ], + "properties": { + "schema": { + "const": 1 + }, + "lane": { + "const": "qualify" + }, + "status": { + "const": "PASS" + }, + "sourceCommit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "sourceTag": { + "type": "string", + "pattern": "^terraphim_lsp-v[0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$" + }, + "version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "toolchain": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+(?:\\.[0-9]+)?$" + }, + "recipeSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "signing": { + "const": "not-applicable" + }, + "commands": { + "type": "array", + "prefixItems": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "recipe-tests" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "install-toolchain" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "fmt" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "clippy" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "test" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + } + ], + "items": false, + "minItems": 5, + "maxItems": 5 + }, + "assets": { + "type": "array", + "items": false, + "minItems": 0, + "maxItems": 0 + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "lane", + "status", + "sourceCommit", + "sourceTag", + "version", + "toolchain", + "recipeSha256", + "signing", + "commands", + "assets" + ], + "properties": { + "schema": { + "const": 1 + }, + "lane": { + "const": "x86_64-unknown-linux-gnu" + }, + "status": { + "const": "PASS" + }, + "sourceCommit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "sourceTag": { + "type": "string", + "pattern": "^terraphim_lsp-v[0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$" + }, + "version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "toolchain": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+(?:\\.[0-9]+)?$" + }, + "recipeSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "signing": { + "const": "not-applicable" + }, + "commands": { + "type": "array", + "prefixItems": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "install-toolchain" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "fetch" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "build" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "architecture" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "version" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + } + ], + "items": false, + "minItems": 5, + "maxItems": 5 + }, + "assets": { + "type": "array", + "prefixItems": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "sha256", + "bytes" + ], + "properties": { + "name": { + "const": "terraphim-lsp-x86_64-unknown-linux-gnu" + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "bytes": { + "type": "integer", + "minimum": 1, + "maximum": 134217728 + } + } + } + ], + "items": false, + "minItems": 1, + "maxItems": 1 + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "lane", + "status", + "sourceCommit", + "sourceTag", + "version", + "toolchain", + "recipeSha256", + "signing", + "commands", + "assets" + ], + "properties": { + "schema": { + "const": 1 + }, + "lane": { + "const": "x86_64-unknown-linux-musl" + }, + "status": { + "const": "PASS" + }, + "sourceCommit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "sourceTag": { + "type": "string", + "pattern": "^terraphim_lsp-v[0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$" + }, + "version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "toolchain": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+(?:\\.[0-9]+)?$" + }, + "recipeSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "signing": { + "const": "not-applicable" + }, + "commands": { + "type": "array", + "prefixItems": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "install-toolchain" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "install-cross" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "fetch" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "build" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "architecture" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "version" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + } + ], + "items": false, + "minItems": 6, + "maxItems": 6 + }, + "assets": { + "type": "array", + "prefixItems": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "sha256", + "bytes" + ], + "properties": { + "name": { + "const": "terraphim-lsp-x86_64-unknown-linux-musl" + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "bytes": { + "type": "integer", + "minimum": 1, + "maximum": 134217728 + } + } + } + ], + "items": false, + "minItems": 1, + "maxItems": 1 + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "lane", + "status", + "sourceCommit", + "sourceTag", + "version", + "toolchain", + "recipeSha256", + "signing", + "commands", + "assets" + ], + "properties": { + "schema": { + "const": 1 + }, + "lane": { + "const": "aarch64-unknown-linux-musl" + }, + "status": { + "const": "PASS" + }, + "sourceCommit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "sourceTag": { + "type": "string", + "pattern": "^terraphim_lsp-v[0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$" + }, + "version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "toolchain": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+(?:\\.[0-9]+)?$" + }, + "recipeSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "signing": { + "const": "not-applicable" + }, + "commands": { + "type": "array", + "prefixItems": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "install-toolchain" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "install-cross" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "fetch" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "build" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "architecture" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + } + ], + "items": false, + "minItems": 5, + "maxItems": 5 + }, + "assets": { + "type": "array", + "prefixItems": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "sha256", + "bytes" + ], + "properties": { + "name": { + "const": "terraphim-lsp-aarch64-unknown-linux-musl" + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "bytes": { + "type": "integer", + "minimum": 1, + "maximum": 134217728 + } + } + } + ], + "items": false, + "minItems": 1, + "maxItems": 1 + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "lane", + "status", + "sourceCommit", + "sourceTag", + "version", + "toolchain", + "recipeSha256", + "signing", + "commands", + "assets" + ], + "properties": { + "schema": { + "const": 1 + }, + "lane": { + "const": "macos" + }, + "status": { + "const": "PASS" + }, + "sourceCommit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "sourceTag": { + "type": "string", + "pattern": "^terraphim_lsp-v[0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$" + }, + "version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "toolchain": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+(?:\\.[0-9]+)?$" + }, + "recipeSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "signing": { + "const": "unsigned" + }, + "commands": { + "type": "array", + "prefixItems": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "install-toolchain" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "fetch" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "build-arm64" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "build-x86_64" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "lipo" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "architecture-arm64" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "architecture-x86_64" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "architecture-universal" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "version-native" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "exitCode", + "logSha256" + ], + "properties": { + "name": { + "const": "version-universal" + }, + "exitCode": { + "const": 0 + }, + "logSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + } + ], + "items": false, + "minItems": 10, + "maxItems": 10 + }, + "assets": { + "type": "array", + "prefixItems": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "sha256", + "bytes" + ], + "properties": { + "name": { + "const": "terraphim-lsp-aarch64-apple-darwin" + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "bytes": { + "type": "integer", + "minimum": 1, + "maximum": 134217728 + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "sha256", + "bytes" + ], + "properties": { + "name": { + "const": "terraphim-lsp-universal-apple-darwin" + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "bytes": { + "type": "integer", + "minimum": 1, + "maximum": 134217728 + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "sha256", + "bytes" + ], + "properties": { + "name": { + "const": "terraphim-lsp-x86_64-apple-darwin" + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "bytes": { + "type": "integer", + "minimum": 1, + "maximum": 134217728 + } + } + } + ], + "items": false, + "minItems": 3, + "maxItems": 3 + } + } + } + ] + } + } +} diff --git a/docs/contracts/schemas/lsp-release-handoff.schema.json b/docs/contracts/schemas/lsp-release-handoff.schema.json new file mode 100644 index 000000000..c2b166865 --- /dev/null +++ b/docs/contracts/schemas/lsp-release-handoff.schema.json @@ -0,0 +1,90 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://terraphim.ai/schemas/lsp-release-handoff.schema.json", + "title": "LSP release-handoff", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "tag", + "requestSha256", + "manifestSha256", + "producer", + "artifact" + ], + "properties": { + "schema": { + "const": 1 + }, + "tag": { + "type": "string", + "pattern": "^terraphim_lsp-v[0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$" + }, + "requestSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "manifestSha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "producer": { + "type": "object", + "additionalProperties": false, + "required": [ + "repository", + "workflow", + "controlCommit", + "runId", + "runAttempt", + "event" + ], + "properties": { + "repository": { + "const": "terraphim/terraphim-ai" + }, + "workflow": { + "const": ".github/workflows/lsp-producer.yml" + }, + "controlCommit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "runId": { + "type": "integer", + "minimum": 1 + }, + "runAttempt": { + "type": "integer", + "minimum": 1 + }, + "event": { + "const": "workflow_dispatch" + } + } + }, + "artifact": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "name", + "digest" + ], + "properties": { + "id": { + "type": "integer", + "minimum": 1 + }, + "name": { + "type": "string", + "pattern": "^lsp-frozen-terraphim_lsp-v[0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?-attempt[1-9][0-9]*$" + }, + "digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + } + } + } + } +} diff --git a/docs/contracts/schemas/lsp-source-request.schema.json b/docs/contracts/schemas/lsp-source-request.schema.json new file mode 100644 index 000000000..5b808c145 --- /dev/null +++ b/docs/contracts/schemas/lsp-source-request.schema.json @@ -0,0 +1,82 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://terraphim.ai/schemas/lsp-source-request.schema.json", + "title": "LSP source-request", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "component", + "version", + "tag", + "toolchain", + "source", + "recipe" + ], + "properties": { + "schema": { + "const": 1 + }, + "component": { + "const": "terraphim-lsp" + }, + "version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "tag": { + "type": "string", + "pattern": "^terraphim_lsp-v[0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$" + }, + "toolchain": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+(?:\\.[0-9]+)?$" + }, + "source": { + "type": "object", + "additionalProperties": false, + "required": [ + "repository", + "commit", + "tag", + "requiredChecks" + ], + "properties": { + "repository": { + "const": "https://git.terraphim.cloud/terraphim/terraphim-ai" + }, + "commit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "tag": { + "type": "string", + "pattern": "^terraphim_lsp-v[0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$" + }, + "requiredChecks": { + "const": [ + "native-ci / build (push)", + "native-clippy / lint (push)" + ] + } + } + }, + "recipe": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "sha256" + ], + "properties": { + "path": { + "const": ".github/scripts/release/build_lsp_lane.py" + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + } + } +} diff --git a/docs/plans/lsp-provenance-handoff.md b/docs/plans/lsp-provenance-handoff.md new file mode 100644 index 000000000..b01d505bf --- /dev/null +++ b/docs/plans/lsp-provenance-handoff.md @@ -0,0 +1,35 @@ +# Private-source LSP binary releases + +This control path releases six frozen LSP binaries from reviewed private source. It does not mirror private source or history. Existing same-source publication remains unchanged in `publish_lsp_release.py`; `lsp_handoff.py` explicitly separates the public control commit from the private build commit. + +## One-time bindings + +First create the organization runner group `lsp-private`, restricted to this repository and exactly `terraphim/terraphim-ai/.github/workflows/lsp-producer.yml@refs/heads/main`. Run its Linux/macOS agents under dedicated OS accounts/workspaces inaccessible to untrusted PR jobs; a custom label alone does not provide that isolation. Existing physical hosts may be reused only after this isolation is verified. Independently read back group policy and runner account/workspace configuration before binding source credentials. The workflows require this group and contain no generic-runner fallback. The initial metadata probe returned HTTP 403 for the current account, so the group and this organization's selected-workflow feature availability are **not verified**. [GitHub runner-group access documentation](https://docs.github.com/en/enterprise-cloud@latest/actions/how-tos/manage-runners/self-hosted-runners/manage-access) describes selected-workflow restrictions. + +Create the GitHub environment `lsp-producer` with a custom deployment policy containing exactly the `main` branch. Bind `GITEA_LSP_SOURCE_TOKEN` to a dedicated read-only token covering Git/API reads for `terraphim/terraphim-ai` and the pinned `terraphim/terraphim-editor` dependency. Bind `CARGO_REGISTRIES_TERRAPHIM_TOKEN` with its required `Bearer ` prefix. The workflows never fall back to broad Gitea or 1Password credentials. The producer runtime verifies source/tag/required Gitea CI and the private build-recipe hash. Environment and credential availability must be verified before a real run; checked-in code is not evidence those bindings exist. + +Create `lsp-releases` with a custom deployment policy containing exactly the `terraphim_lsp-v*` tag pattern. Only its hosted publisher has `contents: write`. Its `actions: read` permission reads producer and artifact metadata. The publisher receives no private-source credentials. The producer's resolve job checks the environment policy with `actions: read`; build jobs have only `contents: read`. + +The producer runs only from reviewed public main using `workflow_dispatch`, on isolated self-hosted Linux and macOS agents in `lsp-private`. No PR-triggered job reads private source. The initial recipe deliberately builds unsigned macOS binaries; it does not claim Developer ID signing or notarization. + +## Reviewed request, build and handoff + +1. Normal-merge the private build recipe, including its tests. Choose a new version absent from private core tags, internal LSP releases and public releases. `rc.3` already identifies an older internal build; do not reuse it. Create the exact private source tag only after source review and required CI. This document creates no tag. +2. Add `release/lsp/requests/.json` through a normal public PR, following `docs/contracts/schemas/lsp-source-request.schema.json`. It pins the private source SHA/tag, toolchain, CI contexts and the exact build-recipe path/hash. No production request or invented source identity is included with this framework. +3. Dispatch **Build private-source LSP artifact** at public main using that tag. Qualification runs the private recipe tests, pinned Rust formatting/Clippy/core and LSP tests before any binary build lane. Each lane fetches the reviewed private source, verifies the recipe hash, uses an isolated Cargo cache, and produces only expected binaries plus a strict receipt. The supported matrix is Linux GNU x86_64, Linux musl x86_64/aarch64, and macOS x86_64/aarch64/universal. Native architectures verify `--version`; cross architectures receive explicit architecture checks. +4. Raw checkout, compiler/test logs and private failure receipts stay under the runner owner's mode-0700 `~/.local/state/terraphim-lsp-private/producer-*`. The ephemeral askpass script is removed. Public outcomes use fixed error codes; only hashes and known receipt fields leave the private lane. These logs need deliberate owner retention/cleanup after release review. They are not uploaded as public Actions artifacts. +5. Each public lane receipt binds the producer repository/workflow/control commit/run/attempt. Assembly refuses mixed attempts, extra files, altered bytes or failed qualification. Re-run **all jobs** after a failure; artifacts from earlier attempts cannot satisfy later attempts. The final immutable artifact contains exactly six raw binaries, `checksums.txt` and `lsp-release-manifest.json`. +6. Download and independently review that artifact, then add `release/lsp/handoffs/.json` in a separate normal PR. The handoff pins its artifact ID/digest, producer run/attempt/control commit, and manifest hash. `requestSha256` is SHA-256 of canonical JSON (sorted keys, compact separators); duplicate JSON keys are rejected. The artifact digest is GitHub's ZIP SHA-256. The manifest and every binary also have independent byte hashes. Metadata fixtures in tests are not a claim of successful remote publishing. +7. Normal-merge the handoff, then create the public release tag on that reviewed public main commit. The tag-only publisher requires exact main ancestry, tag target, workflow identity, completed successful producer and exact artifact. Expired artifacts, reruns, moved tags and mismatched hashes fail closed. It downloads the frozen ZIP with bounded extraction, executes no private source, and uses the unchanged draft/resume publisher. Existing published bytes are never replaced. +8. Inspect the publication receipt and anonymously download every public asset. Only exact byte readback is a successful release. Install that exact released LSP in stock Zed and rerun host acceptance before updating the extension's immutable provisioning pin and preparing its upstream catalog handoff. + +## Verification + +Use the existing pinned requirements in `scripts/requirements-release-manifest.txt` and `scripts/requirements-release-workflow-tests.txt`. Run: + +```sh +python -m unittest -v tests.lsp_handoff_test tests.lsp_private_producer_test tests.workflow_lsp_handoff_test tests.workflow_release_recovery_contract_test +actionlint .github/workflows/lsp-producer.yml .github/workflows/lsp-publish.yml .github/workflows/release-manifest-contract.yml +``` + +These tests use real Git histories, files, subprocesses and ZIP archives. Metadata inputs are explicit fixtures. They prove refusal and orchestration contracts; six-target compilation, source credential access, native host acceptance and public release readback remain separate actual-output gates. diff --git a/tests/lsp_handoff_test.py b/tests/lsp_handoff_test.py new file mode 100644 index 000000000..ab49c3398 --- /dev/null +++ b/tests/lsp_handoff_test.py @@ -0,0 +1,196 @@ +"""Real files/Git/ZIP tests; metadata samples do not prove remote publication.""" +import copy +import importlib.util +import json +from pathlib import Path +import stat +import sys +import subprocess +import tempfile +import unittest +import zipfile +import warnings + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('handoff', ROOT / '.github/scripts/release/lsp_handoff.py') +h = importlib.util.module_from_spec(spec) +spec.loader.exec_module(h) + + +def request(): + return {'schema': 1, 'component': 'terraphim-lsp', 'version': '1.22.0', + 'tag': 'terraphim_lsp-v1.22.0-rc.3', 'toolchain': '1.96', + 'source': {'repository': h.PRIVATE_REPO, 'commit': '1' * 40, + 'tag': 'terraphim_lsp-v1.22.0-rc.3', 'requiredChecks': list(h.SOURCE_CHECKS)}, + 'recipe': {'path': h.RECIPE, 'sha256': '2' * 64}} + + +def producer(): + return {'repository': h.REPO, 'workflow': h.WORKFLOW, 'controlCommit': '3' * 40, + 'runId': 100, 'runAttempt': 1, 'event': 'workflow_dispatch'} + + +def lane(name, req, assets=()): + return {'schema': 1, 'lane': name, 'status': 'PASS', 'sourceCommit': req['source']['commit'], + 'sourceTag': req['tag'], 'version': req['version'], 'toolchain': req['toolchain'], + 'recipeSha256': req['recipe']['sha256'], 'signing': 'unsigned' if name == 'macos' else 'not-applicable', + 'commands': [{'name': n, 'exitCode': 0, 'logSha256': '4' * 64} for n in h.COMMAND_NAMES[name]], + 'assets': list(assets)} + + +class ContractTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + self.addCleanup(self.temp.cleanup) + + def test_request_exact_and_extra_private_data_refused(self): + self.assertEqual(h.validate_request(request()), request()) + for path, value in [('toolchain', 'stable'), ('tag', '../escape'), ('extra', 'private log')]: + bad = request(); bad[path] = value + with self.subTest(path=path), self.assertRaises(ValueError): h.validate_request(bad) + for key, value in [('repository', 'https://example.com/source'), ('commit', 'main'), ('tag', 'other'), ('requiredChecks', [])]: + bad = request(); bad['source'][key] = value + with self.subTest(key=key), self.assertRaises(ValueError): h.validate_request(bad) + for value in ['../secret.py', '.github/scripts/release/other.py']: + bad = request(); bad['recipe']['path'] = value + with self.assertRaises(ValueError): h.validate_request(bad) + + def test_lane_refuses_missing_failed_extra_or_source_changes(self): + good = lane('qualify', request()) + h.validate_lane(good, request()) + variants = [] + for key, value in [('sourceCommit', '5' * 40), ('toolchain', 'nightly'), ('privateLog', 'secret'), ('status', 'FAIL')]: + bad = copy.deepcopy(good); bad[key] = value; variants.append(bad) + bad = copy.deepcopy(good); bad['commands'][0]['exitCode'] = 1; variants.append(bad) + bad = copy.deepcopy(good); bad['commands'][0]['stderr'] = 'private'; variants.append(bad) + bad = copy.deepcopy(good); bad['commands'].pop(); variants.append(bad) + for bad in variants: + with self.assertRaises(ValueError): h.validate_lane(bad, request()) + + def bundle(self): + req = request(); folders = [] + for name in h.LANES: + folder = self.root / name; folder.mkdir(); folders.append(folder) + for asset in h.LANE_ASSETS[name]: (folder / asset).write_bytes(b'fixture-not-a-released-binary:' + asset.encode()) + receipt = lane(name, req, h.pub.asset_records([folder / n for n in h.LANE_ASSETS[name]])) + h.pub.write_new(folder / 'lane.json', {'schema': 1, 'producer': producer(), 'receipt': receipt}) + out = self.root / 'bundle' + manifest = h.assemble_public_bundle(folders, req, producer(), out) + return req, out, manifest + + def test_bundle_checks_exact_bytes_and_no_private_logs(self): + req, out, manifest = self.bundle() + h.verify_bundle(out, req, producer()) + self.assertEqual(set(p.name for p in out.iterdir()), set(h.pub.RELEASE_FILES)) + self.assertEqual(manifest['provenanceMode'], 'private-source-handoff-v1') + (out / h.pub.RAW_ASSETS[0]).write_bytes(b'changed') + with self.assertRaises(ValueError): h.verify_bundle(out, req, producer()) + + def test_lane_directory_extra_and_symlink_refused(self): + req, out, manifest = self.bundle() + folder = self.root / 'qualify'; (folder / 'private.log').write_text('private') + with self.assertRaises(ValueError): h.assemble_public_bundle([self.root / n for n in h.LANES], req, producer(), self.root / 'out2') + (folder / 'private.log').unlink() + asset = self.root / 'macos' / h.LANE_ASSETS['macos'][0] + data = asset.read_bytes(); asset.unlink(); (self.root / 'outside').write_bytes(data); asset.symlink_to(self.root / 'outside') + with self.assertRaises(ValueError): h.assemble_public_bundle([self.root / n for n in h.LANES], req, producer(), self.root / 'out3') + + def test_mixed_attempt_lane_receipt_refused(self): + req, out, manifest = self.bundle() + path = self.root / 'qualify/lane.json'; envelope = h.pub.read_json(path) + envelope['producer']['runAttempt'] = 2; path.write_text(json.dumps(envelope)) + with self.assertRaises(ValueError): h.assemble_public_bundle([self.root/n for n in h.LANES], req, producer(), self.root/'mixed') + + def test_cli_missing_request_saves_only_fixed_code(self): + receipt = self.root/'failure.json' + result = subprocess.run([sys.executable, str(ROOT/'.github/scripts/release/lsp_handoff.py'), 'check-request', + '--root', str(self.root), '--tag', request()['tag'], '--receipt', str(receipt)], + text=True, capture_output=True, check=False, timeout=30) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stderr.strip(), 'LSP_HANDOFF_REJECTED') + self.assertEqual(h.pub.read_json(receipt), {'schema': 1, 'status': 'FAIL', 'errorCode': 'LSP_HANDOFF_REJECTED'}) + self.assertNotIn(str(self.root), result.stderr) + + def test_zip_limits_paths_duplicates_and_symlinks(self): + for index, names in enumerate([['../secret'], ['/secret'], ['folder/file'], ['checksums.txt', 'checksums.txt'], ['unexpected']]): + z = self.root / f'bad{index}.zip' + with warnings.catch_warnings(), zipfile.ZipFile(z, 'w') as archive: + warnings.simplefilter('ignore', UserWarning) + for name in names: archive.writestr(name, b'x') + with self.subTest(names=names), self.assertRaises(ValueError): h.extract_bundle(z, self.root / f'extracted{index}') + z = self.root / 'symlink.zip' + with zipfile.ZipFile(z, 'w') as archive: + for name in h.pub.RELEASE_FILES: + info = zipfile.ZipInfo(name) + info.create_system = 3; info.external_attr = (stat.S_IFLNK | 0o777) << 16 + archive.writestr(info, b'elsewhere') + with self.assertRaises(ValueError): h.extract_bundle(z, self.root / 'symlink-out') + + def test_zip_real_bundle_roundtrip(self): + req, out, manifest = self.bundle(); z = self.root / 'bundle.zip' + with zipfile.ZipFile(z, 'w') as archive: + for path in out.iterdir(): archive.write(path, path.name) + dest = self.root / 'unpacked'; h.extract_bundle(z, dest) + h.verify_bundle(dest, req, producer()) + with self.assertRaises(ValueError): h.extract_bundle(z, dest) + with self.assertRaises(ValueError): h.extract_bundle(z, self.root / 'small', max_bytes=1) + + def metadata(self): + req, out, manifest = self.bundle() + proof = {'schema': 1, 'tag': req['tag'], 'requestSha256': h.canonical_digest(req), + 'manifestSha256': h.pub.digest(out / h.pub.MANIFEST), 'producer': producer(), + 'artifact': {'id': 200, 'name': h.artifact_name(req['tag'], 1), 'digest': 'sha256:' + '6' * 64}} + run = {'id': 100, 'run_attempt': 1, 'repository': {'full_name': h.REPO}, 'head_repository': {'full_name': h.REPO}, + 'head_sha': '3' * 40, 'head_branch': 'main', 'path': h.WORKFLOW, 'event': 'workflow_dispatch', + 'status': 'completed', 'conclusion': 'success'} + art = {'id': 200, 'name': proof['artifact']['name'], 'digest': proof['artifact']['digest'], 'expired': False, 'size_in_bytes': 1000, + 'workflow_run': {'id': 100, 'head_sha': '3' * 40, 'head_branch': 'main'}} + return req, out, proof, run, art + + def test_producer_proof_refuses_every_identity_failure(self): + req, out, proof, run, art = self.metadata(); h.verify_producer(proof, run, art) + for key, val in [('id', 101), ('run_attempt', 2), ('head_sha', '7' * 40), ('head_branch', 'feature'), ('path', 'other.yml'), ('event', 'pull_request'), ('status', 'in_progress'), ('conclusion', 'failure'), ('repository', {'full_name': 'fork/repo'}), ('head_repository', {'full_name': 'fork/repo'})]: + bad = copy.deepcopy(run); bad[key] = val + with self.subTest(key=key), self.assertRaises(ValueError): h.verify_producer(proof, bad, art) + for key, val in [('id', 201), ('digest', 'sha256:' + '7' * 64), ('expired', True), ('name', 'other'), ('workflow_run', {'id': 101})]: + bad = copy.deepcopy(art); bad[key] = val + with self.subTest(key=key), self.assertRaises(ValueError): h.verify_producer(proof, run, bad) + + def test_malformed_remote_shapes_raise_safe_validation_error(self): + req, out, proof, run, art = self.metadata() + for malformed in (None, [], 'wrong', 4): + with self.subTest(top=malformed), self.assertRaises(ValueError): h.verify_producer(proof, malformed, art) + with self.assertRaises(ValueError): h.verify_producer(proof, run, malformed) + for field in ('repository', 'head_repository'): + bad = copy.deepcopy(run); bad[field] = malformed + with self.subTest(field=field, value=malformed), self.assertRaises(ValueError): h.verify_producer(proof, bad, art) + bad = copy.deepcopy(art); bad['workflow_run'] = malformed + with self.assertRaises(ValueError): h.verify_producer(proof, run, bad) + with self.assertRaises(ValueError): h.validate_environment(malformed, {}, 'lsp-producer') + + def test_real_git_public_control_tag_main_and_dirty_guards(self): + repo = self.root / 'git'; repo.mkdir() + def git(*args): return subprocess.check_output(['git', *args], cwd=repo, text=True, stderr=subprocess.DEVNULL, timeout=30).strip() + git('init', '-b', 'main'); git('config', 'user.name', 'Fixture'); git('config', 'user.email', 'fixture@example.invalid') + git('remote', 'add', 'origin', 'https://github.com/' + h.REPO + '.git') + (repo / 'control').write_text('reviewed'); git('add', '.'); git('commit', '-m', 'fixture') + sha = git('rev-parse', 'HEAD'); git('update-ref', 'refs/remotes/origin/main', sha); git('tag', request()['tag']) + h.public_control(repo, sha, request()['tag']) + with self.assertRaises(ValueError): h.public_control(repo, '0' * 40, request()['tag']) + (repo / 'control').write_text('changed') + with self.assertRaises(ValueError): h.public_control(repo, sha, request()['tag']) + git('checkout', '--', 'control'); git('tag', '-d', request()['tag']) + with self.assertRaises(ValueError): h.public_control(repo, sha, request()['tag']) + + def test_handoff_changed_manifest_or_request_refused(self): + req, out, proof, run, art = self.metadata(); h.validate_handoff(proof) + h.verify_handoff_bundle(proof, req, out) + bad = copy.deepcopy(proof); bad['requestSha256'] = '0' * 64 + with self.assertRaises(ValueError): h.verify_handoff_bundle(bad, req, out) + bad = copy.deepcopy(proof); bad['manifestSha256'] = '0' * 64 + with self.assertRaises(ValueError): h.verify_handoff_bundle(bad, req, out) + bad = copy.deepcopy(proof); bad['privateLog'] = 'oops' + with self.assertRaises(ValueError): h.validate_handoff(bad) + +if __name__ == '__main__': unittest.main() diff --git a/tests/lsp_private_producer_test.py b/tests/lsp_private_producer_test.py new file mode 100644 index 000000000..9bb60a82d --- /dev/null +++ b/tests/lsp_private_producer_test.py @@ -0,0 +1,85 @@ +"""Credential boundary and metadata validators; no remote-success simulation.""" +import copy +import importlib.util +from pathlib import Path +import tempfile +import unittest +from tests.lsp_handoff_test import request + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('private_producer', ROOT/'.github/scripts/release/lsp_private_producer.py') +p = importlib.util.module_from_spec(spec); spec.loader.exec_module(p) + +class BoundaryTest(unittest.TestCase): + def test_dedicated_binding_required_no_broad_token_fallback(self): + with self.assertRaisesRegex(ValueError, 'LSP_SOURCE_CREDENTIAL_MISSING'): + p.credentials({'GITEA_TOKEN': 'broad-fixture', 'OP_SERVICE_ACCOUNT_TOKEN': 'broad-fixture', + 'CARGO_REGISTRIES_TERRAPHIM_TOKEN': 'Bearer fixture'}) + with self.assertRaisesRegex(ValueError, 'LSP_REGISTRY_CREDENTIAL_MISSING'): + p.credentials({'GITEA_LSP_SOURCE_TOKEN': 'scoped-fixture'}) + env = p.credentials({'GITEA_LSP_SOURCE_TOKEN': 'scoped-fixture', 'CARGO_REGISTRIES_TERRAPHIM_TOKEN': 'Bearer fixture', + 'GH_TOKEN': 'not-allowed', 'GITEA_TOKEN': 'not-allowed', 'OP_SERVICE_ACCOUNT_TOKEN': 'not-allowed', + 'GIT_TRACE': 'not-allowed', 'HOME': '/fixture'}) + self.assertEqual(set(env), {'GITEA_LSP_SOURCE_TOKEN', 'CARGO_REGISTRIES_TERRAPHIM_TOKEN', 'HOME'}) + + def test_askpass_exact_host_repository_and_prompt(self): + canary = 'fixture-scoped-canary' + self.assertEqual(p.askpass_response("Username for 'https://git.terraphim.cloud/terraphim/terraphim-ai.git': ", canary), 'token') + self.assertEqual(p.askpass_response("Password for 'https://token@git.terraphim.cloud/terraphim/terraphim-editor.git': ", canary), canary) + for url in ('https://git.terraphim.cloud.attacker.invalid/terraphim/terraphim-ai.git', + 'https://attacker.invalid/https://git.terraphim.cloud', + 'https://git.terraphim.cloud@attacker.invalid/terraphim/terraphim-ai.git', + 'https://evil@git.terraphim.cloud/terraphim/terraphim-ai.git', + 'http://git.terraphim.cloud/terraphim/terraphim-ai.git', + 'https://git.terraphim.cloud/terraphim/other.git', + 'https://git.terraphim.cloud:443/terraphim/terraphim-ai.git', + 'https://git.terraphim.cloud/terraphim/terraphim-ai.git?secret=1'): + with self.subTest(url=url), self.assertRaises(ValueError): + p.askpass_response("Password for '" + url + "': ", canary) + + def test_metadata_source_tag_and_ci_required(self): + req = request(); repo = {'full_name': p.h.REPO, 'default_branch': 'main', 'private': True, 'archived': False} + editor = {'full_name': 'terraphim/terraphim-editor', 'archived': False} + tag = {'name': req['tag'], 'commit': {'sha': req['source']['commit']}} + status = {'sha': req['source']['commit'], 'state': 'success', 'statuses': [ + {'context': name, 'status': 'success', 'target_url': '/terraphim/terraphim-ai/actions/runs/1/jobs/2'} for name in p.h.SOURCE_CHECKS]} + result = p.verify_source_metadata(req, repo, tag, status, editor) + self.assertNotIn('statuses', result); self.assertEqual(len(result['metadataSha256']), 64) + for value in ('0'*40, None): + bad = copy.deepcopy(tag); bad['commit']['sha'] = value + with self.assertRaises(ValueError): p.verify_source_metadata(req, repo, bad, status, editor) + for value in ('pending', 'failure'): + bad = copy.deepcopy(status); bad['statuses'][0]['status'] = value + with self.assertRaises(ValueError): p.verify_source_metadata(req, repo, tag, bad, editor) + bad = copy.deepcopy(status); bad['statuses'][0]['target_url'] = '' + with self.assertRaises(ValueError): p.verify_source_metadata(req, repo, tag, bad, editor) + + def test_source_metadata_malformed_shapes_fail_validation(self): + req = request() + repo = {'full_name': p.h.REPO, 'default_branch': 'main', 'private': True, 'archived': False} + editor = {'full_name': 'terraphim/terraphim-editor', 'archived': False} + tag = {'name': req['tag'], 'commit': {'sha': req['source']['commit']}} + status = {'sha': req['source']['commit'], 'state': 'success', 'statuses': []} + for malformed in (None, [], 'wrong', 4): + for index in range(4): + args = [repo, tag, status, editor]; args[index] = malformed + with self.subTest(index=index, value=malformed), self.assertRaises(ValueError): p.verify_source_metadata(req, *args) + bad = copy.deepcopy(tag); bad['commit'] = malformed + with self.assertRaises(ValueError): p.verify_source_metadata(req, repo, bad, status, editor) + bad = copy.deepcopy(status); bad['statuses'] = malformed + with self.assertRaises(ValueError): p.verify_source_metadata(req, repo, tag, bad, editor) + bad = copy.deepcopy(status); bad['statuses'] = [malformed] + with self.assertRaises(ValueError): p.verify_source_metadata(req, repo, tag, bad, editor) + + def test_real_private_directory_permissions_and_symlink_refusal(self): + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary); lane = p.protected_directory(home) + self.assertEqual(lane.stat().st_mode & 0o077, 0) + self.assertNotIn('runner_temp', str(lane)) + lane.parent.chmod(0o755) + with self.assertRaises(ValueError): p.protected_directory(home) + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary); (home/'target').mkdir(); (home/'.local').symlink_to(home/'target') + with self.assertRaises(ValueError): p.protected_directory(home) + +if __name__ == '__main__': unittest.main() diff --git a/tests/workflow_lsp_handoff_test.py b/tests/workflow_lsp_handoff_test.py new file mode 100644 index 000000000..919d345f3 --- /dev/null +++ b/tests/workflow_lsp_handoff_test.py @@ -0,0 +1,100 @@ +"""Workflow/schema boundaries; real release execution remains a separate gate.""" +import copy +import hashlib +import json +from pathlib import Path +import unittest +import yaml +import jsonschema +from tests.lsp_handoff_test import h, request, producer, lane +from tests.workflow_release_recovery_contract_test import GITHUB_RELEASE_WRITER_PATTERN + +ROOT = Path(__file__).resolve().parents[1] + +class WorkflowTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.producer = yaml.safe_load((ROOT/'.github/workflows/lsp-producer.yml').read_text()) + cls.publisher = yaml.safe_load((ROOT/'.github/workflows/lsp-publish.yml').read_text()) + + def test_producer_no_pr_event_no_release_writer_no_broad_credentials(self): + value = self.producer + self.assertEqual(set(value.get('on', value.get(True))), {'workflow_dispatch'}) + self.assertEqual(value['permissions'], {'contents': 'read'}) + text = (ROOT/'.github/workflows/lsp-producer.yml').read_text() + self.assertFalse(GITHUB_RELEASE_WRITER_PATTERN.search(text)) + for forbidden in ('pull_request_target', 'OP_SERVICE_ACCOUNT_TOKEN', 'secrets.GITEA_TOKEN', 'contents: write'): + self.assertNotIn(forbidden, text) + self.assertIn("github.ref == 'refs/heads/main'", value['jobs']['resolve']['if']) + for name in ('qualify', 'linux', 'macos'): + job = value['jobs'][name]; self.assertEqual(job['environment'], 'lsp-producer') + self.assertEqual(job['runs-on']['group'], 'lsp-private') + self.assertIn('self-hosted', job['runs-on']['labels']) + self.assertIn('resolve', job['needs'] if isinstance(job['needs'], list) else [job['needs']]) + if name != 'qualify': self.assertIn('qualify', job['needs']) + private_steps = [s for s in job['steps'] if 'GITEA_LSP_SOURCE_TOKEN' in s.get('env', {})] + self.assertEqual(len(private_steps), 1) + self.assertIn('lsp_private_producer.py', private_steps[0]['run']) + for step in job['steps']: + if 'upload-artifact@' in step.get('uses', ''): + self.assertIn(step['with']['path'], ('public-lane/*', 'producer-receipt.json')) + + def test_publisher_separate_hosted_tag_only_and_no_build_or_private_credentials(self): + value = self.publisher; self.assertEqual(value.get('on', value.get(True)), {'push': {'tags': ['terraphim_lsp-v*']}}) + job = value['jobs']['publish']; self.assertEqual(job['runs-on'], 'ubuntu-latest') + self.assertEqual(job['environment'], 'lsp-releases') + self.assertEqual(job['permissions'], {'contents': 'write', 'actions': 'read'}) + text = (ROOT/'.github/workflows/lsp-publish.yml').read_text() + for forbidden in ('secrets.', 'cargo', 'GITEA_', 'OP_SERVICE', 'workflow_dispatch'): + self.assertNotIn(forbidden, text) + self.assertIn('lsp_handoff.py publish', text) + self.assertTrue(GITHUB_RELEASE_WRITER_PATTERN.search(text)) + + def test_exact_checkouts_and_pinned_actions(self): + for value in (self.producer, self.publisher): + for job in value['jobs'].values(): + for step in job['steps']: + if 'uses' in step: + self.assertRegex(step['uses'], r'^[\w/-]+@[a-f0-9]{40}$') + if step.get('uses', '').startswith('actions/checkout@'): + self.assertEqual(step['with']['ref'], '${{ github.sha }}') + self.assertEqual(step['with']['fetch-depth'], 0) + self.assertIs(step['with']['persist-credentials'], False) + + def test_schemas_valid_and_runtime_examples_agree(self): + definitions = {} + for kind in ('source-request', 'lane-receipt', 'release-handoff'): + schema = json.loads((ROOT/f'docs/contracts/schemas/lsp-{kind}.schema.json').read_text()) + jsonschema.Draft202012Validator.check_schema(schema) + definitions[kind] = jsonschema.Draft202012Validator(schema) + definitions['source-request'].validate(request()) + definitions['lane-receipt'].validate({'schema': 1, 'producer': producer(), 'receipt': lane('qualify', request())}) + handoff = {'schema': 1, 'tag': request()['tag'], 'requestSha256': 'a'*64, 'manifestSha256': 'b'*64, + 'producer': producer(), 'artifact': {'id': 2, 'name': h.artifact_name(request()['tag'], 1), 'digest': 'sha256:'+'c'*64}} + definitions['release-handoff'].validate(handoff); h.validate_handoff(handoff) + for kind, value in [('source-request', request()), ('release-handoff', handoff)]: + bad = copy.deepcopy(value); bad['privateLog'] = 'must not be public' + with self.assertRaises(jsonschema.ValidationError): definitions[kind].validate(bad) + + def test_environment_exact_branch_or_tag_policy_required(self): + for name, branch, kind in [('lsp-producer', 'main', 'branch'), ('lsp-releases', 'terraphim_lsp-v*', 'tag')]: + metadata = {'name': name, 'deployment_branch_policy': {'protected_branches': False, 'custom_branch_policies': True}} + policy = {'branch_policies': [{'name': branch, 'type': kind}]} + h.validate_environment(metadata, policy, name) + for bad in ({'branch_policies': []}, {'branch_policies': [{'name': '*', 'type': kind}]}, + {'branch_policies': [{'name': branch, 'type': kind}, {'name': 'feature', 'type': 'branch'}]}): + with self.assertRaises(ValueError): h.validate_environment(metadata, bad, name) + with self.assertRaises(ValueError): h.validate_environment({'name': name, 'deployment_branch_policy': None}, policy, name) + + def test_same_source_transport_hash_preserved(self): + actual = hashlib.sha256((ROOT/'.github/scripts/release/publish_lsp_release.py').read_bytes()).hexdigest() + self.assertEqual(actual, 'c87fdaa79a4901b039bce51c6a440a42bd950d1079968b97de17dab4752016c0') + + def test_contract_ci_runs_handoff_and_schema_tests(self): + text = (ROOT/'.github/workflows/release-manifest-contract.yml').read_text() + for name in ('tests.lsp_handoff_test', 'tests.lsp_private_producer_test', 'tests.workflow_lsp_handoff_test'): + self.assertIn(name, text) + for path in ('.github/workflows/lsp-producer.yml', '.github/workflows/lsp-publish.yml', 'docs/contracts/schemas/lsp-*.schema.json', 'tests/lsp_handoff_test.py', 'tests/lsp_private_producer_test.py', 'tests/workflow_lsp_handoff_test.py'): + self.assertGreaterEqual(text.count(path), 2) + +if __name__ == '__main__': unittest.main() diff --git a/tests/workflow_release_recovery_contract_test.py b/tests/workflow_release_recovery_contract_test.py index 657b8c56a..407988c7b 100644 --- a/tests/workflow_release_recovery_contract_test.py +++ b/tests/workflow_release_recovery_contract_test.py @@ -32,7 +32,9 @@ r"softprops/action-gh-release|gh release (?:create|edit|upload)|" r"repos\.(?:create|update)Release|createRelease|updateRelease|" r"github-publish-release\.sh|" - r"\bpython[0-9.]*\s+[\"']?(?:[\w./-]+/)?publish_release\.py\b", + r"\bpython[0-9.]*\s+[\"']?(?:[\w./-]+/)?publish_release\.py\b|" + r"\bpython[0-9.]*\s+[\"']?(?:[\w./-]+/)?publish_lsp_release\.py\b|" + r"\bpython[0-9.]*\s+[\"']?(?:[\w./-]+/)?lsp_handoff\.py[\"']?\s+publish\b", re.IGNORECASE, ) @@ -252,6 +254,7 @@ def test_exhaustive_github_release_writer_inventory_is_single_publisher_safe( "release-coordinator.yml", "continue-integration.yml", "vscode-release.yml", + "lsp-publish.yml", }, "every workflow that writes GitHub releases must be classified", )