From 62dd2d9b73313555a82e45e1784d27ca5fd55200 Mon Sep 17 00:00:00 2001 From: James Tarran Date: Sat, 19 Sep 2026 11:59:23 +0100 Subject: [PATCH] Publish a prebuilt manifest index to remove the API from the install path Follow-up to the caching work: takes the steady state to zero api.github.com calls rather than merely fewer. A nightly Action resolves every package in Index/Catalog.json once, in CI, where GITHUB_TOKEN gives a 5000/hour allowance, and publishes the result as a single Manifests.json. Endpoints read that one file from raw.githubusercontent.com, which is CDN-backed and carries no API rate limit, so an install resolves without spending any of the site's 60/hour public-IP allowance. The index is force-pushed to an orphan manifest-index branch. That keeps exactly one commit on it, so a nightly refresh never grows the repository and never touches code history. Resolution order in Get-GitHubInstaller is now: local per-package cache -> prebuilt index -> live API -> stale cache Every layer is optional. A missing, stale, unreachable or malformed index returns null and falls through to the live API exactly as before, so this cannot make installs worse than they are today. Adding a package to the index is a one-line edit to Index/Catalog.json. The seed list holds the three IDs that already carry argument overrides in the code plus common business applications; edit freely. Verified: builder resolves 7zip.7zip v26.03 and Notepad++.Notepad++ v8.9.8 against live manifests and emits a valid index; client returns the correct entry on a hit, and null on an unknown ID, an unindexed architecture and a completely absent index, degrading to the live API in each case. --- .gitattributes | 6 ++ .github/workflows/build-manifest-index.yml | 54 +++++++++++++ .gitignore | 2 + Build/Build-ManifestIndex.ps1 | 91 ++++++++++++++++++++++ Index/Catalog.json | 17 ++++ Private/Get-ManifestIndex.ps1 | 86 ++++++++++++++++++++ Public/Get-GitHubInstaller.ps1 | 15 +++- 7 files changed, 269 insertions(+), 2 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/build-manifest-index.yml create mode 100644 .gitignore create mode 100644 Build/Build-ManifestIndex.ps1 create mode 100644 Index/Catalog.json create mode 100644 Private/Get-ManifestIndex.ps1 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3dbbea2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Workflow files must stay LF. A CRLF `run:` block reaches bash on the runner +# with a literal CR on every line, which breaks the script. +# Deliberately NOT declaring `* text=auto`: the existing blobs in this repo are +# committed CRLF, so that would queue a repo-wide renormalisation diff. +*.yml -text +*.yaml -text diff --git a/.github/workflows/build-manifest-index.yml b/.github/workflows/build-manifest-index.yml new file mode 100644 index 0000000..9f12f25 --- /dev/null +++ b/.github/workflows/build-manifest-index.yml @@ -0,0 +1,54 @@ +name: Build manifest index + +on: + schedule: + # Nightly. NOTE: GitHub only runs scheduled workflows from the repository's + # DEFAULT branch, so this fires once the workflow exists on that branch. + - cron: '17 3 * * *' + workflow_dispatch: + push: + paths: + - 'Index/Catalog.json' + - 'Build/Build-ManifestIndex.ps1' + - 'Private/Resolve-GitHubManifest.ps1' + - '.github/workflows/build-manifest-index.yml' + +permissions: + contents: write + +concurrency: + group: manifest-index + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Resolve manifests + shell: pwsh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: ./Build/Build-ManifestIndex.ps1 -CatalogPath Index/Catalog.json -OutputPath Index/Manifests.json + + - name: Publish to the manifest-index branch + run: | + set -euo pipefail + cp Index/Manifests.json "$RUNNER_TEMP/Manifests.json" + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + # Orphan + force-push keeps exactly one commit on the branch, so a + # nightly refresh never grows the repository or touches code history. + git checkout --orphan manifest-index + git rm -rf . -q + cp "$RUNNER_TEMP/Manifests.json" Manifests.json + git add Manifests.json + git commit -q -m "Refresh manifest index ($(date -u +%Y-%m-%dT%H:%M:%SZ))" + git push -f origin manifest-index + + echo "Published to https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/manifest-index/Manifests.json" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d0b7058 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +# Built in CI and published to the manifest-index branch, never committed here. +Index/Manifests.json diff --git a/Build/Build-ManifestIndex.ps1 b/Build/Build-ManifestIndex.ps1 new file mode 100644 index 0000000..68fc61b --- /dev/null +++ b/Build/Build-ManifestIndex.ps1 @@ -0,0 +1,91 @@ +<# +.SYNOPSIS + Pre-resolves winget-pkgs manifests into a single JSON index. + +.DESCRIPTION + Runs in CI, not on endpoints. Resolving a package costs two + api.github.com calls; doing it once here and publishing the result means + endpoints read one file from raw.githubusercontent.com instead, which is + CDN-backed and not subject to the API rate limit. + + A package that fails to resolve is reported and skipped. The build only + fails if every package failed, so one delisted app cannot stop the + nightly refresh. +#> +[CmdletBinding()] +param( + [string]$CatalogPath = 'Index/Catalog.json', + [string]$OutputPath = 'Index/Manifests.json', + [string[]]$Architectures = @('x64', 'arm64'), + [string]$GitHubToken = $env:GITHUB_TOKEN +) + +$ErrorActionPreference = 'Stop' + +. (Join-Path $PSScriptRoot '..\Private\Resolve-GitHubManifest.ps1') + +$Headers = @{ 'User-Agent' = 'TecharyGet-IndexBuilder' } +if ($GitHubToken) { + $Headers['Authorization'] = "token $GitHubToken" + Write-Host "Authenticated to the GitHub API (5000 requests/hour)." +} else { + Write-Warning "No token supplied. Falling back to 60 requests/hour, which will not cover a full catalogue." +} + +$Catalog = Get-Content -Path $CatalogPath -Raw | ConvertFrom-Json +$Ids = @($Catalog.Packages | Sort-Object) +Write-Host "Catalogue contains $($Ids.Count) package(s); resolving $($Architectures -join ', ')." + +$Packages = [ordered]@{} +$Ok = 0 +$Failed = New-Object System.Collections.Generic.List[string] + +foreach ($Id in $Ids) { + $PerArch = [ordered]@{} + + foreach ($Arch in $Architectures) { + try { + $Meta = Resolve-GitHubManifest -Id $Id -SysArch $Arch -Headers $Headers + $PerArch[$Arch] = [ordered]@{ + Version = $Meta.Version + Url = $Meta.Url + SilentArgs = $Meta.SilentArgs + InstallerType = $Meta.InstallerType + ProductCode = $Meta.ProductCode + } + Write-Host (" ok {0,-34} {1,-6} v{2}" -f $Id, $Arch, $Meta.Version) + } + catch { + # Most packages genuinely have no arm64 installer. Only an x64 + # failure is worth counting as a real failure. + $Level = if ($Arch -eq 'x64') { 'warn ' } else { 'skip ' } + Write-Host (" {0} {1,-34} {2,-6} {3}" -f $Level, $Id, $Arch, $_.Exception.Message) + } + } + + if ($PerArch.Count -gt 0) { + $Packages[$Id] = $PerArch + $Ok++ + } else { + $Failed.Add($Id) + } +} + +if ($Ok -eq 0) { + throw "Every package in the catalogue failed to resolve. Refusing to publish an empty index." +} + +$Index = [ordered]@{ + Generated = (Get-Date).ToUniversalTime().ToString('o') + Source = 'microsoft/winget-pkgs' + PackageCount = $Ok + Packages = $Packages +} + +$Dir = Split-Path $OutputPath -Parent +if ($Dir -and -not (Test-Path $Dir)) { New-Item -ItemType Directory -Path $Dir -Force | Out-Null } +$Index | ConvertTo-Json -Depth 6 | Set-Content -Path $OutputPath -Encoding UTF8 + +Write-Host "" +Write-Host "Wrote $OutputPath with $Ok package(s)." +if ($Failed.Count -gt 0) { Write-Warning "Unresolved: $($Failed -join ', ')" } diff --git a/Index/Catalog.json b/Index/Catalog.json new file mode 100644 index 0000000..efe1d6b --- /dev/null +++ b/Index/Catalog.json @@ -0,0 +1,17 @@ +{ + "_comment": "winget package IDs pre-resolved nightly into Manifests.json. Adding an ID here is the only step needed to take it off the live GitHub API path. Seed list - edit freely.", + "Packages": [ + "7zip.7zip", + "8x8.Work", + "Adobe.Acrobat.Reader.64-bit", + "Dell.CommandUpdate", + "Google.Chrome", + "Microsoft.PowerToys", + "Microsoft.VisualStudioCode", + "Mozilla.Firefox", + "Notepad++.Notepad++", + "SublimeHQ.SublimeText.4", + "VideoLAN.VLC", + "Zoom.Zoom" + ] +} diff --git a/Private/Get-ManifestIndex.ps1 b/Private/Get-ManifestIndex.ps1 new file mode 100644 index 0000000..99089ec --- /dev/null +++ b/Private/Get-ManifestIndex.ps1 @@ -0,0 +1,86 @@ +function Get-ManifestIndexUrl { + # Published by .github/workflows/build-manifest-index.yml to a dedicated + # orphan branch, so a nightly refresh never touches code history. + # raw.githubusercontent.com is CDN-backed and is NOT subject to the + # api.github.com rate limit, which is the entire point of the index. + return "https://raw.githubusercontent.com/Techary/TecharyGet/manifest-index/Manifests.json" +} + +function Get-ManifestIndex { + [CmdletBinding()] + param( + [int]$CacheHours = 12, + [switch]$Force + ) + + $CacheDir = "$env:ProgramData\TecharyGet" + $CachePath = Join-Path $CacheDir 'ManifestIndex.json' + + try { + if (-not (Test-Path $CacheDir)) { New-Item -ItemType Directory -Path $CacheDir -Force -ErrorAction Stop | Out-Null } + + $NeedUpdate = $true + if (-not $Force -and (Test-Path $CachePath)) { + $Age = (Get-Date) - (Get-Item $CachePath).LastWriteTime + if ($Age.TotalHours -lt $CacheHours) { $NeedUpdate = $false } + } + + if ($NeedUpdate) { + try { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 } catch {} + + # Stage and parse before promoting, so a captive portal or proxy + # page returning HTTP 200 with HTML cannot poison the cache. + $StagePath = "$CachePath.tmp" + Invoke-WebRequest -Uri (Get-ManifestIndexUrl) -OutFile $StagePath -UseBasicParsing -ErrorAction Stop + + $Parsed = Get-Content -Path $StagePath -Raw | ConvertFrom-Json + if (-not $Parsed.Packages) { throw "Index downloaded but contains no Packages block." } + + Move-Item -Path $StagePath -Destination $CachePath -Force -ErrorAction Stop + Write-PackagerLog -Message "Manifest index refreshed ($($Parsed.PackageCount) packages, generated $($Parsed.Generated))." + } + } + catch { + # An index miss is not an error: the live API path still works. + Write-PackagerLog -Message "Could not refresh manifest index ($($_.Exception.Message)). Using local copy if present." -Severity Warning + Remove-Item "$CachePath.tmp" -Force -ErrorAction SilentlyContinue + } + + if (-not (Test-Path $CachePath)) { return $null } + + try { return (Get-Content -Path $CachePath -Raw | ConvertFrom-Json) } + catch { + Write-PackagerLog -Message "Manifest index cache is unreadable: $($_.Exception.Message)" -Severity Warning + return $null + } +} + +function Get-IndexedManifest { + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)][string]$Id, + [Parameter(Mandatory=$true)][string]$SysArch + ) + + $Index = Get-ManifestIndex + if (-not $Index -or -not $Index.Packages) { return $null } + + $Entry = $Index.Packages.$Id + if (-not $Entry) { return $null } + + $ForArch = $Entry.$SysArch + if (-not $ForArch -or -not $ForArch.Url) { return $null } + + # Shaped identically to Resolve-GitHubManifest so callers can use either. + return [PSCustomObject]@{ + Id = $Id + Version = $ForArch.Version + Arch = $SysArch + Url = $ForArch.Url + SilentArgs = $ForArch.SilentArgs + InstallerType = $ForArch.InstallerType + ProductCode = $ForArch.ProductCode + ResolvedUtc = $Index.Generated + Source = 'index' + } +} diff --git a/Public/Get-GitHubInstaller.ps1 b/Public/Get-GitHubInstaller.ps1 index 83c10a6..85f424f 100644 --- a/Public/Get-GitHubInstaller.ps1 +++ b/Public/Get-GitHubInstaller.ps1 @@ -10,8 +10,9 @@ function Get-GitHubInstaller { # without the token appearing in a command line or an RMM job log. [string]$GitHubToken = $env:TECHARYGET_GITHUB_TOKEN, - # Resolved manifests are cached on disk. Repeat and retry installs of - # the same app then cost no API calls at all. + # Resolved manifests are cached on disk, and the prebuilt index covers + # the catalogue centrally. Repeat and retry installs of the same app + # then cost no API calls at all. [int]$CacheHours = 24, [switch]$NoCache ) @@ -53,6 +54,16 @@ function Get-GitHubInstaller { } } + # Prebuilt index: one CDN file covering the whole catalogue, refreshed + # nightly in CI. Costs no api.github.com allowance at all. + if (-not $Meta -and -not $NoCache) { + $Indexed = Get-IndexedManifest -Id $Id -SysArch $SysArch + if ($Indexed) { + Write-PackagerLog -Message "Resolved $Id v$($Indexed.Version) from the prebuilt index. No API call needed." + $Meta = $Indexed + } + } + if (-not $Meta) { try { $Meta = Resolve-GitHubManifest -Id $Id -SysArch $SysArch -Headers $Headers