Publish a prebuilt manifest index to remove the API from the install path

.gitattributes

@@ -0,0 +1,6 @@
+# Workflow files must stay LF. A CRLF `run:` block reaches bash on the runner
+# with a literal CR on every line, which breaks the script.
+# Deliberately NOT declaring `* text=auto`: the existing blobs in this repo are
+# committed CRLF, so that would queue a repo-wide renormalisation diff.
+*.yml -text
+*.yaml -text

.github/workflows/build-manifest-index.yml

@@ -0,0 +1,54 @@
+name: Build manifest index
+
+on:
+  schedule:
+    # Nightly. NOTE: GitHub only runs scheduled workflows from the repository's
+    # DEFAULT branch, so this fires once the workflow exists on that branch.
+    - cron: '17 3 * * *'
+  workflow_dispatch:
+  push:
+    paths:
+      - 'Index/Catalog.json'
+      - 'Build/Build-ManifestIndex.ps1'
+      - 'Private/Resolve-GitHubManifest.ps1'
+      - '.github/workflows/build-manifest-index.yml'
+
+permissions:
+  contents: write
+
+concurrency:
+  group: manifest-index
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Resolve manifests
+        shell: pwsh
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: ./Build/Build-ManifestIndex.ps1 -CatalogPath Index/Catalog.json -OutputPath Index/Manifests.json
+
+      - name: Publish to the manifest-index branch
+        run: |
+          set -euo pipefail
+          cp Index/Manifests.json "$RUNNER_TEMP/Manifests.json"
+
+          git config user.name  "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+          # Orphan + force-push keeps exactly one commit on the branch, so a
+          # nightly refresh never grows the repository or touches code history.
+          git checkout --orphan manifest-index
+          git rm -rf . -q
+          cp "$RUNNER_TEMP/Manifests.json" Manifests.json
+          git add Manifests.json
+          git commit -q -m "Refresh manifest index ($(date -u +%Y-%m-%dT%H:%M:%SZ))"
+          git push -f origin manifest-index
+
+          echo "Published to https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/manifest-index/Manifests.json"

.gitignore

@@ -0,0 +1,2 @@
+# Built in CI and published to the manifest-index branch, never committed here.
+Index/Manifests.json

Build/Build-ManifestIndex.ps1

@@ -0,0 +1,91 @@
+<#
+.SYNOPSIS
+    Pre-resolves winget-pkgs manifests into a single JSON index.
+
+.DESCRIPTION
+    Runs in CI, not on endpoints. Resolving a package costs two
+    api.github.com calls; doing it once here and publishing the result means
+    endpoints read one file from raw.githubusercontent.com instead, which is
+    CDN-backed and not subject to the API rate limit.
+
+    A package that fails to resolve is reported and skipped. The build only
+    fails if every package failed, so one delisted app cannot stop the
+    nightly refresh.
+#>
+[CmdletBinding()]
+param(
+    [string]$CatalogPath = 'Index/Catalog.json',
+    [string]$OutputPath = 'Index/Manifests.json',
+    [string[]]$Architectures = @('x64', 'arm64'),
+    [string]$GitHubToken = $env:GITHUB_TOKEN
+)
+
+$ErrorActionPreference = 'Stop'
+
+. (Join-Path $PSScriptRoot '..\Private\Resolve-GitHubManifest.ps1')
+
+$Headers = @{ 'User-Agent' = 'TecharyGet-IndexBuilder' }
+if ($GitHubToken) {
+    $Headers['Authorization'] = "token $GitHubToken"
+    Write-Host "Authenticated to the GitHub API (5000 requests/hour)."
+} else {
+    Write-Warning "No token supplied. Falling back to 60 requests/hour, which will not cover a full catalogue."
+}
+
+$Catalog = Get-Content -Path $CatalogPath -Raw | ConvertFrom-Json
+$Ids = @($Catalog.Packages | Sort-Object)
+Write-Host "Catalogue contains $($Ids.Count) package(s); resolving $($Architectures -join ', ')."
+
+$Packages = [ordered]@{}
+$Ok = 0
+$Failed = New-Object System.Collections.Generic.List[string]
+
+foreach ($Id in $Ids) {
+    $PerArch = [ordered]@{}
+
+    foreach ($Arch in $Architectures) {
+        try {
+            $Meta = Resolve-GitHubManifest -Id $Id -SysArch $Arch -Headers $Headers
+            $PerArch[$Arch] = [ordered]@{
+                Version       = $Meta.Version
+                Url           = $Meta.Url
+                SilentArgs    = $Meta.SilentArgs
+                InstallerType = $Meta.InstallerType
+                ProductCode   = $Meta.ProductCode
+            }
+            Write-Host ("  ok    {0,-34} {1,-6} v{2}" -f $Id, $Arch, $Meta.Version)
+        }
+        catch {
+            # Most packages genuinely have no arm64 installer. Only an x64
+            # failure is worth counting as a real failure.
+            $Level = if ($Arch -eq 'x64') { 'warn ' } else { 'skip ' }
+            Write-Host ("  {0} {1,-34} {2,-6} {3}" -f $Level, $Id, $Arch, $_.Exception.Message)
+        }
+    }
+
+    if ($PerArch.Count -gt 0) {
+        $Packages[$Id] = $PerArch
+        $Ok++
+    } else {
+        $Failed.Add($Id)
+    }
+}
+
+if ($Ok -eq 0) {
+    throw "Every package in the catalogue failed to resolve. Refusing to publish an empty index."
+}
+
+$Index = [ordered]@{
+    Generated    = (Get-Date).ToUniversalTime().ToString('o')
+    Source       = 'microsoft/winget-pkgs'
+    PackageCount = $Ok
+    Packages     = $Packages
+}
+
+$Dir = Split-Path $OutputPath -Parent
+if ($Dir -and -not (Test-Path $Dir)) { New-Item -ItemType Directory -Path $Dir -Force | Out-Null }
+$Index | ConvertTo-Json -Depth 6 | Set-Content -Path $OutputPath -Encoding UTF8
+
+Write-Host ""
+Write-Host "Wrote $OutputPath with $Ok package(s)."
+if ($Failed.Count -gt 0) { Write-Warning "Unresolved: $($Failed -join ', ')" }

Index/Catalog.json

@@ -0,0 +1,17 @@
+{
+    "_comment": "winget package IDs pre-resolved nightly into Manifests.json. Adding an ID here is the only step needed to take it off the live GitHub API path. Seed list - edit freely.",
+    "Packages": [
+        "7zip.7zip",
+        "8x8.Work",
+        "Adobe.Acrobat.Reader.64-bit",
+        "Dell.CommandUpdate",
+        "Google.Chrome",
+        "Microsoft.PowerToys",
+        "Microsoft.VisualStudioCode",
+        "Mozilla.Firefox",
+        "Notepad++.Notepad++",
+        "SublimeHQ.SublimeText.4",
+        "VideoLAN.VLC",
+        "Zoom.Zoom"
+    ]
+}

Private/Get-ManifestIndex.ps1

@@ -0,0 +1,86 @@
+function Get-ManifestIndexUrl {
+    # Published by .github/workflows/build-manifest-index.yml to a dedicated
+    # orphan branch, so a nightly refresh never touches code history.
+    # raw.githubusercontent.com is CDN-backed and is NOT subject to the
+    # api.github.com rate limit, which is the entire point of the index.
+    return "https://raw.githubusercontent.com/Techary/TecharyGet/manifest-index/Manifests.json"
+}
+
+function Get-ManifestIndex {
+    [CmdletBinding()]
+    param(
+        [int]$CacheHours = 12,
+        [switch]$Force
+    )
+
+    $CacheDir  = "$env:ProgramData\TecharyGet"
+    $CachePath = Join-Path $CacheDir 'ManifestIndex.json'
+
+    try {
+        if (-not (Test-Path $CacheDir)) { New-Item -ItemType Directory -Path $CacheDir -Force -ErrorAction Stop | Out-Null }
+
+        $NeedUpdate = $true
+        if (-not $Force -and (Test-Path $CachePath)) {
+            $Age = (Get-Date) - (Get-Item $CachePath).LastWriteTime
+            if ($Age.TotalHours -lt $CacheHours) { $NeedUpdate = $false }
+        }
+
+        if ($NeedUpdate) {
+            try { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 } catch {}
+
+            # Stage and parse before promoting, so a captive portal or proxy
+            # page returning HTTP 200 with HTML cannot poison the cache.
+            $StagePath = "$CachePath.tmp"
+            Invoke-WebRequest -Uri (Get-ManifestIndexUrl) -OutFile $StagePath -UseBasicParsing -ErrorAction Stop
+
+            $Parsed = Get-Content -Path $StagePath -Raw | ConvertFrom-Json
+            if (-not $Parsed.Packages) { throw "Index downloaded but contains no Packages block." }
+
+            Move-Item -Path $StagePath -Destination $CachePath -Force -ErrorAction Stop
+            Write-PackagerLog -Message "Manifest index refreshed ($($Parsed.PackageCount) packages, generated $($Parsed.Generated))."
+        }
+    }
+    catch {
+        # An index miss is not an error: the live API path still works.
+        Write-PackagerLog -Message "Could not refresh manifest index ($($_.Exception.Message)). Using local copy if present." -Severity Warning
+        Remove-Item "$CachePath.tmp" -Force -ErrorAction SilentlyContinue
+    }
+
+    if (-not (Test-Path $CachePath)) { return $null }
+
+    try { return (Get-Content -Path $CachePath -Raw | ConvertFrom-Json) }
+    catch {
+        Write-PackagerLog -Message "Manifest index cache is unreadable: $($_.Exception.Message)" -Severity Warning
+        return $null
+    }
+}
+
+function Get-IndexedManifest {
+    [CmdletBinding()]
+    param(
+        [Parameter(Mandatory=$true)][string]$Id,
+        [Parameter(Mandatory=$true)][string]$SysArch
+    )
+
+    $Index = Get-ManifestIndex
+    if (-not $Index -or -not $Index.Packages) { return $null }
+
+    $Entry = $Index.Packages.$Id
+    if (-not $Entry) { return $null }
+
+    $ForArch = $Entry.$SysArch
+    if (-not $ForArch -or -not $ForArch.Url) { return $null }
+
+    # Shaped identically to Resolve-GitHubManifest so callers can use either.
+    return [PSCustomObject]@{
+        Id            = $Id
+        Version       = $ForArch.Version
+        Arch          = $SysArch
+        Url           = $ForArch.Url
+        SilentArgs    = $ForArch.SilentArgs
+        InstallerType = $ForArch.InstallerType
+        ProductCode   = $ForArch.ProductCode
+        ResolvedUtc   = $Index.Generated
+        Source        = 'index'
+    }
+}

Public/Get-GitHubInstaller.ps1

@@ -10,8 +10,9 @@ function Get-GitHubInstaller {
         # without the token appearing in a command line or an RMM job log.
         [string]$GitHubToken = $env:TECHARYGET_GITHUB_TOKEN,
 
-        # Resolved manifests are cached on disk. Repeat and retry installs of
-        # the same app then cost no API calls at all.
+        # Resolved manifests are cached on disk, and the prebuilt index covers
+        # the catalogue centrally. Repeat and retry installs of the same app
+        # then cost no API calls at all.
         [int]$CacheHours = 24,
         [switch]$NoCache
     )
@@ -53,6 +54,16 @@ function Get-GitHubInstaller {
         }
     }
 
+    # Prebuilt index: one CDN file covering the whole catalogue, refreshed
+    # nightly in CI. Costs no api.github.com allowance at all.
+    if (-not $Meta -and -not $NoCache) {
+        $Indexed = Get-IndexedManifest -Id $Id -SysArch $SysArch
+        if ($Indexed) {
+            Write-PackagerLog -Message "Resolved $Id v$($Indexed.Version) from the prebuilt index. No API call needed."
+            $Meta = $Indexed
+        }
+    }
+
     if (-not $Meta) {
         try {
             $Meta = Resolve-GitHubManifest -Id $Id -SysArch $SysArch -Headers $Headers