From 930ef746dcb108af4d53fe94aea316ba2b803431 Mon Sep 17 00:00:00 2001 From: James Tarran Date: Sat, 19 Sep 2026 12:59:23 +0100 Subject: [PATCH 1/3] Index every winget package for detection, not just the curated catalogue Two problems with detection as merged. The index was never fetched on an endpoint that only runs detection. -NoRefresh was implemented as "never download", but a detection-only machine installs nothing, so nothing else would ever fetch the index for it and ProductCode detection could never work there. It now means "do not re-download a copy we already have": absent means fetch, present means use what is there. Coverage was limited to the twelve packages in Index/Catalog.json. Detection of anything else fell back to name matching, which cannot resolve a winget package ID at all. The detection index is now built from Microsoft's own published winget source, which already carries the ARP product codes and MSIX package family names that detection matches on, for every package in the repository. That is one 3.5 MB CDN download in CI, with no api.github.com calls and no clone of winget-pkgs. Packages carrying neither a product code nor a package family name are omitted, because they cannot be identified this way. Detection now tries, in order: product codes and package family names from the full index, the product code from the curated index or the local manifest cache, an exact display name, a substring, then an MSIX name. Package family name matching uses -AllUsers when elevated, for the same reason as the uninstall path. The curated Index/Catalog.json stays, and still carries installer URLs and silent arguments so a common install needs no API call either. It was never an allow-list: Install-TecharyApp resolves any winget package live. --- .github/workflows/build-manifest-index.yml | 64 ++++++++++++++- .gitignore | 1 + Private/Get-DetectionIndex.ps1 | 91 ++++++++++++++++++++++ Private/Get-ManifestIndex.ps1 | 15 ++-- Public/Test-TecharyApp.ps1 | 61 ++++++++++++--- 5 files changed, 213 insertions(+), 19 deletions(-) create mode 100644 Private/Get-DetectionIndex.ps1 diff --git a/.github/workflows/build-manifest-index.yml b/.github/workflows/build-manifest-index.yml index 9f12f25..1fc54d0 100644 --- a/.github/workflows/build-manifest-index.yml +++ b/.github/workflows/build-manifest-index.yml @@ -4,6 +4,7 @@ on: schedule: # Nightly. NOTE: GitHub only runs scheduled workflows from the repository's # DEFAULT branch, so this fires once the workflow exists on that branch. + # Push and manual dispatch work from any branch. - cron: '17 3 * * *' workflow_dispatch: push: @@ -28,7 +29,62 @@ jobs: with: fetch-depth: 0 - - name: Resolve manifests + # --------------------------------------------------------------- + # Detection index: EVERY package in the winget source, not just the + # curated catalogue. Built from Microsoft's own published source + # index, which already carries the ARP product codes and MSIX package + # family names that detection matches on. One 3.5 MB CDN download, no + # api.github.com calls, no clone of winget-pkgs. + # --------------------------------------------------------------- + - name: Build detection index from the winget source + run: | + set -euo pipefail + curl -sSL --retry 3 --max-time 300 \ + -o source2.msix https://cdn.winget.microsoft.com/cache/source2.msix + ls -l source2.msix + + unzip -o -q source2.msix -d wingetsrc + DB=wingetsrc/Public/index.db + test -f "$DB" + + # Packages with neither a product code nor a package family name + # cannot be identified this way, so including them is dead weight. + # char(31) as the separator: a unit separator cannot occur inside a + # product code, whereas a pipe or comma plausibly could. + sqlite3 "$DB" -json " + SELECT p.id AS Id, + p.name AS Name, + p.latest_version AS Version, + (SELECT group_concat(productcode, char(31)) FROM productcodes2 WHERE package = p.rowid) AS ProductCodes, + (SELECT group_concat(pfn, char(31)) FROM pfns2 WHERE package = p.rowid) AS Pfns + FROM packages p + WHERE EXISTS (SELECT 1 FROM productcodes2 WHERE package = p.rowid) + OR EXISTS (SELECT 1 FROM pfns2 WHERE package = p.rowid) + ORDER BY p.id;" > packages.json + + NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) + jq --arg now "$NOW" ' + { Generated: $now, + Source: "winget source2 index", + PackageCount: length, + Packages: (map({ + key: .Id, + value: { + Name: .Name, + Version: .Version, + ProductCodes: ((.ProductCodes // "") | if . == "" then [] else split("") end), + Pfns: ((.Pfns // "") | if . == "" then [] else split("") end) + }}) | from_entries) } + ' packages.json > Index/Detection.json + + echo "detection index: $(jq -r .PackageCount Index/Detection.json) packages, $(du -h Index/Detection.json | cut -f1)" + + # --------------------------------------------------------------- + # Curated index: full install metadata (installer URL, silent args) for + # the packages in Index/Catalog.json, so a common install needs no API + # call either. Anything not listed still installs, resolved live. + # --------------------------------------------------------------- + - name: Resolve curated manifests shell: pwsh env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -38,6 +94,7 @@ jobs: run: | set -euo pipefail cp Index/Manifests.json "$RUNNER_TEMP/Manifests.json" + cp Index/Detection.json "$RUNNER_TEMP/Detection.json" git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" @@ -47,8 +104,9 @@ jobs: git checkout --orphan manifest-index git rm -rf . -q cp "$RUNNER_TEMP/Manifests.json" Manifests.json - git add Manifests.json + cp "$RUNNER_TEMP/Detection.json" Detection.json + git add Manifests.json Detection.json git commit -q -m "Refresh manifest index ($(date -u +%Y-%m-%dT%H:%M:%SZ))" git push -f origin manifest-index - echo "Published to https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/manifest-index/Manifests.json" + echo "Published to https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/manifest-index/" diff --git a/.gitignore b/.gitignore index d0b7058..4aadb73 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ # Built in CI and published to the manifest-index branch, never committed here. Index/Manifests.json +Index/Detection.json diff --git a/Private/Get-DetectionIndex.ps1 b/Private/Get-DetectionIndex.ps1 new file mode 100644 index 0000000..327c8e9 --- /dev/null +++ b/Private/Get-DetectionIndex.ps1 @@ -0,0 +1,91 @@ +function Get-DetectionIndexUrl { + # Built nightly from Microsoft's own published winget source index, so it + # covers EVERY package in the winget repository rather than the curated + # Index/Catalog.json. Served from raw.githubusercontent.com, which is + # CDN-backed and carries no api.github.com rate limit. + return "https://raw.githubusercontent.com/Techary/TecharyGet/manifest-index/Detection.json" +} + +function Get-DetectionIndex { + <# + .SYNOPSIS + The full package-to-ARP mapping used to identify installed software. + #> + [CmdletBinding()] + param( + [int]$CacheHours = 12, + [switch]$NoRefresh, + [switch]$Force + ) + + $CacheDir = "$env:ProgramData\TecharyGet" + $CachePath = Join-Path $CacheDir 'DetectionIndex.json' + $HaveCache = Test-Path $CachePath + + try { + if (-not (Test-Path $CacheDir)) { New-Item -ItemType Directory -Path $CacheDir -Force -ErrorAction Stop | Out-Null } + + # -NoRefresh means "do not re-download a copy we already have", NOT + # "never download". An endpoint that only runs detection installs + # nothing, so nothing else would ever fetch this for it. + $NeedUpdate = $true + if ($HaveCache -and -not $Force) { + $Age = (Get-Date) - (Get-Item $CachePath).LastWriteTime + if ($NoRefresh -or $Age.TotalHours -lt $CacheHours) { $NeedUpdate = $false } + } + + if ($NeedUpdate) { + try { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 } catch {} + + # Stage and parse before promoting, so a captive portal or proxy + # page returning HTTP 200 with HTML cannot poison the cache. + $StagePath = "$CachePath.tmp" + Invoke-WebRequest -Uri (Get-DetectionIndexUrl) -OutFile $StagePath -UseBasicParsing -ErrorAction Stop + + $Parsed = Get-Content -Path $StagePath -Raw | ConvertFrom-Json + if (-not $Parsed.Packages) { throw "Detection index downloaded but contains no Packages block." } + + Move-Item -Path $StagePath -Destination $CachePath -Force -ErrorAction Stop + Write-PackagerLog -Message "Detection index refreshed ($($Parsed.PackageCount) packages, generated $($Parsed.Generated))." + } + } + catch { + # A miss is not an error: detection falls back to name matching. + Write-PackagerLog -Message "Could not refresh the detection index ($($_.Exception.Message)). Using local copy if present." -Severity Warning + Remove-Item "$CachePath.tmp" -Force -ErrorAction SilentlyContinue + } + + if (-not (Test-Path $CachePath)) { return $null } + + try { return (Get-Content -Path $CachePath -Raw | ConvertFrom-Json) } + catch { + Write-PackagerLog -Message "Detection index cache is unreadable: $($_.Exception.Message)" -Severity Warning + return $null + } +} + +function Get-DetectionEntry { + <# + .SYNOPSIS + ARP product codes and MSIX package family names for one package ID. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)][string]$Id, + [switch]$NoRefresh + ) + + $Index = Get-DetectionIndex -NoRefresh:$NoRefresh + if (-not $Index -or -not $Index.Packages) { return $null } + + $Entry = $Index.Packages.$Id + if (-not $Entry) { return $null } + + return [PSCustomObject]@{ + Id = $Id + Name = $Entry.Name + Version = $Entry.Version + ProductCodes = @($Entry.ProductCodes) + Pfns = @($Entry.Pfns) + } +} diff --git a/Private/Get-ManifestIndex.ps1 b/Private/Get-ManifestIndex.ps1 index 5a747fc..69ce5f6 100644 --- a/Private/Get-ManifestIndex.ps1 +++ b/Private/Get-ManifestIndex.ps1 @@ -17,15 +17,20 @@ function Get-ManifestIndex { $CacheDir = "$env:ProgramData\TecharyGet" $CachePath = Join-Path $CacheDir 'ManifestIndex.json' + $HaveCache = Test-Path $CachePath + try { if (-not (Test-Path $CacheDir)) { New-Item -ItemType Directory -Path $CacheDir -Force -ErrorAction Stop | Out-Null } - # Detection runs on a schedule on every endpoint, so it must never make - # a network call. -NoRefresh reads whatever is already cached. - $NeedUpdate = -not $NoRefresh - if (-not $Force -and (Test-Path $CachePath)) { + # -NoRefresh means "do not re-download a copy we already have", NOT + # "never download". An endpoint that only ever runs detection installs + # nothing, so nothing else would ever fetch the index for it: treating + # NoRefresh as "never download" left those machines permanently without + # one, and ProductCode detection could never work there. + $NeedUpdate = $true + if ($HaveCache -and -not $Force) { $Age = (Get-Date) - (Get-Item $CachePath).LastWriteTime - if ($Age.TotalHours -lt $CacheHours) { $NeedUpdate = $false } + if ($NoRefresh -or $Age.TotalHours -lt $CacheHours) { $NeedUpdate = $false } } if ($NeedUpdate) { diff --git a/Public/Test-TecharyApp.ps1 b/Public/Test-TecharyApp.ps1 index 8f81a88..323197f 100644 --- a/Public/Test-TecharyApp.ps1 +++ b/Public/Test-TecharyApp.ps1 @@ -58,32 +58,71 @@ function Test-TecharyApp { elseif ([Environment]::Is64BitOperatingSystem) { $SysArch = "x64" } else { $SysArch = "x86" } - # --- 1. PRODUCT CODE (definitive) --------------------------------- - $ProductCode = $null + # --- 1. PRODUCT CODE / PACKAGE FAMILY (definitive) ----------------- + # Three sources, most complete first: the full detection index covers + # every package in the winget repository; the curated manifest index and + # the local manifest cache cover what this machine has installed before. + $ProductCodes = New-Object System.Collections.Generic.List[string] + $Pfns = New-Object System.Collections.Generic.List[string] + + try { + $Entry = Get-DetectionEntry -Id $Name -NoRefresh + if ($Entry) { + foreach ($Code in $Entry.ProductCodes) { if ($Code) { $ProductCodes.Add($Code) } } + foreach ($Family in $Entry.Pfns) { if ($Family) { $Pfns.Add($Family) } } + Write-Verbose "Detection index: $($ProductCodes.Count) product code(s), $($Pfns.Count) package family name(s) for '$Name'" + } + } catch {} + try { $Indexed = Get-IndexedManifest -Id $Name -SysArch $SysArch -NoRefresh - if ($Indexed) { $ProductCode = $Indexed.ProductCode } + if ($Indexed -and $Indexed.ProductCode) { $ProductCodes.Add($Indexed.ProductCode) } } catch {} - if (-not $ProductCode) { - $CachedManifest = Join-Path $env:ProgramData ("TecharyGet\ManifestCache\" + ($Name -replace '[\\/:*?"<>|]', '_') + ".json") - if (Test-Path $CachedManifest) { - try { $ProductCode = (Get-Content $CachedManifest -Raw | ConvertFrom-Json).ProductCode } catch {} - } + $CachedManifest = Join-Path $env:ProgramData ("TecharyGet\ManifestCache\" + ($Name -replace '[\\/:*?"<>|]', '_') + ".json") + if (Test-Path $CachedManifest) { + try { + $Local = (Get-Content $CachedManifest -Raw | ConvertFrom-Json).ProductCode + if ($Local) { $ProductCodes.Add($Local) } + } catch {} } - if ($ProductCode) { + foreach ($Code in $ProductCodes) { foreach ($Hive in $Hives) { - $Key = Join-Path $Hive $ProductCode + $Key = Join-Path $Hive $Code if (Test-Path $Key) { $Item = Get-ItemProperty -Path $Key -ErrorAction SilentlyContinue - Write-Verbose "Matched on ProductCode '$ProductCode' at $Key" + Write-Verbose "Matched on ProductCode '$Code' at $Key" $R = New-Result $true 'ProductCode' $Item.DisplayName $Item.DisplayVersion $Key if ($Detailed) { return $R } else { return $true } } } } + if ($Pfns.Count -gt 0) { + $Elevated = $false + try { + $Ident = [Security.Principal.WindowsIdentity]::GetCurrent() + $Elevated = (New-Object Security.Principal.WindowsPrincipal($Ident)).IsInRole( + [Security.Principal.WindowsBuiltInRole]::Administrator) + } catch {} + + try { + # -AllUsers when we can: SYSTEM sees almost none of its own packages. + if ($Elevated) { $Installed = @(Get-AppxPackage -AllUsers -ErrorAction Stop) } + else { $Installed = @(Get-AppxPackage -ErrorAction SilentlyContinue) } + + foreach ($Family in $Pfns) { + $Hit = $Installed | Where-Object { $_.PackageFamilyName -eq $Family } | Select-Object -First 1 + if ($Hit) { + Write-Verbose "Matched on PackageFamilyName '$Family'" + $R = New-Result $true 'PackageFamilyName' $Hit.Name $Hit.Version $Hit.PackageFullName + if ($Detailed) { return $R } else { return $true } + } + } + } catch {} + } + # --- 2. EXACT DISPLAY NAME ---------------------------------------- # A custom catalogue entry carries the real ARP DisplayName for its ID. $Candidates = New-Object System.Collections.Generic.List[string] From 8c81eb281eaac0d3c215abbea567ff4af08262dd Mon Sep 17 00:00:00 2001 From: James Tarran Date: Sat, 19 Sep 2026 13:03:33 +0100 Subject: [PATCH 2/3] Emit index pairs and group in jq, avoiding an illegal control character The separator in the SQL group_concat was char(31), and the matching jq split used a \u001f escape. That escape reached the workflow file as a raw unit separator byte, which YAML does not permit, so the workflow failed to parse and the run produced no jobs at all. Any separator safe inside a product code has to be a control character, so the query now emits one row per identifier and jq groups them. No separator, nothing to escape. --- .github/workflows/build-manifest-index.yml | 55 ++++++++++++---------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build-manifest-index.yml b/.github/workflows/build-manifest-index.yml index 1fc54d0..bb35293 100644 --- a/.github/workflows/build-manifest-index.yml +++ b/.github/workflows/build-manifest-index.yml @@ -47,35 +47,42 @@ jobs: DB=wingetsrc/Public/index.db test -f "$DB" - # Packages with neither a product code nor a package family name - # cannot be identified this way, so including them is dead weight. - # char(31) as the separator: a unit separator cannot occur inside a - # product code, whereas a pipe or comma plausibly could. + # One row per identifier rather than a delimited list. Any separator + # has to be a character that cannot appear inside a product code, and + # the obvious safe choice is a control character, which is not legal + # in a YAML workflow file. Emitting pairs and grouping in jq avoids + # the question entirely. + # + # Packages with neither a product code nor a package family name are + # excluded: they cannot be identified this way, so they are dead + # weight in a file every endpoint downloads. sqlite3 "$DB" -json " - SELECT p.id AS Id, - p.name AS Name, - p.latest_version AS Version, - (SELECT group_concat(productcode, char(31)) FROM productcodes2 WHERE package = p.rowid) AS ProductCodes, - (SELECT group_concat(pfn, char(31)) FROM pfns2 WHERE package = p.rowid) AS Pfns - FROM packages p - WHERE EXISTS (SELECT 1 FROM productcodes2 WHERE package = p.rowid) - OR EXISTS (SELECT 1 FROM pfns2 WHERE package = p.rowid) - ORDER BY p.id;" > packages.json + SELECT p.id AS Id, p.name AS Name, p.latest_version AS Version, + 'P' AS Kind, c.productcode AS Value + FROM packages p JOIN productcodes2 c ON c.package = p.rowid + UNION ALL + SELECT p.id, p.name, p.latest_version, + 'F', f.pfn + FROM packages p JOIN pfns2 f ON f.package = p.rowid + ORDER BY 1;" > pairs.json NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) jq --arg now "$NOW" ' - { Generated: $now, - Source: "winget source2 index", - PackageCount: length, - Packages: (map({ - key: .Id, + group_by(.Id) + | map({ + key: .[0].Id, value: { - Name: .Name, - Version: .Version, - ProductCodes: ((.ProductCodes // "") | if . == "" then [] else split("") end), - Pfns: ((.Pfns // "") | if . == "" then [] else split("") end) - }}) | from_entries) } - ' packages.json > Index/Detection.json + Name: .[0].Name, + Version: .[0].Version, + ProductCodes: [ .[] | select(.Kind == "P") | .Value ], + Pfns: [ .[] | select(.Kind == "F") | .Value ] + }}) + | from_entries + | { Generated: $now, + Source: "winget source2 index", + PackageCount: length, + Packages: . } + ' pairs.json > Index/Detection.json echo "detection index: $(jq -r .PackageCount Index/Detection.json) packages, $(du -h Index/Detection.json | cut -f1)" From b8d09773b7363937acd94b01d5da9b39631bbee8 Mon Sep 17 00:00:00 2001 From: James Tarran Date: Sat, 19 Sep 2026 13:06:47 +0100 Subject: [PATCH 3/3] Hash-look-up product codes instead of probing the registry per code The winget source carries every product code a package has ever shipped: Mozilla.Firefox alone has 5205, one per locale and version. Probing those across three hives is 15,615 registry reads, measured at ~384 seconds, which would exceed an N-central scan interval on its own. The uninstall key names are now enumerated once into a case-insensitive dictionary and each candidate is a hash lookup, so the cost is ~230 registry reads regardless of how many codes a package carries. Firefox drops from ~384s to 0.47s. Ordinal-ignore-case is required, not cosmetic: the source index stores codes normalised to lower case ("7-zip") while the real key is "7-Zip". --- Public/Test-TecharyApp.ps1 | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/Public/Test-TecharyApp.ps1 b/Public/Test-TecharyApp.ps1 index 323197f..7c0e335 100644 --- a/Public/Test-TecharyApp.ps1 +++ b/Public/Test-TecharyApp.ps1 @@ -87,13 +87,31 @@ function Test-TecharyApp { } catch {} } - foreach ($Code in $ProductCodes) { + if ($ProductCodes.Count -gt 0) { + # Enumerate the machine's uninstall key NAMES once and hash-look-up each + # candidate, rather than probing the registry per code. + # + # The winget source carries every product code a package has ever + # shipped: Mozilla.Firefox alone has 5205, one per locale and version. + # Probing those across three hives is 15,615 registry reads and took + # ~384 seconds measured, which would exceed an N-central scan interval + # on its own. This is ~230 reads regardless of how many codes a package + # has. Ordinal-ignore-case because the index stores codes normalised to + # lower case ("7-zip") while the real key is "7-Zip", and the registry + # itself is case-insensitive. + $ArpKeys = New-Object 'System.Collections.Generic.Dictionary[string,string]' ([StringComparer]::OrdinalIgnoreCase) foreach ($Hive in $Hives) { - $Key = Join-Path $Hive $Code - if (Test-Path $Key) { - $Item = Get-ItemProperty -Path $Key -ErrorAction SilentlyContinue - Write-Verbose "Matched on ProductCode '$Code' at $Key" - $R = New-Result $true 'ProductCode' $Item.DisplayName $Item.DisplayVersion $Key + foreach ($Key in (Get-ChildItem -Path $Hive -ErrorAction SilentlyContinue)) { + if (-not $ArpKeys.ContainsKey($Key.PSChildName)) { $ArpKeys[$Key.PSChildName] = $Key.PSPath } + } + } + + foreach ($Code in $ProductCodes) { + $Path = $null + if ($ArpKeys.TryGetValue($Code, [ref]$Path)) { + $Item = Get-ItemProperty -Path $Path -ErrorAction SilentlyContinue + Write-Verbose "Matched on ProductCode '$Code' at $Path" + $R = New-Result $true 'ProductCode' $Item.DisplayName $Item.DisplayVersion $Path if ($Detailed) { return $R } else { return $true } } }