Index every winget package for detection, not just the curated catalogue

.github/workflows/build-manifest-index.yml

@@ -4,6 +4,7 @@ on:
   schedule:
     # Nightly. NOTE: GitHub only runs scheduled workflows from the repository's
     # DEFAULT branch, so this fires once the workflow exists on that branch.
+    # Push and manual dispatch work from any branch.
     - cron: '17 3 * * *'
   workflow_dispatch:
   push:
@@ -28,7 +29,69 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Resolve manifests
+      # ---------------------------------------------------------------
+      # Detection index: EVERY package in the winget source, not just the
+      # curated catalogue. Built from Microsoft's own published source
+      # index, which already carries the ARP product codes and MSIX package
+      # family names that detection matches on. One 3.5 MB CDN download, no
+      # api.github.com calls, no clone of winget-pkgs.
+      # ---------------------------------------------------------------
+      - name: Build detection index from the winget source
+        run: |
+          set -euo pipefail
+          curl -sSL --retry 3 --max-time 300 \
+            -o source2.msix https://cdn.winget.microsoft.com/cache/source2.msix
+          ls -l source2.msix
+
+          unzip -o -q source2.msix -d wingetsrc
+          DB=wingetsrc/Public/index.db
+          test -f "$DB"
+
+          # One row per identifier rather than a delimited list. Any separator
+          # has to be a character that cannot appear inside a product code, and
+          # the obvious safe choice is a control character, which is not legal
+          # in a YAML workflow file. Emitting pairs and grouping in jq avoids
+          # the question entirely.
+          #
+          # Packages with neither a product code nor a package family name are
+          # excluded: they cannot be identified this way, so they are dead
+          # weight in a file every endpoint downloads.
+          sqlite3 "$DB" -json "
+            SELECT p.id AS Id, p.name AS Name, p.latest_version AS Version,
+                   'P' AS Kind, c.productcode AS Value
+            FROM packages p JOIN productcodes2 c ON c.package = p.rowid
+            UNION ALL
+            SELECT p.id, p.name, p.latest_version,
+                   'F', f.pfn
+            FROM packages p JOIN pfns2 f ON f.package = p.rowid
+            ORDER BY 1;" > pairs.json
+
+          NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+          jq --arg now "$NOW" '
+            group_by(.Id)
+            | map({
+                key: .[0].Id,
+                value: {
+                  Name: .[0].Name,
+                  Version: .[0].Version,
+                  ProductCodes: [ .[] | select(.Kind == "P") | .Value ],
+                  Pfns:         [ .[] | select(.Kind == "F") | .Value ]
+                }})
+            | from_entries
+            | { Generated: $now,
+                Source: "winget source2 index",
+                PackageCount: length,
+                Packages: . }
+          ' pairs.json > Index/Detection.json
+
+          echo "detection index: $(jq -r .PackageCount Index/Detection.json) packages, $(du -h Index/Detection.json | cut -f1)"
+
+      # ---------------------------------------------------------------
+      # Curated index: full install metadata (installer URL, silent args) for
+      # the packages in Index/Catalog.json, so a common install needs no API
+      # call either. Anything not listed still installs, resolved live.
+      # ---------------------------------------------------------------
+      - name: Resolve curated manifests
         shell: pwsh
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -38,6 +101,7 @@ jobs:
         run: |
           set -euo pipefail
           cp Index/Manifests.json "$RUNNER_TEMP/Manifests.json"
+          cp Index/Detection.json "$RUNNER_TEMP/Detection.json"
 
           git config user.name  "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
@@ -47,8 +111,9 @@ jobs:
           git checkout --orphan manifest-index
           git rm -rf . -q
           cp "$RUNNER_TEMP/Manifests.json" Manifests.json
-          git add Manifests.json
+          cp "$RUNNER_TEMP/Detection.json" Detection.json
+          git add Manifests.json Detection.json
           git commit -q -m "Refresh manifest index ($(date -u +%Y-%m-%dT%H:%M:%SZ))"
           git push -f origin manifest-index
 
-          echo "Published to https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/manifest-index/Manifests.json"
+          echo "Published to https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/manifest-index/"

.gitignore

@@ -1,2 +1,3 @@
 # Built in CI and published to the manifest-index branch, never committed here.
 Index/Manifests.json
+Index/Detection.json

Private/Get-DetectionIndex.ps1

@@ -0,0 +1,91 @@
+function Get-DetectionIndexUrl {
+    # Built nightly from Microsoft's own published winget source index, so it
+    # covers EVERY package in the winget repository rather than the curated
+    # Index/Catalog.json. Served from raw.githubusercontent.com, which is
+    # CDN-backed and carries no api.github.com rate limit.
+    return "https://raw.githubusercontent.com/Techary/TecharyGet/manifest-index/Detection.json"
+}
+
+function Get-DetectionIndex {
+    <#
+    .SYNOPSIS
+        The full package-to-ARP mapping used to identify installed software.
+    #>
+    [CmdletBinding()]
+    param(
+        [int]$CacheHours = 12,
+        [switch]$NoRefresh,
+        [switch]$Force
+    )
+
+    $CacheDir  = "$env:ProgramData\TecharyGet"
+    $CachePath = Join-Path $CacheDir 'DetectionIndex.json'
+    $HaveCache = Test-Path $CachePath
+
+    try {
+        if (-not (Test-Path $CacheDir)) { New-Item -ItemType Directory -Path $CacheDir -Force -ErrorAction Stop | Out-Null }
+
+        # -NoRefresh means "do not re-download a copy we already have", NOT
+        # "never download". An endpoint that only runs detection installs
+        # nothing, so nothing else would ever fetch this for it.
+        $NeedUpdate = $true
+        if ($HaveCache -and -not $Force) {
+            $Age = (Get-Date) - (Get-Item $CachePath).LastWriteTime
+            if ($NoRefresh -or $Age.TotalHours -lt $CacheHours) { $NeedUpdate = $false }
+        }
+
+        if ($NeedUpdate) {
+            try { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 } catch {}
+
+            # Stage and parse before promoting, so a captive portal or proxy
+            # page returning HTTP 200 with HTML cannot poison the cache.
+            $StagePath = "$CachePath.tmp"
+            Invoke-WebRequest -Uri (Get-DetectionIndexUrl) -OutFile $StagePath -UseBasicParsing -ErrorAction Stop
+
+            $Parsed = Get-Content -Path $StagePath -Raw | ConvertFrom-Json
+            if (-not $Parsed.Packages) { throw "Detection index downloaded but contains no Packages block." }
+
+            Move-Item -Path $StagePath -Destination $CachePath -Force -ErrorAction Stop
+            Write-PackagerLog -Message "Detection index refreshed ($($Parsed.PackageCount) packages, generated $($Parsed.Generated))."
+        }
+    }
+    catch {
+        # A miss is not an error: detection falls back to name matching.
+        Write-PackagerLog -Message "Could not refresh the detection index ($($_.Exception.Message)). Using local copy if present." -Severity Warning
+        Remove-Item "$CachePath.tmp" -Force -ErrorAction SilentlyContinue
+    }
+
+    if (-not (Test-Path $CachePath)) { return $null }
+
+    try { return (Get-Content -Path $CachePath -Raw | ConvertFrom-Json) }
+    catch {
+        Write-PackagerLog -Message "Detection index cache is unreadable: $($_.Exception.Message)" -Severity Warning
+        return $null
+    }
+}
+
+function Get-DetectionEntry {
+    <#
+    .SYNOPSIS
+        ARP product codes and MSIX package family names for one package ID.
+    #>
+    [CmdletBinding()]
+    param(
+        [Parameter(Mandatory=$true)][string]$Id,
+        [switch]$NoRefresh
+    )
+
+    $Index = Get-DetectionIndex -NoRefresh:$NoRefresh
+    if (-not $Index -or -not $Index.Packages) { return $null }
+
+    $Entry = $Index.Packages.$Id
+    if (-not $Entry) { return $null }
+
+    return [PSCustomObject]@{
+        Id           = $Id
+        Name         = $Entry.Name
+        Version      = $Entry.Version
+        ProductCodes = @($Entry.ProductCodes)
+        Pfns         = @($Entry.Pfns)
+    }
+}

Private/Get-ManifestIndex.ps1

@@ -17,15 +17,20 @@ function Get-ManifestIndex {
     $CacheDir  = "$env:ProgramData\TecharyGet"
     $CachePath = Join-Path $CacheDir 'ManifestIndex.json'
 
+    $HaveCache = Test-Path $CachePath
+
     try {
         if (-not (Test-Path $CacheDir)) { New-Item -ItemType Directory -Path $CacheDir -Force -ErrorAction Stop | Out-Null }
 
-        # Detection runs on a schedule on every endpoint, so it must never make
-        # a network call. -NoRefresh reads whatever is already cached.
-        $NeedUpdate = -not $NoRefresh
-        if (-not $Force -and (Test-Path $CachePath)) {
+        # -NoRefresh means "do not re-download a copy we already have", NOT
+        # "never download". An endpoint that only ever runs detection installs
+        # nothing, so nothing else would ever fetch the index for it: treating
+        # NoRefresh as "never download" left those machines permanently without
+        # one, and ProductCode detection could never work there.
+        $NeedUpdate = $true
+        if ($HaveCache -and -not $Force) {
             $Age = (Get-Date) - (Get-Item $CachePath).LastWriteTime
-            if ($Age.TotalHours -lt $CacheHours) { $NeedUpdate = $false }
+            if ($NoRefresh -or $Age.TotalHours -lt $CacheHours) { $NeedUpdate = $false }
         }
 
         if ($NeedUpdate) {

Public/Test-TecharyApp.ps1

@@ -58,32 +58,89 @@ function Test-TecharyApp {
     elseif ([Environment]::Is64BitOperatingSystem) { $SysArch = "x64" }
     else { $SysArch = "x86" }
 
-    # --- 1. PRODUCT CODE (definitive) ---------------------------------
-    $ProductCode = $null
+    # --- 1. PRODUCT CODE / PACKAGE FAMILY (definitive) -----------------
+    # Three sources, most complete first: the full detection index covers
+    # every package in the winget repository; the curated manifest index and
+    # the local manifest cache cover what this machine has installed before.
+    $ProductCodes = New-Object System.Collections.Generic.List[string]
+    $Pfns         = New-Object System.Collections.Generic.List[string]
+
+    try {
+        $Entry = Get-DetectionEntry -Id $Name -NoRefresh
+        if ($Entry) {
+            foreach ($Code in $Entry.ProductCodes) { if ($Code) { $ProductCodes.Add($Code) } }
+            foreach ($Family in $Entry.Pfns) { if ($Family) { $Pfns.Add($Family) } }
+            Write-Verbose "Detection index: $($ProductCodes.Count) product code(s), $($Pfns.Count) package family name(s) for '$Name'"
+        }
+    } catch {}
+
     try {
         $Indexed = Get-IndexedManifest -Id $Name -SysArch $SysArch -NoRefresh
-        if ($Indexed) { $ProductCode = $Indexed.ProductCode }
+        if ($Indexed -and $Indexed.ProductCode) { $ProductCodes.Add($Indexed.ProductCode) }
     } catch {}
 
-    if (-not $ProductCode) {
-        $CachedManifest = Join-Path $env:ProgramData ("TecharyGet\ManifestCache\" + ($Name -replace '[\\/:*?"<>|]', '_') + ".json")
-        if (Test-Path $CachedManifest) {
-            try { $ProductCode = (Get-Content $CachedManifest -Raw | ConvertFrom-Json).ProductCode } catch {}
-        }
+    $CachedManifest = Join-Path $env:ProgramData ("TecharyGet\ManifestCache\" + ($Name -replace '[\\/:*?"<>|]', '_') + ".json")
+    if (Test-Path $CachedManifest) {
+        try {
+            $Local = (Get-Content $CachedManifest -Raw | ConvertFrom-Json).ProductCode
+            if ($Local) { $ProductCodes.Add($Local) }
+        } catch {}
     }
 
-    if ($ProductCode) {
+    if ($ProductCodes.Count -gt 0) {
+        # Enumerate the machine's uninstall key NAMES once and hash-look-up each
+        # candidate, rather than probing the registry per code.
+        #
+        # The winget source carries every product code a package has ever
+        # shipped: Mozilla.Firefox alone has 5205, one per locale and version.
+        # Probing those across three hives is 15,615 registry reads and took
+        # ~384 seconds measured, which would exceed an N-central scan interval
+        # on its own. This is ~230 reads regardless of how many codes a package
+        # has. Ordinal-ignore-case because the index stores codes normalised to
+        # lower case ("7-zip") while the real key is "7-Zip", and the registry
+        # itself is case-insensitive.
+        $ArpKeys = New-Object 'System.Collections.Generic.Dictionary[string,string]' ([StringComparer]::OrdinalIgnoreCase)
         foreach ($Hive in $Hives) {
-            $Key = Join-Path $Hive $ProductCode
-            if (Test-Path $Key) {
-                $Item = Get-ItemProperty -Path $Key -ErrorAction SilentlyContinue
-                Write-Verbose "Matched on ProductCode '$ProductCode' at $Key"
-                $R = New-Result $true 'ProductCode' $Item.DisplayName $Item.DisplayVersion $Key
+            foreach ($Key in (Get-ChildItem -Path $Hive -ErrorAction SilentlyContinue)) {
+                if (-not $ArpKeys.ContainsKey($Key.PSChildName)) { $ArpKeys[$Key.PSChildName] = $Key.PSPath }
+            }
+        }
+
+        foreach ($Code in $ProductCodes) {
+            $Path = $null
+            if ($ArpKeys.TryGetValue($Code, [ref]$Path)) {
+                $Item = Get-ItemProperty -Path $Path -ErrorAction SilentlyContinue
+                Write-Verbose "Matched on ProductCode '$Code' at $Path"
+                $R = New-Result $true 'ProductCode' $Item.DisplayName $Item.DisplayVersion $Path
                 if ($Detailed) { return $R } else { return $true }
             }
         }
     }
 
+    if ($Pfns.Count -gt 0) {
+        $Elevated = $false
+        try {
+            $Ident = [Security.Principal.WindowsIdentity]::GetCurrent()
+            $Elevated = (New-Object Security.Principal.WindowsPrincipal($Ident)).IsInRole(
+                [Security.Principal.WindowsBuiltInRole]::Administrator)
+        } catch {}
+
+        try {
+            # -AllUsers when we can: SYSTEM sees almost none of its own packages.
+            if ($Elevated) { $Installed = @(Get-AppxPackage -AllUsers -ErrorAction Stop) }
+            else { $Installed = @(Get-AppxPackage -ErrorAction SilentlyContinue) }
+
+            foreach ($Family in $Pfns) {
+                $Hit = $Installed | Where-Object { $_.PackageFamilyName -eq $Family } | Select-Object -First 1
+                if ($Hit) {
+                    Write-Verbose "Matched on PackageFamilyName '$Family'"
+                    $R = New-Result $true 'PackageFamilyName' $Hit.Name $Hit.Version $Hit.PackageFullName
+                    if ($Detailed) { return $R } else { return $true }
+                }
+            }
+        } catch {}
+    }
+
     # --- 2. EXACT DISPLAY NAME ----------------------------------------
     # A custom catalogue entry carries the real ARP DisplayName for its ID.
     $Candidates = New-Object System.Collections.Generic.List[string]