Correlate installed software the way winget does

.github/workflows/build-manifest-index.yml

@@ -58,40 +58,74 @@ jobs:
           # but the entry still carries the canonical display name, and that is
           # what bridges an ID to its ARP entry: "Valve.Steam" never matches
           # "Steam" on its own. A name-only row costs about 80 bytes.
+          # Kinds: P product code, F package family name, M normalised name,
+          # B normalised publisher, N name-only placeholder.
+          #
+          # M and B are winget's OWN correlation keys, from norm_names2 and
+          # norm_publishers2. They are what makes this work for the 6,443
+          # packages that declare no product code and no package family name
+          # -- Microsoft.Office among them. A name match is only valid when a
+          # publisher matches too, so both must ship.
           sqlite3 "$DB" -json "
             SELECT p.id AS Id, p.name AS Name, p.latest_version AS Version,
                    'P' AS Kind, c.productcode AS Value
             FROM packages p JOIN productcodes2 c ON c.package = p.rowid
             UNION ALL
-            SELECT p.id, p.name, p.latest_version,
-                   'F', f.pfn
+            SELECT p.id, p.name, p.latest_version, 'F', f.pfn
             FROM packages p JOIN pfns2 f ON f.package = p.rowid
             UNION ALL
-            SELECT p.id, p.name, p.latest_version,
-                   'N', NULL
+            SELECT p.id, p.name, p.latest_version, 'M', n.norm_name
+            FROM packages p JOIN norm_names2 n ON n.package = p.rowid
+            UNION ALL
+            SELECT p.id, p.name, p.latest_version, 'B', b.norm_publisher
+            FROM packages p JOIN norm_publishers2 b ON b.package = p.rowid
+            UNION ALL
+            SELECT p.id, p.name, p.latest_version, 'U', u.upgradecode
+            FROM packages p JOIN upgradecodes2 u ON u.package = p.rowid
+            UNION ALL
+            SELECT p.id, p.name, p.latest_version, 'N', NULL
             FROM packages p
-            WHERE NOT EXISTS (SELECT 1 FROM productcodes2 WHERE package = p.rowid)
-              AND NOT EXISTS (SELECT 1 FROM pfns2 WHERE package = p.rowid)
             ORDER BY 1;" > pairs.json
 
+          # Name+publisher pairs carried by more than one package. winget
+          # resolves these with a reverse-correlation veto; without the full
+          # reverse index we suppress the weak match instead, which is the
+          # safe direction. mozillathunderbird+mozilla alone is shared by 133
+          # package ids, so skipping this over-reports badly.
+          sqlite3 "$DB" -json "
+            SELECT n.norm_name AS N, b.norm_publisher AS B, COUNT(DISTINCT p.rowid) AS C
+            FROM packages p
+            JOIN norm_names2 n      ON n.package = p.rowid
+            JOIN norm_publishers2 b ON b.package = p.rowid
+            GROUP BY n.norm_name, b.norm_publisher
+            HAVING COUNT(DISTINCT p.rowid) > 1;" > ambiguous.json
+
           NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
-          jq --arg now "$NOW" '
+          jq --arg now "$NOW" --slurpfile amb ambiguous.json '
             group_by(.Id)
             | map({
                 key: .[0].Id,
                 value: {
                   Name: .[0].Name,
                   Version: .[0].Version,
-                  ProductCodes: [ .[] | select(.Kind == "P") | .Value ],
-                  Pfns:         [ .[] | select(.Kind == "F") | .Value ]
+                  ProductCodes:   [ .[] | select(.Kind == "P") | .Value ],
+                  Pfns:           [ .[] | select(.Kind == "F") | .Value ],
+                  NormNames:      [ .[] | select(.Kind == "M") | .Value ],
+                  NormPublishers: [ .[] | select(.Kind == "B") | .Value ],
+                  UpgradeCodes:   [ .[] | select(.Kind == "U") | .Value ]
                 }})
             | from_entries
             | { Generated: $now,
                 Source: "winget source2 index",
+                Normalizer: "Initial",
                 PackageCount: length,
+                AmbiguousPairs: ( $amb[0] | map(.N + "|" + .B) ),
                 Packages: . }
           ' pairs.json > Index/Detection.json
 
+          echo "ambiguous name+publisher pairs: $(jq -r '.AmbiguousPairs | length' Index/Detection.json)"
+          echo "packages with a norm name:      $(jq -r '[.Packages[] | select(.NormNames | length > 0)] | length' Index/Detection.json)"
+
           echo "detection index: $(jq -r .PackageCount Index/Detection.json) packages, $(du -h Index/Detection.json | cut -f1)"
 
       # ---------------------------------------------------------------

.github/workflows/ci.yml

@@ -0,0 +1,20 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Structural checks
+        shell: pwsh
+        run: ./Tests/Test-Module.ps1
+
+      - name: Name normaliser against winget's own corpus
+        shell: pwsh
+        run: ./Tests/Test-Normalizer.ps1

Private/Get-ArpEntry.ps1

@@ -0,0 +1,172 @@
+function Get-ArpEntry {
+    <#
+    .SYNOPSIS
+        The machine's Add/Remove Programs entries, enumerated as winget does.
+
+    .DESCRIPTION
+        Mirrors ARPHelper::PopulateIndexFromARP. Three views, and the skip
+        rules matter: a divergence here means our inventory differs from
+        winget's before any matching happens.
+
+          machine  HKLM\...\Uninstall            64-bit view
+          machine  HKLM\WOW6432Node\...\Uninstall 32-bit view
+          user     HKCU\...\Uninstall             native view only
+
+        There is deliberately no 32-bit view for user scope -- the
+        KEY_WOW64_32KEY branch is gated on machine scope (ARPHelper.cpp:167).
+
+        ProductCode is the SUBKEY NAME, not a value (ARPHelper.cpp:396).
+
+        Note the skip rules are exactly three. The source also appears to
+        skip entries with no version, but DetermineVersion returns
+        "Unknown" rather than empty, so that branch never fires; adding it
+        here would drop entries winget keeps.
+    #>
+    [CmdletBinding()]
+    param()
+
+    $Hives = @(
+        'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall'
+        'HKLM:\SOFTWARE\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall'
+        'HKCU:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall'
+    )
+
+    foreach ($Hive in $Hives) {
+        foreach ($Key in (Get-ChildItem -LiteralPath $Hive -ErrorAction SilentlyContinue)) {
+            $Item = $null
+            try { $Item = Get-ItemProperty -LiteralPath $Key.PSPath -ErrorAction Stop } catch { continue }
+
+            # 1. SystemComponent non-zero
+            if ($Item.PSObject.Properties['SystemComponent'] -and [int]$Item.SystemComponent -ne 0) { continue }
+            # 2 and 3. DisplayName absent, or empty
+            $Display = $Item.PSObject.Properties['DisplayName'] | ForEach-Object { $_.Value }
+            if ([string]::IsNullOrEmpty($Display)) { continue }
+
+            $Publisher = $null
+            if ($Item.PSObject.Properties['Publisher']) { $Publisher = [string]$Item.Publisher }
+
+            # UpgradeCode is only looked up for MSI entries (ARPHelper.cpp:472).
+            $IsMsi = $false
+            if ($Item.PSObject.Properties['WindowsInstaller']) {
+                try { $IsMsi = ([int]$Item.WindowsInstaller -ne 0) } catch { }
+            }
+
+            [PSCustomObject]@{
+                ProductCode      = $Key.PSChildName
+                DisplayName      = [string]$Display
+                Publisher        = $Publisher
+                DisplayVersion   = $(if ($Item.PSObject.Properties['DisplayVersion']) { [string]$Item.DisplayVersion } else { $null })
+                WindowsInstaller = $IsMsi
+                Path             = $Key.PSPath
+            }
+        }
+    }
+}
+
+function ConvertTo-PackedGuid {
+    <#
+        MSI's packed GUID form: the first three groups are reversed whole,
+        the last two are reversed bytewise. ARPHelper.cpp:15-60.
+    #>
+    param([Parameter(Mandatory)][string]$Guid)
+    $g = $Guid.Trim().Trim('{', '}')
+    $p = $g.Split('-')
+    if ($p.Count -ne 5) { return $null }
+    $r  = ($p[0][7,6,5,4,3,2,1,0] -join '')
+    $r += ($p[1][3,2,1,0] -join '')
+    $r += ($p[2][3,2,1,0] -join '')
+    $r += ($p[3][1,0] -join '') + ($p[3][3,2] -join '')
+    for ($i = 0; $i -lt 12; $i += 2) { $r += ($p[4][($i + 1), $i] -join '') }
+    return $r.ToUpperInvariant()
+}
+
+function ConvertFrom-PackedGuid {
+    param([Parameter(Mandatory)][string]$Packed)
+    if ($Packed.Length -ne 32) { return $null }
+    $a = ($Packed[7,6,5,4,3,2,1,0] -join '')
+    $b = ($Packed[11,10,9,8] -join '')
+    $c = ($Packed[15,14,13,12] -join '')
+    $d = ($Packed[17,16] -join '') + ($Packed[19,18] -join '')
+    $e = ''
+    for ($i = 20; $i -lt 32; $i += 2) { $e += ($Packed[($i + 1), $i] -join '') }
+    return ('{{{0}-{1}-{2}-{3}-{4}}}' -f $a, $b, $c, $d, $e).ToLowerInvariant()
+}
+
+function Get-UpgradeCodeMap {
+    <#
+    .SYNOPSIS
+        Packed product code -> upgrade code, for MSI entries.
+
+    .DESCRIPTION
+        UpgradeCode is not a value on the uninstall key. It lives under
+        Installer\UpgradeCodes keyed the opposite way round: the key name is
+        the packed upgrade code and its VALUES are the packed product codes
+        it covers (ARPHelper.cpp:63-105). Built once and inverted, because
+        walking it per entry would be far slower.
+
+        There is no UpgradeCodes key in the 32-bit view (ARPHelper.cpp:82).
+    #>
+    [CmdletBinding()]
+    param()
+
+    $Map = @{}
+    $Root = 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UpgradeCodes'
+    foreach ($Key in (Get-ChildItem -LiteralPath $Root -ErrorAction SilentlyContinue)) {
+        $Upgrade = ConvertFrom-PackedGuid $Key.PSChildName
+        if (-not $Upgrade) { continue }
+        $Props = $null
+        try { $Props = Get-ItemProperty -LiteralPath $Key.PSPath -ErrorAction Stop } catch { continue }
+        foreach ($p in $Props.PSObject.Properties) {
+            if ($p.Name -like 'PS*') { continue }
+            if (-not $Map.ContainsKey($p.Name)) { $Map[$p.Name] = $Upgrade }
+        }
+    }
+    return $Map
+}
+
+function Get-ArpCorrelationSet {
+    <#
+    .SYNOPSIS
+        ARP entries with winget's correlation keys precomputed.
+
+    .DESCRIPTION
+        Normalising every entry costs real time, so this is done once per
+        call to Test-TecharyApp rather than once per candidate key.
+    #>
+    [CmdletBinding()]
+    param()
+
+    $Upgrades = @{}
+    try { $Upgrades = Get-UpgradeCodeMap } catch { }
+
+    foreach ($e in Get-ArpEntry) {
+        $n = $null
+        try { $n = Get-WgNormalizedName -Name $e.DisplayName } catch { }
+        if (-not $n) { continue }
+
+        $upgrade = $null
+        if ($e.WindowsInstaller) {
+            $packed = ConvertTo-PackedGuid $e.ProductCode
+            if ($packed -and $Upgrades.ContainsKey($packed)) { $upgrade = $Upgrades[$packed] }
+        }
+
+        # Both forms: winget stores an extra arch-suffixed row for ARP
+        # display names, and matches against whichever the package carries.
+        $names = [Collections.Generic.List[string]]::new()
+        if ($n.Name) { $names.Add($n.Name) }
+        $withArch = Get-WgNameWithArchitecture $n
+        if ($withArch -and $withArch -ne $n.Name) { $names.Add($withArch) }
+
+        $pub = ''
+        if (-not [string]::IsNullOrEmpty($e.Publisher)) {
+            try { $pub = Get-WgNormalizedPublisher -Publisher $e.Publisher } catch { $pub = '' }
+        }
+
+        [PSCustomObject]@{
+            Entry         = $e
+            UpgradeCode   = $upgrade
+            NormNames     = $names
+            NormPublisher = $pub
+        }
+    }
+}

Private/Get-DetectionIndex.ps1

@@ -82,10 +82,37 @@ function Get-DetectionEntry {
     if (-not $Entry) { return $null }
 
     return [PSCustomObject]@{
-        Id           = $Id
-        Name         = $Entry.Name
-        Version      = $Entry.Version
-        ProductCodes = @($Entry.ProductCodes)
-        Pfns         = @($Entry.Pfns)
+        Id             = $Id
+        Name           = $Entry.Name
+        Version        = $Entry.Version
+        ProductCodes   = @($Entry.ProductCodes)
+        Pfns           = @($Entry.Pfns)
+        UpgradeCodes   = @($Entry.UpgradeCodes)
+        NormNames      = @($Entry.NormNames)
+        NormPublishers = @($Entry.NormPublishers)
     }
 }
+
+function Get-AmbiguousPairSet {
+    <#
+    .SYNOPSIS
+        Name+publisher pairs carried by more than one package.
+
+    .DESCRIPTION
+        winget drops a weak name match when the installed entry correlates to
+        more than one available package. Reproducing that faithfully needs a
+        full reverse index; this is the cheap half of it. Suppressing the
+        match is the safe direction: mozillathunderbird+mozilla is shared by
+        133 package ids, so one Thunderbird entry would otherwise report all
+        133 as installed.
+    #>
+    [CmdletBinding()]
+    param([switch]$NoRefresh)
+
+    $Index = Get-DetectionIndex -NoRefresh:$NoRefresh
+    $Set = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal)
+    if ($Index -and $Index.AmbiguousPairs) {
+        foreach ($p in $Index.AmbiguousPairs) { if ($p) { [void]$Set.Add($p) } }
+    }
+    return $Set
+}