diff --git a/.github/workflows/build-manifest-index.yml b/.github/workflows/build-manifest-index.yml index 9f12f25..bb35293 100644 --- a/.github/workflows/build-manifest-index.yml +++ b/.github/workflows/build-manifest-index.yml @@ -4,6 +4,7 @@ on: schedule: # Nightly. NOTE: GitHub only runs scheduled workflows from the repository's # DEFAULT branch, so this fires once the workflow exists on that branch. + # Push and manual dispatch work from any branch. - cron: '17 3 * * *' workflow_dispatch: push: @@ -28,7 +29,69 @@ jobs: with: fetch-depth: 0 - - name: Resolve manifests + # --------------------------------------------------------------- + # Detection index: EVERY package in the winget source, not just the + # curated catalogue. Built from Microsoft's own published source + # index, which already carries the ARP product codes and MSIX package + # family names that detection matches on. One 3.5 MB CDN download, no + # api.github.com calls, no clone of winget-pkgs. + # --------------------------------------------------------------- + - name: Build detection index from the winget source + run: | + set -euo pipefail + curl -sSL --retry 3 --max-time 300 \ + -o source2.msix https://cdn.winget.microsoft.com/cache/source2.msix + ls -l source2.msix + + unzip -o -q source2.msix -d wingetsrc + DB=wingetsrc/Public/index.db + test -f "$DB" + + # One row per identifier rather than a delimited list. Any separator + # has to be a character that cannot appear inside a product code, and + # the obvious safe choice is a control character, which is not legal + # in a YAML workflow file. Emitting pairs and grouping in jq avoids + # the question entirely. + # + # Packages with neither a product code nor a package family name are + # excluded: they cannot be identified this way, so they are dead + # weight in a file every endpoint downloads. + sqlite3 "$DB" -json " + SELECT p.id AS Id, p.name AS Name, p.latest_version AS Version, + 'P' AS Kind, c.productcode AS Value + FROM packages p JOIN productcodes2 c ON c.package = p.rowid + UNION ALL + SELECT p.id, p.name, p.latest_version, + 'F', f.pfn + FROM packages p JOIN pfns2 f ON f.package = p.rowid + ORDER BY 1;" > pairs.json + + NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) + jq --arg now "$NOW" ' + group_by(.Id) + | map({ + key: .[0].Id, + value: { + Name: .[0].Name, + Version: .[0].Version, + ProductCodes: [ .[] | select(.Kind == "P") | .Value ], + Pfns: [ .[] | select(.Kind == "F") | .Value ] + }}) + | from_entries + | { Generated: $now, + Source: "winget source2 index", + PackageCount: length, + Packages: . } + ' pairs.json > Index/Detection.json + + echo "detection index: $(jq -r .PackageCount Index/Detection.json) packages, $(du -h Index/Detection.json | cut -f1)" + + # --------------------------------------------------------------- + # Curated index: full install metadata (installer URL, silent args) for + # the packages in Index/Catalog.json, so a common install needs no API + # call either. Anything not listed still installs, resolved live. + # --------------------------------------------------------------- + - name: Resolve curated manifests shell: pwsh env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -38,6 +101,7 @@ jobs: run: | set -euo pipefail cp Index/Manifests.json "$RUNNER_TEMP/Manifests.json" + cp Index/Detection.json "$RUNNER_TEMP/Detection.json" git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" @@ -47,8 +111,9 @@ jobs: git checkout --orphan manifest-index git rm -rf . -q cp "$RUNNER_TEMP/Manifests.json" Manifests.json - git add Manifests.json + cp "$RUNNER_TEMP/Detection.json" Detection.json + git add Manifests.json Detection.json git commit -q -m "Refresh manifest index ($(date -u +%Y-%m-%dT%H:%M:%SZ))" git push -f origin manifest-index - echo "Published to https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/manifest-index/Manifests.json" + echo "Published to https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/manifest-index/" diff --git a/.gitignore b/.gitignore index d0b7058..4aadb73 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ # Built in CI and published to the manifest-index branch, never committed here. Index/Manifests.json +Index/Detection.json diff --git a/Private/Get-DetectionIndex.ps1 b/Private/Get-DetectionIndex.ps1 new file mode 100644 index 0000000..327c8e9 --- /dev/null +++ b/Private/Get-DetectionIndex.ps1 @@ -0,0 +1,91 @@ +function Get-DetectionIndexUrl { + # Built nightly from Microsoft's own published winget source index, so it + # covers EVERY package in the winget repository rather than the curated + # Index/Catalog.json. Served from raw.githubusercontent.com, which is + # CDN-backed and carries no api.github.com rate limit. + return "https://raw.githubusercontent.com/Techary/TecharyGet/manifest-index/Detection.json" +} + +function Get-DetectionIndex { + <# + .SYNOPSIS + The full package-to-ARP mapping used to identify installed software. + #> + [CmdletBinding()] + param( + [int]$CacheHours = 12, + [switch]$NoRefresh, + [switch]$Force + ) + + $CacheDir = "$env:ProgramData\TecharyGet" + $CachePath = Join-Path $CacheDir 'DetectionIndex.json' + $HaveCache = Test-Path $CachePath + + try { + if (-not (Test-Path $CacheDir)) { New-Item -ItemType Directory -Path $CacheDir -Force -ErrorAction Stop | Out-Null } + + # -NoRefresh means "do not re-download a copy we already have", NOT + # "never download". An endpoint that only runs detection installs + # nothing, so nothing else would ever fetch this for it. + $NeedUpdate = $true + if ($HaveCache -and -not $Force) { + $Age = (Get-Date) - (Get-Item $CachePath).LastWriteTime + if ($NoRefresh -or $Age.TotalHours -lt $CacheHours) { $NeedUpdate = $false } + } + + if ($NeedUpdate) { + try { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 } catch {} + + # Stage and parse before promoting, so a captive portal or proxy + # page returning HTTP 200 with HTML cannot poison the cache. + $StagePath = "$CachePath.tmp" + Invoke-WebRequest -Uri (Get-DetectionIndexUrl) -OutFile $StagePath -UseBasicParsing -ErrorAction Stop + + $Parsed = Get-Content -Path $StagePath -Raw | ConvertFrom-Json + if (-not $Parsed.Packages) { throw "Detection index downloaded but contains no Packages block." } + + Move-Item -Path $StagePath -Destination $CachePath -Force -ErrorAction Stop + Write-PackagerLog -Message "Detection index refreshed ($($Parsed.PackageCount) packages, generated $($Parsed.Generated))." + } + } + catch { + # A miss is not an error: detection falls back to name matching. + Write-PackagerLog -Message "Could not refresh the detection index ($($_.Exception.Message)). Using local copy if present." -Severity Warning + Remove-Item "$CachePath.tmp" -Force -ErrorAction SilentlyContinue + } + + if (-not (Test-Path $CachePath)) { return $null } + + try { return (Get-Content -Path $CachePath -Raw | ConvertFrom-Json) } + catch { + Write-PackagerLog -Message "Detection index cache is unreadable: $($_.Exception.Message)" -Severity Warning + return $null + } +} + +function Get-DetectionEntry { + <# + .SYNOPSIS + ARP product codes and MSIX package family names for one package ID. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)][string]$Id, + [switch]$NoRefresh + ) + + $Index = Get-DetectionIndex -NoRefresh:$NoRefresh + if (-not $Index -or -not $Index.Packages) { return $null } + + $Entry = $Index.Packages.$Id + if (-not $Entry) { return $null } + + return [PSCustomObject]@{ + Id = $Id + Name = $Entry.Name + Version = $Entry.Version + ProductCodes = @($Entry.ProductCodes) + Pfns = @($Entry.Pfns) + } +} diff --git a/Private/Get-ManifestIndex.ps1 b/Private/Get-ManifestIndex.ps1 index 5a747fc..69ce5f6 100644 --- a/Private/Get-ManifestIndex.ps1 +++ b/Private/Get-ManifestIndex.ps1 @@ -17,15 +17,20 @@ function Get-ManifestIndex { $CacheDir = "$env:ProgramData\TecharyGet" $CachePath = Join-Path $CacheDir 'ManifestIndex.json' + $HaveCache = Test-Path $CachePath + try { if (-not (Test-Path $CacheDir)) { New-Item -ItemType Directory -Path $CacheDir -Force -ErrorAction Stop | Out-Null } - # Detection runs on a schedule on every endpoint, so it must never make - # a network call. -NoRefresh reads whatever is already cached. - $NeedUpdate = -not $NoRefresh - if (-not $Force -and (Test-Path $CachePath)) { + # -NoRefresh means "do not re-download a copy we already have", NOT + # "never download". An endpoint that only ever runs detection installs + # nothing, so nothing else would ever fetch the index for it: treating + # NoRefresh as "never download" left those machines permanently without + # one, and ProductCode detection could never work there. + $NeedUpdate = $true + if ($HaveCache -and -not $Force) { $Age = (Get-Date) - (Get-Item $CachePath).LastWriteTime - if ($Age.TotalHours -lt $CacheHours) { $NeedUpdate = $false } + if ($NoRefresh -or $Age.TotalHours -lt $CacheHours) { $NeedUpdate = $false } } if ($NeedUpdate) { diff --git a/Public/Test-TecharyApp.ps1 b/Public/Test-TecharyApp.ps1 index 8f81a88..7c0e335 100644 --- a/Public/Test-TecharyApp.ps1 +++ b/Public/Test-TecharyApp.ps1 @@ -58,32 +58,89 @@ function Test-TecharyApp { elseif ([Environment]::Is64BitOperatingSystem) { $SysArch = "x64" } else { $SysArch = "x86" } - # --- 1. PRODUCT CODE (definitive) --------------------------------- - $ProductCode = $null + # --- 1. PRODUCT CODE / PACKAGE FAMILY (definitive) ----------------- + # Three sources, most complete first: the full detection index covers + # every package in the winget repository; the curated manifest index and + # the local manifest cache cover what this machine has installed before. + $ProductCodes = New-Object System.Collections.Generic.List[string] + $Pfns = New-Object System.Collections.Generic.List[string] + + try { + $Entry = Get-DetectionEntry -Id $Name -NoRefresh + if ($Entry) { + foreach ($Code in $Entry.ProductCodes) { if ($Code) { $ProductCodes.Add($Code) } } + foreach ($Family in $Entry.Pfns) { if ($Family) { $Pfns.Add($Family) } } + Write-Verbose "Detection index: $($ProductCodes.Count) product code(s), $($Pfns.Count) package family name(s) for '$Name'" + } + } catch {} + try { $Indexed = Get-IndexedManifest -Id $Name -SysArch $SysArch -NoRefresh - if ($Indexed) { $ProductCode = $Indexed.ProductCode } + if ($Indexed -and $Indexed.ProductCode) { $ProductCodes.Add($Indexed.ProductCode) } } catch {} - if (-not $ProductCode) { - $CachedManifest = Join-Path $env:ProgramData ("TecharyGet\ManifestCache\" + ($Name -replace '[\\/:*?"<>|]', '_') + ".json") - if (Test-Path $CachedManifest) { - try { $ProductCode = (Get-Content $CachedManifest -Raw | ConvertFrom-Json).ProductCode } catch {} - } + $CachedManifest = Join-Path $env:ProgramData ("TecharyGet\ManifestCache\" + ($Name -replace '[\\/:*?"<>|]', '_') + ".json") + if (Test-Path $CachedManifest) { + try { + $Local = (Get-Content $CachedManifest -Raw | ConvertFrom-Json).ProductCode + if ($Local) { $ProductCodes.Add($Local) } + } catch {} } - if ($ProductCode) { + if ($ProductCodes.Count -gt 0) { + # Enumerate the machine's uninstall key NAMES once and hash-look-up each + # candidate, rather than probing the registry per code. + # + # The winget source carries every product code a package has ever + # shipped: Mozilla.Firefox alone has 5205, one per locale and version. + # Probing those across three hives is 15,615 registry reads and took + # ~384 seconds measured, which would exceed an N-central scan interval + # on its own. This is ~230 reads regardless of how many codes a package + # has. Ordinal-ignore-case because the index stores codes normalised to + # lower case ("7-zip") while the real key is "7-Zip", and the registry + # itself is case-insensitive. + $ArpKeys = New-Object 'System.Collections.Generic.Dictionary[string,string]' ([StringComparer]::OrdinalIgnoreCase) foreach ($Hive in $Hives) { - $Key = Join-Path $Hive $ProductCode - if (Test-Path $Key) { - $Item = Get-ItemProperty -Path $Key -ErrorAction SilentlyContinue - Write-Verbose "Matched on ProductCode '$ProductCode' at $Key" - $R = New-Result $true 'ProductCode' $Item.DisplayName $Item.DisplayVersion $Key + foreach ($Key in (Get-ChildItem -Path $Hive -ErrorAction SilentlyContinue)) { + if (-not $ArpKeys.ContainsKey($Key.PSChildName)) { $ArpKeys[$Key.PSChildName] = $Key.PSPath } + } + } + + foreach ($Code in $ProductCodes) { + $Path = $null + if ($ArpKeys.TryGetValue($Code, [ref]$Path)) { + $Item = Get-ItemProperty -Path $Path -ErrorAction SilentlyContinue + Write-Verbose "Matched on ProductCode '$Code' at $Path" + $R = New-Result $true 'ProductCode' $Item.DisplayName $Item.DisplayVersion $Path if ($Detailed) { return $R } else { return $true } } } } + if ($Pfns.Count -gt 0) { + $Elevated = $false + try { + $Ident = [Security.Principal.WindowsIdentity]::GetCurrent() + $Elevated = (New-Object Security.Principal.WindowsPrincipal($Ident)).IsInRole( + [Security.Principal.WindowsBuiltInRole]::Administrator) + } catch {} + + try { + # -AllUsers when we can: SYSTEM sees almost none of its own packages. + if ($Elevated) { $Installed = @(Get-AppxPackage -AllUsers -ErrorAction Stop) } + else { $Installed = @(Get-AppxPackage -ErrorAction SilentlyContinue) } + + foreach ($Family in $Pfns) { + $Hit = $Installed | Where-Object { $_.PackageFamilyName -eq $Family } | Select-Object -First 1 + if ($Hit) { + Write-Verbose "Matched on PackageFamilyName '$Family'" + $R = New-Result $true 'PackageFamilyName' $Hit.Name $Hit.Version $Hit.PackageFullName + if ($Detailed) { return $R } else { return $true } + } + } + } catch {} + } + # --- 2. EXACT DISPLAY NAME ---------------------------------------- # A custom catalogue entry carries the real ARP DisplayName for its ID. $Candidates = New-Object System.Collections.Generic.List[string]