diff --git a/docs/reference/authentication.md b/docs/reference/authentication.md index bf82eff107..9e0ff21449 100644 --- a/docs/reference/authentication.md +++ b/docs/reference/authentication.md @@ -98,6 +98,31 @@ and `codeload.` subdomains your catalog/extension URLs use. A `*.ghes.example.com` wildcard matches subdomains but **not** the bare host, so always include the bare host explicitly. +### GitHub Enterprise Cloud with data residency (GHE.com) + +GHE.com tenants use separate web and REST API hostnames. List both hosts so +Specify can authenticate the release-metadata lookup and asset download: + +```json +{ + "providers": [ + { + "hosts": ["tenant.ghe.com", "api.tenant.ghe.com"], + "provider": "github", + "auth": "bearer", + "token_env": "GH_ENTERPRISE_TOKEN" + } + ] +} +``` + +For a release URL on `tenant.ghe.com`, Specify resolves metadata and release +assets through `api.tenant.ghe.com`. The returned asset must identify the same +owner and repository and use the exact numeric +`/repos/.../releases/assets/...` endpoint. A wildcard such as +`*.tenant.ghe.com` matches the API hostname but not the bare tenant web +hostname, so list the web hostname explicitly. + ### Azure DevOps (`azure-devops`) | Scheme | Header | Use for | diff --git a/src/specify_cli/authentication/github_http.py b/src/specify_cli/authentication/github_http.py index 402a147d61..d05c312abd 100644 --- a/src/specify_cli/authentication/github_http.py +++ b/src/specify_cli/authentication/github_http.py @@ -25,6 +25,8 @@ "api.github.com", "codeload.github.com", }) +_GHE_COM_SUFFIX = ".ghe.com" +_GHE_COM_API_PREFIX = "api." _MAX_RELEASE_METADATA_BYTES = 5 * 1024 * 1024 @@ -41,6 +43,18 @@ def _has_valid_percent_escapes(value: str) -> bool: return True +def _has_valid_asset_url_spelling(value: str) -> bool: + """Return whether an asset URL uses an unambiguous raw spelling.""" + return ( + not any( + ord(character) <= 0x20 or ord(character) == 0x7F + for character in value + ) + and not any(delimiter in value for delimiter in ("?", "#", ";")) + and _has_valid_percent_escapes(value) + ) + + def build_github_request(url: str) -> urllib.request.Request: """Build a urllib Request, adding a GitHub auth header when available. @@ -81,6 +95,27 @@ def _host_matches(hostname: str, patterns: tuple[str, ...]) -> bool: return any(p == hostname or fnmatch(hostname, p) for p in patterns) +def _ghe_com_api_hostname(web_hostname: str) -> str | None: + """Return the paired GHE.com API hostname for a tenant web hostname.""" + if ( + web_hostname == "ghe.com" + or web_hostname.startswith(_GHE_COM_API_PREFIX) + or not web_hostname.endswith(_GHE_COM_SUFFIX) + ): + return None + return f"{_GHE_COM_API_PREFIX}{web_hostname}" + + +def _ghe_com_web_hostname(api_hostname: str) -> str | None: + """Return the paired GHE.com web hostname for a tenant API hostname.""" + if not api_hostname.startswith(_GHE_COM_API_PREFIX): + return None + web_hostname = api_hostname.removeprefix(_GHE_COM_API_PREFIX) + if web_hostname == "ghe.com" or not web_hostname.endswith(_GHE_COM_SUFFIX): + return None + return web_hostname + + def resolve_github_release_asset_api_url( download_url: str, open_url_fn: Callable, @@ -91,18 +126,20 @@ def resolve_github_release_asset_api_url( ) -> Optional[str]: """Resolve a GitHub release browser-download URL to its REST API asset URL. - Works for public ``github.com`` and for GitHub Enterprise Server (GHES) - hosts. A host is treated as GHES when it matches one of *github_hosts* - (exact hostname or ``*.suffix``) — supply the hosts the user has trusted - under a ``github`` provider in ``auth.json``. This allowlist is the - security gate: unlisted hosts never receive GHES API treatment, so a - malicious catalog cannot induce an API request to an arbitrary host. + Works for public ``github.com``, GitHub Enterprise Cloud with data + residency (GHE.com), and GitHub Enterprise Server (GHES). Enterprise hosts + must match *github_hosts* (exact hostname or ``*.suffix``), which should be + the hosts the user trusted under a ``github`` provider in ``auth.json``. + GHE.com additionally requires both the tenant web hostname and its paired + ``api.`` hostname to be trusted. - For a public URL the API base is ``https://api.github.com``; for a GHES - host it is ``{scheme}://{host[:port]}/api/v3``. Returns the API asset URL + Public GitHub uses ``https://api.github.com``; a GHE.com tenant + ``tenant.ghe.com`` uses ``https://api.tenant.ghe.com``; GHES uses + ``{scheme}://{host[:port]}/api/v3``. Returns the API asset URL (downloadable with ``Accept: application/octet-stream`` + a token), the - input unchanged if it is already an API asset URL, or ``None`` when the - URL is not a resolvable GitHub release download or the lookup fails. + input unchanged if it is already a recognized API asset URL, or ``None`` + when the URL is not a resolvable GitHub release download or the lookup + fails. Args: download_url: The URL to resolve. @@ -110,7 +147,7 @@ def resolve_github_release_asset_api_url( :func:`specify_cli.authentication.http.open_url` used for the authenticated release-metadata lookup. timeout: Per-request timeout in seconds. - github_hosts: Host patterns to treat as GitHub Enterprise Server. + github_hosts: Host patterns trusted as GitHub Enterprise deployments. redirect_validator: Optional policy applied to metadata redirects. max_metadata_bytes: Maximum release-metadata response size. """ @@ -128,13 +165,23 @@ def resolve_github_release_asset_api_url( try: parsed = urlparse(download_url) hostname = (parsed.hostname or "").lower() + parsed_port = parsed.port except ValueError: return None parts = [unquote(part) for part in parsed.path.strip("/").split("/")] + ghe_com_api_hostname = _ghe_com_api_hostname(hostname) + is_ghe_com = ( + ghe_com_api_hostname is not None + and parsed.scheme == "https" + and parsed_port in (None, 443) + and _host_matches(hostname, github_hosts) + and _host_matches(ghe_com_api_hostname, github_hosts) + ) is_ghes = ( bool(hostname) and hostname not in GITHUB_HOSTS + and not hostname.endswith(_GHE_COM_SUFFIX) and _host_matches(hostname, github_hosts) ) @@ -145,15 +192,46 @@ def _is_asset_path(segments: list[str]) -> bool: and segments[3:5] == ["releases", "assets"] ) - # Already a REST API asset URL — use it directly. Pure passthrough induces - # no new request: the caller fetches this same URL regardless, so it is - # gated on path shape alone rather than the GHES allowlist. The token stays - # independently gated by auth.json in the download helper, and only the - # resolving path below (which issues a tag-lookup request) needs the - # allowlist as its anti-SSRF gate. + def _is_exact_raw_asset_path(path: str) -> bool: + segments = path.split("/") + return ( + len(segments) == 7 + and segments[:2] == ["", "repos"] + and bool(segments[2]) + and bool(segments[3]) + and segments[4:6] == ["releases", "assets"] + and segments[-1].isascii() + and segments[-1].isdigit() + ) + + # Already a REST API asset URL — use it directly. Existing GitHub.com and + # GHES passthrough behavior remains path-gated because it induces no new + # request; the caller would fetch the same URL regardless. GHE.com is new + # here and uses its stricter tenant-pair trust check below. if hostname == "api.github.com" and _is_asset_path(parts): return download_url - if hostname and parts[:2] == ["api", "v3"] and _is_asset_path(parts[2:]): + if ( + hostname + and not hostname.endswith(_GHE_COM_SUFFIX) + and parts[:2] == ["api", "v3"] + and _is_asset_path(parts[2:]) + ): + return download_url + ghe_com_web_hostname = _ghe_com_web_hostname(hostname) + if ( + ghe_com_web_hostname is not None + and _has_valid_asset_url_spelling(download_url) + and parsed.scheme == "https" + and parsed_port in (None, 443) + and parsed.username is None + and parsed.password is None + and not parsed.query + and not parsed.fragment + and not parsed.params + and _host_matches(hostname, github_hosts) + and _host_matches(ghe_com_web_hostname, github_hosts) + and _is_exact_raw_asset_path(parsed.path) + ): return download_url # Browser download URLs must be usable HTTP(S) URLs before they can cause @@ -161,27 +239,23 @@ def _is_asset_path(segments: list[str]) -> bool: # retains its existing path-only behavior. if parsed.scheme not in {"http", "https"}: return None - try: - _browser_port = parsed.port - except ValueError: - return None # Determine the REST API base for browser release-download URLs. if hostname == "github.com": api_base = "https://api.github.com" + expected_asset_prefix = ["", "repos"] + elif is_ghe_com: + api_base = f"https://{ghe_com_api_hostname}" + expected_asset_prefix = ["", "repos"] elif is_ghes: - # ``parsed.port`` raises ValueError on a malformed port (e.g. - # ``host:notaport``); the function's contract is to return None for - # anything it can't resolve, not to raise. - try: - port = parsed.port - except ValueError: - return None # ``urlparse().hostname`` removes IPv6 brackets. Restore them when # constructing an authority so the derived API base remains a URL. authority_host = f"[{hostname}]" if ":" in hostname else hostname - authority = authority_host if port is None else f"{authority_host}:{port}" + authority = ( + authority_host if parsed_port is None else f"{authority_host}:{parsed_port}" + ) api_base = f"{parsed.scheme}://{authority}/api/v3" + expected_asset_prefix = ["", "api", "v3", "repos"] else: return None @@ -202,14 +276,7 @@ def _is_expected_asset_url(asset_url: object) -> bool: # ``urlparse`` tolerates some raw spellings (for example whitespace) # even though the original metadata value is returned to the caller. # Reject those spellings before parsing rather than normalizing them. - if ( - any( - ord(character) <= 0x20 or ord(character) == 0x7F - for character in asset_url - ) - or any(delimiter in asset_url for delimiter in ("?", "#", ";")) - or not _has_valid_percent_escapes(asset_url) - ): + if not _has_valid_asset_url_spelling(asset_url): return False try: asset_parsed = urlparse(asset_url) @@ -250,15 +317,10 @@ def _origin(parsed_url, host: str, port: int | None) -> tuple[str, str, int]: return False asset_parts = asset_parsed.path.split("/") - owner_index = 2 if api_base == "https://api.github.com" else 4 - expected_prefix = ( - ["", "repos"] - if api_base == "https://api.github.com" - else ["", "api", "v3", "repos"] - ) + owner_index = len(expected_asset_prefix) return ( len(asset_parts) == owner_index + 5 - and asset_parts[:owner_index] == expected_prefix + and asset_parts[:owner_index] == expected_asset_prefix and unquote(asset_parts[owner_index]).casefold() == owner.casefold() and unquote(asset_parts[owner_index + 1]).casefold() == repo.casefold() and asset_parts[owner_index + 2:owner_index + 4] == ["releases", "assets"] diff --git a/src/specify_cli/authentication/http.py b/src/specify_cli/authentication/http.py index 32a6ed67c7..7dabc752aa 100644 --- a/src/specify_cli/authentication/http.py +++ b/src/specify_cli/authentication/http.py @@ -154,9 +154,9 @@ def build_request(url: str, extra_headers: dict[str, str] | None = None) -> urll def github_provider_hosts() -> tuple[str, ...]: """Return host patterns from every ``github`` provider entry in ``auth.json``. - Used to classify which hosts are GitHub Enterprise Server instances when - resolving release-asset download URLs. Returns an empty tuple when no - ``auth.json`` exists or it contains no ``github`` entries. + Used to classify trusted GitHub Enterprise Cloud and GitHub Enterprise + Server hosts when resolving release-asset download URLs. Returns an empty + tuple when no ``auth.json`` exists or it contains no ``github`` entries. """ hosts: list[str] = [] for entry in _load_config(): diff --git a/tests/specify_cli/authentication/test_github_http.py b/tests/specify_cli/authentication/test_github_http.py index ad6440f79d..8eae8d0a05 100644 --- a/tests/specify_cli/authentication/test_github_http.py +++ b/tests/specify_cli/authentication/test_github_http.py @@ -315,6 +315,151 @@ def capturing_open(url, timeout=None, extra_headers=None): assert len(captured_urls) == 1 assert "releases/tags/v1%23beta" in captured_urls[0] + # --- GHE.com (GitHub Enterprise Cloud with data residency) --- + + def test_resolves_ghecom_browser_url_to_api_url(self): + """A GHE.com browser URL resolves through its paired API subdomain.""" + asset_url = "https://api.msft.ghe.com/repos/org/repo/releases/assets/42" + captured = [] + + @contextmanager + def capturing_open(url, timeout=None, extra_headers=None): + captured.append(url) + resp = MagicMock() + resp.read.side_effect = io.BytesIO( + json.dumps( + {"assets": [{"name": "bundle.zip", "url": asset_url}]} + ).encode() + ).read + yield resp + + result = resolve_github_release_asset_api_url( + "https://msft.ghe.com/org/repo/releases/download/v1.0/bundle.zip", + capturing_open, + github_hosts=("msft.ghe.com", "api.msft.ghe.com"), + ) + + assert result == asset_url + assert captured == [ + "https://api.msft.ghe.com/repos/org/repo/releases/tags/v1.0" + ] + + def test_passthrough_for_trusted_ghecom_api_asset_url(self): + """A trusted direct GHE.com API asset URL receives asset treatment.""" + url = "https://api.msft.ghe.com/repos/org/repo/releases/assets/42" + result = resolve_github_release_asset_api_url( + url, + lambda *a, **kw: None, + github_hosts=("msft.ghe.com", "api.msft.ghe.com"), + ) + assert result == url + + @pytest.mark.parametrize( + "url", + [ + "https://api.msft.ghe.com/repos/org/repo/releases/assets/42?", + "https://api.msft.ghe.com/repos/org/repo/releases/assets/42#", + "https://api.msft.ghe.com/repos/org/repo/releases/assets/42/", + "https://api.msft.ghe.com/repos/org/repo/releases/assets/%34%32", + "https://api.msft.ghe.com/repos/org/repo/releases/assets/42 ", + "https://api.msft.ghe.com/repos/org/repo/releases/assets/42;", + "https://api.msft.ghe.com/repos/org/repo/releases/assets/%ZZ", + ], + ) + def test_rejects_noncanonical_direct_ghecom_api_asset_url(self, url): + """Direct GHE.com asset URLs retain strict raw-spelling validation.""" + result = resolve_github_release_asset_api_url( + url, + lambda *a, **kw: None, + github_hosts=("msft.ghe.com", "api.msft.ghe.com"), + ) + assert result is None + + def test_rejects_direct_ghecom_api_asset_url_without_trusted_pair(self): + """A direct GHE.com API URL needs both trusted tenant hostnames.""" + url = "https://api.msft.ghe.com/repos/org/repo/releases/assets/42" + result = resolve_github_release_asset_api_url( + url, + lambda *a, **kw: None, + github_hosts=("api.msft.ghe.com",), + ) + assert result is None + + @pytest.mark.parametrize( + "github_hosts", + [ + (), + ("msft.ghe.com", "api.msft.ghe.com"), + ], + ) + def test_rejects_direct_ghecom_api_asset_url_with_ghes_path( + self, github_hosts + ): + """A GHE.com API host never accepts the legacy GHES /api/v3 path.""" + url = ( + "https://api.msft.ghe.com/api/v3/repos/org/repo/" + "releases/assets/42" + ) + result = resolve_github_release_asset_api_url( + url, + lambda *a, **kw: None, + github_hosts=github_hosts, + ) + assert result is None + + @pytest.mark.parametrize( + "github_hosts", + [ + ("msft.ghe.com",), + ("api.msft.ghe.com",), + ("other.ghe.com", "api.other.ghe.com"), + ], + ) + def test_ghecom_resolution_requires_trusted_web_and_api_hosts(self, github_hosts): + """GHE.com resolution requires both members of the tenant host pair.""" + called = [] + + @contextmanager + def recording_open(url, timeout=None, extra_headers=None): + called.append(url) + resp = MagicMock() + resp.read.side_effect = io.BytesIO(b"{}").read + yield resp + + result = resolve_github_release_asset_api_url( + "https://msft.ghe.com/org/repo/releases/download/v1.0/bundle.zip", + recording_open, + github_hosts=github_hosts, + ) + + assert result is None + assert called == [] + + @pytest.mark.parametrize( + "asset_url", + [ + "https://api.other.ghe.com/repos/org/repo/releases/assets/42", + "http://api.msft.ghe.com/repos/org/repo/releases/assets/42", + "https://msft.ghe.com/api/v3/repos/org/repo/releases/assets/42", + "https://api.msft.ghe.com/repos/other/repo/releases/assets/42", + "https://api.msft.ghe.com/repos/org/other/releases/assets/42", + "https://api.msft.ghe.com/repos/org/repo/releases/assets/not-a-number", + "https://api.msft.ghe.com/repos/org/repo/releases/assets/42?download=1", + ], + ) + def test_rejects_wrong_origin_or_path_for_ghecom_metadata_asset_url( + self, asset_url + ): + """GHE.com metadata must identify the paired tenant and repository.""" + result = resolve_github_release_asset_api_url( + "https://msft.ghe.com/org/repo/releases/download/v1.0/bundle.zip", + self._make_open_url_fn( + {"assets": [{"name": "bundle.zip", "url": asset_url}]} + ), + github_hosts=("msft.ghe.com", "api.msft.ghe.com"), + ) + assert result is None + # --- GHES (GitHub Enterprise Server) --- def test_resolves_ghes_browser_url_to_api_url(self): diff --git a/tests/specify_cli/bundles/test_command_info.py b/tests/specify_cli/bundles/test_command_info.py index 6d9f6905a9..673e004ea0 100644 --- a/tests/specify_cli/bundles/test_command_info.py +++ b/tests/specify_cli/bundles/test_command_info.py @@ -578,6 +578,60 @@ def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None assert payload["id"] == "demo-bundle" +def test_bundle_info_resolves_ghecom_browser_release_url_zip(project: Path): + """bundle info resolves a GHE.com release ZIP through its API subdomain.""" + import zipfile + + web_host = "msft.ghe.com" + api_host = f"api.{web_host}" + browser_url = f"https://{web_host}/org/repo/releases/download/v2.0/bundle.zip" + api_asset_url = f"https://{api_host}/repos/org/repo/releases/assets/42" + + archive = io.BytesIO() + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("bundle.yml", yaml.safe_dump(valid_manifest_dict())) + zip_bytes = archive.getvalue() + captured = [] + + def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None): + captured.append((url, extra_headers)) + if "releases/tags/" in url: + return FakeBundleResponse( + json.dumps( + {"assets": [{"name": "bundle.zip", "url": api_asset_url}]} + ).encode(), + url=url, + ) + return FakeBundleResponse(zip_bytes, url=api_asset_url) + + catalog = project / "catalog.json" + write_catalog_file( + catalog, + {"demo-bundle": catalog_entry_dict("demo-bundle", download_url=browser_url)}, + ) + _make_catalog_config(catalog, project) + + with ( + patch("specify_cli.authentication.http.open_url", side_effect=fake_open_url), + patch( + "specify_cli.authentication.http.github_provider_hosts", + return_value=(web_host, api_host), + ), + ): + result = runner.invoke(app, ["bundle", "info", "demo-bundle", "--json"]) + + assert result.exit_code == 0, result.output + + tag_calls = [url for url, _ in captured if "releases/tags/" in url] + assert tag_calls == [f"https://{api_host}/repos/org/repo/releases/tags/v2.0"] + + asset_calls = [(url, headers) for url, headers in captured if url == api_asset_url] + assert asset_calls == [(api_asset_url, {"Accept": "application/octet-stream"})] + + payload = json.loads(result.output) + assert payload["id"] == "demo-bundle" + + def test_bundle_download_rejects_oversized_response(project: Path, monkeypatch): """Bundle download rejects responses exceeding MAX_DOWNLOAD_BYTES.""" # Monkeypatch to a small limit so the test is fast and low-memory.