feat(digest): show Semgrep coverage for repos without Code Security

docs/STATUS.md

@@ -601,15 +601,17 @@ or be explicitly parked with a resume note, by then — see the tracker.
     "without Code Security" (8 private NOS repos and
     `claude-config-backup`). The digest App now has Administration: Read on
     all three installations.
-  - `dev-env#166` stays open only for the Code Security purchase decision.
+  - Code Security is not bought (Andrew, 2026-10-21). Semgrep CE through
+    `github-workflows` `semgrep-scan.yml` is the substitute, piloted on
+    `nightowlstudiollc/reliquarist`; its check fails on new findings but is
+    not required. The digest now shows which repos have Semgrep coverage.
 - **Fleet:** `jordan-paddock-pitch` is a full fleet member with a site
   check and the monitor (`dev-env#212`, `claude-config#764`). Its first
   `site-down` issue (`dev-env#213`) closed once Andrew added
   `NETLIFY_AUTH_TOKEN`. `kebab-tax#1405` and `kebab-tax-netlify#325` remove
   the dead `claude.yml`, so no fleet repo has a Claude assistant workflow.
-- **Open decisions:** `dev-env#166` (Code Security purchase),
-  `claude-config#750` gap 1 (accept a false positive on `grep 'git commit'`
-  to catch `sh -c 'git commit'`).
+- **Open decisions:** `claude-config#750` gap 1 (accept a false positive
+  on `grep 'git commit'` to catch `sh -c 'git commit'`).
 
 **Kanban tracker: GitHub Issues + a Project board in `smartwatermelon/dev-env`**,
 using the org's `Priority`/`Target date`/`Effort` issue fields. Board columns

scripts/dependabot-digest/render.sh

@@ -348,24 +348,51 @@ render_protection() {
   fi
 }
 
+# render_sg_list <state> <count> <heading> <blurb>: one Semgrep list.
+# Reads recs from render_codeql.
+render_sg_list() {
+  local state="$1" count="$2" heading="$3" blurb="$4" list
+  [[ "${count}" -gt 0 ]] || return 0
+  list="$(jq -r --arg s "${state}" '
+    def cell: tostring | gsub("[\r\n]+"; " ");
+    sort_by(.repo)[] | select(.s.state == $s)
+    | "- [" + .repo + "](https://github.com/" + .repo + ")"
+      + (if .s.detail then ": " + (.s.detail | cell) else "" end)' <<<"${recs}" 2>/dev/null)"
+  echo "#### ${heading}"
+  echo
+  if [[ -n "${blurb}" ]]; then
+    echo "${blurb}"
+    echo
+  fi
+  echo "${list:-**${count} repositories, but listing them failed.** See the run log.}"
+  echo
+}
+
 # dev-env#166. As for protection: unreadable is never on or off, and an
 # all-unreadable survey says so.
 render_codeql() {
   local file="$1" recs surveyed configured exempt flagged unreadable by_class owners rows
+  local nocs sg_cov sg_none sg_unk
   echo "## CodeQL"
   echo
   if [[ ! -r "${file}" ]] || ! jq -e -s 'type == "array"' "${file}" >/dev/null 2>&1; then
     echo "**Not checked.** The survey produced no readable result, so no repository's CodeQL setup was verified."
     echo
     return 0
   fi
-  # A record without a codeql object (an older or crashed survey) is
-  # unreadable, not clean.
+  # No codeql object (older or crashed survey) is unreadable, not clean.
+  # .s, the Semgrep substitute, is unreadable when missing or unknown.
   if ! recs="$(jq -cs '
     [.[] | select(type == "object" and (.repo | type == "string") and .state != "archived")
      | .c = (if (.codeql | type) == "object" and (.codeql.state | type) == "string"
              then .codeql
-             else {state: "unreadable", detail: "no CodeQL result recorded", exemptClass: null} end)]
+             else {state: "unreadable", detail: "no CodeQL result recorded", exemptClass: null} end)
+     | .s = (if .c.state != "no-code-security" then null
+             elif .c.substitute == "semgrep" or .c.substitute == "none"
+             then {state: .c.substitute, detail: (.c.substituteDetail // null)}
+             elif .c.substitute == null then {state: "unreadable", detail: "no Semgrep result recorded"}
+             else {state: "unreadable",
+                   detail: (.c.substituteDetail // ("unknown Semgrep state: " + (.c.substitute | tostring)))} end)]
   ' "${file}" 2>/dev/null)" || [[ -z "${recs}" ]]; then
     echo "**Not checked.** The survey output could not be summarized, so no repository's CodeQL setup was verified."
     echo
@@ -377,6 +404,9 @@ render_codeql() {
   exempt="$(jq '[.[] | select(.c.state == "exempt")] | length' <<<"${recs}")"
   flagged="$(jq '[.[] | select(.c.state == "not-configured")] | length' <<<"${recs}")"
   nocs="$(jq '[.[] | select(.c.state == "no-code-security")] | length' <<<"${recs}")"
+  sg_cov="$(jq '[.[] | select(.s.state == "semgrep")] | length' <<<"${recs}")"
+  sg_none="$(jq '[.[] | select(.s.state == "none")] | length' <<<"${recs}")"
+  sg_unk=$((nocs - sg_cov - sg_none))
   unreadable=$((surveyed - configured - exempt - flagged - nocs))
 
   echo "Whether each repository has CodeQL default setup turned on. Exempt repositories are left off by a recorded decision for their class; they are counted, not listed."
@@ -389,7 +419,7 @@ render_codeql() {
     echo "**CodeQL setup could not be verified for any of ${surveyed} repositories.** Every read failed or returned no usable state. The likely cause is a token without \`Administration: Read\`, which the default-setup endpoint needs. Nothing here means CodeQL is on or off anywhere."
     echo
   else
-    echo "**${surveyed} repositories surveyed**: ${configured} configured, ${exempt} exempt, **${flagged} not configured, ${nocs} without Code Security, ${unreadable} unreadable.**"
+    echo "**${surveyed} repositories surveyed**: ${configured} configured, ${exempt} exempt, ${sg_cov} covered by Semgrep, **${flagged} not configured, ${sg_none} with no code scanning, ${sg_unk} with Semgrep coverage unreadable, ${unreadable} unreadable.**"
     echo
     if [[ "${exempt}" -gt 0 ]]; then
       by_class="$(jq -r '[.[] | select(.c.state == "exempt")] | group_by(.c.exemptClass)
@@ -422,16 +452,16 @@ render_codeql() {
       echo
     fi
   fi
-  # Listed by name, not exempt: the cost decision is open (dev-env#166).
+  # Listed by name, not exempt: Semgrep coverage is per repo and can lapse.
   if [[ "${nocs}" -gt 0 ]]; then
-    rows="$(jq -r '[.[] | select(.c.state == "no-code-security") | .repo] | sort[]
-      | "- [" + . + "](https://github.com/" + . + ")"' <<<"${recs}")"
-    echo "### Scanning off: paid Code Security not bought"
-    echo
-    echo "GitHub refuses code scanning on these private repositories until the paid GitHub Code Security add-on is enabled. Whether to buy it is an open cost decision (dev-env#166)."
+    echo "### Without paid Code Security"
     echo
-    echo "${rows:-**${nocs} repositories, but listing them failed.** See the run log.}"
+    echo "GitHub refuses code scanning on these private repositories until the paid GitHub Code Security add-on is enabled. Code Security is not bought, by decision (2026-10-21, dev-env#166). Semgrep CE is the substitute: a caller at \`.github/workflows/semgrep-scan.yml\` runs the reusable \`semgrep-scan.yml\` workflow from smartwatermelon/github-workflows."
     echo
+    render_sg_list semgrep "${sg_cov}" "Covered by Semgrep (CodeQL substitute)" ""
+    render_sg_list none "${sg_none}" "No code scanning: paid Code Security not bought, no Semgrep caller" ""
+    render_sg_list unreadable "${sg_unk}" "Semgrep coverage unreadable" \
+      "The Semgrep caller of these repositories could not be read, so their coverage is unknown: not present, not absent."
   fi
   if [[ -n "${owners}" ]]; then
     echo "**Not listed, so not surveyed:** ${owners}. CodeQL setup of their repositories is unknown."

scripts/dependabot-digest/standards.sh

@@ -237,27 +237,74 @@ JQ
 
 CQ_JSON=""
 set_cq() {
-  # set_cq <state> <detail> [exempt class]
+  # set_cq <state> <detail> [exempt class] [substitute] [substitute detail]
   CQ_JSON="$(jq -cn --arg s "$1" --arg d "$2" --arg c "${3-}" \
+    --arg sub "${4-}" --arg subd "${5-}" \
     '{state: $s, detail: (if $d == "" then null else $d end),
-      exemptClass: (if $c == "" then null else $c end)}' 2>/dev/null)"
+      exemptClass: (if $c == "" then null else $c end),
+      substitute: (if $sub == "" then null else $sub end),
+      substituteDetail: (if $subd == "" then null else $subd end)}' 2>/dev/null)"
   if [[ -z "${CQ_JSON}" ]]; then
-    CQ_JSON='{"state":"unreadable","detail":"could not encode the CodeQL result","exemptClass":null}'
+    CQ_JSON='{"state":"unreadable","detail":"could not encode the CodeQL result","exemptClass":null,"substitute":null,"substituteDetail":null}'
   fi
 }
 set_cq_exempt() { set_cq exempt "${CQ_EXEMPT_REASON[$1]-no reason recorded}" "$1"; }
 
+# Code Security is not bought (2026-10-21, dev-env#166). Semgrep CE,
+# called from this path, is the substitute.
+SG_PATH=".github/workflows/semgrep-scan.yml"
+# A job-level or step-level uses: line, so a commented-out one does not count.
+SG_USES='(^|\n)[ \t]*(-[ \t]+)?uses:[ \t]*["'"'"']?smartwatermelon/github-workflows/\.github/workflows/semgrep-scan\.yml@'
+
+# Prints "semgrep|none|unreadable<TAB>detail". Only a 404 is none.
+read -r -d '' sg_verdict <<'JQ'
+if type != "object" then "unreadable\tthe contents response is malformed"
+elif .type != "file" then "unreadable\tthe path is not a file: " + (.type | tostring)
+elif .encoding != "base64" or (.content | type) != "string"
+then "unreadable\tthe file content is not readable (encoding: " + (.encoding | tostring) + ")"
+else (try (.content | gsub("\\s"; "") | @base64d) catch null) as $text
+  | if ($text | type) != "string" then "unreadable\tthe file content could not be decoded"
+    elif ($text | test($uses)) then "semgrep\t"
+    else "none\tthe file exists but does not call the reusable semgrep-scan.yml workflow"
+    end
+end
+JQ
+
+probe_semgrep() {
+  local nwo="$1" body why verdict
+  if ! body="$(api "repos/${nwo}/contents/${SG_PATH}")"; then
+    why="$(last_err "no detail")"
+    why="${why%"${why##*[![:space:]]}"}"
+    # The repo just answered the default-setup read, so 404 means no file.
+    # Both signals, as for the paywall 403: status line and body.
+    if [[ "${why}" == *"(HTTP 404)"* ]] \
+      && jq -e 'type == "object" and .status == "404"' >/dev/null 2>&1 <<<"${body}"; then
+      printf 'none\t'
+    else
+      printf 'unreadable\tcannot read %s: %s' "${SG_PATH}" "${why}"
+    fi
+    return 0
+  fi
+  verdict="$(jq -r --arg uses "${SG_USES}" "${sg_verdict}" <<<"${body}" 2>/dev/null)"
+  case "${verdict%%$'\t'*}" in
+    semgrep | none | unreadable) printf '%s' "${verdict}" ;;
+    *) printf 'unreadable\tthe contents response is malformed' ;;
+  esac
+}
+
 # Sets CQ_JSON for one repo. Never fails: a failed read is a state.
 probe_codeql() {
-  local nwo="$1" fork="$2" private="$3" body why msg verdict state detail
+  local nwo="$1" fork="$2" private="$3" body why msg verdict state detail sg
   if ! body="$(api "repos/${nwo}/code-scanning/default-setup")"; then
     why="$(last_err "no detail")"
     msg="$(jq -r 'if type == "object" then .message // "" else "" end' <<<"${body}" 2>/dev/null)"
     # Only the exact product message, on a repo the listing says is
     # private: a public repo has no paywall to explain it.
     if [[ "${why}" == *"(HTTP 403)"* && "${msg}" =~ ${CQ_NO_PRODUCT} \
       && "${private}" == "true" ]]; then
-      set_cq no-code-security "${msg}"
+      sg="$(probe_semgrep "${nwo}")"
+      [[ "${sg}" == *$'\t'* ]] || sg=$'unreadable\tthe Semgrep probe returned no result'
+      set_cq no-code-security "${msg}" "" "${sg%%$'\t'*}" "${sg#*$'\t'}"
     else
       set_cq unreadable "cannot read default setup: ${why}"
     fi

scripts/dependabot-digest/tests/fixtures/codeql-semgrep.ndjson

@@ -0,0 +1,6 @@
+{"owner":"o","repo":"o/configured","state":"ok","annotations":[],"protection":{"state":"required","detail":null,"exemptClass":null},"codeql":{"state":"configured","detail":null,"exemptClass":null,"substitute":null,"substituteDetail":null}}
+{"owner":"o","repo":"o/sg-covered","state":"ok","annotations":[],"protection":{"state":"required","detail":null,"exemptClass":null},"codeql":{"state":"no-code-security","detail":"Code Security must be enabled for this repository to use code scanning.","exemptClass":null,"substitute":"semgrep","substituteDetail":null}}
+{"owner":"o","repo":"o/sg-none","state":"ok","annotations":[],"protection":{"state":"required","detail":null,"exemptClass":null},"codeql":{"state":"no-code-security","detail":"Code Security must be enabled for this repository to use code scanning.","exemptClass":null,"substitute":"none","substituteDetail":null}}
+{"owner":"o","repo":"o/sg-unreadable","state":"ok","annotations":[],"protection":{"state":"required","detail":null,"exemptClass":null},"codeql":{"state":"no-code-security","detail":"Code Security must be enabled for this repository to use code scanning.","exemptClass":null,"substitute":"unreadable","substituteDetail":"cannot read .github/workflows/semgrep-scan.yml: gh: Server Error (HTTP 502)"}}
+{"owner":"o","repo":"o/sg-older","state":"ok","annotations":[],"protection":{"state":"required","detail":null,"exemptClass":null},"codeql":{"state":"no-code-security","detail":"Code Security must be enabled for this repository to use code scanning.","exemptClass":null}}
+{"owner":"o","repo":"o/sg-odd","state":"ok","annotations":[],"protection":{"state":"required","detail":null,"exemptClass":null},"codeql":{"state":"no-code-security","detail":"Code Security must be enabled for this repository to use code scanning.","exemptClass":null,"substitute":"maybe","substituteDetail":null}}

scripts/dependabot-digest/tests/fixtures/standards/installation_repositories.json.nightowlstudiollc

@@ -1,6 +1,6 @@
 [
  {
-  "total_count": 5,
+  "total_count": 8,
   "repositories": [
    {
     "full_name": "nightowlstudiollc/tnjcleaning",
@@ -36,6 +36,27 @@
     "archived": false,
     "fork": false,
     "private": true
+   },
+   {
+    "full_name": "nightowlstudiollc/sg-refused",
+    "default_branch": "main",
+    "archived": false,
+    "fork": false,
+    "private": true
+   },
+   {
+    "full_name": "nightowlstudiollc/sg-other",
+    "default_branch": "main",
+    "archived": false,
+    "fork": false,
+    "private": true
+   },
+   {
+    "full_name": "nightowlstudiollc/sg-badenc",
+    "default_branch": "main",
+    "archived": false,
+    "fork": false,
+    "private": true
    }
   ]
  }

scripts/dependabot-digest/tests/fixtures/standards/repos_nightowlstudiollc_cleanroom_contents_.github_workflows_semgrep-scan.yml.json

@@ -0,0 +1,10 @@
+{
+  "name": "semgrep-scan.yml",
+  "path": ".github/workflows/semgrep-scan.yml",
+  "sha": "5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e",
+  "size": 160,
+  "html_url": "https://github.com/nightowlstudiollc/cleanroom/blob/main/.github/workflows/semgrep-scan.yml",
+  "type": "file",
+  "content": "bmFtZTogU2VtZ3JlcApvbjoKICBwdWxsX3JlcXVlc3Q6CnBlcm1pc3Npb25z\nOgogIGNvbnRlbnRzOiByZWFkCmpvYnM6CiAgc2VtZ3JlcDoKICAgIHVzZXM6\nIHNtYXJ0d2F0ZXJtZWxvbi9naXRodWItd29ya2Zsb3dzLy5naXRodWIvd29y\na2Zsb3dzL3NlbWdyZXAtc2Nhbi55bWxAc2VtZ3JlcC1zY2FuLXYxCg==\n",
+  "encoding": "base64"
+}

scripts/dependabot-digest/tests/fixtures/standards/repos_nightowlstudiollc_sg-badenc_branches_main.json

@@ -0,0 +1,21 @@
+{
+  "name": "main",
+  "protected": true,
+  "protection": {
+    "enabled": true,
+    "required_status_checks": {
+      "enforcement_level": "non_admins",
+      "contexts": ["standards-check / run-standards-check", "test"],
+      "checks": [
+        {
+          "context": "standards-check / run-standards-check",
+          "app_id": null
+        },
+        {
+          "context": "test",
+          "app_id": 15368
+        }
+      ]
+    }
+  }
+}

scripts/dependabot-digest/tests/fixtures/standards/repos_nightowlstudiollc_sg-badenc_code-scanning_default-setup.403.json

@@ -0,0 +1,5 @@
+{
+  "message": "Code Security must be enabled for this repository to use code scanning.",
+  "documentation_url": "https://docs.github.com/rest/code-scanning/code-scanning#get-a-code-scanning-default-setup-configuration",
+  "status": "403"
+}

scripts/dependabot-digest/tests/fixtures/standards/repos_nightowlstudiollc_sg-badenc_contents_.github_workflows_semgrep-scan.yml.json

@@ -0,0 +1,10 @@
+{
+  "name": "semgrep-scan.yml",
+  "path": ".github/workflows/semgrep-scan.yml",
+  "sha": "5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e",
+  "size": 160,
+  "html_url": "https://github.com/nightowlstudiollc/sg-badenc/blob/main/.github/workflows/semgrep-scan.yml",
+  "type": "file",
+  "content": "",
+  "encoding": "none"
+}

scripts/dependabot-digest/tests/fixtures/standards/repos_nightowlstudiollc_sg-other_branches_main.json

@@ -0,0 +1,21 @@
+{
+  "name": "main",
+  "protected": true,
+  "protection": {
+    "enabled": true,
+    "required_status_checks": {
+      "enforcement_level": "non_admins",
+      "contexts": ["standards-check / run-standards-check", "test"],
+      "checks": [
+        {
+          "context": "standards-check / run-standards-check",
+          "app_id": null
+        },
+        {
+          "context": "test",
+          "app_id": 15368
+        }
+      ]
+    }
+  }
+}

scripts/dependabot-digest/tests/fixtures/standards/repos_nightowlstudiollc_sg-other_code-scanning_default-setup.403.json

@@ -0,0 +1,5 @@
+{
+  "message": "Code Security must be enabled for this repository to use code scanning.",
+  "documentation_url": "https://docs.github.com/rest/code-scanning/code-scanning#get-a-code-scanning-default-setup-configuration",
+  "status": "403"
+}

scripts/dependabot-digest/tests/fixtures/standards/repos_nightowlstudiollc_sg-other_contents_.github_workflows_semgrep-scan.yml.json

@@ -0,0 +1,10 @@
+{
+  "name": "semgrep-scan.yml",
+  "path": ".github/workflows/semgrep-scan.yml",
+  "sha": "5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e",
+  "size": 160,
+  "html_url": "https://github.com/nightowlstudiollc/sg-other/blob/main/.github/workflows/semgrep-scan.yml",
+  "type": "file",
+  "content": "bmFtZTogU2VtZ3JlcApvbjoKICBwdWxsX3JlcXVlc3Q6CnBlcm1pc3Npb25z\nOgogIGNvbnRlbnRzOiByZWFkCmpvYnM6CiAgc2VtZ3JlcDoKICAgICMgdXNl\nczogc21hcnR3YXRlcm1lbG9uL2dpdGh1Yi13b3JrZmxvd3MvLmdpdGh1Yi93\nb3JrZmxvd3Mvc2VtZ3JlcC1zY2FuLnltbEBzZW1ncmVwLXNjYW4tdjEKICAg\nIHJ1bnMtb246IHVidW50dS1sYXRlc3QKICAgIHN0ZXBzOgogICAgICAtIHJ1\nbjogZWNobyAibm90IHRoZSByZXVzYWJsZSB3b3JrZmxvdyIK\n",
+  "encoding": "base64"
+}

scripts/dependabot-digest/tests/fixtures/standards/repos_nightowlstudiollc_sg-refused_branches_main.json

@@ -0,0 +1,21 @@
+{
+  "name": "main",
+  "protected": true,
+  "protection": {
+    "enabled": true,
+    "required_status_checks": {
+      "enforcement_level": "non_admins",
+      "contexts": ["standards-check / run-standards-check", "test"],
+      "checks": [
+        {
+          "context": "standards-check / run-standards-check",
+          "app_id": null
+        },
+        {
+          "context": "test",
+          "app_id": 15368
+        }
+      ]
+    }
+  }
+}

scripts/dependabot-digest/tests/fixtures/standards/repos_nightowlstudiollc_sg-refused_code-scanning_default-setup.403.json

@@ -0,0 +1,5 @@
+{
+  "message": "Code Security must be enabled for this repository to use code scanning.",
+  "documentation_url": "https://docs.github.com/rest/code-scanning/code-scanning#get-a-code-scanning-default-setup-configuration",
+  "status": "403"
+}

scripts/dependabot-digest/tests/fixtures/standards/repos_smartwatermelon_claude-config-backup_contents_.github_workflows_semgrep-scan.yml.404.json

@@ -0,0 +1,5 @@
+{
+  "message": "Not Found",
+  "documentation_url": "https://docs.github.com/rest/repos/contents#get-repository-content",
+  "status": "404"
+}