diff --git a/.github/workflows/check-workflow-status.yml b/.github/workflows/check-workflow-status.yml new file mode 100644 index 000000000..e5a15d2a1 --- /dev/null +++ b/.github/workflows/check-workflow-status.yml @@ -0,0 +1,52 @@ +name: Check Workflow Status Dashboard + +# Fails when a workflow file is added/renamed without a corresponding row +# in docs/ci/workflow-status.md. Pure structural check β€” no GitHub API +# calls, no live data β€” so it runs fast and doesn't need any auth. +# +# To fix a failure: regenerate the dashboard with +# `pdm run python scripts/generate_workflow_status.py` +# This requires `gh` authenticated against the repo. If you can't run it +# locally, ping a maintainer to regenerate, or add a temporary placeholder +# `\`.yml\`` mention in the file's manual-edit region +# to unblock the PR. +on: + pull_request: + paths: + - '.github/workflows/**' + - 'docs/ci/workflow-status.md' + - 'scripts/generate_workflow_status.py' + workflow_dispatch: + +permissions: + contents: read + +jobs: + check-structure: + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1 + with: + egress-policy: audit + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + + - name: Install PyYAML + # Pinned to match pdm.lock; the rest of the repo uses exact + # `==` pins for ad-hoc workflow installs (see e.g. + # validate-image-pinning.yml). Floating `~=` ranges can pick up + # yanked / replaced patch versions silently. + run: pip install pyyaml==6.0.3 + + - name: Verify dashboard structure + run: python scripts/generate_workflow_status.py --check-structure diff --git a/.github/workflows/version_check.yml b/.github/workflows/version_check.yml index d52d700bd..d2cdd7897 100644 --- a/.github/workflows/version_check.yml +++ b/.github/workflows/version_check.yml @@ -43,6 +43,7 @@ jobs: permissions: contents: write pull-requests: write + actions: read # for generate_workflow_status.py to read run history steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1 @@ -125,6 +126,29 @@ jobs: if: steps.check.outputs.needs_bump != 'false' run: python scripts/generate_config_docs.py + # Refreshes docs/ci/workflow-status.md so the version-bump PR + # carries the current snapshot. Output uses coarse "last week / + # last month" buckets so within-day reruns produce zero diff β€” + # the diff that lands in this PR only shows workflows whose + # bucket has actually shifted since the previous release. + # + # ~340 GitHub API calls per run (well under the GITHUB_TOKEN + # 1000/hr workflow-runs limit). Needs `actions: read` on the + # job permissions block above. + - name: Regenerate workflow status dashboard + if: steps.check.outputs.needs_bump != 'false' + # Don't block the version bump if the dashboard refresh fails. + # The regen calls ~340 GitHub API endpoints; a transient outage + # or rate-limit hit would otherwise prevent the version-bump PR + # from being created at all. On failure the dashboard just stays + # at its previous snapshot until the next successful run. + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + pip install pyyaml==6.0.3 + python scripts/generate_workflow_status.py + - name: Generate PR body if: steps.check.outputs.needs_bump != 'false' id: pr_body @@ -157,7 +181,7 @@ jobs: fi echo "Approve and merge to trigger a new release." echo "" - echo "Configuration docs (docs/CONFIGURATION.md) have been regenerated." + echo "Configuration docs (docs/CONFIGURATION.md) and the workflow status dashboard (docs/ci/workflow-status.md) have been regenerated." } > "$body_file" echo "body_file=$body_file" >> "$GITHUB_OUTPUT" diff --git a/README.md b/README.md index 453bdfdbd..3853c659a 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,8 @@ [![🐳 Docker Publish](https://github.com/LearningCircuit/local-deep-research/actions/workflows/docker-publish.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/docker-publish.yml) [![πŸ“¦ PyPI Publish](https://github.com/LearningCircuit/local-deep-research/actions/workflows/publish.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/publish.yml) +[All workflow status β†’](docs/ci/workflow-status.md) + [![Discord](https://img.shields.io/discord/1352043059562680370?style=for-the-badge&logo=discord)](https://discord.gg/ttcqQeFcJ3) [![Reddit](https://img.shields.io/badge/Reddit-r/LocalDeepResearch-FF4500?style=for-the-badge&logo=reddit)](https://www.reddit.com/r/LocalDeepResearch/) [![YouTube](https://img.shields.io/badge/YouTube-Channel-red?style=for-the-badge&logo=youtube)](https://www.youtube.com/@local-deep-research) diff --git a/docs/CI_CD_INFRASTRUCTURE.md b/docs/CI_CD_INFRASTRUCTURE.md index 9eb6cb4e8..6decc7e2d 100644 --- a/docs/CI_CD_INFRASTRUCTURE.md +++ b/docs/CI_CD_INFRASTRUCTURE.md @@ -6,6 +6,8 @@ This document describes the continuous integration, security scanning, and devel The project uses many GitHub Actions workflows and 20+ pre-commit hooks to ensure code quality, security, and reliability. +> **At-a-glance health**: see [`docs/ci/workflow-status.md`](ci/workflow-status.md) β€” an auto-generated dashboard with live badges for every workflow, surfacing disabled, manual-only, and stale (silently-failing) ones at the top. Regenerate with `pdm run python scripts/generate_workflow_status.py`. + ``` β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ Developer Workflow β”‚ diff --git a/docs/ci/workflow-status.md b/docs/ci/workflow-status.md new file mode 100644 index 000000000..97a568903 --- /dev/null +++ b/docs/ci/workflow-status.md @@ -0,0 +1,137 @@ +# Workflow Status + +> **Live status of every GitHub Actions workflow in this repo.** +> Auto-generated by [`scripts/generate_workflow_status.py`](../../scripts/generate_workflow_status.py). +> Do not edit between the generated markers β€” regenerate with +> `pdm run python scripts/generate_workflow_status.py`. Anything outside +> the markers is preserved on regeneration. + +## How to read this page + +- **Live badges** (right column on active gates) re-render on every page + view and reflect the current head-of-default-branch status from + GitHub. Click one to land on that workflow's runs page. +- The **Status** emoji column captures the run conclusion at generation + time. It survives broken badge rendering (corporate proxies, anonymous + viewers). +- **Disabled** = a caller has the `uses:` line commented out, or the + workflow is disabled in the GitHub UI. **Stale** = scheduled trigger + but no successful run within 2Γ— its cron cadence (and β‰₯60 days). The + three top sections are the action items. +- Reusable workflows (those triggered only by `workflow_call:`) show + their **gated** run β€” the most recent run of their parent (release.yml, + release-gate.yml, ci-gate.yml) that included them β€” not their own + empty direct-run history. + + + +**63 workflows:** 1 disabled Β· 1 stale Β· 2 manual-only Β· 59 active + +## ⚠ Disabled workflows + +| Workflow | Disabled where | Last direct run | +|---|---|---| +| `nuclei.yml` | `release-gate.yml:177` (commented) | never | + +## ⚠ Stale (scheduled but no recent successful run) + +| Workflow | Cron | Last successful run | +|---|---|---| +| `update-precommit-hooks.yml` | `0 8 * * 5` | never | + +## β„Ή Manual-only by design + +| Workflow | Last manual run | Trigger | +|---|---|---| +| `check-config-docs.yml` | 2 months ago | manual | +| `sync-main-to-dev.yml` | 2 months ago | manual | + +## Release-blocking gates β€” daily (release-gate cron 02:00 UTC) + +| Workflow | Status | Last activity | Trigger | Live badge | +|---|---|---|---|---| +| `backwards-compatibility.yml` | βœ… | this week | workflow_call, PR, push:main, release, schedule(0 2 * * 0), manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/backwards-compatibility.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/backwards-compatibility.yml) | +| `bearer.yml` | βœ… | this week | manual, workflow_call, schedule(0 4 * * *) | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/bearer.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/bearer.yml) | +| `checkov.yml` | βœ… | this week | manual, workflow_call | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/checkov.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/checkov.yml) | +| `codeql.yml` | βœ… | this week | push:main, PR, schedule(45 5 * * 0), workflow_call, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/codeql.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/codeql.yml) | +| `container-security.yml` | βœ… | this week | workflow_call, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/container-security.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/container-security.yml) | +| `devskim.yml` | βœ… | this week | manual, workflow_call, schedule(0 10 * * *) | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/devskim.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/devskim.yml) | +| `docker-multiarch-test.yml` | βœ… | this week | workflow_call, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/docker-multiarch-test.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/docker-multiarch-test.yml) | +| `dockle.yml` | βœ… | this week | manual, workflow_call, schedule(0 10 * * 2) | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/dockle.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/dockle.yml) | +| `gitleaks-main.yml` | βœ… | this week | workflow_call, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/gitleaks-main.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/gitleaks-main.yml) | +| `grype.yml` | βœ… | this week | workflow_call, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/grype.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/grype.yml) | +| `hadolint.yml` | βœ… | this week | PR, manual, workflow_call, schedule(0 9 * * 2) | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/hadolint.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/hadolint.yml) | +| `journal-data-integration.yml` | βœ… | this week | workflow_call, manual, schedule(0 4 * * 1) | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/journal-data-integration.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/journal-data-integration.yml) | +| `npm-audit.yml` | βœ… | this week | manual, workflow_call | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/npm-audit.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/npm-audit.yml) | +| `owasp-zap-scan.yml` | βœ… | this week | workflow_call, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/owasp-zap-scan.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/owasp-zap-scan.yml) | +| `retirejs.yml` | βœ… | this week | manual, workflow_call, schedule(0 4 * * 1) | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/retirejs.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/retirejs.yml) | +| `security-headers-validation.yml` | βœ… | this week | workflow_call, schedule(0 3 * * *), manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/security-headers-validation.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/security-headers-validation.yml) | +| `security-tests.yml` | βœ… | this week | workflow_call, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/security-tests.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/security-tests.yml) | +| `semgrep.yml` | βœ… | this week | workflow_call, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/semgrep.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/semgrep.yml) | +| `zizmor-security.yml` | βœ… | this week | manual, workflow_call, schedule(0 9 * * 1) | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/zizmor-security.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/zizmor-security.yml) | + +## Release gates β€” release-time only + +| Workflow | Status | Last activity | Trigger | Live badge | +|---|---|---|---|---| +| `check-env-vars.yml` | ⊘ | this week | PR, workflow_call, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/check-env-vars.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/check-env-vars.yml) | +| `ci-gate.yml` | βœ… | this week | workflow_call, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/ci-gate.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/ci-gate.yml) | +| `compose-integration-test.yml` | βœ… | this week | workflow_call, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/compose-integration-test.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/compose-integration-test.yml) | +| `docker-tests.yml` | ⊘ | this week | PR, push:main, workflow_call, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/docker-tests.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/docker-tests.yml) | +| `file-whitelist-check.yml` | ⊘ | this week | PR, workflow_call, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/file-whitelist-check.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/file-whitelist-check.yml) | +| `mypy-type-check.yml` | ⊘ | this week | PR, workflow_call, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/mypy-type-check.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/mypy-type-check.yml) | +| `playwright-webkit-tests.yml` | βœ… | this week | workflow_call, manual, schedule(0 2 * * *) | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/playwright-webkit-tests.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/playwright-webkit-tests.yml) | +| `pre-commit.yml` | ⊘ | this week | PR, workflow_call, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/pre-commit.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/pre-commit.yml) | +| `puppeteer-e2e-tests.yml` | βœ… | this week | PR, workflow_call, manual, schedule(0 2 * * 0) | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/puppeteer-e2e-tests.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/puppeteer-e2e-tests.yml) | +| `responsive-ui-tests-enhanced.yml` | βœ… | this week | workflow_call, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/responsive-ui-tests-enhanced.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/responsive-ui-tests-enhanced.yml) | +| `security-file-write-check.yml` | ⊘ | this week | PR, workflow_call, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/security-file-write-check.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/security-file-write-check.yml) | +| `validate-image-pinning.yml` | ⊘ | this week | PR, workflow_call, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/validate-image-pinning.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/validate-image-pinning.yml) | +| `vulture-dead-code.yml` | βœ… | this week | workflow_call, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/vulture-dead-code.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/vulture-dead-code.yml) | + +## Scheduled (own cron) + +| Workflow | Status | Last activity | Trigger | Live badge | +|---|---|---|---|---| +| `compose-published-smoke.yml` | β€” | never | manual, schedule(0 5 * * 1) | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/compose-published-smoke.yml/badge.svg?event=schedule)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/compose-published-smoke.yml?query=event%3Aschedule) | +| `fuzz.yml` | βœ… | this week | schedule(0 0 * * 0), manual, PR | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/fuzz.yml/badge.svg?event=schedule)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/fuzz.yml?query=event%3Aschedule) | +| `gitleaks.yml` | βœ… | this week | PR, manual, schedule(0 3 * * *) | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/gitleaks.yml/badge.svg?event=schedule)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/gitleaks.yml?query=event%3Aschedule) | +| `ossf-scorecard.yml` | βœ… | this week | branch_protection_rule, schedule(0 8 * * 1), manual, push:main | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/ossf-scorecard.yml/badge.svg?event=schedule)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/ossf-scorecard.yml?query=event%3Aschedule) | +| `osv-scanner-scheduled.yml` | βœ… | this week | push:main, schedule(41 21 * * 1), manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/osv-scanner-scheduled.yml/badge.svg?event=schedule)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/osv-scanner-scheduled.yml?query=event%3Aschedule) | +| `osv-scanner.yml` | βœ… | this week | PR, merge_group, schedule(39 12 * * 1), manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/osv-scanner.yml/badge.svg?event=schedule)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/osv-scanner.yml?query=event%3Aschedule) | +| `release-gate.yml` | βœ… | this week | workflow_call, manual, schedule(0 2 * * *) | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/release-gate.yml/badge.svg?event=schedule)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/release-gate.yml?query=event%3Aschedule) | +| `sbom.yml` | βœ… | this week | manual, schedule(0 10 * * 3), release | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/sbom.yml/badge.svg?event=schedule)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/sbom.yml?query=event%3Aschedule) | +| `update-dependencies.yml` | βœ… | this week | workflow_call, manual, schedule(0 8 * * 3) | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/update-dependencies.yml/badge.svg?event=schedule)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/update-dependencies.yml?query=event%3Aschedule) | +| `update-npm-dependencies.yml` | ❌ | this week | workflow_call, manual, schedule(0 8 * * 4) | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/update-npm-dependencies.yml/badge.svg?event=schedule)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/update-npm-dependencies.yml?query=event%3Aschedule) | + +## PR / push checks + +| Workflow | Status | Last activity | Trigger | Live badge | +|---|---|---|---|---| +| `advanced-search-reminder.yml` | βœ… | this week | PR | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/advanced-search-reminder.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/advanced-search-reminder.yml) | +| `ai-code-reviewer.yml` | βœ… | this week | PR | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/ai-code-reviewer.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/ai-code-reviewer.yml) | +| `check-workflow-status.yml` | βœ… | this week | PR, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/check-workflow-status.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/check-workflow-status.yml) | +| `claude-code-review.yml` | Β· | this week | PR | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/claude-code-review.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/claude-code-review.yml) | +| `danger-zone-alert.yml` | βœ… | this week | PR | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/danger-zone-alert.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/danger-zone-alert.yml) | +| `dependency-review.yml` | βœ… | this week | PR, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/dependency-review.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/dependency-review.yml) | +| `e2e-research-test.yml` | Β· | this week | PR | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/e2e-research-test.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/e2e-research-test.yml) | +| `label-fixed-in-dev.yml` | Β· | 3 weeks ago | PR | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/label-fixed-in-dev.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/label-fixed-in-dev.yml) | +| `labels-sync.yml` | βœ… | this week | push:main, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/labels-sync.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/labels-sync.yml) | +| `mcp-tests.yml` | βœ… | this week | push:main,dev, PR, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/mcp-tests.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/mcp-tests.yml) | +| `pr-triage.yml` | βœ… | this week | PR, pull_request_review | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/pr-triage.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/pr-triage.yml) | +| `release.yml` | ⏳ | this week | push:main, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/release.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/release.yml) | +| `version_check.yml` | βœ… | this week | push:main, manual | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/version_check.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/version_check.yml) | +| `welcome-first-time.yml` | βœ… | this week | PR-target | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/welcome-first-time.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/welcome-first-time.yml) | + +## Repository-dispatch publishers + +| Workflow | Status | Last activity | Trigger | Live badge | +|---|---|---|---|---| +| `docker-publish.yml` | βœ… | last week | repo_dispatch | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/docker-publish.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/docker-publish.yml) | +| `prerelease-docker.yml` | ❌ | this week | repo_dispatch | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/prerelease-docker.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/prerelease-docker.yml) | +| `publish.yml` | βœ… | last week | repo_dispatch | [![status](https://github.com/LearningCircuit/local-deep-research/actions/workflows/publish.yml/badge.svg)](https://github.com/LearningCircuit/local-deep-research/actions/workflows/publish.yml) | + +## Other + +_None._ + + diff --git a/scripts/generate_workflow_status.py b/scripts/generate_workflow_status.py new file mode 100644 index 000000000..f9217c682 --- /dev/null +++ b/scripts/generate_workflow_status.py @@ -0,0 +1,1157 @@ +#!/usr/bin/env python3 +""" +Generate docs/ci/workflow-status.md β€” a live dashboard of every GitHub Actions +workflow in this repo, grouped by role, with disabled / manual-only / stale +items surfaced at the top. + +Usage: + pdm run python scripts/generate_workflow_status.py # Generate + pdm run python scripts/generate_workflow_status.py --output PATH # Custom path + pdm run python scripts/generate_workflow_status.py --check-structure + # CI mode: exit 1 + # if any workflow + # file is missing + # a row in the + # current dashboard + pdm run python scripts/generate_workflow_status.py --verbose # Log API calls + +Why two modes: + Live data (last_run timestamps, badge status) cannot be byte-equality + --check'd against committed output β€” every regeneration changes the + timestamps. Instead, --check-structure does a fast, deterministic check + that every .github/workflows/*.yml file has a row in the dashboard. + Live regeneration is on demand. + +GitHub API: + Uses the `gh` CLI. Requires `gh auth login` with `repo, workflow` scopes. + Owner/repo is read at runtime from `gh repo view`. + +API cost: + ~1 call per workflow (62) + caller-resolution (~30 extra) β‰ˆ 100 calls per + full run. Well within the 5000/hr authenticated rate limit. +""" + +import argparse +import json +import re +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Optional + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent +WORKFLOWS_DIR = REPO_ROOT / ".github" / "workflows" +DEFAULT_OUTPUT = REPO_ROOT / "docs" / "ci" / "workflow-status.md" + +BEGIN_MARKER = "" +END_MARKER = "" + +NEW_FILE_DAYS = 14 +STALE_MULTIPLIER = 2 # last_success > N * cron_period_days β†’ stale +RATE_LIMIT_WARN = 200 +RATE_LIMIT_ABORT = 50 + +GATE_ROOTS = {"release.yml", "release-gate.yml", "ci-gate.yml"} + +# Job names recognized as the release-gate "summary"-shaped roll-up that +# nests other jobs. We list candidates by the named role they fill so we +# don't depend on the exact YAML key. +RELEASE_GATE_FILE = "release-gate.yml" +RELEASE_FILE = "release.yml" +CI_GATE_FILE = "ci-gate.yml" + + +# ============================================================================ +# gh CLI helpers +# ============================================================================ + + +def run_gh(args: list[str], verbose: bool = False) -> str: + """Run `gh ` and return stdout. Raises on nonzero exit.""" + if verbose: + print(f" gh {' '.join(args)}", file=sys.stderr) + result = subprocess.run( + ["gh", *args], capture_output=True, text=True, check=False + ) + if result.returncode != 0: + raise RuntimeError( + f"gh {' '.join(args)} failed (exit {result.returncode}):\n" + f"{result.stderr.strip()}" + ) + return result.stdout + + +def gh_json(args: list[str], verbose: bool = False) -> Any: + """Run `gh` returning a single JSON value (object/array). Use + `gh_json_stream` for `--jq` streamed output.""" + out = run_gh(args, verbose=verbose) + return json.loads(out) if out.strip() else None + + +def gh_json_stream(args: list[str], verbose: bool = False) -> list[Any]: + """Run `gh api ... --jq` whose output is a stream of JSON objects + (one per line, NDJSON). Returns a list.""" + out = run_gh(args, verbose=verbose).strip() + if not out: + return [] + return [json.loads(line) for line in out.splitlines() if line.strip()] + + +def preflight(verbose: bool = False) -> tuple[str, str]: + """Verify gh auth and rate limits; return (owner, repo).""" + # Auth check. + auth = subprocess.run( + ["gh", "auth", "status"], capture_output=True, text=True, check=False + ) + if auth.returncode != 0: + print( + "FAIL: `gh` is not authenticated. Run `gh auth login` and retry.", + file=sys.stderr, + ) + sys.exit(2) + + # Rate limit check. + rl = gh_json( + ["api", "rate_limit", "--jq", ".resources.core"], verbose=verbose + ) + remaining = rl["remaining"] + limit = rl["limit"] + if remaining < RATE_LIMIT_ABORT: + print( + f"FAIL: gh API rate limit too low: {remaining}/{limit}. " + f"Wait until the reset window before regenerating.", + file=sys.stderr, + ) + sys.exit(2) + if remaining < RATE_LIMIT_WARN: + print( + f"WARN: gh API rate limit low: {remaining}/{limit}. " + f"Continuing but consider waiting for reset.", + file=sys.stderr, + ) + + # Repo slug. + repo = gh_json(["repo", "view", "--json", "owner,name"], verbose=verbose) + return repo["owner"]["login"], repo["name"] + + +# ============================================================================ +# Workflow YAML parsing +# ============================================================================ + + +USES_LINE_RE = re.compile( + r"""^(?P\s*) + (?P\#\s*)? + uses:\s*(?P\.\/\.github\/workflows\/[A-Za-z0-9_.-]+\.ya?ml)\s*$ + """, + re.VERBOSE, +) +JOB_KEY_RE = re.compile(r"^ (?:#\s*)?(?P[A-Za-z0-9_-]+):\s*$") + + +def parse_workflow(path: Path) -> dict[str, Any]: + """Extract the structured info we need from one workflow file. + + Returns: + { + "file": "release-gate.yml", + "path": Path(...), + "name": "Release Gate", + "on": {...}, # raw dict from the `on:` block + "uses_calls": [ # parsed line-by-line so we keep + { # comment status (ground truth) AND + "job_key": "nuclei-scan", # job key context + "callee": "nuclei.yml", + "commented": True, + "line": 177, + }, ... + ], + } + """ + text = path.read_text() + # Use yaml.safe_load on the file but with FullLoader behavior: the + # `on:` key gets coerced to True (boolean) by safe_load when YAML 1.1 + # legacy boolean parsing kicks in. The simpler workaround is to load + # then look for both 'on' and True. + data = yaml.safe_load(text) + if data is None: + raise ValueError(f"{path.name}: empty YAML") + + on_block = data.get("on", data.get(True, {})) + name = data.get("name", path.stem) + + uses_calls = _parse_uses_lines(text) + + return { + "file": path.name, + "path": path, + "name": name, + "on": on_block if isinstance(on_block, dict) else {on_block: None}, + "raw_text": text, + "uses_calls": uses_calls, + } + + +def _parse_uses_lines(text: str) -> list[dict[str, Any]]: + """Walk lines once to capture (job_key, callee, commented?, line_no). + + Job context: track the most recent top-level `:` under the + `jobs:` mapping. Lines inside a step's `uses:` (indented further than + a job key) get the most recent job key as context. + """ + calls: list[dict[str, Any]] = [] + in_jobs = False + current_job: Optional[str] = None + for i, line in enumerate(text.splitlines(), start=1): + # Track entering/leaving the top-level `jobs:` mapping. + if re.match(r"^jobs:\s*$", line): + in_jobs = True + current_job = None + continue + if in_jobs: + if line and not line.startswith(" ") and not line.startswith("#"): + # New top-level key; out of `jobs:` block. + in_jobs = False + current_job = None + continue + m = JOB_KEY_RE.match(line) + if m: + current_job = m.group("key") + + m = USES_LINE_RE.match(line) + if m: + target = m.group("target") + callee = target.rsplit("/", 1)[-1] + calls.append( + { + "job_key": current_job or "", + "callee": callee, + "commented": bool(m.group("comment")), + "line": i, + } + ) + return calls + + +# ============================================================================ +# Call graph +# ============================================================================ + + +def build_call_graph( + workflows: dict[str, dict[str, Any]], +) -> tuple[dict[str, list[dict[str, Any]]], dict[str, set[str]]]: + """Return (callers_of, transitive_root_callers). + + callers_of[file] = list of {caller, job_key, commented, line} dicts β€” + every direct caller (including commented-out ones, which we use + to detect "disabled"). + transitive_root_callers[file] = set of root-caller filenames; subset + of GATE_ROOTS, plus possibly the file itself if it has no callers. + """ + callers_of: dict[str, list[dict[str, Any]]] = {f: [] for f in workflows} + + for caller_file, wf in workflows.items(): + for call in wf["uses_calls"]: + callee = call["callee"] + if callee in callers_of: + callers_of[callee].append( + { + "caller": caller_file, + "job_key": call["job_key"], + "commented": call["commented"], + "line": call["line"], + } + ) + + # BFS upward to find which roots ultimately invoke this workflow. + transitive: dict[str, set[str]] = {} + for f in workflows: + roots: set[str] = set() + seen: set[str] = set() + stack = [f] + while stack: + cur = stack.pop() + if cur in seen: + continue + seen.add(cur) + active_callers = [c for c in callers_of[cur] if not c["commented"]] + if not active_callers: + if cur in GATE_ROOTS: + roots.add(cur) + continue + for c in active_callers: + if c["caller"] in GATE_ROOTS: + roots.add(c["caller"]) + stack.append(c["caller"]) + transitive[f] = roots + return callers_of, transitive + + +# ============================================================================ +# GitHub run history +# ============================================================================ + + +def fetch_workflow_meta( + owner: str, repo: str, file: str, verbose: bool = False +) -> Optional[dict[str, Any]]: + """Return {id, state, last_run, last_run_event, last_run_conclusion}. + + Returns None if the workflow isn't registered with GitHub (e.g. very + new file not yet picked up). + """ + try: + meta = gh_json( + [ + "api", + f"repos/{owner}/{repo}/actions/workflows/{file}", + ], + verbose=verbose, + ) + except RuntimeError as e: + if "404" in str(e) or "Not Found" in str(e): + return None + raise + runs_list = gh_json_stream( + [ + "api", + f"repos/{owner}/{repo}/actions/workflows/{file}/runs?per_page=5", + "--jq", + ".workflow_runs[] | " + "{created_at, event, conclusion, status, head_branch, id}", + ], + verbose=verbose, + ) + last = runs_list[0] if runs_list else None + return { + "id": meta["id"], + "state": meta.get("state", "active"), + "last_run": last["created_at"] if last else None, + "last_run_event": last["event"] if last else None, + "last_run_conclusion": last["conclusion"] if last else None, + "last_run_status": last["status"] if last else None, + "recent_runs": runs_list, + } + + +def fetch_last_gated_run( + owner: str, + repo: str, + caller_file: str, + job_keys: set[str], + verbose: bool = False, +) -> Optional[dict[str, Any]]: + """Find the most recent run of caller_file that included any of + `job_keys` as a job, and return that job's status.""" + runs_list = gh_json_stream( + [ + "api", + f"repos/{owner}/{repo}/actions/workflows/{caller_file}/runs?per_page=5", + "--jq", + ".workflow_runs[] | {created_at, event, conclusion, status, id}", + ], + verbose=verbose, + ) + latest_match: Optional[dict[str, Any]] = None + latest_success: Optional[dict[str, Any]] = None + for run in runs_list: + run_id = run["id"] + try: + jobs = gh_json_stream( + [ + "api", + f"repos/{owner}/{repo}/actions/runs/{run_id}/jobs" + "?per_page=100", + "--jq", + ".jobs[] | {name, conclusion, status, completed_at}", + ], + verbose=verbose, + ) + except RuntimeError: + continue + # Reusable workflows show as ` / ` in the + # caller run's jobs list. Match prefix against any of our keys. + for job in jobs: + jname = job["name"] + head = jname.split(" / ", 1)[0] + if head not in job_keys: + continue + entry = { + "caller": caller_file, + "caller_run_id": run_id, + "job_name": jname, + "conclusion": job["conclusion"], + "status": job["status"], + "completed_at": job["completed_at"] or run["created_at"], + "run_created": run["created_at"], + } + if latest_match is None: + latest_match = entry + if latest_success is None and job["conclusion"] == "success": + latest_success = entry + if latest_match and latest_success: + break + if latest_match and latest_success: + break + if latest_match is None: + return None + # Stash the most-recent-success entry alongside the activity entry + # so the caller can use whichever it needs without a second API + # round-trip. `latest_success` is `None` if none of the inspected + # caller runs included a successful invocation of this workflow. + latest_match["latest_success"] = latest_success + return latest_match + + +# ============================================================================ +# Classification +# ============================================================================ + + +def file_added_at(path: Path) -> Optional[str]: + """ISO date of the commit that introduced this file, or None.""" + try: + out = subprocess.run( + [ + "git", + "log", + "--diff-filter=A", + "--format=%aI", + "--", + str(path), + ], + capture_output=True, + text=True, + check=False, + cwd=REPO_ROOT, + ) + if out.returncode != 0: + return None + lines = [ + ln.strip() for ln in out.stdout.strip().splitlines() if ln.strip() + ] + return lines[-1] if lines else None + except Exception: + return None + + +def cron_cadence_days(cron: str) -> Optional[float]: + """Best-effort: convert a cron string to its rough cadence in days. + + Recognises: + `* * * * *` β†’ 1/1440 day (1 min); we cap at 1 hour minimum + `M H * * *` β†’ 1 day + `M H * * D` β†’ 7 days + `M H D * *` β†’ 30 days + `M H D M *` β†’ 365 days + """ + parts = cron.strip().split() + if len(parts) != 5: + return None + minute, hour, dom, month, dow = parts + if month != "*": + return 365.0 + if dom != "*": + return 30.0 + if dow != "*": + return 7.0 + if hour != "*": + return 1.0 + if minute != "*": + return 1.0 / 24 + return 1.0 / 1440 + + +def get_schedules(on_block: dict) -> list[str]: + """Extract cron strings from an on: block.""" + sched = on_block.get("schedule") + if not sched: + return [] + out = [] + if isinstance(sched, list): + for entry in sched: + if isinstance(entry, dict) and "cron" in entry: + out.append(entry["cron"]) + return out + + +def trigger_summary(on_block: dict) -> str: + """Compact human-readable trigger description.""" + if not on_block: + return "β€”" + parts: list[str] = [] + keys = list(on_block.keys()) + for k in keys: + if k == "schedule": + crons = get_schedules(on_block) + if crons: + parts.append(f"schedule({', '.join(crons)})") + elif k == "pull_request": + parts.append("PR") + elif k == "pull_request_target": + parts.append("PR-target") + elif k == "push": + v = on_block[k] + if isinstance(v, dict) and "branches" in v: + parts.append(f"push:{','.join(v['branches'])}") + else: + parts.append("push") + elif k == "workflow_call": + parts.append("workflow_call") + elif k == "workflow_dispatch": + parts.append("manual") + elif k == "repository_dispatch": + parts.append("repo_dispatch") + else: + parts.append(str(k)) + return ", ".join(parts) + + +def primary_role( + workflow_file: str, + on_block: dict, + callers: list[dict[str, Any]], + transitive_roots: set[str], +) -> str: + """Bucket every workflow into exactly one display group. + + Order matters: workflow can satisfy multiple, but lands in the first + matching group. + """ + active_callers = [c for c in callers if not c["commented"]] + + # Disabled supersedes everything else. + if callers and not active_callers: + return "disabled" + + # Manual-only by design. + only_dispatch = set(on_block.keys()) - {True} == { + "workflow_dispatch" + } or list(on_block.keys()) == ["workflow_dispatch"] + if only_dispatch and not active_callers: + return "manual" + + # Gate buckets. + if ( + RELEASE_GATE_FILE in transitive_roots + and workflow_file != RELEASE_GATE_FILE + ): + # The release-gate.yml schedule is daily. + return "gate-daily" + if ( + RELEASE_FILE in transitive_roots + and RELEASE_GATE_FILE not in transitive_roots + and workflow_file not in {RELEASE_FILE, RELEASE_GATE_FILE} + ): + return "gate-release" + if ( + CI_GATE_FILE in transitive_roots + and RELEASE_GATE_FILE not in transitive_roots + and workflow_file not in {CI_GATE_FILE, RELEASE_FILE, RELEASE_GATE_FILE} + ): + return "gate-release" + + # Direct trigger buckets. + has_schedule = "schedule" in on_block + has_pr = "pull_request" in on_block or "pull_request_target" in on_block + has_push = "push" in on_block + has_repo_dispatch = "repository_dispatch" in on_block + + if has_repo_dispatch and not has_pr and not has_push and not has_schedule: + return "repo-dispatch" + if has_schedule: + return "schedule" + if has_pr or has_push: + return "pr-push" + + return "other" + + +def classify_staleness( + workflow: dict[str, Any], + callers: list[dict[str, Any]], + last_success_iso: Optional[str], + file_added_iso: Optional[str], + now: datetime, +) -> Optional[str]: + """Return 'disabled' / 'manual-only' / 'stale' / None. + + `last_success_iso` is the most recent successful run timestamp from + any source (direct or gated), already computed by the caller across + the recent_runs list β€” checking only the most-recent-run conclusion + misses the case where last week failed but the week before passed. + """ + on_block = workflow["on"] + state = workflow.get("state", "active") + active_callers = [c for c in callers if not c["commented"]] + + # 1. Disabled: caller commented OR GitHub UI state non-active. + # Treat "missing" as new (not yet registered with GitHub) β€” handled + # by the new-file check below instead of mis-flagging as disabled. + if callers and not active_callers: + return "disabled" + if state not in ("active", "missing"): + return "disabled" + + # 2. Manual-only: workflow_dispatch only AND no caller. + only_dispatch = list(on_block.keys()) == ["workflow_dispatch"] + if only_dispatch and not active_callers: + return "manual-only" + + # 3. New: skip stale check. + if file_added_iso: + added = datetime.fromisoformat(file_added_iso) + if (now - added).days < NEW_FILE_DAYS: + return None + + # 4. Stale: scheduled trigger AND no successful run within + # STALE_MULTIPLIER * cadence (or 60d, whichever larger). + crons = get_schedules(on_block) + if not crons: + return None + cadences = [c for c in (cron_cadence_days(x) for x in crons) if c] + if not cadences: + return None + threshold_days = max(60.0, STALE_MULTIPLIER * min(cadences)) + + if last_success_iso is None: + return "stale" + last_success = datetime.fromisoformat( + last_success_iso.replace("Z", "+00:00") + ) + if (now - last_success).days > threshold_days: + return "stale" + return None + + +# ============================================================================ +# Markdown rendering +# ============================================================================ + + +GROUP_ORDER = [ + ("disabled", "⚠ Disabled workflows"), + ("stale", "⚠ Stale (scheduled but no recent successful run)"), + ("manual", "β„Ή Manual-only by design"), + ( + "gate-daily", + "Release-blocking gates β€” daily (release-gate cron 02:00 UTC)", + ), + ("gate-release", "Release gates β€” release-time only"), + ("schedule", "Scheduled (own cron)"), + ("pr-push", "PR / push checks"), + ("repo-dispatch", "Repository-dispatch publishers"), + ("other", "Other"), +] + +GLYPH = { + "success": "βœ…", + "failure": "❌", + "timed_out": "❌", + "cancelled": "⊘", + "skipped": "Β·", + "neutral": "Β·", + "stale": "Β·", + "action_required": "⚠️", + "in_progress": "⏳", + "queued": "⏳", + None: "β€”", +} + + +def conclusion_glyph(conclusion: Optional[str], status: Optional[str]) -> str: + if status in ("queued", "in_progress"): + return "⏳" + # GitHub's documented conclusion enum: success, failure, neutral, + # cancelled, skipped, timed_out, action_required, stale. Unknown + # values fall through to em-dash rather than "?" so a future + # GitHub-side enum addition doesn't introduce false-positive diffs. + return GLYPH.get(conclusion, "β€”") + + +# Coarse calendar-day buckets for "Last activity"-type columns. Goal: +# regenerations only produce a diff when a workflow drifts between +# buckets β€” exact timestamps would change every run and drown the signal. +# Boundaries are inclusive on the high side. No "yesterday" β€” anything +# 0-7 days old reads as "this week" so a daily-run healthy workflow +# stays in one bucket forever. +ACTIVITY_BUCKETS = [ + (7, "this week"), + (14, "last week"), + (21, "2 weeks ago"), + (30, "3 weeks ago"), + (60, "last month"), + (90, "2 months ago"), + (180, "3+ months ago"), +] + + +def relative_bucket(iso: Optional[str], now: datetime) -> str: + """Return a coarse 'this week / last month / …' label, or 'never'. + + Uses calendar-day delta (no time-of-day jitter) so two regenerations + on the same date produce identical labels.""" + if not iso: + return "never" + dt = datetime.fromisoformat(iso.replace("Z", "+00:00")) + days = (now.date() - dt.date()).days + for threshold, label in ACTIVITY_BUCKETS: + if days <= threshold: + return label + return "long ago" + + +def badge_md( + owner: str, repo: str, file: str, event_filter: Optional[str] = None +) -> str: + """`[![badge](svg)](runs)` β€” live status, links to runs page. + + `event_filter` (e.g. "schedule") appends `?event=...` to the badge + URL, which makes GitHub render the status of the most recent run + *for that event* rather than the most recent run regardless of + trigger. Useful for workflows that fire on multiple events (e.g. + PR + cron) where we want the badge to reflect the cron health + specifically. Verified effective by SHA-comparing badge bodies for + workflows with multi-event run history. + """ + badge = ( + f"https://github.com/{owner}/{repo}/actions/workflows/{file}/badge.svg" + ) + runs = f"https://github.com/{owner}/{repo}/actions/workflows/{file}" + if event_filter: + badge = f"{badge}?event={event_filter}" + runs = f"{runs}?query=event%3A{event_filter}" + return f"[![status]({badge})]({runs})" + + +def render( + workflows: list[dict[str, Any]], + owner: str, + repo: str, + now: datetime, +) -> str: + """Render the generated section. Caller wraps with markers.""" + lines: list[str] = [] + + by_group: dict[str, list[dict[str, Any]]] = {k: [] for k, _ in GROUP_ORDER} + for wf in workflows: + by_group.setdefault(wf["primary_role"], []).append(wf) + + # Aggregated health banner. Counts only change when a workflow shifts + # between {disabled, stale, manual, active}, so this line is as + # diff-stable as the per-row buckets. + total = len(workflows) + n_disabled = len(by_group.get("disabled", [])) + n_stale = len(by_group.get("stale", [])) + n_manual = len(by_group.get("manual", [])) + n_active = total - n_disabled - n_stale - n_manual + parts = [f"**{total} workflows:**"] + if n_disabled: + parts.append(f"{n_disabled} disabled") + if n_stale: + parts.append(f"{n_stale} stale") + if n_manual: + parts.append(f"{n_manual} manual-only") + parts.append(f"{n_active} active") + lines.append(" ".join(parts[:1]) + " " + " Β· ".join(parts[1:])) + lines.append("") + + for key, heading in GROUP_ORDER: + rows = by_group.get(key, []) + lines.append(f"## {heading}") + lines.append("") + if not rows: + lines.append("_None._") + lines.append("") + continue + + if key == "disabled": + lines.append("| Workflow | Disabled where | Last direct run |") + lines.append("|---|---|---|") + for wf in sorted(rows, key=lambda w: w["file"]): + where = ( + "; ".join( + f"`{c['caller']}:{c['line']}` (commented)" + for c in wf["callers_of"] + if c["commented"] + ) + or f"GitHub UI state = `{wf.get('state', '?')}`" + ) + lines.append( + f"| `{wf['file']}` | {where} | " + f"{relative_bucket(wf['last_direct_run_iso'], now)} |" + ) + lines.append("") + continue + + if key == "manual": + lines.append("| Workflow | Last manual run | Trigger |") + lines.append("|---|---|---|") + for wf in sorted(rows, key=lambda w: w["file"]): + lines.append( + f"| `{wf['file']}` | " + f"{relative_bucket(wf['last_direct_run_iso'], now)} | " + f"{trigger_summary(wf['on'])} |" + ) + lines.append("") + continue + + if key == "stale": + lines.append("| Workflow | Cron | Last successful run |") + lines.append("|---|---|---|") + for wf in sorted(rows, key=lambda w: w["file"]): + crons = ", ".join(get_schedules(wf["on"])) or "β€”" + last_success = wf.get("last_success_iso") + lines.append( + f"| `{wf['file']}` | `{crons}` | " + f"{relative_bucket(last_success, now)} |" + ) + lines.append("") + continue + + # Generic active groups. + # For the "schedule" group (own cron, often combined with PR + # triggers like gitleaks/fuzz/codeql), filter the badge to the + # scheduled event so the rendered status reflects cron health, + # not whichever PR ran last. + event_filter = "schedule" if key == "schedule" else None + lines.append( + "| Workflow | Status | Last activity | Trigger | Live badge |" + ) + lines.append("|---|---|---|---|---|") + for wf in sorted(rows, key=lambda w: w["file"]): + primary_iso, primary_concl, primary_status = _pick_activity(wf, key) + lines.append( + f"| `{wf['file']}` | " + f"{conclusion_glyph(primary_concl, primary_status)} | " + f"{relative_bucket(primary_iso, now)} | " + f"{trigger_summary(wf['on'])} | " + f"{badge_md(owner, repo, wf['file'], event_filter)} |" + ) + lines.append("") + + return "\n".join(lines) + + +def _pick_activity( + wf: dict[str, Any], group: str +) -> tuple[Optional[str], Optional[str], Optional[str]]: + """Choose which run timestamp to display per group.""" + if group in ("gate-daily", "gate-release"): + gated = wf.get("last_gated") + if gated: + return ( + gated.get("completed_at") or gated.get("run_created"), + gated.get("conclusion"), + gated.get("status"), + ) + return ( + wf["last_direct_run_iso"], + wf["last_direct_run_conclusion"], + wf["last_direct_run_status"], + ) + + +# ============================================================================ +# File assembly with markers +# ============================================================================ + + +PAGE_TEMPLATE = """\ +# Workflow Status + +> **Live status of every GitHub Actions workflow in this repo.** +> Auto-generated by [`scripts/generate_workflow_status.py`](../../scripts/generate_workflow_status.py). +> Do not edit between the generated markers β€” regenerate with +> `pdm run python scripts/generate_workflow_status.py`. Anything outside +> the markers is preserved on regeneration. + +## How to read this page + +- **Live badges** (right column on active gates) re-render on every page + view and reflect the current head-of-default-branch status from + GitHub. Click one to land on that workflow's runs page. +- The **Status** emoji column captures the run conclusion at generation + time. It survives broken badge rendering (corporate proxies, anonymous + viewers). +- **Last activity** uses coarse calendar buckets β€” `this week`, + `last week`, `2 weeks ago`, `3 weeks ago`, `last month`, `2 months + ago`, `3+ months ago`, `long ago`, `never`. Exact dates would change + every regeneration; buckets only change when a workflow drifts, which + is the signal worth seeing in version-bump diffs. +- **Disabled** = a caller has the `uses:` line commented out, or the + workflow is disabled in the GitHub UI. **Stale** = scheduled trigger + but no successful run within 2Γ— its cron cadence (and β‰₯60 days). The + three top sections are the action items. +- Reusable workflows (those triggered only by `workflow_call:`) show + their **gated** run β€” the most recent run of their parent (release.yml, + release-gate.yml, ci-gate.yml) that included them β€” not their own + empty direct-run history. + +{begin_marker} + +{generated} + +{end_marker} + +_Regenerate with `pdm run python scripts/generate_workflow_status.py`._ +""" + + +def assemble(generated: str, now: datetime) -> str: + # `now` accepted for API symmetry; no timestamp is rendered into + # the file. Git history is authoritative for "when" β€” embedding a + # date here would cause a single-line diff on every regeneration + # even when nothing else drifted. + del now + return PAGE_TEMPLATE.format( + begin_marker=BEGIN_MARKER, + generated=generated.strip(), + end_marker=END_MARKER, + ) + + +def merge_with_existing(new_full: str, existing_path: Path) -> str: + """Preserve everything outside the marker pair. Inside the markers + (including the timestamp line) is fully owned by the generator.""" + if not existing_path.exists(): + return new_full + existing = existing_path.read_text() + if BEGIN_MARKER not in existing or END_MARKER not in existing: + return new_full + if existing.index(BEGIN_MARKER) > existing.index(END_MARKER): + # Markers out of order (mid-merge-conflict, manual editing + # mistake, etc.) β€” bail to a clean overwrite rather than try + # to splice and produce interleaved garbage. + return new_full + new_inner = _between(new_full, BEGIN_MARKER, END_MARKER) + head = existing.split(BEGIN_MARKER, 1)[0] + tail = existing.split(END_MARKER, 1)[1] + return f"{head}{BEGIN_MARKER}{new_inner}{END_MARKER}{tail}" + + +def _between(text: str, start: str, end: str) -> str: + return text.split(start, 1)[1].split(end, 1)[0] + + +# ============================================================================ +# Modes +# ============================================================================ + + +def cmd_generate(output: Path, verbose: bool) -> int: + print("Preflight: checking gh auth and rate limit…", file=sys.stderr) + owner, repo = preflight(verbose=verbose) + print(f"Target: {owner}/{repo}", file=sys.stderr) + + paths = sorted(WORKFLOWS_DIR.glob("*.yml")) + if not paths: + print(f"No workflows found in {WORKFLOWS_DIR}", file=sys.stderr) + return 1 + print(f"Discovered {len(paths)} workflows", file=sys.stderr) + + workflows: dict[str, dict[str, Any]] = {} + for p in paths: + try: + workflows[p.name] = parse_workflow(p) + except Exception as e: + print(f"WARN: failed to parse {p.name}: {e}", file=sys.stderr) + + callers_of, transitive_roots = build_call_graph(workflows) + + now = datetime.now(timezone.utc) + + # Pre-compute the (caller_file -> {job_keys for each callee}) map. + job_keys_for: dict[tuple[str, str], set[str]] = {} + for caller_file, wf in workflows.items(): + for call in wf["uses_calls"]: + if call["commented"]: + continue + key = (caller_file, call["callee"]) + job_keys_for.setdefault(key, set()).add(call["job_key"]) + + print("Fetching workflow metadata + runs…", file=sys.stderr) + for i, (file, wf) in enumerate(workflows.items(), start=1): + if verbose: + print(f"[{i}/{len(workflows)}] {file}", file=sys.stderr) + meta = fetch_workflow_meta(owner, repo, file, verbose=verbose) + wf["state"] = meta["state"] if meta else "missing" + wf["id"] = meta["id"] if meta else None + wf["last_direct_run_iso"] = meta["last_run"] if meta else None + wf["last_direct_run_event"] = meta["last_run_event"] if meta else None + wf["last_direct_run_conclusion"] = ( + meta["last_run_conclusion"] if meta else None + ) + wf["last_direct_run_status"] = meta["last_run_status"] if meta else None + wf["callers_of"] = callers_of[file] + wf["transitive_roots"] = transitive_roots[file] + + # Resolve gated activity. + wf["last_gated"] = None + for caller_file in wf["transitive_roots"]: + if caller_file == file: + continue + keys = job_keys_for.get((caller_file, file), set()) + if not keys: + # Walk one level deeper: a transitive root reaches `file` + # via some intermediate caller β€” collect any job keys for + # `file` from any active caller, then check the root for + # those keys. + for intermediate in workflows: + keys |= job_keys_for.get((intermediate, file), set()) + if not keys: + continue + gated = fetch_last_gated_run( + owner, repo, caller_file, keys, verbose=verbose + ) + if gated: + cur = wf["last_gated"] + if cur is None or (gated["completed_at"] or "") > ( + cur["completed_at"] or "" + ): + wf["last_gated"] = gated + + # File-added timestamp for new-file exception. + wf["file_added_iso"] = file_added_at(wf["path"]) + + # Compute "last successful" by walking recent runs (not just the + # most recent β€” a workflow that ran red yesterday and green a + # week ago is not stale). + last_success = None + recent = (meta["recent_runs"] if meta else []) or [] + for r in recent: + if r.get("conclusion") == "success": + last_success = r["created_at"] + break + # Pull the most-recent-success gated entry (computed alongside + # the latest match by `fetch_last_gated_run`). This is needed + # because the *latest* gated run is often `in_progress` during + # an active release, which would otherwise mask a real success + # from the previous release and trip the stale-flag false. + gated = wf["last_gated"] or {} + gated_success = gated.get("latest_success") or ( + gated if gated.get("conclusion") == "success" else None + ) + if gated_success: + cand = gated_success.get("completed_at") or gated_success.get( + "run_created" + ) + if cand and (last_success is None or cand > last_success): + last_success = cand + wf["last_success_iso"] = last_success + + # Staleness verdict (None if active). Uses the success timestamp + # we just computed, not the most-recent-run conclusion. + wf["stale_verdict"] = classify_staleness( + wf, + callers_of[file], + wf["last_success_iso"], + wf["file_added_iso"], + now, + ) + + # Primary role for grouping. + if wf["stale_verdict"] == "disabled": + wf["primary_role"] = "disabled" + elif wf["stale_verdict"] == "manual-only": + wf["primary_role"] = "manual" + elif wf["stale_verdict"] == "stale": + wf["primary_role"] = "stale" + else: + wf["primary_role"] = primary_role( + file, + wf["on"], + callers_of[file], + transitive_roots[file], + ) + + generated = render(list(workflows.values()), owner, repo, now) + full = assemble(generated, now) + + output.parent.mkdir(parents=True, exist_ok=True) + final = merge_with_existing(full, output) + output.write_text(final) + + counts = {k: 0 for k, _ in GROUP_ORDER} + for wf in workflows.values(): + counts[wf["primary_role"]] = counts.get(wf["primary_role"], 0) + 1 + print( + f"Wrote {output} ({len(workflows)} workflows: " + + ", ".join(f"{n} {k}" for k, n in counts.items() if n) + + ")" + ) + return 0 + + +def cmd_check_structure(output: Path) -> int: + """Verify every workflow file has a row in the dashboard.""" + if not output.exists(): + print( + f"FAIL: {output} does not exist. Generate it with " + f"`pdm run python scripts/generate_workflow_status.py`.", + file=sys.stderr, + ) + return 1 + text = output.read_text() + missing = [] + for p in sorted(WORKFLOWS_DIR.glob("*.yml")): + if f"`{p.name}`" not in text: + missing.append(p.name) + if missing: + print( + f"FAIL: {output} is missing rows for {len(missing)} workflow(s):", + file=sys.stderr, + ) + for m in missing: + print(f" - {m}", file=sys.stderr) + print( + "Regenerate with `pdm run python scripts/generate_workflow_status.py`.", + file=sys.stderr, + ) + return 1 + print( + f"OK: {output} has rows for all {len(list(WORKFLOWS_DIR.glob('*.yml')))} workflows." + ) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Generate docs/ci/workflow-status.md from .github/workflows/*.yml." + ) + ) + parser.add_argument( + "--output", + "-o", + type=Path, + default=DEFAULT_OUTPUT, + help=f"Output path (default: {DEFAULT_OUTPUT.relative_to(REPO_ROOT)})", + ) + parser.add_argument( + "--check-structure", + action="store_true", + help=( + "Verify every workflow file has a row in the dashboard. " + "Exits 1 if any is missing. No API calls." + ), + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Print one line per gh API call.", + ) + args = parser.parse_args() + + if args.check_structure: + return cmd_check_structure(args.output) + return cmd_generate(args.output, args.verbose) + + +if __name__ == "__main__": + sys.exit(main())