diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be861ca..8275142 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -227,17 +227,23 @@ jobs: with: fetch-depth: 0 persist-credentials: false + # Also writes "What this run does" to the run's summary page: for each + # playbook, the files that made this run touch it, and whether it gets a + # dry run, an apply or nothing. Skipped jobs on the run page say no more. - id: diff env: EVENT: ${{ github.event_name }} PLAYBOOK: ${{ inputs.playbook }} + RUN: ${{ inputs.run }} + LIMIT: ${{ inputs.limit }} + FORK: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} RANGE: >- ${{ github.event_name == 'pull_request' && format('{0}...{1}', github.event.pull_request.base.sha, github.event.pull_request.head.sha) || format('{0}..{1}', github.event.before, github.sha) }} run: | set -euo pipefail - shared='group_vars/|inventory/|ansible\.cfg|scripts/ci-playbook\.sh|\.github/workflows/ci\.yml' + shared='group_vars/|inventory/|ansible\.cfg|scripts/ci-(playbook\.sh|summary\.py)|\.github/workflows/ci\.yml' declare -A paths=( [site]="^(site\.yml|roles/(common|hardening|k3s|storage)/|host_vars/|keys/|$shared)" [platform]="^(platform\.yml|roles/(argocd|monitoring|ufw_allowlist)/|roles/route_origin/defaults/main\.yml|$shared)" @@ -245,6 +251,7 @@ jobs: if [[ "$EVENT" != workflow_dispatch && "$RANGE" != 0000000000000000000000000000000000000000..* ]]; then changed=$(git diff --name-only "$RANGE") fi + rows="" for playbook in site platform; do if [[ "$EVENT" == workflow_dispatch ]]; then hit=$([[ "$PLAYBOOK" == "$playbook" ]] && echo true || echo false) @@ -256,8 +263,48 @@ jobs: hit=false fi echo "$playbook=$hit" | tee -a "$GITHUB_OUTPUT" + + if [[ "$EVENT" == workflow_dispatch ]]; then + why=$([[ "$hit" == true ]] && echo "Chosen in a manual run" || echo "Not chosen") + elif [[ "$RANGE" == 0000000000000000000000000000000000000000..* ]]; then + why="First push of the branch: everything" + else + files=$(grep -E "${paths[$playbook]}" <<< "${changed:-}" || true) + why=$(head -n 10 <<< "$files" | awk 'NF { printf "%s`%s`", (n++ ? "
" : ""), $0 }') + count=$(grep -c . <<< "$files" || true) + if (( count > 10 )); then why+="
and $(( count - 10 )) more"; fi + if [[ -z "$why" ]]; then why="None"; fi + fi + if [[ "$hit" != true ]]; then + does="Nothing" + elif [[ "$EVENT" == pull_request && "$FORK" == true ]]; then + does="Nothing: pull requests from forks get no secrets" + elif [[ "$EVENT" == pull_request ]]; then + does="Dry run only. Nothing is applied from a pull request" + elif [[ "$EVENT" == push ]]; then + does="Dry run, then, if it would change anything, **the apply, once a required reviewer approves it** in \`production\`" + elif [[ "$RUN" == check ]]; then + does="Dry run only" + elif [[ "$RUN" == apply ]]; then + does="Dry run, then **the apply, once a required reviewer approves it** in \`production\`" + else + does="Nothing: checks only (run: none)" + fi + [[ "$hit" == true && -n "$LIMIT" ]] && does+=", limited to \`$LIMIT\`" + rows+="| \`$playbook.yml\` | $why | $does |"$'\n' done + { + echo "### What this run does" + echo + echo "| Playbook | Changed files it applies | This run |" + echo "|---|---|---|" + printf '%s' "$rows" + echo + echo "Dry runs start once every check has passed, and site.yml goes before platform.yml." + echo "Each dry run's summary below lists exactly what it would change." + } >> "$GITHUB_STEP_SUMMARY" + # --- site.yml -------------------------------------------------------------- # What site.yml would change, for this exact commit: on a merge, this is what @@ -265,6 +312,8 @@ jobs: # the run's page. Runs only once every check has passed. site-dry-run: needs: [ansible, manifests, generated, shell, changes] + outputs: + changes: ${{ steps.dry-run.outputs.changes }} if: >- needs.changes.outputs.site == 'true' && ( (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || @@ -296,6 +345,7 @@ jobs: pip install ansible-core ansible-galaxy collection install community.general ansible.posix kubernetes.core - name: Dry run site.yml + id: dry-run env: ANSIBLE_SSH_PRIVATE_KEY: ${{ secrets.ANSIBLE_SSH_PRIVATE_KEY }} LIMIT: ${{ inputs.limit }} @@ -303,7 +353,9 @@ jobs: # Needs every check and the dry run of this same commit: a commit that fails # either is never offered for approval. No status function in `if`, so - # GitHub adds success(). + # GitHub adds success(). On a merge, only if the dry run would change + # something: nothing to approve otherwise. A manual `apply` always runs, in + # case a dry run ever misses a change. # # site.yml can lock everyone out (sshd, ufw, the admin keys) or stop k3s, so # besides the approval it rolls one box at a time (production_rollout), and @@ -314,7 +366,7 @@ jobs: needs: [ansible, manifests, generated, shell, changes, site-dry-run] if: >- needs.changes.outputs.site == 'true' && ( - github.event_name == 'push' || + (github.event_name == 'push' && needs.site-dry-run.outputs.changes == 'true') || (github.event_name == 'workflow_dispatch' && inputs.run == 'apply')) runs-on: ubuntu-latest # One at a time with main's dry runs, never cancelled half-way. A newer @@ -356,6 +408,8 @@ jobs: # also means GitHub no longer adds success(), so every check is named. platform-dry-run: needs: [ansible, manifests, generated, shell, changes, site-dry-run, site-apply] + outputs: + changes: ${{ steps.dry-run.outputs.changes }} if: >- !cancelled() && needs.ansible.result == 'success' && needs.manifests.result == 'success' && @@ -386,6 +440,7 @@ jobs: pip install ansible-core ansible-galaxy collection install community.general ansible.posix kubernetes.core - name: Dry run platform.yml + id: dry-run env: ANSIBLE_SSH_PRIVATE_KEY: ${{ secrets.ANSIBLE_SSH_PRIVATE_KEY }} ARGOCD_AGE_KEY: ${{ secrets.ARGOCD_AGE_KEY }} @@ -399,13 +454,14 @@ jobs: # success(): that also counts the dry run's own needs, and site-dry-run and # site-apply are skipped on every merge that leaves site.yml alone, which # skipped this job too. !cancelled() and the dry run's result instead. + # Like site-apply, skipped on a merge whose dry run would change nothing. platform-apply: needs: [ansible, manifests, generated, shell, changes, platform-dry-run] if: >- !cancelled() && needs.platform-dry-run.result == 'success' && needs.changes.outputs.platform == 'true' && ( - github.event_name == 'push' || + (github.event_name == 'push' && needs.platform-dry-run.outputs.changes == 'true') || (github.event_name == 'workflow_dispatch' && inputs.run == 'apply')) runs-on: ubuntu-latest concurrency: diff --git a/scripts/ci-playbook.sh b/scripts/ci-playbook.sh index cea8aff..d207ffc 100755 --- a/scripts/ci-playbook.sh +++ b/scripts/ci-playbook.sh @@ -66,51 +66,29 @@ args=("$playbook.yml" -e "@$secrets") [[ $mode == check ]] && args+=(--check --diff) [[ -n $limit ]] && args+=(--limit "$limit") -if [[ $mode == apply || -z ${GITHUB_STEP_SUMMARY:-} ]]; then +if [[ -z ${GITHUB_STEP_SUMMARY:-} ]]; then ansible-playbook "${args[@]}" exit fi -# A dry run in CI also writes what it would change to the run's summary page: -# on a merge, that is what the approver reads before approving the apply. +# In CI, both modes also write what they change to the run's summary page: +# each box, task, loop item and diff. On a merge, the dry run's summary is what +# the approver reads before approving the apply; the apply's says what it did. log=$(mktemp) trap 'rm -f "$secrets" "$log"' EXIT set +e ansible-playbook "${args[@]}" 2>&1 | tee "$log" status=${PIPESTATUS[0]} set -e -python3 - "$playbook" "$log" "$status" >> "$GITHUB_STEP_SUMMARY" <<'PY' -import collections, re, sys -playbook, log, status = sys.argv[1], sys.argv[2], int(sys.argv[3]) -ansi = re.compile(r"\x1b\[[0-9;]*m") -lines = [ansi.sub("", l.rstrip("\n")) for l in open(log, errors="replace")] -task, changes, failures, recap, in_recap = "", collections.OrderedDict(), [], [], False -for line in lines: - if m := re.match(r"TASK \[(.*)\]", line): - task = m.group(1) - elif m := re.match(r"changed: \[([^\]]+)\]", line): - changes.setdefault((m.group(1).split(" -> ")[0], task), 0) - changes[(m.group(1).split(" -> ")[0], task)] += 1 - elif m := re.match(r"fatal: \[([^\]]+)\]", line): - failures.append(f"{m.group(1)}: {task}") - elif line.startswith("PLAY RECAP"): - in_recap = True - elif in_recap: - if not line.strip(): - in_recap = False - else: - recap.append(line) -print(f"### {playbook}.yml dry run (`--check --diff`)\n") -if failures: - print("**Failed:** " + "; ".join(failures) + "\n") -if changes: - print("Would change (approve only if this is what you expect):\n") - print("| Box | Task | Changes |\n|---|---|---|") - for (host, name), n in changes.items(): - print(f"| {host} | {name} | {n} |") -elif status == 0: - print("Nothing would change on the boxes.") -print("\n
PLAY RECAP\n\n```\n" + "\n".join(recap) + "\n```\n
\n") -print("Full output, including the diffs: this job's log.") -PY +python3 scripts/ci-summary.py "$playbook" "$mode" "$log" "$status" >> "$GITHUB_STEP_SUMMARY" + +# Whether the dry run would change anything (any `changed=` above 0 in the +# PLAY RECAP). On a merge, the apply jobs are skipped when it would not. +if [[ $mode == check && -n ${GITHUB_OUTPUT:-} ]]; then + if sed 's/\x1b\[[0-9;]*m//g' "$log" | grep -qE '^[^ ]+ +: ok=[0-9]+ +changed=[1-9]'; then + echo "changes=true" >> "$GITHUB_OUTPUT" + else + echo "changes=false" >> "$GITHUB_OUTPUT" + fi +fi exit "$status" diff --git a/scripts/ci-summary.py b/scripts/ci-summary.py new file mode 100644 index 0000000..a0e6d0c --- /dev/null +++ b/scripts/ci-summary.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Turn an ansible-playbook log into the Markdown for the run's summary page. + + scripts/ci-summary.py site|platform check|apply + +Called by scripts/ci-playbook.sh in CI. For each box and task that changed (or +would), it lists the loop items, for example the addresses a firewall rule +was added for, and puts any --diff output in a collapsed block. Tasks with +no_log print neither, so nothing here is not already in the job's log. +""" +import re +import sys +from collections import OrderedDict + +MAX_ITEM = 120 # characters of one loop item +MAX_DIFF = 200 # lines of diff per box and task + +playbook, mode, log, status = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]) +check = mode == "check" + +ansi = re.compile(r"\x1b\[[0-9;]*m") +result = re.compile(r"(ok|changed|skipping|fatal|failed): \[([^\]]+)\](?: => \(item=(.*)\)\s*$)?") + +task, diff, in_diff, in_recap = "", [], False, False +changes = OrderedDict() # (box, task) -> {"count", "items", "diff"} +failures, recap = [], [] + +for raw in open(log, errors="replace"): + line = ansi.sub("", raw.rstrip("\n")) + if m := re.match(r"TASK \[(.*)\]", line): + task, diff, in_diff = m.group(1), [], False + elif line.startswith("PLAY RECAP"): + in_recap, in_diff = True, False + elif in_recap: + if line.strip(): + recap.append(line) + else: + in_recap = False + elif m := result.match(line): + state, box, item = m.group(1), m.group(2).split(" -> ")[0], m.group(3) + if state == "changed": + entry = changes.setdefault((box, task), {"count": 0, "items": [], "diff": []}) + entry["count"] += 1 + if item and item != "None": + entry["items"].append(item) + entry["diff"] += diff + elif state in ("fatal", "failed"): + failures.append(f"{box}: {task}") + diff, in_diff = [], False + elif line.startswith("--- before"): + in_diff = True + diff.append(line) + elif in_diff: + diff.append(line) + + +def cell(text): + text = text if len(text) <= MAX_ITEM else text[: MAX_ITEM - 1] + "…" + return "`" + text.replace("`", "'").replace("|", "\\|") + "`" + + +if check: + print(f"### {playbook}.yml dry run (`--check --diff`)\n") +else: + print(f"### {playbook}.yml apply\n") + +if failures: + print("**Failed:** " + "; ".join(failures) + "\n") +elif status != 0: + print(f"**ansible-playbook exited with {status}**; see this job's log.\n") + +if changes: + if check: + print("Would change on the boxes. Approve the apply only if this is what you expect:\n") + else: + print("Changed on the boxes:\n") + print("| Box | Task | What |\n|---|---|---|") + for (box, name), entry in changes.items(): + if entry["items"]: + what = "
".join(cell(i) for i in entry["items"]) + elif entry["diff"]: + what = "See the diff below" + else: + what = "1 change" if entry["count"] == 1 else f"{entry['count']} changes" + name = name.replace("|", "\\|") + print(f"| {box} | {name} | {what} |") + print() + for (box, name), entry in changes.items(): + if not entry["diff"]: + continue + lines = entry["diff"] + while lines and not lines[-1].strip(): + lines = lines[:-1] + more = len(lines) - MAX_DIFF + body = "\n".join(lines[:MAX_DIFF]).replace("```", "'''") + print(f"
Diff: {box}, {name}\n\n```diff\n{body}\n```\n") + if more > 0: + print(f"{more} more lines in this job's log.\n") + print("
\n") +elif status == 0: + if check: + print("Nothing would change on the boxes, so on a merge the apply is skipped.\n") + else: + print("Nothing changed on the boxes.\n") + +if recap: + print("
PLAY RECAP\n\n```\n" + "\n".join(recap) + "\n```\n
\n") +print("Full output: this job's log.")