Say on each run what it does and what it changes; skip applies with nothing to do

.github/workflows/ci.yml

@@ -227,24 +227,31 @@ jobs:
         with:
           fetch-depth: 0
           persist-credentials: false
+      # Also writes "What this run does" to the run's summary page: for each
+      # playbook, the files that made this run touch it, and whether it gets a
+      # dry run, an apply or nothing. Skipped jobs on the run page say no more.
       - id: diff
         env:
           EVENT: ${{ github.event_name }}
           PLAYBOOK: ${{ inputs.playbook }}
+          RUN: ${{ inputs.run }}
+          LIMIT: ${{ inputs.limit }}
+          FORK: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}
           RANGE: >-
             ${{ github.event_name == 'pull_request'
                 && format('{0}...{1}', github.event.pull_request.base.sha, github.event.pull_request.head.sha)
                 || format('{0}..{1}', github.event.before, github.sha) }}
         run: |
           set -euo pipefail
-          shared='group_vars/|inventory/|ansible\.cfg|scripts/ci-playbook\.sh|\.github/workflows/ci\.yml'
+          shared='group_vars/|inventory/|ansible\.cfg|scripts/ci-(playbook\.sh|summary\.py)|\.github/workflows/ci\.yml'
           declare -A paths=(
             [site]="^(site\.yml|roles/(common|hardening|k3s|storage)/|host_vars/|keys/|$shared)"
             [platform]="^(platform\.yml|roles/(argocd|monitoring|ufw_allowlist)/|roles/route_origin/defaults/main\.yml|$shared)"
           )
           if [[ "$EVENT" != workflow_dispatch && "$RANGE" != 0000000000000000000000000000000000000000..* ]]; then
             changed=$(git diff --name-only "$RANGE")
           fi
+          rows=""
           for playbook in site platform; do
             if [[ "$EVENT" == workflow_dispatch ]]; then
               hit=$([[ "$PLAYBOOK" == "$playbook" ]] && echo true || echo false)
@@ -256,15 +263,57 @@ jobs:
               hit=false
             fi
             echo "$playbook=$hit" | tee -a "$GITHUB_OUTPUT"
+
+            if [[ "$EVENT" == workflow_dispatch ]]; then
+              why=$([[ "$hit" == true ]] && echo "Chosen in a manual run" || echo "Not chosen")
+            elif [[ "$RANGE" == 0000000000000000000000000000000000000000..* ]]; then
+              why="First push of the branch: everything"
+            else
+              files=$(grep -E "${paths[$playbook]}" <<< "${changed:-}" || true)
+              why=$(head -n 10 <<< "$files" | awk 'NF { printf "%s`%s`", (n++ ? "<br>" : ""), $0 }')
+              count=$(grep -c . <<< "$files" || true)
+              if (( count > 10 )); then why+="<br>and $(( count - 10 )) more"; fi
+              if [[ -z "$why" ]]; then why="None"; fi
+            fi
+            if [[ "$hit" != true ]]; then
+              does="Nothing"
+            elif [[ "$EVENT" == pull_request && "$FORK" == true ]]; then
+              does="Nothing: pull requests from forks get no secrets"
+            elif [[ "$EVENT" == pull_request ]]; then
+              does="Dry run only. Nothing is applied from a pull request"
+            elif [[ "$EVENT" == push ]]; then
+              does="Dry run, then, if it would change anything, **the apply, once a required reviewer approves it** in \`production\`"
+            elif [[ "$RUN" == check ]]; then
+              does="Dry run only"
+            elif [[ "$RUN" == apply ]]; then
+              does="Dry run, then **the apply, once a required reviewer approves it** in \`production\`"
+            else
+              does="Nothing: checks only (run: none)"
+            fi
+            [[ "$hit" == true && -n "$LIMIT" ]] && does+=", limited to \`$LIMIT\`"
+            rows+="| \`$playbook.yml\` | $why | $does |"$'\n'
           done
 
+          {
+            echo "### What this run does"
+            echo
+            echo "| Playbook | Changed files it applies | This run |"
+            echo "|---|---|---|"
+            printf '%s' "$rows"
+            echo
+            echo "Dry runs start once every check has passed, and site.yml goes before platform.yml."
+            echo "Each dry run's summary below lists exactly what it would change."
+          } >> "$GITHUB_STEP_SUMMARY"
+
   # --- site.yml --------------------------------------------------------------
 
   # What site.yml would change, for this exact commit: on a merge, this is what
   # the approver reads before approving the apply below. It writes a summary on
   # the run's page. Runs only once every check has passed.
   site-dry-run:
     needs: [ansible, manifests, generated, shell, changes]
+    outputs:
+      changes: ${{ steps.dry-run.outputs.changes }}
     if: >-
       needs.changes.outputs.site == 'true' && (
         (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) ||
@@ -296,14 +345,17 @@ jobs:
           pip install ansible-core
           ansible-galaxy collection install community.general ansible.posix kubernetes.core
       - name: Dry run site.yml
+        id: dry-run
         env:
           ANSIBLE_SSH_PRIVATE_KEY: ${{ secrets.ANSIBLE_SSH_PRIVATE_KEY }}
           LIMIT: ${{ inputs.limit }}
         run: scripts/ci-playbook.sh site check "$LIMIT"
 
   # Needs every check and the dry run of this same commit: a commit that fails
   # either is never offered for approval. No status function in `if`, so
-  # GitHub adds success().
+  # GitHub adds success(). On a merge, only if the dry run would change
+  # something: nothing to approve otherwise. A manual `apply` always runs, in
+  # case a dry run ever misses a change.
   #
   # site.yml can lock everyone out (sshd, ufw, the admin keys) or stop k3s, so
   # besides the approval it rolls one box at a time (production_rollout), and
@@ -314,7 +366,7 @@ jobs:
     needs: [ansible, manifests, generated, shell, changes, site-dry-run]
     if: >-
       needs.changes.outputs.site == 'true' && (
-        github.event_name == 'push' ||
+        (github.event_name == 'push' && needs.site-dry-run.outputs.changes == 'true') ||
         (github.event_name == 'workflow_dispatch' && inputs.run == 'apply'))
     runs-on: ubuntu-latest
     # One at a time with main's dry runs, never cancelled half-way. A newer
@@ -356,6 +408,8 @@ jobs:
   # also means GitHub no longer adds success(), so every check is named.
   platform-dry-run:
     needs: [ansible, manifests, generated, shell, changes, site-dry-run, site-apply]
+    outputs:
+      changes: ${{ steps.dry-run.outputs.changes }}
     if: >-
       !cancelled() &&
       needs.ansible.result == 'success' && needs.manifests.result == 'success' &&
@@ -386,6 +440,7 @@ jobs:
           pip install ansible-core
           ansible-galaxy collection install community.general ansible.posix kubernetes.core
       - name: Dry run platform.yml
+        id: dry-run
         env:
           ANSIBLE_SSH_PRIVATE_KEY: ${{ secrets.ANSIBLE_SSH_PRIVATE_KEY }}
           ARGOCD_AGE_KEY: ${{ secrets.ARGOCD_AGE_KEY }}
@@ -399,13 +454,14 @@ jobs:
   # success(): that also counts the dry run's own needs, and site-dry-run and
   # site-apply are skipped on every merge that leaves site.yml alone, which
   # skipped this job too. !cancelled() and the dry run's result instead.
+  # Like site-apply, skipped on a merge whose dry run would change nothing.
   platform-apply:
     needs: [ansible, manifests, generated, shell, changes, platform-dry-run]
     if: >-
       !cancelled() &&
       needs.platform-dry-run.result == 'success' &&
       needs.changes.outputs.platform == 'true' && (
-        github.event_name == 'push' ||
+        (github.event_name == 'push' && needs.platform-dry-run.outputs.changes == 'true') ||
         (github.event_name == 'workflow_dispatch' && inputs.run == 'apply'))
     runs-on: ubuntu-latest
     concurrency:

scripts/ci-playbook.sh

@@ -66,51 +66,29 @@ args=("$playbook.yml" -e "@$secrets")
 [[ $mode == check ]] && args+=(--check --diff)
 [[ -n $limit ]] && args+=(--limit "$limit")
 
-if [[ $mode == apply || -z ${GITHUB_STEP_SUMMARY:-} ]]; then
+if [[ -z ${GITHUB_STEP_SUMMARY:-} ]]; then
   ansible-playbook "${args[@]}"
   exit
 fi
 
-# A dry run in CI also writes what it would change to the run's summary page:
-# on a merge, that is what the approver reads before approving the apply.
+# In CI, both modes also write what they change to the run's summary page:
+# each box, task, loop item and diff. On a merge, the dry run's summary is what
+# the approver reads before approving the apply; the apply's says what it did.
 log=$(mktemp)
 trap 'rm -f "$secrets" "$log"' EXIT
 set +e
 ansible-playbook "${args[@]}" 2>&1 | tee "$log"
 status=${PIPESTATUS[0]}
 set -e
-python3 - "$playbook" "$log" "$status" >> "$GITHUB_STEP_SUMMARY" <<'PY'
-import collections, re, sys
-playbook, log, status = sys.argv[1], sys.argv[2], int(sys.argv[3])
-ansi = re.compile(r"\x1b\[[0-9;]*m")
-lines = [ansi.sub("", l.rstrip("\n")) for l in open(log, errors="replace")]
-task, changes, failures, recap, in_recap = "", collections.OrderedDict(), [], [], False
-for line in lines:
-    if m := re.match(r"TASK \[(.*)\]", line):
-        task = m.group(1)
-    elif m := re.match(r"changed: \[([^\]]+)\]", line):
-        changes.setdefault((m.group(1).split(" -> ")[0], task), 0)
-        changes[(m.group(1).split(" -> ")[0], task)] += 1
-    elif m := re.match(r"fatal: \[([^\]]+)\]", line):
-        failures.append(f"{m.group(1)}: {task}")
-    elif line.startswith("PLAY RECAP"):
-        in_recap = True
-    elif in_recap:
-        if not line.strip():
-            in_recap = False
-        else:
-            recap.append(line)
-print(f"### {playbook}.yml dry run (`--check --diff`)\n")
-if failures:
-    print("**Failed:** " + "; ".join(failures) + "\n")
-if changes:
-    print("Would change (approve only if this is what you expect):\n")
-    print("| Box | Task | Changes |\n|---|---|---|")
-    for (host, name), n in changes.items():
-        print(f"| {host} | {name} | {n} |")
-elif status == 0:
-    print("Nothing would change on the boxes.")
-print("\n<details><summary>PLAY RECAP</summary>\n\n```\n" + "\n".join(recap) + "\n```\n</details>\n")
-print("Full output, including the diffs: this job's log.")
-PY
+python3 scripts/ci-summary.py "$playbook" "$mode" "$log" "$status" >> "$GITHUB_STEP_SUMMARY"
+
+# Whether the dry run would change anything (any `changed=` above 0 in the
+# PLAY RECAP). On a merge, the apply jobs are skipped when it would not.
+if [[ $mode == check && -n ${GITHUB_OUTPUT:-} ]]; then
+  if sed 's/\x1b\[[0-9;]*m//g' "$log" | grep -qE '^[^ ]+ +: ok=[0-9]+ +changed=[1-9]'; then
+    echo "changes=true" >> "$GITHUB_OUTPUT"
+  else
+    echo "changes=false" >> "$GITHUB_OUTPUT"
+  fi
+fi
 exit "$status"

scripts/ci-summary.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""Turn an ansible-playbook log into the Markdown for the run's summary page.
+
+    scripts/ci-summary.py site|platform check|apply <log> <exit status>
+
+Called by scripts/ci-playbook.sh in CI. For each box and task that changed (or
+would), it lists the loop items, for example the addresses a firewall rule
+was added for, and puts any --diff output in a collapsed block. Tasks with
+no_log print neither, so nothing here is not already in the job's log.
+"""
+import re
+import sys
+from collections import OrderedDict
+
+MAX_ITEM = 120  # characters of one loop item
+MAX_DIFF = 200  # lines of diff per box and task
+
+playbook, mode, log, status = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4])
+check = mode == "check"
+
+ansi = re.compile(r"\x1b\[[0-9;]*m")
+result = re.compile(r"(ok|changed|skipping|fatal|failed): \[([^\]]+)\](?: => \(item=(.*)\)\s*$)?")
+
+task, diff, in_diff, in_recap = "", [], False, False
+changes = OrderedDict()  # (box, task) -> {"count", "items", "diff"}
+failures, recap = [], []
+
+for raw in open(log, errors="replace"):
+    line = ansi.sub("", raw.rstrip("\n"))
+    if m := re.match(r"TASK \[(.*)\]", line):
+        task, diff, in_diff = m.group(1), [], False
+    elif line.startswith("PLAY RECAP"):
+        in_recap, in_diff = True, False
+    elif in_recap:
+        if line.strip():
+            recap.append(line)
+        else:
+            in_recap = False
+    elif m := result.match(line):
+        state, box, item = m.group(1), m.group(2).split(" -> ")[0], m.group(3)
+        if state == "changed":
+            entry = changes.setdefault((box, task), {"count": 0, "items": [], "diff": []})
+            entry["count"] += 1
+            if item and item != "None":
+                entry["items"].append(item)
+            entry["diff"] += diff
+        elif state in ("fatal", "failed"):
+            failures.append(f"{box}: {task}")
+        diff, in_diff = [], False
+    elif line.startswith("--- before"):
+        in_diff = True
+        diff.append(line)
+    elif in_diff:
+        diff.append(line)
+
+
+def cell(text):
+    text = text if len(text) <= MAX_ITEM else text[: MAX_ITEM - 1] + "…"
+    return "`" + text.replace("`", "'").replace("|", "\\|") + "`"
+
+
+if check:
+    print(f"### {playbook}.yml dry run (`--check --diff`)\n")
+else:
+    print(f"### {playbook}.yml apply\n")
+
+if failures:
+    print("**Failed:** " + "; ".join(failures) + "\n")
+elif status != 0:
+    print(f"**ansible-playbook exited with {status}**; see this job's log.\n")
+
+if changes:
+    if check:
+        print("Would change on the boxes. Approve the apply only if this is what you expect:\n")
+    else:
+        print("Changed on the boxes:\n")
+    print("| Box | Task | What |\n|---|---|---|")
+    for (box, name), entry in changes.items():
+        if entry["items"]:
+            what = "<br>".join(cell(i) for i in entry["items"])
+        elif entry["diff"]:
+            what = "See the diff below"
+        else:
+            what = "1 change" if entry["count"] == 1 else f"{entry['count']} changes"
+        name = name.replace("|", "\\|")
+        print(f"| {box} | {name} | {what} |")
+    print()
+    for (box, name), entry in changes.items():
+        if not entry["diff"]:
+            continue
+        lines = entry["diff"]
+        while lines and not lines[-1].strip():
+            lines = lines[:-1]
+        more = len(lines) - MAX_DIFF
+        body = "\n".join(lines[:MAX_DIFF]).replace("```", "'''")
+        print(f"<details><summary>Diff: {box}, {name}</summary>\n\n```diff\n{body}\n```\n")
+        if more > 0:
+            print(f"{more} more lines in this job's log.\n")
+        print("</details>\n")
+elif status == 0:
+    if check:
+        print("Nothing would change on the boxes, so on a merge the apply is skipped.\n")
+    else:
+        print("Nothing changed on the boxes.\n")
+
+if recap:
+    print("<details><summary>PLAY RECAP</summary>\n\n```\n" + "\n".join(recap) + "\n```\n</details>\n")
+print("Full output: this job's log.")