From 7664c357dfddf2291281b055bc510394e0d3cac7 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 15 Oct 2026 07:20:42 +0100 Subject: [PATCH 1/7] test(arc): run a node-cache capability's init container in kind under a non-root pod context The runner user writes to the node-cache directory the init container prepared, in a pod whose securityContext is non-root, as a cluster may impose. --- tests/capabilities/run.sh | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tests/capabilities/run.sh b/tests/capabilities/run.sh index 9b92f50..89f1a5b 100755 --- a/tests/capabilities/run.sh +++ b/tests/capabilities/run.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Integration test for scale-set profile capabilities: builds the reference Node.js tool image from tool-images/node and the reference libpq system-library image from tool-images/libpq, renders a profile carrying them (the tool with a path, so the job-started hook puts it on PATH; the library with an extra variable) and a hook capability through the role's own tasks, renders the gha-runner-scale-set chart with those values, and checks the AutoscalingRunnerSet against the ARC CRDs with a server-side dry run in a kind cluster. It then starts a pod from the chart's own runner pod template in that cluster, with the runner's command replaced by what the runner does at job start (run the ACTIONS_RUNNER_HOOK_JOB_STARTED hook with GITHUB_PATH and GITHUB_ENV) followed by a step that asserts the cached Node.js is on PATH and that @actions/tool-cache's find, the lookup actions/setup-node makes before downloading, returns the copy the init container put in the tool cache, and that libpq's programs run from the sysroot its init container filled, with the search paths and the extra variable the hook set. +# Integration test for scale-set profile capabilities: builds the reference Node.js tool image from tool-images/node and the reference libpq system-library image from tool-images/libpq, renders a profile carrying them (the tool with a path, so the job-started hook puts it on PATH; the library with an extra variable) and a hook capability through the role's own tasks, renders the gha-runner-scale-set chart with those values, and checks the AutoscalingRunnerSet against the ARC CRDs with a server-side dry run in a kind cluster. It then starts a pod from the chart's own runner pod template in that cluster, with the runner's command replaced by what the runner does at job start (run the ACTIONS_RUNNER_HOOK_JOB_STARTED hook with GITHUB_PATH and GITHUB_ENV) followed by a step that asserts the cached Node.js is on PATH and that @actions/tool-cache's find, the lookup actions/setup-node makes before downloading, returns the copy the init container put in the tool cache, and that libpq's programs run from the sysroot its init container filled, with the search paths and the extra variable the hook set, and that the runner user can write to the node-cache directory its init container prepared, under a pod-level non-root securityContext. # # Usage: tests/capabilities/run.sh. Needs Docker, kind, kubectl, helm, python3 with PyYAML, and ansible-playbook (ANSIBLE_PLAYBOOK overrides which); reaches the network for the chart, the runner image and the npm package. GRTEST_SCALESET_CHART_VERSION pins the scale-set chart and the controller chart its CRDs come from (default: the latest). Creates a kind cluster named grtest-capabilities and removes it on exit unless GRTEST_KEEP=1. set -euo pipefail @@ -10,6 +10,8 @@ cluster=grtest-capabilities namespace=arc-runners-example tool_image=grtest/toolcache-node:test library_image=grtest/sysroot-libpq:test +# A node-cache capability's init image, pinned by digest as the role requires. +busybox_image=docker.io/library/busybox:1.37.0@sha256:bdf57e528e45e4433820e045b29b4597825a1c9e38353532d90a01445013f82e sysroot=/opt/sysroot runner_image=ghcr.io/actions/actions-runner:latest charts=oci://ghcr.io/actions/actions-runner-controller-charts @@ -50,17 +52,18 @@ kind load docker-image "$tool_image" "$library_image" --name "$cluster" >/dev/nu log "Rendering the profile's values through the role" # The profile as arc_profiles expands it, and a catalogue with the tool image (put on PATH through its bin directory), the library image (with PGSYSCONFDIR, where libpq looks for its service file, as its extra variable) and a hook capability that records it ran. -python3 - "$repo_root/plugins/filter/arc.py" "$work/extra.json" "$tool_image" "$node_version" "$work/rendered.json" "$library_image" "$sysroot" <<'EOF' +python3 - "$repo_root/plugins/filter/arc.py" "$work/extra.json" "$tool_image" "$node_version" "$work/rendered.json" "$library_image" "$sysroot" "$busybox_image" <<'EOF' import importlib.util, json, sys -plugin, extra, image, version, output, library_image, sysroot = sys.argv[1:] +plugin, extra, image, version, output, library_image, sysroot, busybox_image = sys.argv[1:] spec = importlib.util.spec_from_file_location("arc", plugin) arc = importlib.util.module_from_spec(spec) spec.loader.exec_module(arc) -result = arc.arc_profiles([{"name": "Example", "image": "unused:1", "scale_set_profiles": [{"max_runners": 1, "capabilities": ["node", "libpq", "marker"]}]}], require_app_id=False) +result = arc.arc_profiles([{"name": "Example", "image": "unused:1", "scale_set_profiles": [{"max_runners": 1, "capabilities": ["node", "libpq", "marker", "pnpm-store"]}]}], require_app_id=False) assert result["errors"] == [], result["errors"] catalogue = { "node": {"image": image, "path": f"node/{version}/bin"}, "libpq": {"sysroot_image": library_image, "env": {"PGSYSCONFDIR": f"{sysroot}/etc"}}, + "pnpm-store": {"node_cache": {"init_image": busybox_image, "dirs": {"pnpm_config_store_dir": "pnpm"}}}, "marker": {"job_started_hook": 'echo "CAPABILITY_HOOK_RAN=yes" >> "$GITHUB_ENV"\n'}, } with open(extra, "w") as out: @@ -114,6 +117,11 @@ echo "psql on PATH: \$(command -v psql) \$(psql --version)" [ "\${LD_LIBRARY_PATH%%:*}" = "${sysroot}/lib" ] [ "\$PGSYSCONFDIR" = "${sysroot}/etc" ] echo "system library ran from ${sysroot}" +[ "\$(id -u)" = 1001 ] +[ "\$pnpm_config_store_dir" = /node-cache/pnpm ] +touch "\$pnpm_config_store_dir/probe" +[ "\$(stat -c %u "\$pnpm_config_store_dir/probe")" = 1001 ] +echo "node cache writable by the runner user" cd "\$(mktemp -d)" npm install --silent --no-audit --no-fund "${tool_cache_package}" >/dev/null node --input-type=module -e "import * as tc from '@actions/tool-cache'; const found = tc.find('node', '${node_version%%.*}', process.arch); if (!found) { console.error('not found in ' + process.env.RUNNER_TOOL_CACHE); process.exit(1); } console.log('tool-cache find: ' + found);" @@ -127,6 +135,8 @@ spec = template["spec"] spec.pop("serviceAccountName", None) spec.pop("imagePullSecrets", None) spec["restartPolicy"] = "Never" +# A pod-level non-root context, as a cluster may impose on every pod: the init containers must set their own to do their work. +spec["securityContext"] = {"runAsNonRoot": True, "runAsUser": 1001, "runAsGroup": 1001} runner = next(container for container in spec["containers"] if container["name"] == "runner") runner["command"] = ["bash", "-c", open(sys.argv[2]).read()] yaml.safe_dump({"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "runner", "labels": template.get("metadata", {}).get("labels", {})}, "spec": spec}, open(sys.argv[3], "w")) @@ -146,4 +156,6 @@ init_state="$(kubectl -n "$namespace" get pod runner -o jsonpath='{.status.initC [ "$init_state" = 0 ] || fail "the capability-node init container did not succeed (exit code '${init_state}')" init_state="$(kubectl -n "$namespace" get pod runner -o jsonpath='{.status.initContainerStatuses[?(@.name=="capability-libpq")].state.terminated.exitCode}')" [ "$init_state" = 0 ] || fail "the capability-libpq init container did not succeed (exit code '${init_state}')" -log "PASS: the init containers populated the tool cache and the sysroot, the hook ran, put the tool on PATH and the library on its search paths, and @actions/tool-cache found the tool" +init_state="$(kubectl -n "$namespace" get pod runner -o jsonpath='{.status.initContainerStatuses[?(@.name=="capability-pnpm-store")].state.terminated.exitCode}')" +[ "$init_state" = 0 ] || fail "the capability-pnpm-store init container did not succeed (exit code '${init_state}')" +log "PASS: the init containers populated the tool cache and the sysroot, the hook ran, put the tool on PATH and the library on its search paths, and @actions/tool-cache found the tool, and the runner user could write to the node cache" From ec873445180716ecc84618927b7f225a708edd9a Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 15 Oct 2026 07:24:25 +0100 Subject: [PATCH 2/7] test(arc): show every container's log when the capabilities test pod fails before its step --- tests/capabilities/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/capabilities/run.sh b/tests/capabilities/run.sh index 89f1a5b..01a4820 100755 --- a/tests/capabilities/run.sh +++ b/tests/capabilities/run.sh @@ -149,7 +149,8 @@ until phase="$(kubectl -n "$namespace" get pod runner -o jsonpath='{.status.phas [ "$(date +%s)" -lt "$deadline" ] || fail "the pod did not finish" sleep 5 done -kubectl -n "$namespace" logs runner -c runner +# A pod that failed in an init container has no runner log to print; fail dumps every container's. +kubectl -n "$namespace" logs runner -c runner || fail "the pod failed before the step ran" [ "$phase" = Succeeded ] || fail "the step failed" kubectl -n "$namespace" logs runner -c runner | grep -qx CAPABILITIES-OK || fail "the step did not reach its end" init_state="$(kubectl -n "$namespace" get pod runner -o jsonpath='{.status.initContainerStatuses[?(@.name=="capability-node")].state.terminated.exitCode}')" From c05d76c7431cd9abe827c9b8325fdcf95d36d9fd Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 15 Oct 2026 07:27:36 +0100 Subject: [PATCH 3/7] fix(arc): keep DAC_OVERRIDE and FOWNER on the node-cache init container The command gives the node directory to the runner user and then creates a subdirectory in it. Root with only CHOWN is an ordinary user of a directory it no longer owns, so mkdir was denied and the pod stopped in Init:Error. --- plugins/filter/arc.py | 4 ++-- tests/unit/test_capabilities.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 8287848..47f494e 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1016,7 +1016,7 @@ def _node_cache_env(node_cache_path: str, capabilities: Sequence[tuple[str, Mapp def _node_cache_init_container(name: str, capability: Mapping[str, Any], node_cache_path: str, runner_user: int, runner_group: int) -> dict[str, Any]: - """Return the init container that creates a node-cache capability's subdirectories, owned by the runner's user. It runs as root, since the node's directory belongs to root until a pod claims it, and keeps only the one capability that needs: chown.""" + """Return the init container that creates a node-cache capability's subdirectories, owned by the runner's user. It runs as root, since the node's directory belongs to root until a pod claims it. Its capabilities are the three a root process needs once it has handed a directory to the runner user: CHOWN to hand it over, and DAC_OVERRIDE and FOWNER because root without them is an ordinary user of a directory it no longer owns, and cannot create the next subdirectory in it.""" root = node_cache_path.rstrip("/") # The mount root is the profile's own directory, created by the kubelet as root. Owning it lets a job recreate a subdirectory the pruner has set aside, instead of failing on a directory it cannot write to. directories = [root] + [f"{root}/{subdirectory}" for subdirectory in capability["node_cache"]["dirs"].values()] @@ -1025,7 +1025,7 @@ def _node_cache_init_container(name: str, capability: Mapping[str, Any], node_ca "image": capability["node_cache"]["init_image"], # The image contract: a POSIX sh, mkdir and chown. chown is not recursive: what a pod creates under a directory is already the runner user's. "command": ["/bin/sh", "-c", 'owner=$1; shift; for directory in "$@"; do mkdir -p "$directory" && chown "$owner" "$directory"; done', "prepare-node-cache", f"{int(runner_user)}:{int(runner_group)}", *directories], - "securityContext": {"runAsUser": 0, "runAsNonRoot": False, "allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"], "add": ["CHOWN"]}}, + "securityContext": {"runAsUser": 0, "runAsNonRoot": False, "allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"], "add": ["CHOWN", "DAC_OVERRIDE", "FOWNER"]}}, "volumeMounts": [{"name": _NODE_CACHE_VOLUME, "mountPath": node_cache_path}], } diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index ecde264..79e8c86 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -343,8 +343,8 @@ def test_a_node_cache_capability_mounts_one_node_directory_and_prepares_its_subd self.assertEqual(init["name"], "capability-pnpm-store") self.assertEqual(init["image"], BUSYBOX) self.assertEqual(init["command"][-3:], ["1001:1001", "/node-cache", "/node-cache/pnpm"]) - self.assertEqual(init["securityContext"]["runAsUser"], 0) - self.assertEqual(init["securityContext"]["capabilities"], {"drop": ["ALL"], "add": ["CHOWN"]}) + # Explicit, so a pod-level non-root securityContext is overridden. Root needs DAC_OVERRIDE and FOWNER as well as CHOWN: the command gives /node-cache to the runner user first, and root without them cannot then create /node-cache/pnpm in it. + self.assertEqual(init["securityContext"], {"runAsUser": 0, "runAsNonRoot": False, "allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"], "add": ["CHOWN", "DAC_OVERRIDE", "FOWNER"]}}) self.assertEqual(init["volumeMounts"], [{"name": "node-cache", "mountPath": NODE_CACHE}]) container = runner(result["values"]) self.assertEqual(container["volumeMounts"], [{"name": "node-cache", "mountPath": NODE_CACHE}]) From cd891c2c6e7c22a09905e8d1564993f655052594 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 15 Oct 2026 07:47:00 +0100 Subject: [PATCH 4/7] test(arc): start a node-cache pod on a directory the runner user already owns --- tests/capabilities/run.sh | 63 +++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 19 deletions(-) diff --git a/tests/capabilities/run.sh b/tests/capabilities/run.sh index 01a4820..9a96f09 100755 --- a/tests/capabilities/run.sh +++ b/tests/capabilities/run.sh @@ -25,8 +25,8 @@ log() { echo "==> $*"; } fail() { echo "FAIL: $*" >&2 kubectl -n "$namespace" get pods -o wide >&2 || true - kubectl -n "$namespace" describe pod runner >&2 || true - kubectl -n "$namespace" logs runner --all-containers >&2 || true + kubectl -n "$namespace" describe pods >&2 || true + for pod in $(kubectl -n "$namespace" get pods -o name 2>/dev/null); do kubectl -n "$namespace" logs "$pod" --all-containers >&2 || true; done exit 1 } cleanup() { @@ -97,8 +97,17 @@ kubectl wait --for condition=established --timeout=60s crd/autoscalingrunnersets kubectl create namespace "$namespace" >/dev/null kubectl apply --dry-run=server -f "$work/autoscalingrunnerset.yaml" >/dev/null || fail "the API server rejected the rendered AutoscalingRunnerSet" -log "Starting a pod from the chart's runner pod template" kubectl -n "$namespace" apply -f "$work/configmap.yaml" >/dev/null +# What the runner user must find in the node cache once the init container has run: the directory the profile names, writable, and the cache's root writable too, since a job recreates a subdirectory the pruner has set aside directly under it. +cat >"$work/cache-step.sh" <<'EOF' +[ "$(id -u)" = 1001 ] +[ "$pnpm_config_store_dir" = /node-cache/pnpm ] +touch "$pnpm_config_store_dir/probe" +[ "$(stat -c %u "$pnpm_config_store_dir/probe")" = 1001 ] +mkdir /node-cache/recreated-by-a-job +[ "$(stat -c %u /node-cache/recreated-by-a-job)" = 1001 ] +echo "node cache writable by the runner user" +EOF # The step: what the runner does at job start (it runs the hook named by ACTIONS_RUNNER_HOOK_JOB_STARTED with bash -e and fresh environment files, then applies them), then a job step that uses the tool and asks @actions/tool-cache for it as actions/setup-node does. cat >"$work/step.sh" </dev/null node --input-type=module -e "import * as tc from '@actions/tool-cache'; const found = tc.find('node', '${node_version%%.*}', process.arch); if (!found) { console.error('not found in ' + process.env.RUNNER_TOOL_CACHE); process.exit(1); } console.log('tool-cache find: ' + found);" echo CAPABILITIES-OK EOF -python3 - "$work/autoscalingrunnerset.yaml" "$work/step.sh" "$work/pod.yaml" <<'EOF' +cat >"$work/make-pod.py" <<'EOF' import sys, yaml +# Usage: make-pod.py AUTOSCALINGRUNNERSET STEP_SCRIPT POD_NAME OUTPUT template = yaml.safe_load(open(sys.argv[1]))["spec"]["template"] spec = template["spec"] # The chart's service account and pull Secret belong to a real release, which this test does not install. @@ -139,19 +145,38 @@ spec["restartPolicy"] = "Never" spec["securityContext"] = {"runAsNonRoot": True, "runAsUser": 1001, "runAsGroup": 1001} runner = next(container for container in spec["containers"] if container["name"] == "runner") runner["command"] = ["bash", "-c", open(sys.argv[2]).read()] -yaml.safe_dump({"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "runner", "labels": template.get("metadata", {}).get("labels", {})}, "spec": spec}, open(sys.argv[3], "w")) +yaml.safe_dump({"apiVersion": "v1", "kind": "Pod", "metadata": {"name": sys.argv[3], "labels": template.get("metadata", {}).get("labels", {})}, "spec": spec}, open(sys.argv[4], "w")) EOF +# The runner image is large, so the first pull dominates; a pod that has not finished within the deadline is reported with its events. +wait_for_pod() { + local deadline phase + deadline=$(($(date +%s) + 900)) + until phase="$(kubectl -n "$namespace" get pod "$1" -o jsonpath='{.status.phase}')" && [ "$phase" = Succeeded ] || [ "$phase" = Failed ]; do + [ "$(date +%s)" -lt "$deadline" ] || fail "the pod $1 did not finish" + sleep 5 + done + # A pod that failed in an init container has no runner log to print; fail dumps every container's. + kubectl -n "$namespace" logs "$1" -c runner || fail "the pod $1 failed before the step ran" + [ "$phase" = Succeeded ] || fail "the step of pod $1 failed" +} + +# The state a node is really in: the directory already given to the runner user, with no subdirectory in it, which is what the pruner leaves after it empties the cache and what an earlier pod leaves behind. The init container must still create the subdirectory there, as root over a directory it no longer owns. +cache_host_path="$(python3 -c 'import sys, yaml; spec = yaml.safe_load(open(sys.argv[1]))["spec"]["template"]["spec"]; print(next(volume["hostPath"]["path"] for volume in spec["volumes"] if volume["name"] == "node-cache"))' "$work/autoscalingrunnerset.yaml")" +log "Starting a pod on a node cache already owned by the runner user, with no subdirectory" +docker exec "${cluster}-control-plane" sh -c 'mkdir -p "$1" && chown 1001:1001 "$1"' sh "$cache_host_path" +{ cat "$work/cache-step.sh"; echo CACHE-OK; } >"$work/seeded-step.sh" +python3 "$work/make-pod.py" "$work/autoscalingrunnerset.yaml" "$work/seeded-step.sh" seeded "$work/seeded-pod.yaml" +kubectl -n "$namespace" apply -f "$work/seeded-pod.yaml" >/dev/null +wait_for_pod seeded +kubectl -n "$namespace" logs seeded -c runner | grep -qx CACHE-OK || fail "the step of pod seeded did not reach its end" +kubectl -n "$namespace" delete pod seeded --wait >/dev/null +docker exec "${cluster}-control-plane" rm -rf "$cache_host_path" + +log "Starting a pod from the chart's runner pod template" +python3 "$work/make-pod.py" "$work/autoscalingrunnerset.yaml" "$work/step.sh" runner "$work/pod.yaml" kubectl -n "$namespace" apply -f "$work/pod.yaml" >/dev/null +wait_for_pod runner -# The runner image is large, so the first pull dominates; a pod that has not finished within the deadline is reported with its events. -deadline=$(($(date +%s) + 900)) -until phase="$(kubectl -n "$namespace" get pod runner -o jsonpath='{.status.phase}')" && [ "$phase" = Succeeded ] || [ "$phase" = Failed ]; do - [ "$(date +%s)" -lt "$deadline" ] || fail "the pod did not finish" - sleep 5 -done -# A pod that failed in an init container has no runner log to print; fail dumps every container's. -kubectl -n "$namespace" logs runner -c runner || fail "the pod failed before the step ran" -[ "$phase" = Succeeded ] || fail "the step failed" kubectl -n "$namespace" logs runner -c runner | grep -qx CAPABILITIES-OK || fail "the step did not reach its end" init_state="$(kubectl -n "$namespace" get pod runner -o jsonpath='{.status.initContainerStatuses[?(@.name=="capability-node")].state.terminated.exitCode}')" [ "$init_state" = 0 ] || fail "the capability-node init container did not succeed (exit code '${init_state}')" From 62de190f245247931a29bb1b83ad4e8414c5d414 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 15 Oct 2026 07:47:02 +0100 Subject: [PATCH 5/7] fix(arc): drop FOWNER from the node-cache init container and document its capabilities mkdir and chown need only CHOWN and DAC_OVERRIDE. The README and the docstring now give the reason: the node directory is already the runner user's when an earlier pod or the pruner left it. --- plugins/filter/arc.py | 6 +++--- roles/github_runner_arc/README.md | 2 +- tests/unit/test_capabilities.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 47f494e..edec2d9 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1016,16 +1016,16 @@ def _node_cache_env(node_cache_path: str, capabilities: Sequence[tuple[str, Mapp def _node_cache_init_container(name: str, capability: Mapping[str, Any], node_cache_path: str, runner_user: int, runner_group: int) -> dict[str, Any]: - """Return the init container that creates a node-cache capability's subdirectories, owned by the runner's user. It runs as root, since the node's directory belongs to root until a pod claims it. Its capabilities are the three a root process needs once it has handed a directory to the runner user: CHOWN to hand it over, and DAC_OVERRIDE and FOWNER because root without them is an ordinary user of a directory it no longer owns, and cannot create the next subdirectory in it.""" + """Return the init container that creates a node-cache capability's subdirectories, owned by the runner's user. It runs as root, since the node's directory belongs to root until a pod claims it. Its capabilities are CHOWN, to hand a directory to the runner user, and DAC_OVERRIDE, because root without it is an ordinary user of a directory it has handed over, or finds already owned by the runner user from an earlier pod or the pruner's emptying, and cannot create the next subdirectory in it.""" root = node_cache_path.rstrip("/") - # The mount root is the profile's own directory, created by the kubelet as root. Owning it lets a job recreate a subdirectory the pruner has set aside, instead of failing on a directory it cannot write to. + # The mount root is the profile's own directory, created by the kubelet as root. The runner user owns it so that a job can recreate a subdirectory the pruner has set aside; this container recreates the capability's own subdirectories at the start of every pod. directories = [root] + [f"{root}/{subdirectory}" for subdirectory in capability["node_cache"]["dirs"].values()] return { "name": f"{_CAPABILITY_CONTAINER_PREFIX}{name}", "image": capability["node_cache"]["init_image"], # The image contract: a POSIX sh, mkdir and chown. chown is not recursive: what a pod creates under a directory is already the runner user's. "command": ["/bin/sh", "-c", 'owner=$1; shift; for directory in "$@"; do mkdir -p "$directory" && chown "$owner" "$directory"; done', "prepare-node-cache", f"{int(runner_user)}:{int(runner_group)}", *directories], - "securityContext": {"runAsUser": 0, "runAsNonRoot": False, "allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"], "add": ["CHOWN", "DAC_OVERRIDE", "FOWNER"]}}, + "securityContext": {"runAsUser": 0, "runAsNonRoot": False, "allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"], "add": ["CHOWN", "DAC_OVERRIDE"]}}, "volumeMounts": [{"name": _NODE_CACHE_VOLUME, "mountPath": node_cache_path}], } diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 6b1e804..3982cad 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -122,7 +122,7 @@ A system-library capability, `{sysroot_image, env?}`, brings shared libraries, t A hook capability, `{job_started_hook}`, is a bash script the runner runs before each job's first step, for anything small: configuring git, writing a proxy setting to `GITHUB_ENV`, checking a mount. -A node-cache capability, `{node_cache: {init_image, dirs}, env?}`, gives the profile's runner pods on a node one directory of that node to share, for what is expensive to rebuild in every pod and safe to share: a package manager's content-addressed store. **Every job of the profile shares that directory and can write to it, so give it only to a profile whose workflows are all of one trust level, and never to one that runs pull requests from forks.** The runner pod gets a `hostPath` volume, `//` on the node (so profiles never share a cache), mounted once on the runner container at `github_runner_arc_node_cache_path` (`/node-cache` by default). An init container named `capability-`, run as root from the `init_image` (pinned by digest, since it runs as root beside a hostPath volume; it needs a POSIX `sh`, `mkdir` and `chown`, and keeps only the `CHOWN` capability), gives the runner's user the cache's directory and each subdirectory `dirs` names. `dirs` maps an environment variable to a subdirectory, and the variable is set on the runner container to its path, so `pnpm_config_store_dir: pnpm` makes pnpm keep its store in `/node-cache/pnpm`; `env` sets further variables literally. All the node-cache capabilities of a profile share the one volume. The host path root, `github_runner_arc_node_cache_host_path`, is checked whether or not any profile has a node cache, because the pruner mounts it read-write and empties it as root. It must be an absolute path with no colon, no repeated slash and no trailing slash (the pods' volume paths and the pruner's comparison are built from that one string), at least three levels deep, with a last component ending in `node-cache` (so `/var/lib/github-runner/node-cache` is valid and `/var/lib/etcd` is not), and not inside a directory the node itself uses (`/etc`, `/var/lib/kubelet`, `/tmp` and similar). A symbolic link at or above the path is not detected and defeats this validation. +A node-cache capability, `{node_cache: {init_image, dirs}, env?}`, gives the profile's runner pods on a node one directory of that node to share, for what is expensive to rebuild in every pod and safe to share: a package manager's content-addressed store. **Every job of the profile shares that directory and can write to it, so give it only to a profile whose workflows are all of one trust level, and never to one that runs pull requests from forks.** The runner pod gets a `hostPath` volume, `//` on the node (so profiles never share a cache), mounted once on the runner container at `github_runner_arc_node_cache_path` (`/node-cache` by default). An init container named `capability-`, run as root from the `init_image` (pinned by digest, since it runs as root beside a hostPath volume; it needs a POSIX `sh`, `mkdir` and `chown`, and keeps only the `CHOWN` and `DAC_OVERRIDE` capabilities, the second so that root can create a subdirectory in a directory it has handed to the runner user, or finds already theirs from an earlier pod or the pruner), gives the runner's user the cache's directory and each subdirectory `dirs` names. `dirs` maps an environment variable to a subdirectory, and the variable is set on the runner container to its path, so `pnpm_config_store_dir: pnpm` makes pnpm keep its store in `/node-cache/pnpm`; `env` sets further variables literally. All the node-cache capabilities of a profile share the one volume. The host path root, `github_runner_arc_node_cache_host_path`, is checked whether or not any profile has a node cache, because the pruner mounts it read-write and empties it as root. It must be an absolute path with no colon, no repeated slash and no trailing slash (the pods' volume paths and the pruner's comparison are built from that one string), at least three levels deep, with a last component ending in `node-cache` (so `/var/lib/github-runner/node-cache` is valid and `/var/lib/etcd` is not), and not inside a directory the node itself uses (`/etc`, `/var/lib/kubelet`, `/tmp` and similar). A symbolic link at or above the path is not detected and defeats this validation. The cache outlives the pod, so the next job on the node starts with it filled, and a job on a node whose cache is empty fills it as it goes. It is the workflow's own cache step that remains the fallback: a workflow checks that the variable's path exists before relying on it, so it still runs on a runner without the capability, such as a GitHub-hosted one. diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index 79e8c86..673483c 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -343,8 +343,8 @@ def test_a_node_cache_capability_mounts_one_node_directory_and_prepares_its_subd self.assertEqual(init["name"], "capability-pnpm-store") self.assertEqual(init["image"], BUSYBOX) self.assertEqual(init["command"][-3:], ["1001:1001", "/node-cache", "/node-cache/pnpm"]) - # Explicit, so a pod-level non-root securityContext is overridden. Root needs DAC_OVERRIDE and FOWNER as well as CHOWN: the command gives /node-cache to the runner user first, and root without them cannot then create /node-cache/pnpm in it. - self.assertEqual(init["securityContext"], {"runAsUser": 0, "runAsNonRoot": False, "allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"], "add": ["CHOWN", "DAC_OVERRIDE", "FOWNER"]}}) + # Explicit, so a pod-level non-root securityContext is overridden. Root needs DAC_OVERRIDE as well as CHOWN: the node directory is already the runner user's (the command gives it away, and an earlier pod or the pruner leaves it so), and root without it cannot create /node-cache/pnpm in it. + self.assertEqual(init["securityContext"], {"runAsUser": 0, "runAsNonRoot": False, "allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"], "add": ["CHOWN", "DAC_OVERRIDE"]}}) self.assertEqual(init["volumeMounts"], [{"name": "node-cache", "mountPath": NODE_CACHE}]) container = runner(result["values"]) self.assertEqual(container["volumeMounts"], [{"name": "node-cache", "mountPath": NODE_CACHE}]) From e9401e5a06d9dcc6b2ac8578c36f64259b8a765c Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 15 Oct 2026 07:56:25 +0100 Subject: [PATCH 6/7] test(arc): print the seeded pod's end marker from the step --- tests/capabilities/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/capabilities/run.sh b/tests/capabilities/run.sh index 9a96f09..3793c6b 100755 --- a/tests/capabilities/run.sh +++ b/tests/capabilities/run.sh @@ -164,7 +164,7 @@ wait_for_pod() { cache_host_path="$(python3 -c 'import sys, yaml; spec = yaml.safe_load(open(sys.argv[1]))["spec"]["template"]["spec"]; print(next(volume["hostPath"]["path"] for volume in spec["volumes"] if volume["name"] == "node-cache"))' "$work/autoscalingrunnerset.yaml")" log "Starting a pod on a node cache already owned by the runner user, with no subdirectory" docker exec "${cluster}-control-plane" sh -c 'mkdir -p "$1" && chown 1001:1001 "$1"' sh "$cache_host_path" -{ cat "$work/cache-step.sh"; echo CACHE-OK; } >"$work/seeded-step.sh" +{ cat "$work/cache-step.sh"; echo "echo CACHE-OK"; } >"$work/seeded-step.sh" python3 "$work/make-pod.py" "$work/autoscalingrunnerset.yaml" "$work/seeded-step.sh" seeded "$work/seeded-pod.yaml" kubectl -n "$namespace" apply -f "$work/seeded-pod.yaml" >/dev/null wait_for_pod seeded From ca63e130c005704675808a3e8971ab70a3c4b790 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 15 Oct 2026 09:22:20 +0100 Subject: [PATCH 7/7] test(arc): fail the node-cache step on any assertion and explain the single init container The step ran without errexit in the seeded pod, so a wrong owner or an unwritable directory still ended in the pod's success marker. Both steps now stop at the first failed assertion and print the owners and modes they check. --- plugins/filter/arc.py | 1 + tests/capabilities/run.sh | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index edec2d9..b78fedb 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1018,6 +1018,7 @@ def _node_cache_env(node_cache_path: str, capabilities: Sequence[tuple[str, Mapp def _node_cache_init_container(name: str, capability: Mapping[str, Any], node_cache_path: str, runner_user: int, runner_group: int) -> dict[str, Any]: """Return the init container that creates a node-cache capability's subdirectories, owned by the runner's user. It runs as root, since the node's directory belongs to root until a pod claims it. Its capabilities are CHOWN, to hand a directory to the runner user, and DAC_OVERRIDE, because root without it is an ordinary user of a directory it has handed over, or finds already owned by the runner user from an earlier pod or the pruner's emptying, and cannot create the next subdirectory in it.""" root = node_cache_path.rstrip("/") + # One init container with DAC_OVERRIDE rather than two: a root container that only creates the directories, then a non-root one running as the runner user for the rest, would need no capability beyond CHOWN, but costs a container per capability and requires the image to run as a non-root user, which the digest-pinned busybox default does not guarantee. DAC_OVERRIDE is confined to a container that runs a fixed command on one mounted directory. # The mount root is the profile's own directory, created by the kubelet as root. The runner user owns it so that a job can recreate a subdirectory the pruner has set aside; this container recreates the capability's own subdirectories at the start of every pod. directories = [root] + [f"{root}/{subdirectory}" for subdirectory in capability["node_cache"]["dirs"].values()] return { diff --git a/tests/capabilities/run.sh b/tests/capabilities/run.sh index 3793c6b..cd8a516 100755 --- a/tests/capabilities/run.sh +++ b/tests/capabilities/run.sh @@ -100,11 +100,15 @@ kubectl apply --dry-run=server -f "$work/autoscalingrunnerset.yaml" >/dev/null | kubectl -n "$namespace" apply -f "$work/configmap.yaml" >/dev/null # What the runner user must find in the node cache once the init container has run: the directory the profile names, writable, and the cache's root writable too, since a job recreates a subdirectory the pruner has set aside directly under it. cat >"$work/cache-step.sh" <<'EOF' +set -euo pipefail +echo "uid $(id -u); the store directory is $pnpm_config_store_dir" [ "$(id -u)" = 1001 ] [ "$pnpm_config_store_dir" = /node-cache/pnpm ] +stat -c 'owner %u mode %a %n' /node-cache "$pnpm_config_store_dir" touch "$pnpm_config_store_dir/probe" [ "$(stat -c %u "$pnpm_config_store_dir/probe")" = 1001 ] mkdir /node-cache/recreated-by-a-job +stat -c 'owner %u mode %a %n' /node-cache/recreated-by-a-job [ "$(stat -c %u /node-cache/recreated-by-a-job)" = 1001 ] echo "node cache writable by the runner user" EOF