diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 8287848..b78fedb 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1016,16 +1016,17 @@ def _node_cache_env(node_cache_path: str, capabilities: Sequence[tuple[str, Mapp def _node_cache_init_container(name: str, capability: Mapping[str, Any], node_cache_path: str, runner_user: int, runner_group: int) -> dict[str, Any]: - """Return the init container that creates a node-cache capability's subdirectories, owned by the runner's user. It runs as root, since the node's directory belongs to root until a pod claims it, and keeps only the one capability that needs: chown.""" + """Return the init container that creates a node-cache capability's subdirectories, owned by the runner's user. It runs as root, since the node's directory belongs to root until a pod claims it. Its capabilities are CHOWN, to hand a directory to the runner user, and DAC_OVERRIDE, because root without it is an ordinary user of a directory it has handed over, or finds already owned by the runner user from an earlier pod or the pruner's emptying, and cannot create the next subdirectory in it.""" root = node_cache_path.rstrip("/") - # The mount root is the profile's own directory, created by the kubelet as root. Owning it lets a job recreate a subdirectory the pruner has set aside, instead of failing on a directory it cannot write to. + # One init container with DAC_OVERRIDE rather than two: a root container that only creates the directories, then a non-root one running as the runner user for the rest, would need no capability beyond CHOWN, but costs a container per capability and requires the image to run as a non-root user, which the digest-pinned busybox default does not guarantee. DAC_OVERRIDE is confined to a container that runs a fixed command on one mounted directory. + # The mount root is the profile's own directory, created by the kubelet as root. The runner user owns it so that a job can recreate a subdirectory the pruner has set aside; this container recreates the capability's own subdirectories at the start of every pod. directories = [root] + [f"{root}/{subdirectory}" for subdirectory in capability["node_cache"]["dirs"].values()] return { "name": f"{_CAPABILITY_CONTAINER_PREFIX}{name}", "image": capability["node_cache"]["init_image"], # The image contract: a POSIX sh, mkdir and chown. chown is not recursive: what a pod creates under a directory is already the runner user's. "command": ["/bin/sh", "-c", 'owner=$1; shift; for directory in "$@"; do mkdir -p "$directory" && chown "$owner" "$directory"; done', "prepare-node-cache", f"{int(runner_user)}:{int(runner_group)}", *directories], - "securityContext": {"runAsUser": 0, "runAsNonRoot": False, "allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"], "add": ["CHOWN"]}}, + "securityContext": {"runAsUser": 0, "runAsNonRoot": False, "allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"], "add": ["CHOWN", "DAC_OVERRIDE"]}}, "volumeMounts": [{"name": _NODE_CACHE_VOLUME, "mountPath": node_cache_path}], } diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 6b1e804..3982cad 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -122,7 +122,7 @@ A system-library capability, `{sysroot_image, env?}`, brings shared libraries, t A hook capability, `{job_started_hook}`, is a bash script the runner runs before each job's first step, for anything small: configuring git, writing a proxy setting to `GITHUB_ENV`, checking a mount. -A node-cache capability, `{node_cache: {init_image, dirs}, env?}`, gives the profile's runner pods on a node one directory of that node to share, for what is expensive to rebuild in every pod and safe to share: a package manager's content-addressed store. **Every job of the profile shares that directory and can write to it, so give it only to a profile whose workflows are all of one trust level, and never to one that runs pull requests from forks.** The runner pod gets a `hostPath` volume, `//` on the node (so profiles never share a cache), mounted once on the runner container at `github_runner_arc_node_cache_path` (`/node-cache` by default). An init container named `capability-`, run as root from the `init_image` (pinned by digest, since it runs as root beside a hostPath volume; it needs a POSIX `sh`, `mkdir` and `chown`, and keeps only the `CHOWN` capability), gives the runner's user the cache's directory and each subdirectory `dirs` names. `dirs` maps an environment variable to a subdirectory, and the variable is set on the runner container to its path, so `pnpm_config_store_dir: pnpm` makes pnpm keep its store in `/node-cache/pnpm`; `env` sets further variables literally. All the node-cache capabilities of a profile share the one volume. The host path root, `github_runner_arc_node_cache_host_path`, is checked whether or not any profile has a node cache, because the pruner mounts it read-write and empties it as root. It must be an absolute path with no colon, no repeated slash and no trailing slash (the pods' volume paths and the pruner's comparison are built from that one string), at least three levels deep, with a last component ending in `node-cache` (so `/var/lib/github-runner/node-cache` is valid and `/var/lib/etcd` is not), and not inside a directory the node itself uses (`/etc`, `/var/lib/kubelet`, `/tmp` and similar). A symbolic link at or above the path is not detected and defeats this validation. +A node-cache capability, `{node_cache: {init_image, dirs}, env?}`, gives the profile's runner pods on a node one directory of that node to share, for what is expensive to rebuild in every pod and safe to share: a package manager's content-addressed store. **Every job of the profile shares that directory and can write to it, so give it only to a profile whose workflows are all of one trust level, and never to one that runs pull requests from forks.** The runner pod gets a `hostPath` volume, `//` on the node (so profiles never share a cache), mounted once on the runner container at `github_runner_arc_node_cache_path` (`/node-cache` by default). An init container named `capability-`, run as root from the `init_image` (pinned by digest, since it runs as root beside a hostPath volume; it needs a POSIX `sh`, `mkdir` and `chown`, and keeps only the `CHOWN` and `DAC_OVERRIDE` capabilities, the second so that root can create a subdirectory in a directory it has handed to the runner user, or finds already theirs from an earlier pod or the pruner), gives the runner's user the cache's directory and each subdirectory `dirs` names. `dirs` maps an environment variable to a subdirectory, and the variable is set on the runner container to its path, so `pnpm_config_store_dir: pnpm` makes pnpm keep its store in `/node-cache/pnpm`; `env` sets further variables literally. All the node-cache capabilities of a profile share the one volume. The host path root, `github_runner_arc_node_cache_host_path`, is checked whether or not any profile has a node cache, because the pruner mounts it read-write and empties it as root. It must be an absolute path with no colon, no repeated slash and no trailing slash (the pods' volume paths and the pruner's comparison are built from that one string), at least three levels deep, with a last component ending in `node-cache` (so `/var/lib/github-runner/node-cache` is valid and `/var/lib/etcd` is not), and not inside a directory the node itself uses (`/etc`, `/var/lib/kubelet`, `/tmp` and similar). A symbolic link at or above the path is not detected and defeats this validation. The cache outlives the pod, so the next job on the node starts with it filled, and a job on a node whose cache is empty fills it as it goes. It is the workflow's own cache step that remains the fallback: a workflow checks that the variable's path exists before relying on it, so it still runs on a runner without the capability, such as a GitHub-hosted one. diff --git a/tests/capabilities/run.sh b/tests/capabilities/run.sh index 9b92f50..cd8a516 100755 --- a/tests/capabilities/run.sh +++ b/tests/capabilities/run.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Integration test for scale-set profile capabilities: builds the reference Node.js tool image from tool-images/node and the reference libpq system-library image from tool-images/libpq, renders a profile carrying them (the tool with a path, so the job-started hook puts it on PATH; the library with an extra variable) and a hook capability through the role's own tasks, renders the gha-runner-scale-set chart with those values, and checks the AutoscalingRunnerSet against the ARC CRDs with a server-side dry run in a kind cluster. It then starts a pod from the chart's own runner pod template in that cluster, with the runner's command replaced by what the runner does at job start (run the ACTIONS_RUNNER_HOOK_JOB_STARTED hook with GITHUB_PATH and GITHUB_ENV) followed by a step that asserts the cached Node.js is on PATH and that @actions/tool-cache's find, the lookup actions/setup-node makes before downloading, returns the copy the init container put in the tool cache, and that libpq's programs run from the sysroot its init container filled, with the search paths and the extra variable the hook set. +# Integration test for scale-set profile capabilities: builds the reference Node.js tool image from tool-images/node and the reference libpq system-library image from tool-images/libpq, renders a profile carrying them (the tool with a path, so the job-started hook puts it on PATH; the library with an extra variable) and a hook capability through the role's own tasks, renders the gha-runner-scale-set chart with those values, and checks the AutoscalingRunnerSet against the ARC CRDs with a server-side dry run in a kind cluster. It then starts a pod from the chart's own runner pod template in that cluster, with the runner's command replaced by what the runner does at job start (run the ACTIONS_RUNNER_HOOK_JOB_STARTED hook with GITHUB_PATH and GITHUB_ENV) followed by a step that asserts the cached Node.js is on PATH and that @actions/tool-cache's find, the lookup actions/setup-node makes before downloading, returns the copy the init container put in the tool cache, and that libpq's programs run from the sysroot its init container filled, with the search paths and the extra variable the hook set, and that the runner user can write to the node-cache directory its init container prepared, under a pod-level non-root securityContext. # # Usage: tests/capabilities/run.sh. Needs Docker, kind, kubectl, helm, python3 with PyYAML, and ansible-playbook (ANSIBLE_PLAYBOOK overrides which); reaches the network for the chart, the runner image and the npm package. GRTEST_SCALESET_CHART_VERSION pins the scale-set chart and the controller chart its CRDs come from (default: the latest). Creates a kind cluster named grtest-capabilities and removes it on exit unless GRTEST_KEEP=1. set -euo pipefail @@ -10,6 +10,8 @@ cluster=grtest-capabilities namespace=arc-runners-example tool_image=grtest/toolcache-node:test library_image=grtest/sysroot-libpq:test +# A node-cache capability's init image, pinned by digest as the role requires. +busybox_image=docker.io/library/busybox:1.37.0@sha256:bdf57e528e45e4433820e045b29b4597825a1c9e38353532d90a01445013f82e sysroot=/opt/sysroot runner_image=ghcr.io/actions/actions-runner:latest charts=oci://ghcr.io/actions/actions-runner-controller-charts @@ -23,8 +25,8 @@ log() { echo "==> $*"; } fail() { echo "FAIL: $*" >&2 kubectl -n "$namespace" get pods -o wide >&2 || true - kubectl -n "$namespace" describe pod runner >&2 || true - kubectl -n "$namespace" logs runner --all-containers >&2 || true + kubectl -n "$namespace" describe pods >&2 || true + for pod in $(kubectl -n "$namespace" get pods -o name 2>/dev/null); do kubectl -n "$namespace" logs "$pod" --all-containers >&2 || true; done exit 1 } cleanup() { @@ -50,17 +52,18 @@ kind load docker-image "$tool_image" "$library_image" --name "$cluster" >/dev/nu log "Rendering the profile's values through the role" # The profile as arc_profiles expands it, and a catalogue with the tool image (put on PATH through its bin directory), the library image (with PGSYSCONFDIR, where libpq looks for its service file, as its extra variable) and a hook capability that records it ran. -python3 - "$repo_root/plugins/filter/arc.py" "$work/extra.json" "$tool_image" "$node_version" "$work/rendered.json" "$library_image" "$sysroot" <<'EOF' +python3 - "$repo_root/plugins/filter/arc.py" "$work/extra.json" "$tool_image" "$node_version" "$work/rendered.json" "$library_image" "$sysroot" "$busybox_image" <<'EOF' import importlib.util, json, sys -plugin, extra, image, version, output, library_image, sysroot = sys.argv[1:] +plugin, extra, image, version, output, library_image, sysroot, busybox_image = sys.argv[1:] spec = importlib.util.spec_from_file_location("arc", plugin) arc = importlib.util.module_from_spec(spec) spec.loader.exec_module(arc) -result = arc.arc_profiles([{"name": "Example", "image": "unused:1", "scale_set_profiles": [{"max_runners": 1, "capabilities": ["node", "libpq", "marker"]}]}], require_app_id=False) +result = arc.arc_profiles([{"name": "Example", "image": "unused:1", "scale_set_profiles": [{"max_runners": 1, "capabilities": ["node", "libpq", "marker", "pnpm-store"]}]}], require_app_id=False) assert result["errors"] == [], result["errors"] catalogue = { "node": {"image": image, "path": f"node/{version}/bin"}, "libpq": {"sysroot_image": library_image, "env": {"PGSYSCONFDIR": f"{sysroot}/etc"}}, + "pnpm-store": {"node_cache": {"init_image": busybox_image, "dirs": {"pnpm_config_store_dir": "pnpm"}}}, "marker": {"job_started_hook": 'echo "CAPABILITY_HOOK_RAN=yes" >> "$GITHUB_ENV"\n'}, } with open(extra, "w") as out: @@ -94,8 +97,21 @@ kubectl wait --for condition=established --timeout=60s crd/autoscalingrunnersets kubectl create namespace "$namespace" >/dev/null kubectl apply --dry-run=server -f "$work/autoscalingrunnerset.yaml" >/dev/null || fail "the API server rejected the rendered AutoscalingRunnerSet" -log "Starting a pod from the chart's runner pod template" kubectl -n "$namespace" apply -f "$work/configmap.yaml" >/dev/null +# What the runner user must find in the node cache once the init container has run: the directory the profile names, writable, and the cache's root writable too, since a job recreates a subdirectory the pruner has set aside directly under it. +cat >"$work/cache-step.sh" <<'EOF' +set -euo pipefail +echo "uid $(id -u); the store directory is $pnpm_config_store_dir" +[ "$(id -u)" = 1001 ] +[ "$pnpm_config_store_dir" = /node-cache/pnpm ] +stat -c 'owner %u mode %a %n' /node-cache "$pnpm_config_store_dir" +touch "$pnpm_config_store_dir/probe" +[ "$(stat -c %u "$pnpm_config_store_dir/probe")" = 1001 ] +mkdir /node-cache/recreated-by-a-job +stat -c 'owner %u mode %a %n' /node-cache/recreated-by-a-job +[ "$(stat -c %u /node-cache/recreated-by-a-job)" = 1001 ] +echo "node cache writable by the runner user" +EOF # The step: what the runner does at job start (it runs the hook named by ACTIONS_RUNNER_HOOK_JOB_STARTED with bash -e and fresh environment files, then applies them), then a job step that uses the tool and asks @actions/tool-cache for it as actions/setup-node does. cat >"$work/step.sh" </dev/null node --input-type=module -e "import * as tc from '@actions/tool-cache'; const found = tc.find('node', '${node_version%%.*}', process.arch); if (!found) { console.error('not found in ' + process.env.RUNNER_TOOL_CACHE); process.exit(1); } console.log('tool-cache find: ' + found);" echo CAPABILITIES-OK EOF -python3 - "$work/autoscalingrunnerset.yaml" "$work/step.sh" "$work/pod.yaml" <<'EOF' +cat >"$work/make-pod.py" <<'EOF' import sys, yaml +# Usage: make-pod.py AUTOSCALINGRUNNERSET STEP_SCRIPT POD_NAME OUTPUT template = yaml.safe_load(open(sys.argv[1]))["spec"]["template"] spec = template["spec"] # The chart's service account and pull Secret belong to a real release, which this test does not install. spec.pop("serviceAccountName", None) spec.pop("imagePullSecrets", None) spec["restartPolicy"] = "Never" +# A pod-level non-root context, as a cluster may impose on every pod: the init containers must set their own to do their work. +spec["securityContext"] = {"runAsNonRoot": True, "runAsUser": 1001, "runAsGroup": 1001} runner = next(container for container in spec["containers"] if container["name"] == "runner") runner["command"] = ["bash", "-c", open(sys.argv[2]).read()] -yaml.safe_dump({"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "runner", "labels": template.get("metadata", {}).get("labels", {})}, "spec": spec}, open(sys.argv[3], "w")) +yaml.safe_dump({"apiVersion": "v1", "kind": "Pod", "metadata": {"name": sys.argv[3], "labels": template.get("metadata", {}).get("labels", {})}, "spec": spec}, open(sys.argv[4], "w")) EOF +# The runner image is large, so the first pull dominates; a pod that has not finished within the deadline is reported with its events. +wait_for_pod() { + local deadline phase + deadline=$(($(date +%s) + 900)) + until phase="$(kubectl -n "$namespace" get pod "$1" -o jsonpath='{.status.phase}')" && [ "$phase" = Succeeded ] || [ "$phase" = Failed ]; do + [ "$(date +%s)" -lt "$deadline" ] || fail "the pod $1 did not finish" + sleep 5 + done + # A pod that failed in an init container has no runner log to print; fail dumps every container's. + kubectl -n "$namespace" logs "$1" -c runner || fail "the pod $1 failed before the step ran" + [ "$phase" = Succeeded ] || fail "the step of pod $1 failed" +} + +# The state a node is really in: the directory already given to the runner user, with no subdirectory in it, which is what the pruner leaves after it empties the cache and what an earlier pod leaves behind. The init container must still create the subdirectory there, as root over a directory it no longer owns. +cache_host_path="$(python3 -c 'import sys, yaml; spec = yaml.safe_load(open(sys.argv[1]))["spec"]["template"]["spec"]; print(next(volume["hostPath"]["path"] for volume in spec["volumes"] if volume["name"] == "node-cache"))' "$work/autoscalingrunnerset.yaml")" +log "Starting a pod on a node cache already owned by the runner user, with no subdirectory" +docker exec "${cluster}-control-plane" sh -c 'mkdir -p "$1" && chown 1001:1001 "$1"' sh "$cache_host_path" +{ cat "$work/cache-step.sh"; echo "echo CACHE-OK"; } >"$work/seeded-step.sh" +python3 "$work/make-pod.py" "$work/autoscalingrunnerset.yaml" "$work/seeded-step.sh" seeded "$work/seeded-pod.yaml" +kubectl -n "$namespace" apply -f "$work/seeded-pod.yaml" >/dev/null +wait_for_pod seeded +kubectl -n "$namespace" logs seeded -c runner | grep -qx CACHE-OK || fail "the step of pod seeded did not reach its end" +kubectl -n "$namespace" delete pod seeded --wait >/dev/null +docker exec "${cluster}-control-plane" rm -rf "$cache_host_path" + +log "Starting a pod from the chart's runner pod template" +python3 "$work/make-pod.py" "$work/autoscalingrunnerset.yaml" "$work/step.sh" runner "$work/pod.yaml" kubectl -n "$namespace" apply -f "$work/pod.yaml" >/dev/null +wait_for_pod runner -# The runner image is large, so the first pull dominates; a pod that has not finished within the deadline is reported with its events. -deadline=$(($(date +%s) + 900)) -until phase="$(kubectl -n "$namespace" get pod runner -o jsonpath='{.status.phase}')" && [ "$phase" = Succeeded ] || [ "$phase" = Failed ]; do - [ "$(date +%s)" -lt "$deadline" ] || fail "the pod did not finish" - sleep 5 -done -kubectl -n "$namespace" logs runner -c runner -[ "$phase" = Succeeded ] || fail "the step failed" kubectl -n "$namespace" logs runner -c runner | grep -qx CAPABILITIES-OK || fail "the step did not reach its end" init_state="$(kubectl -n "$namespace" get pod runner -o jsonpath='{.status.initContainerStatuses[?(@.name=="capability-node")].state.terminated.exitCode}')" [ "$init_state" = 0 ] || fail "the capability-node init container did not succeed (exit code '${init_state}')" init_state="$(kubectl -n "$namespace" get pod runner -o jsonpath='{.status.initContainerStatuses[?(@.name=="capability-libpq")].state.terminated.exitCode}')" [ "$init_state" = 0 ] || fail "the capability-libpq init container did not succeed (exit code '${init_state}')" -log "PASS: the init containers populated the tool cache and the sysroot, the hook ran, put the tool on PATH and the library on its search paths, and @actions/tool-cache found the tool" +init_state="$(kubectl -n "$namespace" get pod runner -o jsonpath='{.status.initContainerStatuses[?(@.name=="capability-pnpm-store")].state.terminated.exitCode}')" +[ "$init_state" = 0 ] || fail "the capability-pnpm-store init container did not succeed (exit code '${init_state}')" +log "PASS: the init containers populated the tool cache and the sysroot, the hook ran, put the tool on PATH and the library on its search paths, and @actions/tool-cache found the tool, and the runner user could write to the node cache" diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index ecde264..673483c 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -343,8 +343,8 @@ def test_a_node_cache_capability_mounts_one_node_directory_and_prepares_its_subd self.assertEqual(init["name"], "capability-pnpm-store") self.assertEqual(init["image"], BUSYBOX) self.assertEqual(init["command"][-3:], ["1001:1001", "/node-cache", "/node-cache/pnpm"]) - self.assertEqual(init["securityContext"]["runAsUser"], 0) - self.assertEqual(init["securityContext"]["capabilities"], {"drop": ["ALL"], "add": ["CHOWN"]}) + # Explicit, so a pod-level non-root securityContext is overridden. Root needs DAC_OVERRIDE as well as CHOWN: the node directory is already the runner user's (the command gives it away, and an earlier pod or the pruner leaves it so), and root without it cannot create /node-cache/pnpm in it. + self.assertEqual(init["securityContext"], {"runAsUser": 0, "runAsNonRoot": False, "allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"], "add": ["CHOWN", "DAC_OVERRIDE"]}}) self.assertEqual(init["volumeMounts"], [{"name": "node-cache", "mountPath": NODE_CACHE}]) container = runner(result["values"]) self.assertEqual(container["volumeMounts"], [{"name": "node-cache", "mountPath": NODE_CACHE}])