From 23a9774a9f01cd3425241c1e1bb89d66a9a15c8e Mon Sep 17 00:00:00 2001 From: vumast3r Date: Mon, 7 Sep 2026 13:51:37 -0600 Subject: [PATCH] fix: install etcdctl/etcdutl on the control-plane node for the etcd tasks The Q02/Q102 seeds tried to copy etcdctl out of the running etcd pod with `kubectl exec -- cat`, which cannot work: registry.k8s.io/etcd is distroless (no cat, no sh, no tar). The copy failed, stderr went to /dev/null and seeding still printed a green tick, so the snapshot task had no etcdctl on the node. Q102 was worse: it also ran `exec -- sh -c 'etcdctl ... snapshot save'` inside that container, so its /opt/cka/backup.db was never created at all. - etcd-tools.sh installs both binaries on the control-plane node, taking them from the etcd image already in the node's containerd (offline, exactly the server version), falling back to the upstream release tarball, then apt. - setup.sh runs it at bootstrap; ensure_etcd_tools in the exam prelude installs them through CKA_NODE_ACCESS when a task is seeded, so ssh/vagrant clusters get them too, and both etcd setups now fail with a reason instead of hiding it. - Q102 takes its snapshot with the node's etcdctl; Q02 no longer documents a kubectl cp fallback that cannot work on a distroless image. Co-authored-by: Junie --- CKA_SOLUTIONS_EXPLAINED.md | 10 ++++++ README.md | 6 ++++ etcd-tools.sh | 72 ++++++++++++++++++++++++++++++++++++++ exam | 11 ++++-- setup.sh | 6 ++++ tracks/cka/bank_hard.py | 7 ++-- tracks/cka/questions.py | 12 ++----- tracks/cka/solutions.py | 5 ++- 8 files changed, 112 insertions(+), 17 deletions(-) create mode 100755 etcd-tools.sh diff --git a/CKA_SOLUTIONS_EXPLAINED.md b/CKA_SOLUTIONS_EXPLAINED.md index c3d2d5c..0d62f64 100644 --- a/CKA_SOLUTIONS_EXPLAINED.md +++ b/CKA_SOLUTIONS_EXPLAINED.md @@ -49,6 +49,16 @@ This guide explains how to approach and solve every question in the CKA exam ban ```bash etcdutl snapshot status /opt/cka/etcd-snapshot.db -w table ``` +- **If the node has no `etcdctl`**: run it inside the etcd pod instead. The etcd image is + distroless (no `sh`, no `cat`, no `tar`), so `kubectl cp` cannot copy the file out — write the + snapshot into `/var/lib/etcd`, which is a hostPath, and move it on the node: + ```bash + kubectl -n kube-system exec etcd- -- etcdctl --endpoints=https://127.0.0.1:2379 \ + --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt \ + --key=/etc/kubernetes/pki/etcd/server.key snapshot save /var/lib/etcd/snap.db + # on the node: mv /var/lib/etcd/snap.db /opt/cka/etcd-snapshot.db + ``` + `./etcd-tools.sh` installs both binaries on the control-plane node for good (`setup.sh` runs it). --- diff --git a/README.md b/README.md index d9c3bba..b791f1c 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,12 @@ set -x CKA_CP cp1; set -x CKA_W1 node1; set -x CKA_W2 node2; set -x CKA_W3 no `/opt/cka` and `/etc/kubernetes/manifests`, so the login must be able to. Managed clusters (EKS/AKS/GKE) won't work — no node or control-plane access. Startup preflights all of this. +The etcd tasks (2 and 102) need `etcdctl`/`etcdutl` **on the control-plane node**, as the real +exam has them — they cannot be copied out of the running pod, since the etcd image is distroless. +`setup.sh` installs them on kind; on any cluster, seeding those tasks installs them if they are +missing (from the etcd image in the node's containerd, else the upstream release, else apt) and +fails loudly if it cannot. `./etcd-tools.sh` does the same by hand. + ## CKA bank | Domain | Normal (43) | Hard (22) | diff --git a/etcd-tools.sh b/etcd-tools.sh new file mode 100755 index 0000000..08e988b --- /dev/null +++ b/etcd-tools.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Install etcdctl/etcdutl on the control-plane node (the real exam has them on the node, and +# Q02 (snapshot) / Q102 (restore) need them there). They cannot be copied out of the running +# etcd pod: registry.k8s.io/etcd is distroless, so there is no cat, sh or tar inside it. +# Re-run this after ./setup.sh if the install was skipped or the network was down. +# ./etcd-tools.sh install them on the control-plane node (podman/docker clusters) +# ./etcd-tools.sh --emit print the node-side script; ./exam pipes it through CKA_NODE_ACCESS, +# so ssh/vagrant clusters get the same install when a task is seeded +set -euo pipefail +cd "$(dirname "$0")" + +# Runs on the node, as root, POSIX sh only. Three sources, best first: +# 1. the etcd image already in the node's containerd - offline, exactly the server version +# 2. the matching upstream release tarball - needs network, exactly the server version +# 3. the distro's etcd-client package - needs network, etcdctl only, version skew +NODE_SCRIPT=$(cat <<'EOS' +set -u +PATH="/usr/local/bin:$PATH"; export PATH +BIN=/usr/local/bin +have() { command -v etcdctl >/dev/null 2>&1 && command -v etcdutl >/dev/null 2>&1; } + +have && { echo "already installed: $(etcdctl version | head -1)"; exit 0; } + +img=$(ctr -n k8s.io images ls -q 2>/dev/null | grep -m1 -E '(^|/)etcd:[0-9]' || true) +ver=$(echo "$img" | sed 's/.*etcd://; s/-[0-9]*$//') + +if [ -n "$img" ]; then + mnt=$(mktemp -d) + if ctr -n k8s.io images mount --rw "$img" "$mnt" >/dev/null 2>&1; then + cp "$mnt/usr/local/bin/etcdctl" "$mnt/usr/local/bin/etcdutl" "$BIN/" 2>/dev/null || true + ctr -n k8s.io images unmount --rm "$mnt" >/dev/null 2>&1 || true + fi + rmdir "$mnt" 2>/dev/null || true + chmod +x "$BIN/etcdctl" "$BIN/etcdutl" 2>/dev/null || true + have && { echo "installed $ver from $img"; exit 0; } +fi + +if [ -n "$ver" ] && command -v curl >/dev/null 2>&1; then + arch=$(uname -m | sed 's/x86_64/amd64/; s/aarch64/arm64/') + d="etcd-v$ver-linux-$arch" + curl -sfL --max-time 180 "https://github.com/etcd-io/etcd/releases/download/v$ver/$d.tar.gz" \ + | tar -xzf - -C "$BIN" --strip-components=1 "$d/etcdctl" "$d/etcdutl" 2>/dev/null || true + chmod +x "$BIN/etcdctl" "$BIN/etcdutl" 2>/dev/null || true + have && { echo "installed $ver from the upstream release"; exit 0; } +fi + +if command -v apt-get >/dev/null 2>&1; then + DEBIAN_FRONTEND=noninteractive apt-get update -qq >/dev/null 2>&1 || true + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq etcd-client >/dev/null 2>&1 || true + command -v etcdctl >/dev/null 2>&1 && { + echo "installed $(etcdctl version | head -1 | awk '{print $3}') from apt: no etcdutl (use" + echo "'etcdctl snapshot restore') and it may not match the ${ver:-running} server" + exit 0 + } +fi + +echo "ERROR: could not install etcdctl/etcdutl. Tried the etcd image in containerd" +echo "(${img:-none found}), the upstream release and apt-get (needs outbound network)." +exit 1 +EOS +) + +if [ "${1:-}" = --emit ]; then + printf '%s\n' "$NODE_SCRIPT" + exit 0 +fi + +RT="${CONTAINER_RUNTIME:-$(cat .state/runtime 2>/dev/null || echo podman)}" +[ "$RT" = podman ] && export KIND_EXPERIMENTAL_PROVIDER=podman +CP="${CKA_CP:-cka-sim-control-plane}" +printf '%s: ' "$CP" +printf '%s\n' "$NODE_SCRIPT" | "$RT" exec -i "$CP" sh -s diff --git a/exam b/exam index 47280c3..d49fd8c 100755 --- a/exam +++ b/exam @@ -17,7 +17,7 @@ from collections.abc import Callable from datetime import datetime from typing import Any -VERSION = "2026.09.06.14" +VERSION = "2026.09.07.1" ROOT = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, ROOT) @@ -120,6 +120,13 @@ nx() { *) podman exec -i "$n" sh -c "$*" ;; esac } +# ensure_etcd_tools -> etcdctl and etcdutl on the control-plane node, as on the real exam. +# The etcd image is distroless, so they cannot be copied out of the pod; etcd-tools.sh takes +# them from the image in the node's containerd (and falls back to a download / apt). +ensure_etcd_tools() { + nx "$CP" 'command -v etcdctl >/dev/null 2>&1 && command -v etcdutl >/dev/null 2>&1' Installing etcdctl/etcdutl on the control-plane node..." +printf ' ' +CONTAINER_RUNTIME="$RT" bash ./etcd-tools.sh || echo " warn: no etcd client on the control-plane node; Q02/Q102 will say so when they seed" + # remember the runtime so the CLI uses the matching exec command mkdir -p .state && echo "$RT" > .state/runtime diff --git a/tracks/cka/bank_hard.py b/tracks/cka/bank_hard.py index 2324682..2af5ff5 100644 --- a/tracks/cka/bank_hard.py +++ b/tracks/cka/bank_hard.py @@ -52,12 +52,11 @@ Do not recreate the ConfigMap by hand — its data must come from the snapshot. """, "setup": r""" nx "$CP" 'mkdir -p /opt/cka; rm -f /opt/cka/backup.db' +ensure_etcd_tools || { echo "no etcdctl/etcdutl on $CP - fix with ./etcd-tools.sh, then re-seed"; exit 1; } $K -n default delete cm %{cm} --ignore-not-found >/dev/null $K -n default create cm %{cm} --from-literal=proof=$(date +%s)-%{cm} >/dev/null -POD=$($K -n kube-system get pod -l component=etcd -o jsonpath='{.items[0].metadata.name}') -$K -n kube-system exec "$POD" -- sh -c 'ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key snapshot save /var/lib/etcd/q102-backup.db' >/dev/null 2>&1 -nx "$CP" 'mv /var/lib/etcd/q102-backup.db /opt/cka/backup.db' -for bin in etcdctl etcdutl; do $K -n kube-system exec "$POD" -- cat /usr/local/bin/$bin > /tmp/cka-$bin 2>/dev/null && [ -s /tmp/cka-$bin ] && nx "$CP" "cat > /usr/local/bin/$bin && chmod +x /usr/local/bin/$bin" < /tmp/cka-$bin; rm -f /tmp/cka-$bin; done +out=$(nx "$CP" 'etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key snapshot save /opt/cka/backup.db' 2>&1) \ + || { echo "snapshot save on $CP failed: $out"; exit 1; } sleep 2; $K -n default delete cm %{cm} >/dev/null """, "checks": [ diff --git a/tracks/cka/questions.py b/tracks/cka/questions.py index aca4e4d..65a9cf8 100644 --- a/tracks/cka/questions.py +++ b/tracks/cka/questions.py @@ -79,21 +79,13 @@ Details you need: - etcd listens on `https://127.0.0.1:2379` - certificates are under `/etc/kubernetes/pki/etcd/` (`ca.crt`, `server.crt`, `server.key`) - "ssh to the control-plane node" is: `{SHELL_CP}` -- Setup tried to place `etcdctl` at `/usr/local/bin/etcdctl` on that node. - If it is missing, run etcdctl inside the etcd pod - (`kubectl -n kube-system exec etcd-{CP} -- etcdctl ...`) and copy the file to the node afterwards. +- `etcdctl` and `etcdutl` are installed on that node, in `/usr/local/bin` Verify the snapshot with `etcdutl snapshot status` (or `etcdctl snapshot status`). """, "setup": r""" nx "$CP" 'mkdir -p /opt/cka; rm -f /opt/cka/etcd-snapshot.db' -POD=$($K -n kube-system get pod -l component=etcd -o jsonpath='{.items[0].metadata.name}') -for bin in etcdctl etcdutl; do - if $K -n kube-system exec "$POD" -- cat /usr/local/bin/$bin > /tmp/cka-$bin 2>/dev/null && [ -s /tmp/cka-$bin ]; then - nx "$CP" "cat > /usr/local/bin/$bin && chmod +x /usr/local/bin/$bin" < /tmp/cka-$bin - fi - rm -f /tmp/cka-$bin -done +ensure_etcd_tools || { echo "no etcdctl/etcdutl on $CP - fix with ./etcd-tools.sh, then re-seed"; exit 1; } """, "checks": [ C("/opt/cka/etcd-snapshot.db exists on the control-plane node", 3, r"""nx "$CP" 'test -f /opt/cka/etcd-snapshot.db'"""), diff --git a/tracks/cka/solutions.py b/tracks/cka/solutions.py index 92d882d..e3d000d 100644 --- a/tracks/cka/solutions.py +++ b/tracks/cka/solutions.py @@ -43,9 +43,12 @@ ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 \\ # Step 3: Verify snapshot status and integrity etcdutl snapshot status /opt/cka/etcd-snapshot.db -w table -# Note: If etcdctl is only inside the static pod, run: +# Note: on a cluster where etcdctl is only inside the static pod, run it there. The etcd image +# is distroless (no sh, no cat, no tar), so kubectl cp cannot get the file out - write it into +# /var/lib/etcd instead, which is a hostPath, and move it on the node: # kubectl -n kube-system exec etcd-{CP} -- etcdctl ... snapshot save /var/lib/etcd/snap.db # then on the node: mv /var/lib/etcd/snap.db /opt/cka/etcd-snapshot.db +# ./etcd-tools.sh installs etcdctl/etcdutl on the control-plane node (setup.sh already does). """ S[3] = """# Step 1: Create the overlay directory structure