From 111a6c63a04d5809afb27d5f453166637d829793 Mon Sep 17 00:00:00 2001 From: vumast3r Date: Mon, 14 Sep 2026 22:09:26 -0600 Subject: [PATCH] fix: repair apiserver/staticPodPath leaks, 3 drifted solutions, workspace tracking Remaining findings from the repo audit (1, 2 and 4 landed in df04c56). restore_baseline now undoes two more leaks, both of which cost real debugging time this week: - apiserver (Q123 renames the command to "kube-apiserverr"). This runs FIRST and over podman/ssh rather than kubectl, because it is exactly the failure that makes kubectl unusable - every later step in the block depends on the API being up. It waits for /readyz before continuing. - staticPodPath (Q103 relocates it to /etc/kubernetes/static on the tier=edge node). The existing kubelet repair only fired when /var/lib/kubelet/config.yaml.orig existed, and nothing ever creates that backup, so a relocated path survived every later session and silently broke Q05, whose check hardcodes /etc/kubernetes/manifests. Normalised unconditionally, restarting the kubelet only when the file actually changed. Verified against the live cluster: with both breaks applied the API was unreachable, and one restore_baseline run brought all four nodes Ready, restored the manifest, reset staticPodPath and let the kubelet re-adopt the Q05 mirror pod. Re-running takes 1.1s versus 14.6s when repairing, confirming it is idempotent and does not restart anything it has not changed. etcd (Q102) and certs (Q122) are deliberately left alone, now documented in the block: neither setup damages anything, so any damage comes from a half-finished candidate attempt, and "mid-attempt" is indistinguishable from "done correctly" without guessing - guessing wrong rewrites etcd's data-dir or re-issues certs on a healthy cluster. proxy-rules (Q302) needs no repair at all; its "breaks" key exists for no_conflicts() bookkeeping, not because it touches shared state. Also: - S[30], S[42], S[108] referenced params that do not exist (%{svc}/%{app}, %{np}/%{port}, %{rev}), so they rendered placeholders verbatim. Rewritten against the real task text - S[42] was missing the DNS egress rule entirely and S[108] stopped before the scale/pause/set-image steps, so both would have failed 2 of 3 checks if followed literally. A param-reference scan over the whole bank is now clean. - Q124's check and S[124] used the deprecated v1 Endpoints API; switched to EndpointSlice, verified both forms return 10.89.1.3:6443 on this cluster. - q_by_id printed "Valid: 1-101" for non-contiguous ids; it now derives the real ranges (1-43, 101-124, 201-217, 301-317). - Untracked workspace/q03. Both q03 and q21 setups start with rm -rf and rewrite every file from heredocs, so they are generated output, not seed templates: tracking them only produced phantom diffs, since a re-seed reverted edits to base/ and deleted anything left in overlays/. With nothing tracked there, teardown.sh can go back to wiping workspace/ wholesale. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Fqcio9TXkEXH35UHebTgoL --- .gitignore | 12 +-- exam | 47 ++++++++-- teardown.sh | 2 + tracks/cka/bank_hard.py | 2 +- tracks/cka/solutions.py | 93 ++++++++++++++----- workspace/q03/base/deployment.yaml | 19 ---- workspace/q03/base/kustomization.yaml | 5 - workspace/q03/base/service.yaml | 10 -- .../q03/overlays/prod/kustomization.yaml | 13 --- workspace/q03/overlays/prod/namespace.yaml | 4 - 10 files changed, 118 insertions(+), 89 deletions(-) delete mode 100644 workspace/q03/base/deployment.yaml delete mode 100644 workspace/q03/base/kustomization.yaml delete mode 100644 workspace/q03/base/service.yaml delete mode 100644 workspace/q03/overlays/prod/kustomization.yaml delete mode 100644 workspace/q03/overlays/prod/namespace.yaml diff --git a/.gitignore b/.gitignore index 6fc0e56..e02c0d6 100644 --- a/.gitignore +++ b/.gitignore @@ -4,9 +4,9 @@ __pycache__/ .DS_Store # workspace/ is where task setup scripts write scratch files for whatever question you're -# currently drilling - regenerated (and wiped) every time that question is re-seeded, so it -# doesn't belong in the repo. q03 and q21 are the two exceptions already tracked as seed -# templates. -/workspace/* -!/workspace/q03 -!/workspace/q21 +# currently drilling - regenerated (and wiped) every time that question is re-seeded, so none +# of it belongs in the repo. q03 and q21 used to be excepted as "seed templates", but both +# setups start with `rm -rf "$WS/qNN"` and rewrite every file from heredocs, so they are +# generated output too: tracking them only produced phantom diffs (a re-seed reverted any edit +# to base/ and deleted any answer left in overlays/). +/workspace/ diff --git a/exam b/exam index add7986..ec7a903 100755 --- a/exam +++ b/exam @@ -207,7 +207,16 @@ def q_by_id(n: int) -> dict[str, Any]: for q in QUESTIONS: if q["id"] == n: return q - die(f"No question {n}. Valid: 1-{len(QUESTIONS)}") + # ids are not contiguous (1-43 base, 101-124 hard, 201-217 / 301-317 sim sets), so print the + # real ranges rather than "1-", which named ids that do not exist. + ids = sorted(q["id"] for q in QUESTIONS) + ranges, start, prev = [], ids[0], ids[0] + for i in ids[1:] + [None]: + if i != prev + 1: + ranges.append(str(start) if start == prev else f"{start}-{prev}") + start = i + prev = i + die(f"No question {n}. Valid: {', '.join(ranges)}") return {} def render(text, params): @@ -552,6 +561,15 @@ def _log_msg(*args: Any, **kwargs: Any) -> None: # restores every known "breaks" target to a good baseline before each session seeds, so # leftover damage never leaks across sessions regardless of which questions get sampled. RESTORE_BASELINE = r""" +# apiserver FIRST: Q123 renames the command to "kube-apiserverr", which takes the whole API +# down - every kubectl-based step below would fail until this is undone. The repair goes over +# podman/ssh (nx), not kubectl, so it works precisely when the cluster is unreachable. +if nx "$CP" 'grep -q "^ *- kube-apiserverr$" /etc/kubernetes/manifests/kube-apiserver.yaml' /dev/null; then + nx "$CP" 'sed -i "s|^\( *\)- kube-apiserverr$|\1- kube-apiserver|" /etc/kubernetes/manifests/kube-apiserver.yaml' /dev/null 2>&1 && break; sleep 2; done +fi + # scheduler: undo the "kube-scheduler" -> "kube-schedulerr" command typo (Q20/Q118) nx "$CP" ' f=/etc/kubernetes/manifests/kube-scheduler.yaml @@ -573,10 +591,19 @@ if [ -n "$cf" ] && ! printf '%s' "$cf" | grep -Eq '^[[:space:]]*forward \. /etc/ $K -n kube-system rollout restart deploy coredns >/dev/null 2>&1 fi -# w1/w2/w3 kubelet: undo Q17/Q39-style tampering, then make sure it's enabled and running +# w1/w2/w3 kubelet: undo Q17/Q39-style tampering, then make sure it's enabled and running. +# staticPodPath is normalised unconditionally: Q103 relocates it to /etc/kubernetes/static, and +# the config.yaml.orig copy below only helps if something made that backup first - nothing does, +# so a relocated path used to survive every later session and silently break Q05 (whose check +# looks in /etc/kubernetes/manifests). Only restart the kubelet if we actually changed the file. for n in "$W1" "$W2" "$W3"; do nx "$n" ' [ -f /var/lib/kubelet/config.yaml.orig ] && cp -f /var/lib/kubelet/config.yaml.orig /var/lib/kubelet/config.yaml + if grep -q "^staticPodPath:" /var/lib/kubelet/config.yaml 2>/dev/null \ + && ! grep -q "^staticPodPath: /etc/kubernetes/manifests$" /var/lib/kubelet/config.yaml; then + sed -i "s|^staticPodPath: .*|staticPodPath: /etc/kubernetes/manifests|" /var/lib/kubelet/config.yaml + systemctl restart kubelet >/dev/null 2>&1 + fi systemctl enable kubelet >/dev/null 2>&1 systemctl start kubelet >/dev/null 2>&1 ' /dev/nu # (Q26/Q211 DaemonSet coverage, Q212's one-pod-per-node spread). for n in "$W1" "$W2" "$W3"; do $K uncordon "$n" >/dev/null 2>&1 || true; done -# NOTE: etcd/certs/apiserver corruption (hard-mode only: Q102, Q118 apiserver half, Q122, -# Q123) isn't restored here yet - those are rarer and each needs a more bespoke repair. -# Q103's staticPodPath relocation also only reverts if /var/lib/kubelet/config.yaml.orig -# exists, which it does not unless something made that backup first. +# Deliberately NOT restored, and why: +# etcd (Q102) and certs (Q122). Neither setup damages anything - Q102 only takes a snapshot +# and deletes a ConfigMap, Q122 only records the current notBefore. Any damage comes from a +# half-finished candidate attempt (etcd.yaml left pointing at a restored data-dir, static pod +# manifests moved to /tmp and never moved back), which is not deterministic: there is no way +# to tell "mid-attempt" from "correctly completed" without guessing, and guessing wrong here +# rewrites etcd's data-dir or re-issues certs on a healthy cluster. Re-run ./setup.sh (or +# ./teardown.sh && ./setup.sh) if one of those leaves the control plane wedged. +# +# proxy-rules (Q302) needs nothing: its setup only creates a namespace and deletes a pod and +# service. The "breaks" key is there for no_conflicts() bookkeeping, not because it damages +# shared state. """ def restore_baseline(log: Callable[..., Any] = _log_msg): diff --git a/teardown.sh b/teardown.sh index e9c4f1f..81cdc45 100755 --- a/teardown.sh +++ b/teardown.sh @@ -5,5 +5,7 @@ cd "$(dirname "$0")" RT="${CONTAINER_RUNTIME:-$(cat .state/runtime 2>/dev/null || echo podman)}" [ "$RT" = podman ] && export KIND_EXPERIMENTAL_PROVIDER=podman kind delete cluster --name cka-sim || true +# Safe to wipe wholesale: everything under workspace/ is generated by task setup scripts and +# nothing there is tracked (see .gitignore). Anything you want to keep, copy out first. rm -rf .state workspace echo "cka-sim cluster and local state removed." diff --git a/tracks/cka/bank_hard.py b/tracks/cka/bank_hard.py index 77c7b5f..8e6b5ad 100644 --- a/tracks/cka/bank_hard.py +++ b/tracks/cka/bank_hard.py @@ -233,7 +233,7 @@ want=$(nx "$CP" "openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl r got=$(grep -oE '^CA_HASH=sha256:[0-9a-f]{64}' "$WS/q124/node-join.env" 2>/dev/null | cut -d= -f2 | cut -d: -f2) [ -n "$want" ] && eq "$got" "$want" """), C("API_ENDPOINT is the real in-cluster address, not the local kubeconfig's", 2, r""" -want=$($K get endpoints kubernetes -o jsonpath='{.subsets[0].addresses[0].ip}:{.subsets[0].ports[0].port}' 2>/dev/null) +want=$($K get endpointslices -l kubernetes.io/service-name=kubernetes -o jsonpath='{.items[0].endpoints[0].addresses[0]}:{.items[0].ports[0].port}' 2>/dev/null) got=$(grep -oE '^API_ENDPOINT=.*' "$WS/q124/node-join.env" 2>/dev/null | cut -d= -f2 | tr -d '[:space:]') [ -n "$want" ] && eq "$got" "$want" """), ], diff --git a/tracks/cka/solutions.py b/tracks/cka/solutions.py index 5b5b6ce..ea7346e 100644 --- a/tracks/cka/solutions.py +++ b/tracks/cka/solutions.py @@ -456,7 +456,7 @@ openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | \\ # kind maps the API server to 127.0.0.1: for YOUR machine only; a node # joining over the cluster network needs the address the apiserver actually advertises. exit -kubectl get endpoints kubernetes -o jsonpath='{.subsets[0].addresses[0].ip}:{.subsets[0].ports[0].port}{"\\n"}' +kubectl get endpointslices -l kubernetes.io/service-name=kubernetes -o jsonpath='{.items[0].endpoints[0].addresses[0]}:{.items[0].ports[0].port}{"\\n"}' # Step 4: Assemble the handoff file cat > {WS}/q124/node-join.env <