From 1cd963e8b52f798096764e5402f142a3a4772d86 Mon Sep 17 00:00:00 2001 From: vumast3r Date: Thu, 24 Sep 2026 12:38:59 -0600 Subject: [PATCH] fix: stop the session baseline from restarting the scheduler and controller-manager RESTORE_BASELINE ran `sed -i` on the scheduler and controller-manager manifests before every session, whether anything needed fixing or not. sed writes its temp file (sedXXXXXX) into /etc/kubernetes/manifests, the kubelet loads it as one more static pod and kills the running one, so every session start sent both into CrashLoopBackOff for up to minutes - while that session's tasks were being seeded. New namespaces got no default ServiceAccount, so their bare Pods were rejected, and Pods sat unscheduled. The practice cluster's kubelet journal showed the sed churn at each session start, and kube-controller-manager had restarted 10 times. Manifests are now edited through `medit`: only when an edit changes the content, and via a hidden temp file (the kubelet ignores dot-files) moved into place. mkns waits 90 s for the ServiceAccount instead of 30 and fails loudly instead of silently. Verified on a throwaway cluster: three no-op baselines cause zero restarts; a scheduler command typo, an injected controller-manager flag and an apiserver command typo are all still repaired. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Mhxm4rsUfuE5HoytYbu2YW --- exam | 29 ++++++++++++++++++++--------- tracks/cka/common.py | 6 +++++- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/exam b/exam index b7c32a6..efdaa6a 100755 --- a/exam +++ b/exam @@ -644,11 +644,22 @@ def _log_msg(*args: Any, **kwargs: Any) -> None: # restores every known "breaks" target to a good baseline before each session seeds, so # leftover damage never leaks across sessions regardless of which questions get sampled. RESTORE_BASELINE = r""" +# Static-pod manifests are only ever rewritten when an edit actually changes them, and through a +# hidden temp file the kubelet ignores. `sed -i` writes its temp file (sedXXXXXX) into the SAME +# directory, the kubelet loads that as one more static pod and kills the running one: every session +# start used to knock the scheduler and controller-manager into CrashLoopBackOff for up to minutes +# while the session's tasks were being seeded (new Namespaces got no ServiceAccount, so bare Pods +# were rejected; Pods sat unscheduled). The user's cluster had 10 such restarts by 2026-09-24. +MEDIT='medit() { f=$1; shift; t=$(mktemp "${f%/*}/.medit.XXXXXX") || return 1 + if sed "$@" "$f" > "$t" 2>/dev/null && [ "$(md5sum < "$t")" != "$(md5sum < "$f")" ]; then + chmod --reference="$f" "$t" 2>/dev/null; mv -f "$t" "$f"; else rm -f "$t"; fi; }' + # apiserver FIRST: Q66 renames the command to "kube-apiserverr", which takes the whole API # down - every kubectl-based step below would fail until this is undone. The repair goes over # podman/ssh (nx), not kubectl, so it works precisely when the cluster is unreachable. if nx "$CP" 'grep -q "^ *- kube-apiserverr$" /etc/kubernetes/manifests/kube-apiserver.yaml' /dev/null; then - nx "$CP" 'sed -i "s|^\( *\)- kube-apiserverr$|\1- kube-apiserver|" /etc/kubernetes/manifests/kube-apiserver.yaml' /dev/null 2>&1 && break; sleep 2; done fi @@ -656,22 +667,22 @@ fi # scheduler: undo the "kube-scheduler" -> "kube-schedulerr" command typo (Q20/Q61), and put the # manifest back if a task had the candidate stop the scheduler by moving it out (Q97) and they # never moved it back - that setup keeps a pristine copy outside the static pod directory. -nx "$CP" ' +nx "$CP" "$MEDIT"' f=/etc/kubernetes/manifests/kube-scheduler.yaml [ -f $f ] || { [ -f /etc/kubernetes/.cka-sim/kube-scheduler.yaml ] && cp /etc/kubernetes/.cka-sim/kube-scheduler.yaml $f; } -sed -i "s|^\( *\)- kube-schedulerr$|\1- kube-scheduler|" $f 2>/dev/null +medit $f -e "s|^\( *\)- kube-schedulerr$|\1- kube-scheduler|" ' /dev/null -sed -i "s|\(image: registry.k8s.io/kube-controller-manager:v[0-9.]*\)-broken|\1|" $f 2>/dev/null -sed -i "s|--cluster-signing-cert-file=/etc/kubernetes/pki/ca-signing.crt|--cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt|" $f 2>/dev/null -sed -i "s|,-certificatesigningrequest-signing-controller||; s|,-csrsigning||" $f 2>/dev/null +nx "$CP" "$MEDIT"' +medit /etc/kubernetes/manifests/kube-controller-manager.yaml \ + -e "/--leader-elect-lease-duration=1s/d" \ + -e "s|\(image: registry.k8s.io/kube-controller-manager:v[0-9.]*\)-broken|\1|" \ + -e "s|--cluster-signing-cert-file=/etc/kubernetes/pki/ca-signing.crt|--cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt|" \ + -e "s|,-certificatesigningrequest-signing-controller||; s|,-csrsigning||" ' /default: serviceaccount "default" not found # Setups that create a namespace and immediately apply Pods lose that race under load, # which fails the seed and costs the candidate a whole task. It belongs here, not in each task. - for _ in $(seq 1 30); do $K -n "$1" get sa default >/dev/null 2>&1 && return 0; sleep 1; done + # 90 s, not 30: the ServiceAccount controller runs in kube-controller-manager, and a manager that + # is restarting (it takes ~60 s to be back and leading) left bare Pods rejected with no trace. + for _ in $(seq 1 90); do $K -n "$1" get sa default >/dev/null 2>&1 && return 0; sleep 1; done + echo "namespace $1 still has no default ServiceAccount after 90 s - is kube-controller-manager running?" >&2 + return 1 } """