diff --git a/exam b/exam index b7c32a6..efdaa6a 100755 --- a/exam +++ b/exam @@ -644,11 +644,22 @@ def _log_msg(*args: Any, **kwargs: Any) -> None: # restores every known "breaks" target to a good baseline before each session seeds, so # leftover damage never leaks across sessions regardless of which questions get sampled. RESTORE_BASELINE = r""" +# Static-pod manifests are only ever rewritten when an edit actually changes them, and through a +# hidden temp file the kubelet ignores. `sed -i` writes its temp file (sedXXXXXX) into the SAME +# directory, the kubelet loads that as one more static pod and kills the running one: every session +# start used to knock the scheduler and controller-manager into CrashLoopBackOff for up to minutes +# while the session's tasks were being seeded (new Namespaces got no ServiceAccount, so bare Pods +# were rejected; Pods sat unscheduled). The user's cluster had 10 such restarts by 2026-09-24. +MEDIT='medit() { f=$1; shift; t=$(mktemp "${f%/*}/.medit.XXXXXX") || return 1 + if sed "$@" "$f" > "$t" 2>/dev/null && [ "$(md5sum < "$t")" != "$(md5sum < "$f")" ]; then + chmod --reference="$f" "$t" 2>/dev/null; mv -f "$t" "$f"; else rm -f "$t"; fi; }' + # apiserver FIRST: Q66 renames the command to "kube-apiserverr", which takes the whole API # down - every kubectl-based step below would fail until this is undone. The repair goes over # podman/ssh (nx), not kubectl, so it works precisely when the cluster is unreachable. if nx "$CP" 'grep -q "^ *- kube-apiserverr$" /etc/kubernetes/manifests/kube-apiserver.yaml' /dev/null; then - nx "$CP" 'sed -i "s|^\( *\)- kube-apiserverr$|\1- kube-apiserver|" /etc/kubernetes/manifests/kube-apiserver.yaml' /dev/null 2>&1 && break; sleep 2; done fi @@ -656,22 +667,22 @@ fi # scheduler: undo the "kube-scheduler" -> "kube-schedulerr" command typo (Q20/Q61), and put the # manifest back if a task had the candidate stop the scheduler by moving it out (Q97) and they # never moved it back - that setup keeps a pristine copy outside the static pod directory. -nx "$CP" ' +nx "$CP" "$MEDIT"' f=/etc/kubernetes/manifests/kube-scheduler.yaml [ -f $f ] || { [ -f /etc/kubernetes/.cka-sim/kube-scheduler.yaml ] && cp /etc/kubernetes/.cka-sim/kube-scheduler.yaml $f; } -sed -i "s|^\( *\)- kube-schedulerr$|\1- kube-scheduler|" $f 2>/dev/null +medit $f -e "s|^\( *\)- kube-schedulerr$|\1- kube-scheduler|" ' /dev/null -sed -i "s|\(image: registry.k8s.io/kube-controller-manager:v[0-9.]*\)-broken|\1|" $f 2>/dev/null -sed -i "s|--cluster-signing-cert-file=/etc/kubernetes/pki/ca-signing.crt|--cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt|" $f 2>/dev/null -sed -i "s|,-certificatesigningrequest-signing-controller||; s|,-csrsigning||" $f 2>/dev/null +nx "$CP" "$MEDIT"' +medit /etc/kubernetes/manifests/kube-controller-manager.yaml \ + -e "/--leader-elect-lease-duration=1s/d" \ + -e "s|\(image: registry.k8s.io/kube-controller-manager:v[0-9.]*\)-broken|\1|" \ + -e "s|--cluster-signing-cert-file=/etc/kubernetes/pki/ca-signing.crt|--cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt|" \ + -e "s|,-certificatesigningrequest-signing-controller||; s|,-csrsigning||" ' /default: serviceaccount "default" not found # Setups that create a namespace and immediately apply Pods lose that race under load, # which fails the seed and costs the candidate a whole task. It belongs here, not in each task. - for _ in $(seq 1 30); do $K -n "$1" get sa default >/dev/null 2>&1 && return 0; sleep 1; done + # 90 s, not 30: the ServiceAccount controller runs in kube-controller-manager, and a manager that + # is restarting (it takes ~60 s to be back and leading) left bare Pods rejected with no trace. + for _ in $(seq 1 90); do $K -n "$1" get sa default >/dev/null 2>&1 && return 0; sleep 1; done + echo "namespace $1 still has no default ServiceAccount after 90 s - is kube-controller-manager running?" >&2 + return 1 } """