diff --git a/exam b/exam index d49fd8c..053bfae 100755 --- a/exam +++ b/exam @@ -520,6 +520,57 @@ def _log_msg(*args: Any, **kwargs: Any) -> None: print(*args, **kwargs) +# Disruptive tasks (see "breaks": ... in the question bank) corrupt shared control-plane / +# node state. A session's setup only touches the questions it actually contains, so damage +# from a task in an earlier session lingers if that task isn't picked again - e.g. Q39 wrecks +# W1's kubelet config, and a later session with different questions never undoes it. This +# restores every known "breaks" target to a good baseline before each session seeds, so +# leftover damage never leaks across sessions regardless of which questions get sampled. +RESTORE_BASELINE = r""" +# scheduler: undo the "kube-scheduler" -> "kube-schedulerr" command typo (Q20/Q118) +nx "$CP" ' +f=/etc/kubernetes/manifests/kube-scheduler.yaml +sed -i "s|^\( *\)- kube-schedulerr$|\1- kube-scheduler|" $f 2>/dev/null +' /dev/null +sed -i "s|\(image: registry.k8s.io/kube-controller-manager:v[0-9.]*\)-broken|\1|" $f 2>/dev/null +' /dev/null) +if [ -n "$cf" ] && ! printf '%s' "$cf" | grep -Eq '^[[:space:]]*forward \. /etc/resolv\.conf'; then + printf '%s' "$cf" | sed -E 's|^([[:space:]]*forward \. )[^[:space:]{]+|\1/etc/resolv.conf|' > /tmp/Corefile.restore + $K -n kube-system create cm coredns --from-file=Corefile=/tmp/Corefile.restore --dry-run=client -o yaml | $K apply -f - >/dev/null 2>&1 + $K -n kube-system rollout restart deploy coredns >/dev/null 2>&1 +fi + +# w1/w2/w3 kubelet: undo Q17/Q39-style tampering, then make sure it's enabled and running +for n in "$W1" "$W2" "$W3"; do + nx "$n" ' + [ -f /var/lib/kubelet/config.yaml.orig ] && cp -f /var/lib/kubelet/config.yaml.orig /var/lib/kubelet/config.yaml + systemctl enable kubelet >/dev/null 2>&1 + systemctl start kubelet >/dev/null 2>&1 + ' /dev/null 2>&1 || true + +# NOTE: etcd/certs/apiserver corruption (hard-mode only: Q102, Q118 apiserver half, Q122, +# Q123) isn't restored here yet - those are rarer and each needs a more bespoke repair. +""" + +def restore_baseline(log: Callable[..., Any] = _log_msg): + log(dim(" resetting control-plane / node state to a clean baseline...")) + rc, out = run(RESTORE_BASELINE, timeout=90) + if rc != 0: + log(yellow(f" warning: baseline restore had issues (continuing anyway):\n{dim(out[-500:])}")) + + def seed(qs, log: Callable[..., Any] = _log_msg, params_by_id=None): params_by_id = params_by_id or {} normal = [q for q in qs if not q.get("disruptive")] @@ -556,6 +607,7 @@ def begin_session(qs, mode, duration, force, log: Callable[..., Any] = _log_msg, rng = random.Random() params_by_id = {str(q["id"]): make_params(q, rng) for q in qs} setup_contexts(hard, params_by_id) + restore_baseline(log) log(bold(f"Seeding {len(qs)} scenario(s){' [HARD]' if hard else ''}...")) seed(qs, log, params_by_id) st = {"mode": mode, "track": TRACK_ID, "hard": hard, "untimed": untimed_session, "ids": [q["id"] for q in qs],