fix: reset control-plane/node state to baseline before every session
Disruptive tasks corrupt shared state (static pod manifests, kubelet config, CoreDNS) keyed by a "breaks" target, but a session's setup only touches the questions it actually contains. If a task isn't re-picked in a later session, its damage lingers indefinitely - e.g. Q39 wrecking W1's kubelet config persisted across two unrelated TSHOOT drills that never included Q39 again. Add restore_baseline(), run unconditionally in begin_session() before seeding, that repairs every known "breaks" target (scheduler, controller-manager, CoreDNS, all three worker kubelets, W1's demo taints) regardless of which questions are in the new session. Hard-mode- only breaks (etcd, certs, apiserver) aren't covered yet. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
7dd3763e68
commit
d755c4017c
52
exam
52
exam
@ -520,6 +520,57 @@ def _log_msg(*args: Any, **kwargs: Any) -> None:
|
||||
print(*args, **kwargs)
|
||||
|
||||
|
||||
# Disruptive tasks (see "breaks": ... in the question bank) corrupt shared control-plane /
|
||||
# node state. A session's setup only touches the questions it actually contains, so damage
|
||||
# from a task in an earlier session lingers if that task isn't picked again - e.g. Q39 wrecks
|
||||
# W1's kubelet config, and a later session with different questions never undoes it. This
|
||||
# restores every known "breaks" target to a good baseline before each session seeds, so
|
||||
# leftover damage never leaks across sessions regardless of which questions get sampled.
|
||||
RESTORE_BASELINE = r"""
|
||||
# scheduler: undo the "kube-scheduler" -> "kube-schedulerr" command typo (Q20/Q118)
|
||||
nx "$CP" '
|
||||
f=/etc/kubernetes/manifests/kube-scheduler.yaml
|
||||
sed -i "s|^\( *\)- kube-schedulerr$|\1- kube-scheduler|" $f 2>/dev/null
|
||||
' </dev/null
|
||||
|
||||
# controller-manager: undo Q38's injected flag and Q118's broken image tag
|
||||
nx "$CP" '
|
||||
f=/etc/kubernetes/manifests/kube-controller-manager.yaml
|
||||
sed -i "/--leader-elect-lease-duration=1s/d" $f 2>/dev/null
|
||||
sed -i "s|\(image: registry.k8s.io/kube-controller-manager:v[0-9.]*\)-broken|\1|" $f 2>/dev/null
|
||||
' </dev/null
|
||||
|
||||
# coredns: restore the default upstream forward (Q37/Q114 point it somewhere bogus)
|
||||
cf=$($K -n kube-system get cm coredns -o jsonpath='{.data.Corefile}' 2>/dev/null)
|
||||
if [ -n "$cf" ] && ! printf '%s' "$cf" | grep -Eq '^[[:space:]]*forward \. /etc/resolv\.conf'; then
|
||||
printf '%s' "$cf" | sed -E 's|^([[:space:]]*forward \. )[^[:space:]{]+|\1/etc/resolv.conf|' > /tmp/Corefile.restore
|
||||
$K -n kube-system create cm coredns --from-file=Corefile=/tmp/Corefile.restore --dry-run=client -o yaml | $K apply -f - >/dev/null 2>&1
|
||||
$K -n kube-system rollout restart deploy coredns >/dev/null 2>&1
|
||||
fi
|
||||
|
||||
# w1/w2/w3 kubelet: undo Q17/Q39-style tampering, then make sure it's enabled and running
|
||||
for n in "$W1" "$W2" "$W3"; do
|
||||
nx "$n" '
|
||||
[ -f /var/lib/kubelet/config.yaml.orig ] && cp -f /var/lib/kubelet/config.yaml.orig /var/lib/kubelet/config.yaml
|
||||
systemctl enable kubelet >/dev/null 2>&1
|
||||
systemctl start kubelet >/dev/null 2>&1
|
||||
' </dev/null
|
||||
done
|
||||
|
||||
# w1-taint: clear any of the randomized demo taint keys Q25/Q107 might have left behind
|
||||
$K taint node "$W1" dedicated- workload- team- >/dev/null 2>&1 || true
|
||||
|
||||
# NOTE: etcd/certs/apiserver corruption (hard-mode only: Q102, Q118 apiserver half, Q122,
|
||||
# Q123) isn't restored here yet - those are rarer and each needs a more bespoke repair.
|
||||
"""
|
||||
|
||||
def restore_baseline(log: Callable[..., Any] = _log_msg):
|
||||
log(dim(" resetting control-plane / node state to a clean baseline..."))
|
||||
rc, out = run(RESTORE_BASELINE, timeout=90)
|
||||
if rc != 0:
|
||||
log(yellow(f" warning: baseline restore had issues (continuing anyway):\n{dim(out[-500:])}"))
|
||||
|
||||
|
||||
def seed(qs, log: Callable[..., Any] = _log_msg, params_by_id=None):
|
||||
params_by_id = params_by_id or {}
|
||||
normal = [q for q in qs if not q.get("disruptive")]
|
||||
@ -556,6 +607,7 @@ def begin_session(qs, mode, duration, force, log: Callable[..., Any] = _log_msg,
|
||||
rng = random.Random()
|
||||
params_by_id = {str(q["id"]): make_params(q, rng) for q in qs}
|
||||
setup_contexts(hard, params_by_id)
|
||||
restore_baseline(log)
|
||||
log(bold(f"Seeding {len(qs)} scenario(s){' [HARD]' if hard else ''}..."))
|
||||
seed(qs, log, params_by_id)
|
||||
st = {"mode": mode, "track": TRACK_ID, "hard": hard, "untimed": untimed_session, "ids": [q["id"] for q in qs],
|
||||
|
||||
Loading…
Reference in New Issue
Block a user