"""Hard-difficulty CKA tasks (ids 101-122): multi-part, no hints, traps, exact output,
compound failures, collateral-damage checks, exam-style contexts. Modelled on the kind of
difficulty people report from killer.sh, not on its content."""
from tracks.cka.common import C, NS, ns_pick, name_pick

CTX = "Use context: `kubectl config use-context %{ctx}`\n\n"

def ctx_pick(r):
    return r.choice(["k8s-c1-H", "k8s-c2-AC", "k8s-c3-CCC"])

QUESTIONS = [
# ================================================================ ARCH ====
{
 "id": 101, "domain": "ARCH", "difficulty": "hard", "title": "Exact RBAC with an existing over-grant",
 "params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "ci"), "sa": name_pick(r, "ci-bot")},
 "task": CTX + """
ServiceAccount `%{sa}` in namespace `%{ns}` is used by a CI system. Its permissions must be
**exactly**:
- in `%{ns}`: `get`, `list`, `watch`, `patch` on `deployments` and `replicasets` (apps),
  and `create` on `pods/exec`,
- cluster-wide: `list` on `nodes`.

Nothing more. Audit what it can do now and correct it. Use Role `%{sa}-role`, RoleBinding
`%{sa}-rb`, ClusterRole `%{sa}-nodes`, ClusterRoleBinding `%{sa}-crb`.
""",
 "setup": NS + r"""
mkns %{ns}
$K -n %{ns} create sa %{sa} --dry-run=client -o yaml | $K apply -f - >/dev/null
$K -n %{ns} delete role,rolebinding --all >/dev/null 2>&1
$K delete clusterrole %{sa}-nodes --ignore-not-found >/dev/null; $K delete clusterrolebinding %{sa}-crb %{sa}-legacy-admin --ignore-not-found >/dev/null
# trap: someone bound cluster-admin to this SA "temporarily"
$K create clusterrolebinding %{sa}-legacy-admin --clusterrole=cluster-admin --serviceaccount=%{ns}:%{sa} >/dev/null
""",
 "checks": [
  C("can get/list/watch/patch deployments and replicasets in %{ns}", 2, r"""for res in deployments.apps replicasets.apps; do for v in get list watch patch; do eq "$($K auth can-i $v $res -n %{ns} --as=system:serviceaccount:%{ns}:%{sa} 2>/dev/null)" yes || exit 1; done; done"""),
  C("can create pods/exec in %{ns}; can list nodes", 2, r"""eq "$($K auth can-i create pods/exec -n %{ns} --as=system:serviceaccount:%{ns}:%{sa} 2>/dev/null)" yes && eq "$($K auth can-i list nodes --as=system:serviceaccount:%{ns}:%{sa} 2>/dev/null)" yes"""),
  C("no over-grant: cannot delete deployments, cannot get secrets, cannot get nodes", 3, r"""for t in "delete deployments -n %{ns}" "get secrets -n %{ns}" "get nodes" "create pods -n default"; do eq "$($K auth can-i $t --as=system:serviceaccount:%{ns}:%{sa} 2>/dev/null || true)" no || exit 1; done"""),
  C("the cluster-admin binding is gone and named objects exist", 2, r"""! $K get clusterrolebinding %{sa}-legacy-admin >/dev/null 2>&1 && $K -n %{ns} get role %{sa}-role rolebinding %{sa}-rb >/dev/null 2>&1 && $K get clusterrole %{sa}-nodes clusterrolebinding %{sa}-crb >/dev/null 2>&1"""),
 ],
},
{
 "id": 102, "domain": "ARCH", "breaks": "etcd", "difficulty": "hard", "title": "etcd restore",
 "params": lambda r: {"ctx": "k8s-c1-H", "cm": name_pick(r, "restore-proof")},
 "task": CTX + """
An etcd snapshot taken earlier is at `/opt/cka/backup.db` on the control-plane node.
A ConfigMap `%{cm}` in namespace `default` existed when the snapshot was taken and has since
been deleted.

**Restore the cluster's etcd from that snapshot** so the ConfigMap exists again. The cluster
must be healthy afterwards (API server up, all kube-system pods Running).
Do not recreate the ConfigMap by hand — its data must come from the snapshot.
""",
 "setup": r"""
nx "$CP" 'mkdir -p /opt/cka; rm -f /opt/cka/backup.db'
ensure_etcd_tools || { echo "no etcdctl/etcdutl on $CP - fix with ./etcd-tools.sh, then re-seed"; exit 1; }
$K -n default delete cm %{cm} --ignore-not-found >/dev/null
$K -n default create cm %{cm} --from-literal=proof=$(date +%s)-%{cm} >/dev/null
out=$(nx "$CP" 'etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key snapshot save /opt/cka/backup.db' 2>&1) \
  || { echo "snapshot save on $CP failed: $out"; exit 1; }
sleep 2; $K -n default delete cm %{cm} >/dev/null
""",
 "checks": [
  C("ConfigMap %{cm} exists again with its original data", 6, r"""j -n default cm %{cm} '{.data.proof}' | grep -q -- '-%{cm}$'"""),
  C("etcd uses a restored data directory (manifest changed)", 2, r"""! nx "$CP" 'grep -q "path: /var/lib/etcd$" /etc/kubernetes/manifests/etcd.yaml' """),
  C("control plane healthy: etcd, apiserver, scheduler, controller-manager Running", 2, r"""for c in etcd kube-apiserver kube-scheduler kube-controller-manager; do eq "$(j -n kube-system pod -l component=$c '{.items[0].status.phase}')" Running || exit 1; done"""),
 ],
},
{
 "id": 103, "domain": "ARCH", "breaks": "w3-kubelet", "difficulty": "hard", "title": "Static pod on a relocated manifest path",
 "params": lambda r: {"ctx": "k8s-c1-H", "ns": ns_pick(r, "edge"), "pod": name_pick(r, "edge-proxy")},
 "task": CTX + """
Exactly one worker node is labelled `tier=edge`. On that node create a **static pod**:
- name `%{pod}`, namespace `%{ns}` (create the namespace),
- image `nginx:1.27`, requests cpu 50m / memory 32Mi, label `role=edge`.

The mirror pod must be Running. Note that the kubelet on that node may not be configured the
way you expect.
""",
 "setup": NS + r"""
mkns %{ns}
for n in "$W1" "$W2" "$W3"; do $K label node $n tier- >/dev/null 2>&1 || true; done
$K label node "$W3" tier=edge --overwrite >/dev/null
nx "$W3" '
rm -f /etc/kubernetes/manifests/%{pod}*.yaml /etc/kubernetes/static/%{pod}*.yaml 2>/dev/null; mkdir -p /etc/kubernetes/static
sed -i "s|^staticPodPath: .*|staticPodPath: /etc/kubernetes/static|" /var/lib/kubelet/config.yaml
systemctl restart kubelet'
""",
 "checks": [
  C("mirror pod %{pod}-<edge node> Running in %{ns}", 4, r"""eq "$(j -n %{ns} pod "%{pod}-$W3" '{.status.phase}')" Running"""),
  C("image, requests and label correct", 2, r"""p="$K -n %{ns} get pod %{pod}-$W3"; eq "$($p -o jsonpath='{.spec.containers[0].image}' 2>/dev/null)" nginx:1.27 && eq "$($p -o jsonpath='{.spec.containers[0].resources.requests.cpu}')" 50m && eq "$($p -o jsonpath='{.metadata.labels.role}')" edge"""),
  C("manifest lives in the kubelet's actual staticPodPath", 2, r"""d=$(nx "$W3" 'grep staticPodPath /var/lib/kubelet/config.yaml' | awk '{print $2}'); nx "$W3" "grep -ls %{pod} $d/*.yaml" >/dev/null 2>&1"""),
 ],
},
{
 "id": 104, "domain": "ARCH", "difficulty": "hard", "title": "User onboarding to a working kubeconfig",
 "params": lambda r: {"ctx": "k8s-c1-H", "user": r.choice(["maya", "theo", "iris"]), "ns": ns_pick(r, "dev")},
 "task": CTX + """
Onboard user `%{user}` (group `developers`) end to end:
1. Key + CSR with CN=%{user}, O=developers; Kubernetes CSR object `%{user}`, approved, certificate issued.
2. A **self-contained** kubeconfig at `{WS}/q104/%{user}.kubeconfig` (certs embedded, not file paths)
   with a context `%{user}@lab` whose default namespace is `%{ns}`.
3. `%{user}` may do everything to pods, deployments and services in `%{ns}` (edit-level), nothing in other namespaces.

`kubectl --kubeconfig {WS}/q104/%{user}.kubeconfig get pods` must work with no extra flags.
""",
 "setup": NS + r"""mkns %{ns}; rm -rf "$WS/q104"; mkdir -p "$WS/q104"; $K delete csr %{user} --ignore-not-found >/dev/null; $K -n %{ns} delete rolebinding --all >/dev/null 2>&1""",
 "checks": [
  C("CSR %{user} approved with certificate, O=developers", 2, r"""[ -n "$(j csr %{user} '{.status.certificate}')" ] && j csr %{user} '{.status.certificate}' | base64 -d | openssl x509 -noout -subject 2>/dev/null | grep -q 'O *= *developers'"""),
  C("kubeconfig is self-contained and works without flags", 3, r"""f="$WS/q104/%{user}.kubeconfig"; [ -f "$f" ] || exit 1; grep -q 'client-certificate-data' "$f" && grep -q 'certificate-authority-data' "$f" && ! grep -q 'client-certificate:' "$f" && kubectl --kubeconfig "$f" get pods >/dev/null 2>&1"""),
  C("context %{user}@lab defaults to %{ns}", 1, r"""eq "$(kubectl --kubeconfig "$WS/q104/%{user}.kubeconfig" config view -o jsonpath='{.contexts[?(@.name=="%{user}@lab")].context.namespace}')" %{ns}"""),
  C("edit rights in %{ns}, none elsewhere", 2, r"""eq "$($K auth can-i create deployments -n %{ns} --as=%{user} 2>/dev/null)" yes && eq "$($K auth can-i delete pods -n %{ns} --as=%{user} 2>/dev/null)" yes && eq "$($K auth can-i get pods -n default --as=%{user} 2>/dev/null || true)" no && eq "$($K auth can-i get secrets -n kube-system --as=%{user} 2>/dev/null || true)" no"""),
 ],
},
{
 "id": 105, "domain": "ARCH", "difficulty": "hard", "title": "Kustomize with generators and patches",
 "params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "kz"), "rep": r.choice([3, 4]), "lvl": r.choice(["debug", "trace", "warn"])},
 "task": CTX + """
Base at `{WS}/q105/base` (Deployment `web`, Service `web`). Build overlay `{WS}/q105/overlays/prod` that:
- targets namespace `%{ns}` and adds label `tier=prod` to all resources (labels only, not selectors),
- sets replicas to %{rep} via a **strategic merge patch** file `replicas.yaml`,
- adds a `configMapGenerator` named `web-config` with literal `LOG_LEVEL=%{lvl}` (hash suffix enabled),
- injects that ConfigMap into the container via `envFrom` (JSON 6902 patch file `envfrom.yaml`).

Apply with `kubectl apply -k`. The Deployment must reference the hashed ConfigMap name.
""",
 "setup": r"""
rm -rf "$WS/q105"; mkdir -p "$WS/q105/base"
cat > "$WS/q105/base/kustomization.yaml" <<'EOF'
resources: [deployment.yaml, service.yaml]
EOF
cat > "$WS/q105/base/deployment.yaml" <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata: {name: web}
spec:
  replicas: 1
  selector: {matchLabels: {app: web}}
  template:
    metadata: {labels: {app: web}}
    spec:
      containers: [{name: web, image: nginx:1.25, ports: [{containerPort: 80}]}]
EOF
cat > "$WS/q105/base/service.yaml" <<'EOF'
apiVersion: v1
kind: Service
metadata: {name: web}
spec: {selector: {app: web}, ports: [{port: 80}]}
EOF
$K delete ns %{ns} --ignore-not-found --wait=false >/dev/null
""",
 "checks": [
  C("overlay has replicas.yaml (SMP) and envfrom.yaml (JSON6902)", 1, r"""[ -f "$WS/q105/overlays/prod/replicas.yaml" ] && [ -f "$WS/q105/overlays/prod/envfrom.yaml" ] && grep -q 'op:' "$WS/q105/overlays/prod/envfrom.yaml" """),
  C("deployed to %{ns}: %{rep} replicas, tier=prod label, selector untouched", 2, r"""d="$K -n %{ns} get deploy web"; eq "$($d -o jsonpath='{.spec.replicas}' 2>/dev/null)" %{rep} && eq "$($d -o jsonpath='{.metadata.labels.tier}')" prod && [ -z "$($d -o jsonpath='{.spec.selector.matchLabels.tier}')" ]"""),
  C("hashed ConfigMap with LOG_LEVEL=%{lvl}, referenced via envFrom", 3, r"""cm=$(j -n %{ns} deploy web '{.spec.template.spec.containers[0].envFrom[0].configMapRef.name}'); [ -n "$cm" ] && echo "$cm" | grep -Eq '^web-config-[a-z0-9]{6,}$' && eq "$(j -n %{ns} cm "$cm" '{.data.LOG_LEVEL}')" %{lvl}"""),
  C("pods available and env visible", 1, r"""eq "$(j -n %{ns} deploy web '{.status.availableReplicas}')" %{rep} && $K -n %{ns} exec deploy/web -- env 2>/dev/null | grep -q '^LOG_LEVEL=%{lvl}$'"""),
 ],
},
{
 "id": 106, "domain": "ARCH", "difficulty": "hard", "title": "CRD with validation and defaults",
 "params": lambda r: {"ctx": ctx_pick(r), "kind": r.choice(["Rollout", "Canary", "Release"]), "env": r.choice(["prod", "stage"])},
 "task": CTX + """
Create CRD `%{kind}` (group `deploy.example.com`, v1, Namespaced, plural lowercase+s, shortName `%{kind}`-initials) with schema:
- `spec.replicas` integer, **required**, minimum 1, maximum 10,
- `spec.environment` string, enum `prod|stage|dev`, **default** `dev`,
- `spec.image` string, required.
Then:
1. Create `%{kind}` `good` in `default` with replicas 3, image `nginx:1.27`, environment `%{env}`.
2. Save the API server's rejection message for a `%{kind}` with `replicas: 0` to `{WS}/q106/rejected.txt`.
3. Create `%{kind}` `defaulted` with only replicas 2 and image set; its environment must be `dev` after creation.
""",
 "setup": r"""rm -rf "$WS/q106"; mkdir -p "$WS/q106"; k=$(echo %{kind} | tr A-Z a-z); $K delete crd ${k}s.deploy.example.com --ignore-not-found >/dev/null""",
 "checks": [
  C("CRD exists, Namespaced, replicas has min/max and is required", 2, r"""k=$(echo %{kind} | tr A-Z a-z); c="$K get crd ${k}s.deploy.example.com"; eq "$($c -o jsonpath='{.spec.scope}' 2>/dev/null)" Namespaced && s=$($c -o jsonpath='{.spec.versions[0].schema.openAPIV3Schema.properties.spec}'); echo "$s" | grep -q '"minimum":1' && echo "$s" | grep -q '"maximum":10' && echo "$s" | grep -q '"required":\[' """),
  C("good created with environment %{env}", 1, r"""k=$(echo %{kind} | tr A-Z a-z); eq "$(j -n default ${k}s.deploy.example.com good '{.spec.environment}')" %{env}"""),
  C("replicas 0 is rejected and the message was saved", 2, r"""grep -qiE 'minimum|greater than|>= 1|invalid' "$WS/q106/rejected.txt" 2>/dev/null"""),
  C("defaulting works: defaulted.spec.environment == dev", 2, r"""k=$(echo %{kind} | tr A-Z a-z); eq "$(j -n default ${k}s.deploy.example.com defaulted '{.spec.environment}')" dev"""),
 ],
},
{
 "id": 122, "domain": "ARCH", "breaks": "certs", "difficulty": "hard", "title": "Renew control-plane certificates",
 "params": lambda r: {"ctx": "k8s-c1-H"},
 "task": CTX + """
Security policy requires the control-plane certificates to be renewed now.
On the control-plane node, renew **all** kubeadm-managed certificates, make every control-plane
component pick up the new certificates, and update the admin kubeconfig on the node
(`/etc/kubernetes/admin.conf`) so it uses the renewed client certificate.
Write the new `apiserver` certificate's expiry (as printed by `kubeadm certs check-expiration`) to `{WS}/q122/expiry.txt`.
""",
 "setup": r"""rm -rf "$WS/q122"; mkdir -p "$WS/q122"; nx "$CP" 'openssl x509 -in /etc/kubernetes/pki/apiserver.crt -noout -startdate' > "$WS/.q122-before" """,
 "checks": [
  C("apiserver certificate was renewed (newer notBefore)", 3, r"""b=$(sed 's/notBefore=//' "$WS/.q122-before"); a=$(nx "$CP" 'openssl x509 -in /etc/kubernetes/pki/apiserver.crt -noout -startdate' | sed 's/notBefore=//'); [ "$(date -d "$a" +%s)" -gt "$(date -d "$b" +%s)" ]"""),
  C("all components restarted and Running with the new certs (apiserver serving cert matches file)", 3, r"""
for c in kube-apiserver kube-scheduler kube-controller-manager etcd; do eq "$(j -n kube-system pod -l component=$c '{.items[0].status.phase}')" Running || exit 1; done
srv=$(nx "$CP" 'echo | openssl s_client -connect 127.0.0.1:6443 2>/dev/null | openssl x509 -noout -serial'); f=$(nx "$CP" 'openssl x509 -in /etc/kubernetes/pki/apiserver.crt -noout -serial'); [ -n "$srv" ] && eq "$srv" "$f" """),
  C("admin.conf uses the renewed client cert", 2, r"""nx "$CP" 'grep client-certificate-data /etc/kubernetes/admin.conf | awk "{print \$2}" | base64 -d | openssl x509 -noout -startdate' | sed 's/notBefore=//' > /tmp/q122a; b=$(sed 's/notBefore=//' "$WS/.q122-before"); [ "$(date -d "$(cat /tmp/q122a)" +%s)" -gt "$(date -d "$b" +%s)" ]"""),
  C("expiry.txt matches kubeadm's report for apiserver", 1, r"""exp=$(nx "$CP" 'kubeadm certs check-expiration 2>/dev/null' | awk '$1=="apiserver"{print $2,$3,$4,$5,$6}'); [ -n "$exp" ] && grep -qF "$exp" "$WS/q122/expiry.txt" 2>/dev/null"""),
 ],
},
{
 "id": 124, "domain": "ARCH", "difficulty": "hard", "title": "Prepare a node join without shell access to the control plane",
 "params": lambda r: {"ctx": "k8s-c1-H"},
 "task": CTX + """
A new worker node is about to join this cluster, but you can only hand the networking team a
**file** — no shell access to the control plane.

1. On the control-plane node, create a **new** bootstrap token with TTL `2h`.
2. Compute the CA certificate's discovery hash **yourself** with openssl — the same
   SHA-256-of-the-SubjectPublicKeyInfo value `kubeadm join --discovery-token-ca-cert-hash`
   checks against. Do not just copy whatever `kubeadm token create --print-join-command` prints.
3. Determine the API server's real address as seen **from inside the cluster network** — not the
   `127.0.0.1:<port>` your local kubeconfig uses to reach it from this machine, which a joining
   node cannot use.
4. Write exactly these three lines to `{WS}/q124/node-join.env`:
```
JOIN_TOKEN=<token>
CA_HASH=sha256:<hash>
API_ENDPOINT=<ip>:<port>
```
Every value must be genuine: plugged into `kubeadm join $API_ENDPOINT --token $JOIN_TOKEN
--discovery-token-ca-cert-hash $CA_HASH` right now, it would work.
""",
 "setup": r"""
rm -rf "$WS/q124"; mkdir -p "$WS/q124"
for t in $($K -n kube-system get secrets -o jsonpath='{.items[?(@.type=="bootstrap.kubernetes.io/token")].metadata.name}'); do $K -n kube-system delete secret $t >/dev/null; done
""",
 "checks": [
  C("a real, unexpired bootstrap token", 2, r"""
tok=$(grep -oE '^JOIN_TOKEN=[a-z0-9]{6}\.[a-z0-9]{16}' "$WS/q124/node-join.env" 2>/dev/null | cut -d= -f2); [ -n "$tok" ] || exit 1
id=${tok%%.*}; exp=$(j -n kube-system secret bootstrap-token-$id '{.data.expiration}' | base64 -d 2>/dev/null); [ -n "$exp" ] || exit 1
[ "$(date -d "$exp" +%s)" -gt "$(date +%s)" ]"""),
  C("CA_HASH computed correctly (matches the real CA's SPKI SHA-256)", 3, r"""
want=$(nx "$CP" "openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'")
got=$(grep -oE '^CA_HASH=sha256:[0-9a-f]{64}' "$WS/q124/node-join.env" 2>/dev/null | cut -d= -f2 | cut -d: -f2)
[ -n "$want" ] && eq "$got" "$want" """),
  C("API_ENDPOINT is the real in-cluster address, not the local kubeconfig's", 2, r"""
want=$($K get endpoints kubernetes -o jsonpath='{.subsets[0].addresses[0].ip}:{.subsets[0].ports[0].port}' 2>/dev/null)
got=$(grep -oE '^API_ENDPOINT=.*' "$WS/q124/node-join.env" 2>/dev/null | cut -d= -f2 | tr -d '[:space:]')
[ -n "$want" ] && eq "$got" "$want" """),
 ],
},
# ================================================================ WORK ====
{
 "id": 107, "domain": "WORK", "breaks": "w1-taint", "difficulty": "hard", "title": "Spread, tolerate, prioritise — under a quota",
 "params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "spread"), "dep": name_pick(r, "gateway"), "key": r.choice(["pool", "lane"]), "val": r.choice(["ingress", "edge"])},
 "task": CTX + """
Node `{W1}` is tainted `%{key}=%{val}:NoSchedule`. In namespace `%{ns}` create Deployment `%{dep}`
(nginx:1.27, **3 replicas**) such that:
- exactly one pod runs on each worker node (required pod anti-affinity on `kubernetes.io/hostname`),
- it tolerates the taint on `{W1}`,
- it uses PriorityClass `%{dep}-prio` (value 50000, create it),
- requests cpu 100m / memory 64Mi.

All 3 pods must be Running. The namespace has a ResourceQuota — respect it or adjust it, but the
quota must still cap pods in the namespace.
""",
 "setup": NS + r"""
mkns %{ns}
$K taint node "$W1" %{key}- >/dev/null 2>&1 || true; $K taint node "$W1" %{key}=%{val}:NoSchedule >/dev/null
$K -n %{ns} delete deploy %{dep} --ignore-not-found >/dev/null; $K delete priorityclass %{dep}-prio --ignore-not-found >/dev/null
$K apply -f - >/dev/null <<EOF
apiVersion: v1
kind: ResourceQuota
metadata: {name: %{ns}-quota, namespace: %{ns}}
spec: {hard: {pods: "2", requests.cpu: "250m"}}
EOF
""",
 "checks": [
  C("3 pods available, one per worker", 3, r"""eq "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" 3 && [ "$(j -n %{ns} pod -l app=%{dep} '{.items[*].spec.nodeName}' | tr ' ' '\n' | sort -u | wc -l)" = 3 ]"""),
  C("required anti-affinity on hostname + toleration for %{key}", 2, r"""t=$(j -n %{ns} deploy %{dep} '{.spec.template.spec}'); echo "$t" | grep -q 'requiredDuringSchedulingIgnoredDuringExecution' && echo "$t" | grep -q '"topologyKey":"kubernetes.io/hostname"' && echo "$t" | grep -q '"key":"%{key}"' """),
  C("PriorityClass %{dep}-prio (50000) in use", 1, r"""eq "$(j priorityclass %{dep}-prio '{.value}')" 50000 && eq "$(j -n %{ns} pod -l app=%{dep} '{.items[0].spec.priority}')" 50000"""),
  C("quota still exists and caps pods (>=3)", 2, r"""p=$(j -n %{ns} quota %{ns}-quota '{.spec.hard.pods}'); [ -n "$p" ] && [ "$p" -ge 3 ]"""),
 ],
},
{
 "id": 108, "domain": "WORK", "difficulty": "hard", "title": "Rollback to a specific revision, then pause",
 "params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "rel"), "dep": name_pick(r, "portal")},
 "task": CTX + """
Deployment `%{dep}` in `%{ns}` has gone through several releases; the current one is broken.
1. Roll it back to the revision that used image `nginx:1.25` (not simply the previous revision).
2. Scale it to 3 replicas and wait until available.
3. **Pause** the rollout, then change the image to `nginx:1.27` and add env `RELEASE=next`.
   Because it is paused, no new ReplicaSet may be created and the running pods must stay on `nginx:1.25`.
Leave it paused.
""",
 "setup": NS + r"""
mkns %{ns}; $K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null
$K -n %{ns} create deploy %{dep} --image=nginx:1.25 >/dev/null
$K -n %{ns} rollout status deploy %{dep} --timeout=90s >/dev/null 2>&1
$K -n %{ns} set image deploy %{dep} nginx=nginx:1.26 >/dev/null; $K -n %{ns} rollout status deploy %{dep} --timeout=90s >/dev/null 2>&1
$K -n %{ns} set image deploy %{dep} nginx=nginx:1.27 >/dev/null; $K -n %{ns} rollout status deploy %{dep} --timeout=90s >/dev/null 2>&1
$K -n %{ns} set image deploy %{dep} nginx=nginx:9.99-broken >/dev/null
""",
 "checks": [
  C("running pods use nginx:1.25 and 3 are available", 3, r"""eq "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" 3 && for i in $(j -n %{ns} pod -l app=%{dep} '{.items[*].spec.containers[0].image}'); do eq "$i" nginx:1.25 || exit 1; done"""),
  C("deployment is paused with template image nginx:1.27 and RELEASE=next", 2, r"""d="$K -n %{ns} get deploy %{dep}"; eq "$($d -o jsonpath='{.spec.paused}' 2>/dev/null)" true && eq "$($d -o jsonpath='{.spec.template.spec.containers[0].image}')" nginx:1.27 && eq "$($d -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="RELEASE")].value}')" next"""),
  C("no ReplicaSet exists for the paused template (1.27 + RELEASE)", 2, r"""! $K -n %{ns} get rs -o json 2>/dev/null | python3 -c '
import sys,json
for rs in json.load(sys.stdin)["items"]:
    c=rs["spec"]["template"]["spec"]["containers"][0]
    if c["image"]=="nginx:1.27" and any(e.get("name")=="RELEASE" for e in c.get("env",[])): sys.exit(0)
sys.exit(1)'"""),
 ],
},
{
 "id": 109, "domain": "WORK", "difficulty": "hard", "title": "Native sidecar with probes and an exact report",
 "params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "side"), "pod": name_pick(r, "collector")},
 "task": CTX + """
Create pod `%{pod}` in `%{ns}`:
- a **native sidecar** (an init container with `restartPolicy: Always`) named `agent` (busybox:1.36)
  that appends `$(date)` to `/var/log/agent.log` every second,
- main container `app` (nginx:1.27) with a readiness probe (HTTP GET / on 80, period 5s)
  and a liveness probe (TCP 80),
- shared `emptyDir` `logs` mounted at `/var/log` in both,
- `terminationGracePeriodSeconds: 5`.
When it is Running, write the names of **all** containers of the pod (init and regular), sorted,
one per line, to `{WS}/q109/containers.txt`, and the pod's `.status.podIP` to `{WS}/q109/ip.txt`.
""",
 "setup": NS + r"""mkns %{ns}; rm -rf "$WS/q109"; mkdir -p "$WS/q109"; $K -n %{ns} delete pod %{pod} --ignore-not-found --wait=false >/dev/null""",
 "checks": [
  C("native sidecar: initContainer agent with restartPolicy Always", 2, r"""eq "$(j -n %{ns} pod %{pod} '{.spec.initContainers[?(@.name=="agent")].restartPolicy}')" Always"""),
  C("probes on app as specified", 2, r"""c='{.spec.containers[?(@.name=="app")]'; eq "$(j -n %{ns} pod %{pod} "$c.readinessProbe.httpGet.path}")" / && eq "$(j -n %{ns} pod %{pod} "$c.readinessProbe.periodSeconds}")" 5 && eq "$(j -n %{ns} pod %{pod} "$c.livenessProbe.tcpSocket.port}")" 80"""),
  C("Running, shared log written by agent, grace period 5", 2, r"""eq "$(j -n %{ns} pod %{pod} '{.status.phase}')" Running && eq "$(j -n %{ns} pod %{pod} '{.spec.terminationGracePeriodSeconds}')" 5 && [ -n "$($K -n %{ns} exec %{pod} -c app -- sh -c 'tail -1 /var/log/agent.log' 2>/dev/null)" ]"""),
  C("report files exact", 2, r"""want=$( (j -n %{ns} pod %{pod} '{.spec.initContainers[*].name}'; echo; j -n %{ns} pod %{pod} '{.spec.containers[*].name}') | tr ' ' '\n' | grep . | sort); [ "$(grep . "$WS/q109/containers.txt" 2>/dev/null | sort)" = "$want" ] && eq "$(tr -d '[:space:]' < "$WS/q109/ip.txt" 2>/dev/null)" "$(j -n %{ns} pod %{pod} '{.status.podIP}')" """),
 ],
},
{
 "id": 110, "domain": "WORK", "difficulty": "hard", "title": "Cluster audit to exact files",
 "params": lambda r: {"ctx": "k8s-c1-H"},
 "task": CTX + """
Produce three files, contents **exactly** as specified (no headers, sorted with `sort`, one per line):
1. `{WS}/q110/no-readiness.txt` — `<namespace>/<pod>` for every pod in the cluster whose first
   container has **no** readiness probe. Exclude namespace `kube-system`.
2. `{WS}/q110/per-node.txt` — `<node> <count>` of Running pods per node, all namespaces.
3. `{WS}/q110/images.txt` — every distinct container image running in namespace `kube-system`.
""",
 "setup": r"""rm -rf "$WS/q110"; mkdir -p "$WS/q110" """,
 "checks": [
  C("no-readiness.txt exact", 3, r"""want=$($K get pods -A -o json 2>/dev/null | python3 -c '
import sys,json
for p in json.load(sys.stdin)["items"]:
    if p["metadata"]["namespace"]=="kube-system": continue
    if not p["spec"]["containers"][0].get("readinessProbe"): print(p["metadata"]["namespace"]+"/"+p["metadata"]["name"])' | sort); [ "$(grep . "$WS/q110/no-readiness.txt" 2>/dev/null | sort)" = "$want" ]"""),
  C("per-node.txt exact", 2, r"""want=$($K get pods -A -o json 2>/dev/null | python3 -c '
import sys,json,collections
c=collections.Counter(p["spec"].get("nodeName") for p in json.load(sys.stdin)["items"] if p["status"]["phase"]=="Running" and p["spec"].get("nodeName"))
for n in sorted(c): print(n, c[n])'); [ "$(grep . "$WS/q110/per-node.txt" 2>/dev/null | sort)" = "$(echo "$want" | sort)" ]"""),
  C("images.txt exact", 2, r"""want=$(j -n kube-system pods '{.items[*].spec.containers[*].image}' | tr ' ' '\n' | sort -u); [ "$(grep . "$WS/q110/images.txt" 2>/dev/null | sort -u)" = "$want" ]"""),
 ],
},
{
 "id": 121, "domain": "WORK", "difficulty": "hard", "title": "The context's namespace",
 "params": lambda r: {"ctx": "k8s-c2-AC", "ns": ns_pick(r, "ac"), "dep": name_pick(r, "ledger")},
 "task": CTX + """
In the **current namespace of that context** (do not pass `-n`), create Deployment `%{dep}`
(busybox:1.36, `sleep 3600`, 2 replicas, label `owner=ac`) and a ClusterIP Service `%{dep}`
on port 9000 targeting port 9000. Then write the namespace name you deployed into, as reported
by `kubectl config view --minify`, to `{WS}/q121/ns.txt`.
""",
 "setup": NS + r"""mkns %{ns}; rm -rf "$WS/q121"; mkdir -p "$WS/q121"; $K -n %{ns} delete deploy,svc %{dep} --ignore-not-found >/dev/null; $K -n default delete deploy,svc %{dep} --ignore-not-found >/dev/null""",
 "checks": [
  C("Deployment %{dep} in %{ns} (the context's namespace), 2 available, owner=ac", 3, r"""eq "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" 2 && eq "$(j -n %{ns} deploy %{dep} '{.spec.template.metadata.labels.owner}')" ac"""),
  C("Service %{dep} port 9000 in the same namespace", 1, r"""eq "$(j -n %{ns} svc %{dep} '{.spec.ports[0].port}')" 9000"""),
  C("nothing was created in namespace default by mistake", 1, r"""! $K -n default get deploy %{dep} >/dev/null 2>&1"""),
  C("ns.txt has the namespace", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q121/ns.txt" 2>/dev/null)" %{ns}"""),
 ],
},
# ================================================================= NET ====
{
 "id": 111, "domain": "NET", "difficulty": "hard", "title": "Zero-trust namespace",
 "params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "zt"), "from": ns_pick(r, "front")},
 "task": CTX + """
Namespace `%{ns}` runs `api` (label `app=api`, port 8080) and `db` (label `app=db`, port 5432).
Implement zero trust with NetworkPolicies:
1. `default-deny`: deny all ingress **and** egress for every pod in `%{ns}`.
2. `allow-dns`: every pod may egress to DNS (UDP/TCP 53) in `kube-system` only.
3. `api-ingress`: `api` accepts ingress on 8080 only from pods in namespaces labelled `team=%{from}`.
4. `api-to-db`: `api` may egress to `db` on 5432; `db` accepts ingress on 5432 from `api` only.
Remove anything that contradicts this. Label namespace `%{from}` appropriately.
""",
 "setup": NS + r"""
mkns %{ns}; mkns %{from}
$K -n %{ns} create deploy api --image=nginx:1.25 --port=8080 --dry-run=client -o yaml | $K apply -f - >/dev/null
$K -n %{ns} create deploy db --image=nginx:1.25 --port=5432 --dry-run=client -o yaml | $K apply -f - >/dev/null
$K -n %{ns} delete netpol --all >/dev/null 2>&1
# trap: an allow-all left behind
$K apply -f - >/dev/null <<EOF
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: temp-allow-all, namespace: %{ns}}
spec: {podSelector: {}, policyTypes: [Ingress, Egress], ingress: [{}], egress: [{}]}
EOF
$K label ns %{from} team- >/dev/null 2>&1 || true
""",
 "checks": [
  C("default-deny covers Ingress and Egress for all pods; allow-all removed", 2, r"""n="$K -n %{ns} get netpol default-deny"; eq "$($n -o jsonpath='{.spec.podSelector}' 2>/dev/null)" '{}' && $n -o jsonpath='{.spec.policyTypes}' | grep -q Ingress && $n -o jsonpath='{.spec.policyTypes}' | grep -q Egress && [ -z "$($n -o jsonpath='{.spec.ingress}')" ] && [ -z "$($n -o jsonpath='{.spec.egress}')" ] && ! $K -n %{ns} get netpol temp-allow-all >/dev/null 2>&1"""),
  C("allow-dns: egress 53 UDP+TCP to kube-system only", 2, r"""$K -n %{ns} get netpol allow-dns -o json 2>/dev/null | python3 -c '
import sys,json; s=json.load(sys.stdin)["spec"]; e=s.get("egress",[])
ok=s.get("podSelector")=={} and any({p.get("protocol") for p in r.get("ports",[]) if str(p.get("port"))=="53"}>={"UDP","TCP"} and all("kube-system" in json.dumps(t.get("namespaceSelector",{})) for t in r.get("to",[])) and r.get("to") for r in e)
sys.exit(0 if ok else 1)'"""),
  C("api-ingress: from namespaces team=%{from} on 8080 only; namespace labelled", 2, r"""eq "$(j ns %{from} '{.metadata.labels.team}')" %{from} && $K -n %{ns} get netpol api-ingress -o json 2>/dev/null | python3 -c '
import sys,json; s=json.load(sys.stdin)["spec"]
ok=s["podSelector"].get("matchLabels",{}).get("app")=="api" and len(s.get("ingress",[]))==1 and all(str(p.get("port"))=="8080" for p in s["ingress"][0].get("ports",[])) and s["ingress"][0].get("ports") and all(t.get("namespaceSelector",{}).get("matchLabels",{}).get("team")=="%{from}" for t in s["ingress"][0].get("from",[])) and s["ingress"][0].get("from")
sys.exit(0 if ok else 1)'"""),
  C("api-to-db: api egress to db:5432 and db ingress from api:5432", 2, r"""$K -n %{ns} get netpol -o json 2>/dev/null | python3 -c '
import sys,json; items=json.load(sys.stdin)["items"]; eg=ing=False
for n in items:
    s=n["spec"]; app=s["podSelector"].get("matchLabels",{}).get("app")
    for r in s.get("egress",[]):
        if app=="api" and any(t.get("podSelector",{}).get("matchLabels",{}).get("app")=="db" for t in r.get("to",[])) and any(str(p.get("port"))=="5432" for p in r.get("ports",[])): eg=True
    for r in s.get("ingress",[]):
        if app=="db" and any(t.get("podSelector",{}).get("matchLabels",{}).get("app")=="api" for t in r.get("from",[])) and any(str(p.get("port"))=="5432" for p in r.get("ports",[])): ing=True
sys.exit(0 if eg and ing else 1)'"""),
 ],
},
{
 "id": 112, "domain": "NET", "difficulty": "hard", "title": "TLS Ingress with a generated certificate",
 "params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "tls"), "host": r.choice(["secure.lab.local", "portal.lab.local", "admin.lab.local"])},
 "task": CTX + """
Namespace `%{ns}` has Deployment+Service `web` (port 80).
1. Generate a self-signed certificate (RSA 2048, 365 days) with CN `%{host}` and store it as a
   Secret `%{host}-tls` of type `kubernetes.io/tls`.
2. Create Ingress `web` (class `nginx`) serving host `%{host}` with that TLS secret, path `/` Prefix → `web:80`,
   and the annotation `nginx.ingress.kubernetes.io/ssl-redirect: "true"`.
3. Write the certificate's SHA-256 fingerprint (`openssl x509 -fingerprint -sha256`, the value only) to `{WS}/q112/fp.txt`.
""",
 "setup": NS + r"""mkns %{ns}; rm -rf "$WS/q112"; mkdir -p "$WS/q112"; $K -n %{ns} create deploy web --image=nginx:1.25 --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null; $K -n %{ns} expose deploy web --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null; $K -n %{ns} delete secret %{host}-tls --ignore-not-found >/dev/null; $K -n %{ns} delete ingress web --ignore-not-found >/dev/null""",
 "checks": [
  C("Secret %{host}-tls is kubernetes.io/tls with CN %{host}", 2, r"""eq "$(j -n %{ns} secret %{host}-tls '{.type}')" kubernetes.io/tls && j -n %{ns} secret %{host}-tls '{.data.tls\.crt}' | base64 -d | openssl x509 -noout -subject 2>/dev/null | grep -q 'CN *= *%{host}'"""),
  C("Ingress web: class nginx, host, TLS secret, ssl-redirect annotation", 3, r"""i="$K -n %{ns} get ingress web"; eq "$($i -o jsonpath='{.spec.ingressClassName}' 2>/dev/null)" nginx && eq "$($i -o jsonpath='{.spec.tls[0].secretName}')" %{host}-tls && eq "$($i -o jsonpath='{.spec.tls[0].hosts[0]}')" %{host} && eq "$($i -o jsonpath='{.spec.rules[0].host}')" %{host} && eq "$($i -o jsonpath='{.spec.rules[0].http.paths[0].backend.service.name}')" web && eq "$($i -o jsonpath='{.metadata.annotations.nginx\.ingress\.kubernetes\.io/ssl-redirect}')" true"""),
  C("fp.txt matches the secret's certificate fingerprint", 2, r"""fp=$(j -n %{ns} secret %{host}-tls '{.data.tls\.crt}' | base64 -d | openssl x509 -noout -fingerprint -sha256 2>/dev/null | sed 's/.*=//'); [ -n "$fp" ] && grep -qiF "$fp" "$WS/q112/fp.txt" 2>/dev/null"""),
 ],
},
{
 "id": 113, "domain": "NET", "difficulty": "hard", "title": "Gateway API canary with header match",
 "params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "canary"), "host": r.choice(["shop", "pay", "cart"]), "w": r.choice([10, 20, 25])},
 "task": CTX + """
Gateway API CRDs are installed. Namespace `%{ns}` has Services `v1` and `v2` (port 80).
Create in `%{ns}`:
- Gateway `%{host}-gw` (class `%{host}-gc`, create the GatewayClass, controllerName `example.com/gw`)
  with two HTTP listeners: `web` for `%{host}.example.com` and `api` for `api.%{host}.example.com`, both port 80.
- HTTPRoute `%{host}-route` attached to listener `web` only (`sectionName`), with:
  - a rule matching header `X-Canary: always` → `v2` (100%),
  - a default rule splitting `v1` %{w2}% / `v2` %{w}% by weight.
""",
 "params_post": None,
 "setup": NS + r"""
mkns %{ns}
$K get crd gateways.gateway.networking.k8s.io >/dev/null 2>&1 || { curl -sfL --max-time 60 https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/standard-install.yaml -o /tmp/gwapi.yaml && $K apply -f /tmp/gwapi.yaml >/dev/null 2>&1; } || echo "warn: could not install Gateway API CRDs"
for v in v1 v2; do $K -n %{ns} create deploy $v --image=nginx:1.25 --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null; $K -n %{ns} expose deploy $v --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null; done
$K delete gatewayclass %{host}-gc --ignore-not-found >/dev/null 2>&1; $K -n %{ns} delete gateway %{host}-gw --ignore-not-found >/dev/null 2>&1; $K -n %{ns} delete httproute %{host}-route --ignore-not-found >/dev/null 2>&1
""",
 "checks": [
  C("Gateway with listeners web and api on the right hostnames", 2, r"""g="$K -n %{ns} get gateway %{host}-gw"; eq "$($g -o jsonpath='{.spec.gatewayClassName}' 2>/dev/null)" %{host}-gc && eq "$($g -o jsonpath='{.spec.listeners[?(@.name=="web")].hostname}')" %{host}.example.com && eq "$($g -o jsonpath='{.spec.listeners[?(@.name=="api")].hostname}')" api.%{host}.example.com"""),
  C("route attaches to listener web only (sectionName)", 1, r"""eq "$(j -n %{ns} httproute %{host}-route '{.spec.parentRefs[0].sectionName}')" web && [ -z "$(j -n %{ns} httproute %{host}-route '{.spec.parentRefs[1]}')" ]"""),
  C("header rule X-Canary: always -> v2", 2, r"""$K -n %{ns} get httproute %{host}-route -o json 2>/dev/null | python3 -c '
import sys,json; r=json.load(sys.stdin)["spec"]["rules"]
ok=any(any(any(h.get("name","").lower()=="x-canary" and h.get("value")=="always" for h in m.get("headers",[])) for m in x.get("matches",[])) and x["backendRefs"][0]["name"]=="v2" and len(x["backendRefs"])==1 for x in r)
sys.exit(0 if ok else 1)'"""),
  C("default rule weights v1 %{w2} / v2 %{w}", 2, r"""$K -n %{ns} get httproute %{host}-route -o json 2>/dev/null | python3 -c '
import sys,json; r=json.load(sys.stdin)["spec"]["rules"]
ok=any(not x.get("matches") or all(not m.get("headers") for m in x["matches"]) and {b["name"]:b.get("weight") for b in x["backendRefs"]}=={"v1":%{w2},"v2":%{w}} for x in r if len(x.get("backendRefs",[]))==2)
sys.exit(0 if ok else 1)'"""),
 ],
},
{
 "id": 114, "domain": "NET", "breaks": "coredns", "difficulty": "hard", "title": "DNS down: three faults", "disruptive": True,
 "params": lambda r: {"ctx": "k8s-c1-H"},
 "task": CTX + """
Nothing in the cluster resolves DNS. Restore it. There is more than one cause.
When done, `nslookup kubernetes.default` and `nslookup kubernetes.io` must succeed from a pod
in `default`, CoreDNS must have 2 available replicas, and no NetworkPolicy may remain in `kube-system`.
""",
 "setup": r"""
$K -n kube-system get cm coredns -o jsonpath='{.data.Corefile}' > /tmp/Corefile.q114
sed -i 's|forward \. /etc/resolv.conf|forward . 10.255.255.254|' /tmp/Corefile.q114
$K -n kube-system create cm coredns --from-file=Corefile=/tmp/Corefile.q114 --dry-run=client -o yaml | $K apply -f - >/dev/null
$K -n kube-system scale deploy coredns --replicas=0 >/dev/null
$K apply -f - >/dev/null <<EOF
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: maintenance-lockdown, namespace: kube-system}
spec: {podSelector: {matchLabels: {k8s-app: kube-dns}}, policyTypes: [Ingress], ingress: []}
EOF
""",
 "checks": [
  C("Corefile forwards to /etc/resolv.conf", 2, r"""j -n kube-system cm coredns '{.data.Corefile}' | grep -q 'forward \. /etc/resolv.conf'"""),
  C("CoreDNS has 2 available replicas", 2, r"""for i in $(seq 1 12); do [ "$(j -n kube-system deploy coredns '{.status.availableReplicas}')" = 2 ] && exit 0; sleep 5; done; exit 1"""),
  C("no NetworkPolicy in kube-system", 1, r"""[ "$($K -n kube-system get netpol --no-headers 2>/dev/null | wc -l)" = 0 ]"""),
  C("resolution works from a pod", 3, r"""
$K -n default delete pod dns-probe --ignore-not-found --wait=true >/dev/null 2>&1
$K -n default run dns-probe --image=busybox:1.36 --restart=Never -- sh -c 'for i in 1 2 3 4 5 6; do nslookup kubernetes.default >/dev/null 2>&1 && nslookup kubernetes.io >/dev/null 2>&1 && exit 0; sleep 5; done; exit 1' >/dev/null 2>&1
for i in $(seq 1 20); do ph=$(j -n default pod dns-probe '{.status.phase}'); [ "$ph" = Succeeded ] && { $K -n default delete pod dns-probe --wait=false >/dev/null; exit 0; }; [ "$ph" = Failed ] && break; sleep 3; done
$K -n default delete pod dns-probe --wait=false >/dev/null; exit 1"""),
 ],
},
# ================================================================ STOR ====
{
 "id": 115, "domain": "STOR", "difficulty": "hard", "title": "Bind the right local volume",
 "params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "lv"), "pvc": name_pick(r, "fast-data")},
 "task": CTX + """
Two PersistentVolumes of class `local-fast` exist. Only one of them lives on the node labelled
`disk=nvme`. Create PVC `%{pvc}` in `%{ns}` (RWO, 1Gi, class `local-fast`) that binds to **that**
PV — use a label selector, do not edit the PVs — and pod `consumer` (busybox:1.36, `sleep 3600`)
mounting it at `/data`. The pod must run on the nvme node. Do not delete either PV.
""",
 "setup": NS + r"""
mkns %{ns}
for n in "$W1" "$W2" "$W3"; do $K label node $n disk- >/dev/null 2>&1 || true; done
$K label node "$W2" disk=nvme --overwrite >/dev/null
$K -n %{ns} delete pod consumer --ignore-not-found --wait=false >/dev/null; $K -n %{ns} delete pvc %{pvc} --ignore-not-found >/dev/null
$K delete pv lv-hdd lv-nvme --ignore-not-found >/dev/null
for pair in "lv-hdd:$W1:hdd:2Gi" "lv-nvme:$W2:nvme:1Gi"; do IFS=: read name node kind size <<<"$pair"
nx "$node" "mkdir -p /mnt/$name"
$K apply -f - >/dev/null <<EOF
apiVersion: v1
kind: PersistentVolume
metadata: {name: $name, labels: {media: $kind}}
spec:
  capacity: {storage: $size}
  accessModes: [ReadWriteOnce]
  persistentVolumeReclaimPolicy: Retain
  storageClassName: local-fast
  local: {path: /mnt/$name}
  nodeAffinity: {required: {nodeSelectorTerms: [{matchExpressions: [{key: kubernetes.io/hostname, operator: In, values: [$node]}]}]}}
EOF
done
""",
 "checks": [
  C("PVC uses a selector and is Bound to lv-nvme", 3, r"""[ -n "$(j -n %{ns} pvc %{pvc} '{.spec.selector}')" ] && eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}')" lv-nvme && eq "$(j -n %{ns} pvc %{pvc} '{.status.phase}')" Bound"""),
  C("consumer Running on the nvme node with /data mounted", 2, r"""eq "$(j -n %{ns} pod consumer '{.status.phase}')" Running && eq "$(j -n %{ns} pod consumer '{.spec.nodeName}')" "$W2" && j -n %{ns} pod consumer '{.spec.containers[0].volumeMounts[*].mountPath}' | grep -qw /data"""),
  C("both PVs still exist, unmodified affinity", 1, r"""eq "$(j pv lv-hdd '{.spec.nodeAffinity.required.nodeSelectorTerms[0].matchExpressions[0].values[0]}')" "$W1" && eq "$(j pv lv-nvme '{.spec.nodeAffinity.required.nodeSelectorTerms[0].matchExpressions[0].values[0]}')" "$W2" """),
 ],
},
{
 "id": 116, "domain": "STOR", "difficulty": "hard", "title": "Migrate data between claims",
 "params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "mig"), "n": r.choice([7, 11, 13])},
 "task": CTX + """
In `%{ns}`, PVC `old-data` holds files a running pod `legacy` wrote (`/data`). Storage is being
replaced:
1. Create PVC `new-data` (default StorageClass, 1Gi, RWO).
2. Copy **everything** from `old-data` to `new-data` using a Job named `migrate` (busybox:1.36).
   The Job must complete; use whatever mounts you need.
3. Re-point pod `legacy` to `new-data` (same name, same namespace, mount at `/data`) and get it Running.
4. Delete PVC `old-data` afterwards. Its PV must be released, not deleted (check its reclaim policy first).
""",
 "setup": NS + r"""
mkns %{ns}
$K -n %{ns} delete pod legacy --ignore-not-found --wait=true >/dev/null; $K -n %{ns} delete job migrate --ignore-not-found >/dev/null; $K -n %{ns} delete pvc old-data new-data --ignore-not-found >/dev/null
$K apply -f - >/dev/null <<EOF
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: old-data, namespace: %{ns}}
spec: {accessModes: [ReadWriteOnce], resources: {requests: {storage: 1Gi}}}
---
apiVersion: v1
kind: Pod
metadata: {name: legacy, namespace: %{ns}}
spec:
  containers: [{name: app, image: busybox:1.36, command: [sh, -c, "for i in \$(seq 1 %{n}); do echo record-\$i > /data/file-\$i.txt; done; sleep 3600"], volumeMounts: [{name: d, mountPath: /data}]}]
  volumes: [{name: d, persistentVolumeClaim: {claimName: old-data}}]
EOF
for i in $(seq 1 30); do [ "$(j -n %{ns} pod legacy '{.status.phase}')" = Running ] && break; sleep 2; done
pv=$(j -n %{ns} pvc old-data '{.spec.volumeName}'); [ -n "$pv" ] && $K patch pv $pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' >/dev/null
echo "$pv" > "$WS/.q116-pv"
""",
 "checks": [
  C("Job migrate completed", 2, r"""eq "$(j -n %{ns} job migrate '{.status.succeeded}')" 1"""),
  C("legacy Running on new-data with all %{n} files intact", 3, r"""eq "$(j -n %{ns} pod legacy '{.status.phase}')" Running && eq "$(j -n %{ns} pod legacy '{.spec.volumes[?(@.name!="kube-api-access")].persistentVolumeClaim.claimName}' | tr ' ' '\n' | grep -v '^$' | head -1)" new-data && eq "$($K -n %{ns} exec legacy -- sh -c 'ls /data/file-*.txt | wc -l' 2>/dev/null | tr -d ' ')" %{n} && eq "$($K -n %{ns} exec legacy -- cat /data/file-%{n}.txt 2>/dev/null)" record-%{n}"""),
  C("old-data deleted; its PV Released (not deleted)", 2, r"""! $K -n %{ns} get pvc old-data >/dev/null 2>&1 && pv=$(cat "$WS/.q116-pv" 2>/dev/null) && eq "$(j pv $pv '{.status.phase}')" Released"""),
 ],
},
# ============================================================== TSHOOT ====
{
 "id": 117, "domain": "TSHOOT", "breaks": "w2-kubelet", "difficulty": "hard", "title": "Worker node: three things wrong", "disruptive": True,
 "params": lambda r: {"ctx": "k8s-c1-H"},
 "task": CTX + """
Node `{W2}` is unusable. Make it fully usable again: `Ready`, schedulable, kubelet running and
enabled, and a new pod pinned to it must reach Running. Nothing else on the cluster may be changed.
""",
 "setup": r"""
$K cordon "$W2" >/dev/null 2>&1 || true
nx "$W2" '
sed -i "s|clientCAFile: /etc/kubernetes/pki/ca.crt|clientCAFile: /etc/kubernetes/pki/ca-wrong.crt|" /var/lib/kubelet/config.yaml
systemctl disable kubelet >/dev/null 2>&1; systemctl stop kubelet'
""",
 "checks": [
  C("node Ready", 3, r"""for i in $(seq 1 12); do [ "$(j node "$W2" '{.status.conditions[?(@.type=="Ready")].status}')" = True ] && exit 0; sleep 5; done; exit 1"""),
  C("node uncordoned (collateral check)", 2, r"""[ "$(j node "$W2" '{.spec.unschedulable}')" != true ]"""),
  C("kubelet active and enabled, config points at a real CA", 2, r"""eq "$(nx "$W2" 'systemctl is-active kubelet')" active && eq "$(nx "$W2" 'systemctl is-enabled kubelet')" enabled && f=$(nx "$W2" 'grep clientCAFile /var/lib/kubelet/config.yaml' | awk '{print $2}') && nx "$W2" "test -f $f" """),
  C("a pod pinned to the node runs", 2, r"""
$K -n default delete pod node-probe --ignore-not-found --wait=true >/dev/null 2>&1
$K -n default run node-probe --image=busybox:1.36 --restart=Never --overrides="{\"spec\":{\"nodeName\":\"$W2\"}}" -- sleep 120 >/dev/null 2>&1
ok=1; for i in $(seq 1 20); do [ "$(j -n default pod node-probe '{.status.phase}')" = Running ] && { ok=0; break; }; sleep 3; done
$K -n default delete pod node-probe --wait=false >/dev/null 2>&1; exit $ok"""),
 ],
},
{
 "id": 118, "domain": "TSHOOT", "breaks": "scheduler", "difficulty": "hard", "title": "Two control-plane components down", "disruptive": True,
 "params": lambda r: {"ctx": "k8s-c1-H"},
 "task": CTX + """
New Deployments create no pods, and pods that do get created stay `Pending`. Two different
control-plane components are failing for two different reasons. Repair both on the control-plane node.
Afterwards a new Deployment must reach Available within a minute.
""",
 "setup": r"""
nx "$CP" '
f=/etc/kubernetes/manifests/kube-scheduler.yaml; grep -q "^ *- kube-scheduler$" $f && sed -i "s|^\( *\)- kube-scheduler$|\1- kube-schedulerr|" $f
g=/etc/kubernetes/manifests/kube-controller-manager.yaml; grep -q "image: registry.k8s.io/kube-controller-manager:v" $g && sed -i "s|\(image: registry.k8s.io/kube-controller-manager:v[0-9.]*\)|\1-broken|" $g
'
""",
 "checks": [
  C("kube-scheduler manifest repaired and Running", 3, r"""nx "$CP" 'grep -q "^ *- kube-scheduler$" /etc/kubernetes/manifests/kube-scheduler.yaml' && eq "$(j -n kube-system pod -l component=kube-scheduler '{.items[0].status.phase}')" Running"""),
  C("kube-controller-manager image repaired and Running", 3, r"""! nx "$CP" 'grep -q "\-broken" /etc/kubernetes/manifests/kube-controller-manager.yaml' && eq "$(j -n kube-system pod -l component=kube-controller-manager '{.items[0].status.phase}')" Running"""),
  C("a new Deployment becomes Available", 3, r"""
$K -n default delete deploy cp-probe --ignore-not-found --wait=true >/dev/null 2>&1
$K -n default create deploy cp-probe --image=busybox:1.36 -- sleep 300 >/dev/null 2>&1
ok=1; for i in $(seq 1 20); do [ "$(j -n default deploy cp-probe '{.status.availableReplicas}')" = 1 ] && { ok=0; break; }; sleep 3; done
$K -n default delete deploy cp-probe --wait=false >/dev/null 2>&1; exit $ok"""),
 ],
},
{
 "id": 119, "domain": "TSHOOT", "difficulty": "hard", "title": "Application down for four reasons",
 "params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "checkout"), "dep": name_pick(r, "checkout")},
 "task": CTX + """
Deployment `%{dep}` in `%{ns}` should have 3 Available replicas and Service `%{dep}` should route
to them on port 80. Right now nothing works. Fix it **in place** — the Deployment must keep its
UID (do not delete and recreate it) and the Service must keep its ClusterIP.
""",
 "setup": NS + r"""
mkns %{ns}
$K -n %{ns} delete deploy,svc,quota,secret --all >/dev/null 2>&1
$K apply -f - >/dev/null <<EOF
apiVersion: v1
kind: ResourceQuota
metadata: {name: tight, namespace: %{ns}}
spec: {hard: {pods: "1"}}
---
apiVersion: v1
kind: Secret
metadata: {name: %{dep}-env, namespace: %{ns}}
stringData: {API_KEY: abc123}
---
apiVersion: apps/v1
kind: Deployment
metadata: {name: %{dep}, namespace: %{ns}}
spec:
  replicas: 3
  selector: {matchLabels: {app: %{dep}}}
  template:
    metadata: {labels: {app: %{dep}}}
    spec:
      containers:
        - name: web
          image: nginx:1.27-nope
          ports: [{containerPort: 80}]
          env: [{name: DB_URL, valueFrom: {secretKeyRef: {name: %{dep}-env, key: DATABASE_URL}}}]
          resources: {requests: {cpu: 10m, memory: 16Mi}}
---
apiVersion: v1
kind: Service
metadata: {name: %{dep}, namespace: %{ns}}
spec:
  selector: {app: %{dep}-web}
  ports: [{port: 80, targetPort: 8080}]
EOF
j -n %{ns} deploy %{dep} '{.metadata.uid}' > "$WS/.q119-uid"; j -n %{ns} svc %{dep} '{.spec.clusterIP}' > "$WS/.q119-ip"
""",
 "checks": [
  C("3 replicas Available", 3, r"""for i in $(seq 1 6); do [ "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" = 3 ] && exit 0; sleep 5; done; exit 1"""),
  C("Service routes: endpoints ready, targetPort 80", 2, r"""eq "$(j -n %{ns} svc %{dep} '{.spec.ports[0].targetPort}')" 80 && [ -n "$(j -n %{ns} endpointslices -l kubernetes.io/service-name=%{dep} '{.items[*].endpoints[?(@.conditions.ready==true)].addresses[*]}')" ]"""),
  C("Deployment UID and Service ClusterIP unchanged (fixed in place)", 2, r"""eq "$(j -n %{ns} deploy %{dep} '{.metadata.uid}')" "$(cat "$WS/.q119-uid")" && eq "$(j -n %{ns} svc %{dep} '{.spec.clusterIP}')" "$(cat "$WS/.q119-ip")" """),
  C("quota still present and permits the 3 pods", 1, r"""p=$(j -n %{ns} quota tight '{.spec.hard.pods}'); [ -n "$p" ] && [ "$p" -ge 3 ]"""),
 ],
},
{
 "id": 120, "domain": "TSHOOT", "difficulty": "hard", "title": "Cluster facts, exactly",
 "params": lambda r: {"ctx": "k8s-c1-H", "node": r.choice(["{W1}", "{W2}", "{W3}"])},
 "task": CTX + """
Write `{WS}/q120/facts.txt` with these lines, values taken from the running cluster:
```
pod-cidr=<cluster pod CIDR as configured on kube-controller-manager>
service-cidr=<service CIDR as configured on kube-apiserver>
dns-ip=<ClusterIP of the kube-dns Service>
proxy-mode=<kube-proxy mode from its ConfigMap (empty means iptables — write iptables)>
%{node}-pod-cidr=<.spec.podCIDR of that node>
```
""",
 "setup": r"""rm -rf "$WS/q120"; mkdir -p "$WS/q120" """,
 "checks": [
  C("pod-cidr and service-cidr", 2, r"""f="$WS/q120/facts.txt"; pc=$(j -n kube-system pod -l component=kube-controller-manager '{.items[0].spec.containers[0].command}' | tr ',' '\n' | grep -o 'cluster-cidr=[^"]*' | cut -d= -f2); sc=$(j -n kube-system pod -l component=kube-apiserver '{.items[0].spec.containers[0].command}' | tr ',' '\n' | grep -o 'service-cluster-ip-range=[^"]*' | cut -d= -f2); grep -qx "pod-cidr=$pc" "$f" 2>/dev/null && grep -qx "service-cidr=$sc" "$f" """),
  C("dns-ip and proxy-mode", 2, r"""f="$WS/q120/facts.txt"; d=$(j -n kube-system svc kube-dns '{.spec.clusterIP}'); m=$(j -n kube-system cm kube-proxy '{.data.config\.conf}' | grep '^mode:' | awk '{print $2}' | tr -d '"'); [ -n "$m" ] || m=iptables; grep -qx "dns-ip=$d" "$f" 2>/dev/null && grep -qx "proxy-mode=$m" "$f" """),
  C("%{node} pod CIDR", 1, r"""grep -qx "%{node}-pod-cidr=$(j node %{node} '{.spec.podCIDR}')" "$WS/q120/facts.txt" 2>/dev/null"""),
 ],
},
{
 "id": 123, "domain": "TSHOOT", "breaks": "apiserver", "difficulty": "hard", "title": "Nothing works: kube-apiserver itself is down", "disruptive": True,
 "params": lambda r: {"ctx": "k8s-c1-H"},
 "task": CTX + """
Every `kubectl` command from every context times out or refuses to connect. This is not a
network problem — a control-plane component itself is failing to start. `kubectl` is unusable
until you fix it, so investigate directly on the control-plane node's containers.

Repair the failing component on the control-plane node (`{SHELL_CP}`). Afterwards, `kubectl get
nodes` must work again, and every other control-plane component must still be healthy.
""",
 "setup": r"""
nx "$CP" '
f=/etc/kubernetes/manifests/kube-apiserver.yaml
grep -q "^ *- kube-apiserver$" $f && sed -i "s|^\( *\)- kube-apiserver$|\1- kube-apiserverr|" $f
'
""",
 "checks": [
  C("kube-apiserver manifest repaired and Running", 4, r"""nx "$CP" 'grep -q "^ *- kube-apiserver$" /etc/kubernetes/manifests/kube-apiserver.yaml' && eq "$(j -n kube-system pod -l component=kube-apiserver '{.items[0].status.phase}')" Running"""),
  C("cluster fully reachable again (get nodes, create a pod)", 4, r"""
$K get nodes >/dev/null 2>&1 || exit 1
$K -n default delete pod api-probe --ignore-not-found --wait=true >/dev/null 2>&1
$K -n default run api-probe --image=busybox:1.36 --restart=Never -- sleep 60 >/dev/null 2>&1
ok=1; for i in $(seq 1 20); do [ "$(j -n default pod api-probe '{.status.phase}')" = Running ] && { ok=0; break; }; sleep 3; done
$K -n default delete pod api-probe --wait=false >/dev/null 2>&1; exit $ok"""),
  C("etcd, scheduler and controller-manager unaffected (no collateral damage)", 2, r"""for c in etcd kube-scheduler kube-controller-manager; do eq "$(j -n kube-system pod -l component=$c '{.items[0].status.phase}')" Running || exit 1; done"""),
 ],
},
]

# derived params: w2 = 100 - w for the canary split
for _q in QUESTIONS:
    if _q["id"] == 113:
        _orig = _q["params"]
        _q["params"] = lambda r, _o=_orig: (lambda p: {**p, "w2": 100 - p["w"]})(_o(r))