cka-sim/tracks/cka/bank_sim_b.py

"""Simulator Set B - 17 tasks, fixed set, killer.sh-grade difficulty.

Original scenarios. Weighted toward node-level and control-plane work: etcd flags, kube-proxy
rules, ServiceCIDR, bound tokens, PDB-blocked drains, PV reclaim, upgrade planning, CSR signing.

Run with:  ./exam sim b
"""
from tracks.cka.common import C, NS

Q = []


def task(**kw):
    kw.setdefault("difficulty", "hard")
    kw["set"] = "b"
    Q.append(kw)


# ------------------------------------------------------------------ 301 ----
task(
    id=301, domain="ARCH", title="etcd configuration facts",
    task="""
The security team wants the following about the etcd running on the control-plane node.
Write `{WS}/q301/etcd-info.txt` with exactly these five lines (values after the colon):

```
data-dir: <etcd data directory>
server-key: <path to the server private key>
trusted-ca: <path to the CA used for client certificate verification>
client-cert-auth: <yes|no>
server-cert-expiry: <notAfter value of the etcd server certificate>
```

Take every value from how etcd is actually started, not from memory.
""",
    setup=r"""rm -rf "$WS/q301"; mkdir -p "$WS/q301" """,
    checks=[
        C("data-dir and server-key are correct", 3, r"""
f="$WS/q301/etcd-info.txt"; [ -f "$f" ] || exit 1
cmd=$(j -n kube-system pod -l component=etcd '{.items[0].spec.containers[0].command}')
dd=$(echo "$cmd" | tr ',' '\n' | grep -o 'data-dir=[^"]*' | cut -d= -f2)
sk=$(echo "$cmd" | tr ',' '\n' | grep -o '\-\-key-file=[^"]*' | cut -d= -f2)
grep -q "data-dir: *$dd" "$f" && grep -q "server-key: *$sk" "$f" """),
        C("trusted-ca and client-cert-auth are correct", 3, r"""
f="$WS/q301/etcd-info.txt"
cmd=$(j -n kube-system pod -l component=etcd '{.items[0].spec.containers[0].command}')
ca=$(echo "$cmd" | tr ',' '\n' | grep -o '\-\-trusted-ca-file=[^"]*' | cut -d= -f2)
grep -q "trusted-ca: *$ca" "$f" && grep -Eqi 'client-cert-auth: *(yes|true)' "$f" """),
        C("server-cert-expiry matches the certificate on disk", 3, r"""
exp=$(nx "$CP" 'openssl x509 -in /etc/kubernetes/pki/etcd/server.crt -noout -enddate' | sed 's/notAfter=//')
yr=$(echo "$exp" | awk '{print $4}'); mon=$(echo "$exp" | awk '{print $1}'); day=$(echo "$exp" | awk '{print $2}')
f="$WS/q301/etcd-info.txt"; grep -q "$yr" "$f" 2>/dev/null && grep -qi "$mon" "$f" && grep -q "$day" "$f" """),
    ],
    solution="""
{SHELL_CP}
cat /etc/kubernetes/manifests/etcd.yaml        # every flag etcd runs with
#   --data-dir=/var/lib/etcd
#   --key-file=/etc/kubernetes/pki/etcd/server.key
#   --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
#   --client-cert-auth=true
openssl x509 -in /etc/kubernetes/pki/etcd/server.crt -noout -enddate
# etcd runs as a static pod, so the manifest in /etc/kubernetes/manifests is the source of truth.
# k -n kube-system get pod -l component=etcd -o yaml would show the same command.
""",
)

# ------------------------------------------------------------------ 302 ----
task(
    id=302, domain="NET", title="Service rules on the node", breaks="proxy-rules",
    task="""
Confirm that kube-proxy is programming this cluster.

1. In Namespace `project-proxy` create Pod `proxy-pod` (image `nginx:1-alpine`) and expose it
   internally as Service `proxy-svc` on port **4000** targeting container port 80.
2. From the node where the Pod runs, capture the proxy rules that mention `proxy-svc` into
   `{WS}/q302/rules.txt` (the node uses iptables; `iptables-save` is available as root).
3. Write the Service's ClusterIP to `{WS}/q302/clusterip.txt`.
4. Write the proxy mode kube-proxy is running in to `{WS}/q302/mode.txt`
   (an empty `mode` in its ConfigMap means `iptables`).
""",
    setup=NS + r"""
mkns project-proxy
rm -rf "$WS/q302"; mkdir -p "$WS/q302"
$K -n project-proxy delete pod proxy-pod --ignore-not-found --wait=false >/dev/null
$K -n project-proxy delete svc proxy-svc --ignore-not-found >/dev/null
""",
    checks=[
        C("Pod and Service exist, 4000 -> 80, with endpoints", 3, r"""
eq "$(j -n project-proxy pod proxy-pod '{.status.phase}')" Running \
 && eq "$(j -n project-proxy svc proxy-svc '{.spec.ports[0].port}')" 4000 && eq "$(j -n project-proxy svc proxy-svc '{.spec.ports[0].targetPort}')" 80 \
 && [ -n "$(j -n project-proxy endpointslices -l kubernetes.io/service-name=proxy-svc '{.items[*].endpoints[*].addresses[*]}')" ]"""),
        C("clusterip.txt matches the Service", 2, r"""eq "$(tr -d '[:space:]' < "$WS/q302/clusterip.txt" 2>/dev/null)" "$(j -n project-proxy svc proxy-svc '{.spec.clusterIP}')" """),
        C("rules.txt contains real proxy rules naming the Service and its ClusterIP", 4, r"""
f="$WS/q302/rules.txt"; grep -q 'proxy-svc' "$f" 2>/dev/null && grep -q "$(j -n project-proxy svc proxy-svc '{.spec.clusterIP}')" "$f" && grep -Eq 'KUBE-(SVC|SEP|SERVICES)' "$f" """),
        C("mode.txt says iptables", 2, r"""grep -qi 'iptables' "$WS/q302/mode.txt" 2>/dev/null"""),
    ],
    solution="""
k -n project-proxy run proxy-pod --image=nginx:1-alpine
k -n project-proxy expose pod proxy-pod --name proxy-svc --port 4000 --target-port 80
k -n project-proxy get svc proxy-svc -o jsonpath='{.spec.clusterIP}' > {WS}/q302/clusterip.txt
k -n project-proxy get pod proxy-pod -o wide          # note the node
{SHELL_W1}                                            # the node it landed on
iptables-save | grep proxy-svc                        # -> copy into {WS}/q302/rules.txt
exit
k -n kube-system get cm kube-proxy -o yaml | grep -E '^ *mode:'   # empty -> iptables
echo iptables > {WS}/q302/mode.txt
# Deleting the Service makes the rules disappear within a second - kube-proxy watches the API
# and rewrites the rules on every Service/EndpointSlice change.
""",
)

# ------------------------------------------------------------------ 303 ----
task(
    id=303, domain="NET", title="A second Service CIDR",
    task="""
The cluster is running out of Service IPs.

1. Write the CIDR the cluster currently allocates Service IPs from to `{WS}/q303/current-cidr.txt`.
2. Add a **second** range `10.180.0.0/16` to the cluster by creating a `ServiceCIDR` object named
   `extra-range` (do not remove the existing one).
3. In Namespace `default` create Deployment `cidr-demo` (image `nginx:1-alpine`) and a ClusterIP
   Service `cidr-demo` on port 80 whose ClusterIP is taken **from the new range**
   (request the address explicitly if you need to).
4. Write that Service's ClusterIP to `{WS}/q303/new-ip.txt`.
""",
    setup=r"""
rm -rf "$WS/q303"; mkdir -p "$WS/q303"
$K delete svc cidr-demo --ignore-not-found >/dev/null 2>&1
$K delete deploy cidr-demo --ignore-not-found >/dev/null 2>&1
$K delete servicecidr extra-range --ignore-not-found >/dev/null 2>&1
$K get servicecidr >/dev/null 2>&1 || echo "warn: this cluster has no ServiceCIDR API (needs MultiCIDRServiceAllocator)"
""",
    checks=[
        C("current-cidr.txt matches the apiserver's service range", 2, r"""
sc=$(j -n kube-system pod -l component=kube-apiserver '{.items[0].spec.containers[0].command}' | tr ',' '\n' | grep -o 'service-cluster-ip-range=[^"]*' | cut -d= -f2)
grep -qF "$sc" "$WS/q303/current-cidr.txt" 2>/dev/null"""),
        C("ServiceCIDR extra-range covers 10.180.0.0/16 and is Ready", 4, r"""
j servicecidr extra-range '{.spec.cidrs[*]}' | grep -q '10.180.0.0/16' || exit 1
for i in $(seq 1 10); do [ "$(j servicecidr extra-range '{.status.conditions[?(@.type=="Ready")].status}')" = True ] && exit 0; sleep 2; done; exit 1"""),
        C("Service cidr-demo has an IP from the new range and ready endpoints", 4, r"""
ip=$(j -n default svc cidr-demo '{.spec.clusterIP}'); case "$ip" in 10.180.*) ;; *) exit 1;; esac
[ -n "$(j -n default endpointslices -l kubernetes.io/service-name=cidr-demo '{.items[*].endpoints[?(@.conditions.ready==true)].addresses[*]}')" ]"""),
        C("new-ip.txt records that ClusterIP", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q303/new-ip.txt" 2>/dev/null)" "$(j -n default svc cidr-demo '{.spec.clusterIP}')" """),
    ],
    solution="""
k -n kube-system get pod -l component=kube-apiserver -o yaml | grep service-cluster-ip-range
#   -> e.g. 10.96.0.0/16 ; write it into {WS}/q303/current-cidr.txt
k get servicecidr                        # the built-in "kubernetes" ServiceCIDR
cat <<EOF | k apply -f -
apiVersion: networking.k8s.io/v1
kind: ServiceCIDR
metadata: {name: extra-range}
spec: {cidrs: ["10.180.0.0/16"]}
EOF
k get servicecidr extra-range -o yaml    # wait for Ready=True
k create deploy cidr-demo --image=nginx:1-alpine
k expose deploy cidr-demo --port=80 --cluster-ip=10.180.0.50
k get svc cidr-demo -o jsonpath='{.spec.clusterIP}' > {WS}/q303/new-ip.txt
# New ranges are additive: existing Services keep their IPs, and allocation from a new range
# happens once the ServiceCIDR is Ready. Requesting --cluster-ip makes it deterministic.
""",
)

# ------------------------------------------------------------------ 304 ----
task(
    id=304, domain="ARCH", title="Bound tokens versus legacy tokens",
    task="""
In Namespace `project-token` there is ServiceAccount `reporter`.

1. Issue a **bound token** for it that is valid for exactly 2 hours and save it to
   `{WS}/q304/token.jwt`.
2. Decode the token's payload and write its `exp` and `iat` claims, as
   `exp=<value>` and `iat=<value>` on two lines, to `{WS}/q304/claims.txt`.
3. Also create a **long-lived** token for the same ServiceAccount, as a Secret named
   `reporter-legacy` of the correct type, and confirm the controller populated it.
""",
    setup=NS + r"""
mkns project-token
rm -rf "$WS/q304"; mkdir -p "$WS/q304"
$K -n project-token create sa reporter --dry-run=client -o yaml | $K apply -f - >/dev/null
$K -n project-token delete secret reporter-legacy --ignore-not-found >/dev/null
""",
    checks=[
        C("token.jwt is a JWT issued for the reporter ServiceAccount", 3, r"""
f="$WS/q304/token.jwt"; [ -s "$f" ] || exit 1
python3 - "$f" <<'PY'
import base64,json,sys
t=open(sys.argv[1]).read().strip().split(".")
if len(t)!=3: sys.exit(1)
p=json.loads(base64.urlsafe_b64decode(t[1]+"=="*3))
sub=p.get("sub",""); ko=p.get("kubernetes.io",{}).get("serviceaccount",{}).get("name","")
sys.exit(0 if "reporter" in sub or ko=="reporter" else 1)
PY"""),
        C("the token's lifetime is 2 hours", 3, r"""
python3 - "$WS/q304/token.jwt" <<'PY'
import base64,json,sys
p=json.loads(base64.urlsafe_b64decode(open(sys.argv[1]).read().strip().split(".")[1]+"=="*3))
d=p["exp"]-p["iat"]
sys.exit(0 if 7000 <= d <= 7400 else 1)
PY"""),
        C("claims.txt records exp and iat matching the token", 2, r"""
python3 - "$WS/q304/token.jwt" "$WS/q304/claims.txt" <<'PY'
import base64,json,sys,re
p=json.loads(base64.urlsafe_b64decode(open(sys.argv[1]).read().strip().split(".")[1]+"=="*3))
try: t=open(sys.argv[2]).read()
except OSError: sys.exit(1)
ok=re.search(r"exp\s*=\s*%d"%p["exp"],t) and re.search(r"iat\s*=\s*%d"%p["iat"],t)
sys.exit(0 if ok else 1)
PY"""),
        C("Secret reporter-legacy is a populated service-account-token for reporter", 3, r"""
eq "$(j -n project-token secret reporter-legacy '{.type}')" kubernetes.io/service-account-token \
 && eq "$(j -n project-token secret reporter-legacy '{.metadata.annotations.kubernetes\.io/service-account\.name}')" reporter \
 && [ -n "$(j -n project-token secret reporter-legacy '{.data.token}')" ]"""),
    ],
    solution="""
k -n project-token create token reporter --duration=2h > {WS}/q304/token.jwt
# decode the middle segment (JWTs are base64url, pad it out):
cut -d. -f2 {WS}/q304/token.jwt | base64 -d 2>/dev/null | python3 -m json.tool
#   -> copy exp= and iat= into {WS}/q304/claims.txt
cat <<EOF | k apply -f -
apiVersion: v1
kind: Secret
metadata:
  name: reporter-legacy
  namespace: project-token
  annotations: {kubernetes.io/service-account.name: reporter}
type: kubernetes.io/service-account-token
EOF
k -n project-token get secret reporter-legacy -o yaml     # the controller fills in .data.token
# Bound tokens expire and are tied to an audience; the Secret form never expires - avoid it
# unless something really needs a static token.
""",
)

# ------------------------------------------------------------------ 305 ----
# {W2}, not {W3}: 309 drains {W3} in this same set, and a drain without --force refuses to touch
# bare Pods - while one with --force deletes them, taking this task's answer with it.
task(
    id=305, domain="WORK", title="Schedule without the scheduler",
    task="""
In Namespace `project-orion`:

1. Create Pod `direct-assign` (image `busybox:1.36`, command `sleep 86400`) that runs on node
   `{W2}` **without the scheduler being involved at all**.
2. Create Pod `scheduled` (same image and command) that lands on `{W2}` **through** the scheduler.
3. Write to `{WS}/q305/proof.txt` the value of the field you used for Pod 1, in the form
   `field=<name>`, and on a second line `scheduler=<the schedulerName of the second Pod>`.
""",
    setup=NS + r"""
mkns project-orion
rm -rf "$WS/q305"; mkdir -p "$WS/q305"
$K -n project-orion delete pod direct-assign scheduled --ignore-not-found --wait=false >/dev/null
""",
    # Who set spec.nodeName is recorded permanently in managedFields: a client that submits it
    # (kubectl create/apply/run) owns f:nodeName, while a scheduler binding goes through the
    # pods/binding subresource and leaves no managedFields entry for it (verified on 1.37). Events
    # were the old signal, but plain `get events` has no SOURCE column and they expire after 1h.
    checks=[
        C("direct-assign Running on {W2}", 3, r"""
eq "$(j -n project-orion pod direct-assign '{.status.phase}')" Running && eq "$(j -n project-orion pod direct-assign '{.spec.nodeName}')" "$W2" """),
        C("direct-assign bypassed the scheduler (spec.nodeName came from your manifest)", 3, r"""
$K -n project-orion get pod direct-assign -o json --show-managed-fields 2>/dev/null | python3 -c '
import sys,json
mf=json.load(sys.stdin)["metadata"].get("managedFields",[])
sys.exit(0 if any("f:nodeName" in json.dumps(m.get("fieldsV1",{})) for m in mf if m.get("subresource","")!="status") else 1)'"""),
        C("scheduled Running on {W2}, placed there by default-scheduler (no nodeName in its manifest)", 3, r"""
eq "$(j -n project-orion pod scheduled '{.status.phase}')" Running && eq "$(j -n project-orion pod scheduled '{.spec.nodeName}')" "$W2" \
 && eq "$(j -n project-orion pod scheduled '{.spec.schedulerName}')" default-scheduler || exit 1
$K -n project-orion get pod scheduled -o json --show-managed-fields 2>/dev/null | python3 -c '
import sys,json
mf=json.load(sys.stdin)["metadata"].get("managedFields",[])
sys.exit(1 if any("f:nodeName" in json.dumps(m.get("fieldsV1",{})) for m in mf) else 0)'"""),
        C("proof.txt names nodeName and default-scheduler", 2, r"""
grep -q 'field= *nodeName' "$WS/q305/proof.txt" 2>/dev/null && grep -q 'scheduler= *default-scheduler' "$WS/q305/proof.txt" """),
    ],
    solution="""
cat <<EOF | k apply -f -
apiVersion: v1
kind: Pod
metadata: {name: direct-assign, namespace: project-orion}
spec:
  nodeName: {W2}            # bypasses the scheduler entirely - the kubelet just picks it up
  containers: [{name: c, image: busybox:1.36, command: [sleep, "86400"]}]
EOF
cat <<EOF | k apply -f -
apiVersion: v1
kind: Pod
metadata: {name: scheduled, namespace: project-orion}
spec:
  nodeSelector: {kubernetes.io/hostname: {W2}}     # the scheduler still makes the decision
  containers: [{name: c, image: busybox:1.36, command: [sleep, "86400"]}]
EOF
k -n project-orion describe pod scheduled | grep -A3 Events      # Scheduled  default-scheduler  Successfully assigned...
k -n project-orion describe pod direct-assign | grep -A3 Events  # no Scheduled event at all
printf 'field=nodeName\\nscheduler=default-scheduler\\n' > {WS}/q305/proof.txt
# `k get events` hides who reported an event; -o wide (or describe) shows the SOURCE column.
# Events expire after an hour, so for lasting proof look at who owns spec.nodeName:
#   k -n project-orion get pod direct-assign -o yaml --show-managed-fields | grep -B2 -A2 nodeName
""",
)

# ------------------------------------------------------------------ 306 ----
task(
    id=306, domain="STOR", title="Grow a volume in place",
    task="""
In Namespace `project-orbit`:

1. Create StorageClass `expandable` using provisioner `rancher.io/local-path`,
   `volumeBindingMode: WaitForFirstConsumer`, and allowing volume expansion.
2. Create PVC `growing` (1Gi, RWO) using that class and Pod `grower` (busybox:1.36, `sleep 86400`)
   mounting it at `/data`, so the claim binds.
3. Then grow the claim to **2Gi** without deleting it, and write both sizes the claim reports
   afterwards to `{WS}/q306/capacity.txt`, one per line:
   `requested=<.spec.resources.requests.storage>` and `status=<.status.capacity.storage>`.

(The local-path provisioner has no volume resizer, so on this cluster the status will not catch up
with the request — that gap is what step 3 asks you to observe. With a CSI driver that supports
expansion, the status would follow within seconds.)
""",
    setup=NS + r"""
mkns project-orbit
rm -rf "$WS/q306"; mkdir -p "$WS/q306"
$K -n project-orbit delete pod grower --ignore-not-found --wait=false >/dev/null
$K -n project-orbit delete pvc growing --ignore-not-found >/dev/null
$K delete sc expandable --ignore-not-found >/dev/null
""",
    checks=[
        C("StorageClass expandable: local-path, WaitForFirstConsumer, expansion allowed", 3, r"""
s="$K get sc expandable"
eq "$($s -o jsonpath='{.provisioner}' 2>/dev/null)" rancher.io/local-path && eq "$($s -o jsonpath='{.volumeBindingMode}')" WaitForFirstConsumer \
 && eq "$($s -o jsonpath='{.allowVolumeExpansion}')" true"""),
        C("PVC growing Bound on that class with grower Running", 3, r"""
eq "$(j -n project-orbit pvc growing '{.spec.storageClassName}')" expandable && eq "$(j -n project-orbit pvc growing '{.status.phase}')" Bound \
 && eq "$(j -n project-orbit pod grower '{.status.phase}')" Running"""),
        # local-path cannot resize (verified: ExternalExpanding, capacity never moves), so a bound PV
        # that is still 1Gi under a claim requesting 2Gi is exactly what "grown in place" looks like
        # here. A claim deleted and recreated at 2Gi would have been provisioned a 2Gi volume instead.
        C("claim grown in place: requests 2Gi on the original 1Gi volume", 4, r"""
eq "$(j -n project-orbit pvc growing '{.spec.resources.requests.storage}')" 2Gi || exit 1
pv=$(j -n project-orbit pvc growing '{.spec.volumeName}'); [ -n "$pv" ] || exit 1
eq "$(j pv "$pv" '{.spec.capacity.storage}')" 1Gi"""),
        C("capacity.txt records requested and status sizes as the cluster reports them", 1, r"""
f="$WS/q306/capacity.txt"
eq "$(sed -n 's/^ *requested *= *//p' "$f" 2>/dev/null | tr -d '[:space:]')" 2Gi \
 && eq "$(sed -n 's/^ *status *= *//p' "$f" | tr -d '[:space:]')" "$(j -n project-orbit pvc growing '{.status.capacity.storage}')" """),
    ],
    solution="""
cat <<EOF | k apply -f -
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata: {name: expandable}
provisioner: rancher.io/local-path
volumeBindingMode: WaitForFirstConsumer
allowVolumeExpansion: true
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: growing, namespace: project-orbit}
spec:
  accessModes: [ReadWriteOnce]
  storageClassName: expandable
  resources: {requests: {storage: 1Gi}}
---
apiVersion: v1
kind: Pod
metadata: {name: grower, namespace: project-orbit}
spec:
  volumes: [{name: d, persistentVolumeClaim: {claimName: growing}}]
  containers: [{name: c, image: busybox:1.36, command: [sleep, "86400"], volumeMounts: [{name: d, mountPath: /data}]}]
EOF
k -n project-orbit patch pvc growing -p '{"spec":{"resources":{"requests":{"storage":"2Gi"}}}}'
k -n project-orbit describe pvc growing      # event ExternalExpanding: waiting for an external controller
k -n project-orbit get pvc growing -o jsonpath='requested={.spec.resources.requests.storage}{"\\n"}status={.status.capacity.storage}{"\\n"}' > {WS}/q306/capacity.txt
cat {WS}/q306/capacity.txt                   # requested=2Gi  status=1Gi
# allowVolumeExpansion is the gate: without it the API rejects the patch outright, and shrinking is
# never allowed. Accepting the request is all the API does - a resizer (the CSI driver's
# external-resizer sidecar) then grows the volume and the kubelet the filesystem, and only then does
# .status.capacity change. local-path has no resizer, so here the status stays at 1Gi forever.
""",
)

# ------------------------------------------------------------------ 307 ----
task(
    id=307, domain="TSHOOT", title="Node accepts nothing new", breaks=["w1-block", "w1-taint"],
    task="""
Node `{W1}` is `Ready` and its kubelet is healthy, but no new Pod will schedule onto it.

Find every reason and remove them, so that a Pod with a `nodeSelector` pinning it to `{W1}`
starts successfully. Do not delete or recreate the node, and do not use `nodeName` to work around it.
""",
    setup=r"""
$K cordon "$W1" >/dev/null 2>&1 || true
$K taint node "$W1" maintenance=inprogress:NoSchedule --overwrite >/dev/null 2>&1 || true
$K label node "$W1" kubernetes.io/os- >/dev/null 2>&1 || true
$K label node "$W1" kubernetes.io/os=linux --overwrite >/dev/null 2>&1 || true
""",
    checks=[
        C("node is schedulable (uncordoned)", 3, r"""[ "$(j node "$W1" '{.spec.unschedulable}')" != true ]"""),
        C("the maintenance taint is gone", 3, r"""! j node "$W1" '{.spec.taints}' | grep -q 'maintenance'"""),
        C("a pinned Pod actually runs there", 4, r"""
$K -n default delete pod pin-probe --ignore-not-found --wait=true >/dev/null 2>&1
cat <<EOF | $K apply -f - >/dev/null 2>&1
apiVersion: v1
kind: Pod
metadata: {name: pin-probe, namespace: default}
spec:
  nodeSelector: {kubernetes.io/hostname: $W1}
  containers: [{name: c, image: busybox:1.36, command: [sleep, "120"]}]
EOF
ok=1; for i in $(seq 1 20); do [ "$(j -n default pod pin-probe '{.status.phase}')" = Running ] && { ok=0; break; }; sleep 3; done
$K -n default delete pod pin-probe --wait=false >/dev/null 2>&1; exit $ok"""),
    ],
    solution="""
k describe node {W1} | head -20
#   Taints:             maintenance=inprogress:NoSchedule
#   Unschedulable:      true
k uncordon {W1}
k taint node {W1} maintenance-           # trailing dash removes the taint
k get node {W1} -o jsonpath='{.spec.taints}{"\\n"}{.spec.unschedulable}{"\\n"}'
# Cordon and taints are two independent mechanisms - `kubectl drain` sets the cordon,
# and a taint may have been left by an operator or added by hand. Both must go.
""",
)

# ------------------------------------------------------------------ 308 ----
task(
    id=308, domain="ARCH", title="API resource reference card",
    task="""
Write `{WS}/q308/resources.txt` with one line per resource below, in this exact format:

```
<resource>|<shortname or ->|<apiVersion as served>|<namespaced true|false>
```

for these four: `networkpolicies`, `persistentvolumes`, `storageclasses`, `certificatesigningrequests`.

Take the values from the cluster (`kubectl api-resources`), not from memory. Use `-` when a
resource has no shortname. Lines in the order given above.
""",
    setup=r"""rm -rf "$WS/q308"; mkdir -p "$WS/q308" """,
    checks=[
        C("all four lines present and correctly formatted", 3, r"""
f="$WS/q308/resources.txt"; [ -f "$f" ] || exit 1
for r in networkpolicies persistentvolumes storageclasses certificatesigningrequests; do grep -q "^$r|" "$f" || exit 1; done
[ "$(grep -c '|' "$f")" -ge 4 ]"""),
        C("shortnames and namespaced flags correct", 4, r"""
f="$WS/q308/resources.txt"
grep -Eq '^networkpolicies\|netpol\|networking.k8s.io/v1\|true$' "$f" || exit 1
grep -Eq '^persistentvolumes\|pv\|v1\|false$' "$f" || exit 1
grep -Eq '^storageclasses\|sc\|storage.k8s.io/v1\|false$' "$f" || exit 1
grep -Eq '^certificatesigningrequests\|csr\|certificates.k8s.io/v1\|false$' "$f" || exit 1"""),
        C("order matches the task", 2, r"""
f="$WS/q308/resources.txt"
[ "$(grep -o '^[a-z]*' "$f" | grep . | head -4 | tr '\n' ' ')" = "networkpolicies persistentvolumes storageclasses certificatesigningrequests " ]"""),
    ],
    solution="""
k api-resources | grep -E '^(networkpolicies|persistentvolumes|storageclasses|certificatesigningrequests) '
# NAME  SHORTNAMES  APIVERSION  NAMESPACED  KIND
cat > {WS}/q308/resources.txt <<'EOF'
networkpolicies|netpol|networking.k8s.io/v1|true
persistentvolumes|pv|v1|false
storageclasses|sc|storage.k8s.io/v1|false
certificatesigningrequests|csr|certificates.k8s.io/v1|false
EOF
# k api-resources --namespaced=false narrows to cluster-scoped resources;
# k explain <resource> shows the same apiVersion when you're unsure.
""",
)

# ------------------------------------------------------------------ 309 ----
task(
    id=309, domain="WORK", title="A PodDisruptionBudget that blocks maintenance", breaks="w3-drain",
    task="""
Namespace `project-atlas` runs Deployment `atlas-api` with 2 replicas, and a PodDisruptionBudget
`atlas-api` that currently makes voluntary eviction impossible.

Node `{W3}` must be drained for maintenance, and `atlas-api` must keep at least one Pod serving
at all times.

1. Adjust the PodDisruptionBudget so the drain can complete while still guaranteeing one
   available Pod. Keep the PDB — do not delete it.
2. Drain `{W3}` (DaemonSet Pods aside), then make it schedulable again.
3. Write the value of the PDB's `.status.disruptionsAllowed` after your change to
   `{WS}/q309/allowed.txt`.

Look before you drain: `k get pods -A -o wide --field-selector spec.nodeName={W3}`. A Pod without a
controller blocks a plain drain, and `--force` deletes it for good — if it belongs to another
task, recreate it once the node is back.
""",
    setup=NS + r"""
mkns project-atlas
rm -rf "$WS/q309"; mkdir -p "$WS/q309"
$K uncordon "$W3" >/dev/null 2>&1 || true
$K -n project-atlas delete deploy atlas-api --ignore-not-found >/dev/null
$K -n project-atlas delete pdb atlas-api --ignore-not-found >/dev/null
$K apply -f - >/dev/null <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata: {name: atlas-api, namespace: project-atlas}
spec:
  replicas: 2
  selector: {matchLabels: {app: atlas-api}}
  template:
    metadata: {labels: {app: atlas-api}}
    spec:
      containers: [{name: api, image: nginx:1.27, resources: {requests: {cpu: 10m, memory: 16Mi}}}]
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata: {name: atlas-api, namespace: project-atlas}
spec:
  minAvailable: 2
  selector: {matchLabels: {app: atlas-api}}
EOF
$K -n project-atlas rollout status deploy atlas-api --timeout=120s >/dev/null 2>&1
""",
    checks=[
        C("PDB still exists and now permits exactly one disruption", 3, r"""
p="$K -n project-atlas get pdb atlas-api"
$p >/dev/null 2>&1 || exit 1
ma=$($p -o jsonpath='{.spec.minAvailable}'); mu=$($p -o jsonpath='{.spec.maxUnavailable}')
{ [ "$ma" = 1 ] || [ "$mu" = 1 ]; }"""),
        C("no atlas-api Pod left on the drained node", 3, r"""
n=$($K -n project-atlas get pod -l app=atlas-api -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' 2>/dev/null | grep -c "^$W3$")
eq "$n" 0"""),
        C("both replicas Available and node schedulable again", 3, r"""
eq "$(j -n project-atlas deploy atlas-api '{.status.availableReplicas}')" 2 && [ "$(j node "$W3" '{.spec.unschedulable}')" != true ]"""),
        C("allowed.txt records disruptionsAllowed", 1, r"""grep -Eq '^[0-9]+$' <(tr -dc '0-9\n' < "$WS/q309/allowed.txt" 2>/dev/null | grep .) """),
    ],
    solution="""
k -n project-atlas get pdb atlas-api
#   ALLOWED DISRUPTIONS: 0  -> minAvailable 2 with 2 replicas means nothing may be evicted;
#   a drain would hang forever waiting for the eviction API.
k -n project-atlas patch pdb atlas-api -p '{"spec":{"minAvailable":1}}'
k -n project-atlas get pdb atlas-api        # ALLOWED DISRUPTIONS: 1
k get pods -A -o wide --field-selector spec.nodeName={W3}    # bare pods here? they need --force and are then gone
k drain {W3} --ignore-daemonsets --delete-emptydir-data
k -n project-atlas rollout status deploy atlas-api
k uncordon {W3}
k -n project-atlas get pdb atlas-api -o jsonpath='{.status.disruptionsAllowed}' > {WS}/q309/allowed.txt
""",
)

# ------------------------------------------------------------------ 310 ----
task(
    id=310, domain="NET", title="Route to something outside the cluster",
    task="""
An external database lives at `10.190.7.25` port 5432. Applications should reach it through a
normal in-cluster Service name.

In Namespace `project-orbit`:
1. Create a Service `external-db` (ClusterIP, port 5432) that has **no selector**.
2. Make it resolve to that external address by creating the matching `EndpointSlice`
   named `external-db-1` (port 5432, TCP, addressType IPv4) with the right labels so the
   Service adopts it.
3. Write the ClusterIP of the Service to `{WS}/q310/ip.txt`.

Do not use an ExternalName Service.
""",
    setup=NS + r"""
mkns project-orbit
rm -rf "$WS/q310"; mkdir -p "$WS/q310"
$K -n project-orbit delete svc external-db --ignore-not-found >/dev/null
$K -n project-orbit delete endpointslice external-db-1 --ignore-not-found >/dev/null 2>&1
""",
    checks=[
        C("Service external-db is ClusterIP on 5432 with no selector", 3, r"""
s="$K -n project-orbit get svc external-db"
eq "$($s -o jsonpath='{.spec.type}' 2>/dev/null)" ClusterIP && eq "$($s -o jsonpath='{.spec.ports[0].port}')" 5432 && [ -z "$($s -o jsonpath='{.spec.selector}')" ]"""),
        C("EndpointSlice external-db-1 points at 10.190.7.25:5432", 4, r"""
e="$K -n project-orbit get endpointslice external-db-1"
eq "$($e -o jsonpath='{.addressType}' 2>/dev/null)" IPv4 && $e -o jsonpath='{.endpoints[*].addresses[*]}' | grep -q '10.190.7.25' \
 && eq "$($e -o jsonpath='{.ports[0].port}')" 5432"""),
        C("the slice is linked to the Service by label", 2, r"""eq "$(j -n project-orbit endpointslice external-db-1 '{.metadata.labels.kubernetes\.io/service-name}')" external-db"""),
        C("ip.txt records the Service ClusterIP", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q310/ip.txt" 2>/dev/null)" "$(j -n project-orbit svc external-db '{.spec.clusterIP}')" """),
    ],
    solution="""
cat <<EOF | k apply -f -
apiVersion: v1
kind: Service
metadata: {name: external-db, namespace: project-orbit}
spec:
  ports: [{name: pg, port: 5432, protocol: TCP, targetPort: 5432}]
  # no selector: nothing populates the endpoints automatically, so we do it ourselves
---
apiVersion: discovery.k8s.io/v1
kind: EndpointSlice
metadata:
  name: external-db-1
  namespace: project-orbit
  labels: {kubernetes.io/service-name: external-db}    # this label is the link to the Service
addressType: IPv4
ports: [{name: pg, port: 5432, protocol: TCP}]
endpoints:
  - addresses: ["10.190.7.25"]
    conditions: {ready: true}
EOF
k -n project-orbit get svc external-db -o jsonpath='{.spec.clusterIP}' > {WS}/q310/ip.txt
k -n project-orbit get endpointslice
# ExternalName would return a CNAME instead and gives you no ClusterIP or port mapping.
""",
)

# ------------------------------------------------------------------ 311 ----
task(
    id=311, domain="ARCH", title="Image inventory across the cluster",
    task="""
Produce `{WS}/q311/images.txt`: for every Deployment in every Namespace, one line

```
<namespace>/<deployment> <image>[,<image>...]
```

with the images of its Pod template (init containers included), comma-separated in the order they
appear, and the file sorted by the first column.

Then write the number of **distinct** images in use across those Deployments to
`{WS}/q311/distinct.txt`.
""",
    setup=r"""rm -rf "$WS/q311"; mkdir -p "$WS/q311" """,
    checks=[
        # Graded against the Deployments that existed when the file was written (its mtime): other
        # set-B tasks create Deployments later (303's cidr-demo), and an exact cluster-wide
        # comparison at submit time failed a correct answer whenever one of them came after this.
        C("images.txt matches the cluster as it was when you wrote it", 5, r"""
f="$WS/q311/images.txt"; [ -s "$f" ] || exit 1
$K get deploy -A -o json 2>/dev/null | python3 -c '
import sys,json,os,datetime
f=sys.argv[1]; T=os.path.getmtime(f)
def ts(d): return datetime.datetime.strptime(d["metadata"]["creationTimestamp"],"%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=datetime.timezone.utc).timestamp()
want=set(); later=set()
for d in json.load(sys.stdin)["items"]:
    key="%s/%s"%(d["metadata"]["namespace"],d["metadata"]["name"]); s=d["spec"]["template"]["spec"]
    if ts(d) > T: later.add(key); continue
    want.add("%s %s"%(key,",".join([c["image"] for c in s.get("initContainers",[])]+[c["image"] for c in s["containers"]])))
got={l.strip() for l in open(f) if l.strip()}
extra={g for g in got-want if g.split(" ")[0] in {w.split(" ")[0] for w in want}}   # a real deployment, wrong images
sys.exit(0 if want<=got and not extra else 1)' "$f" """),
        C("distinct.txt has the right count", 3, r"""
f="$WS/q311/images.txt"; [ -s "$f" ] || exit 1
want=$($K get deploy -A -o json 2>/dev/null | python3 -c '
import sys,json,os,datetime
T=os.path.getmtime(sys.argv[1]); s=set()
for d in json.load(sys.stdin)["items"]:
    if datetime.datetime.strptime(d["metadata"]["creationTimestamp"],"%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=datetime.timezone.utc).timestamp() > T: continue
    sp=d["spec"]["template"]["spec"]
    for c in sp.get("initContainers",[])+sp["containers"]: s.add(c["image"])
print(len(s))' "$f")
eq "$(tr -dc '0-9' < "$WS/q311/distinct.txt" 2>/dev/null)" "$want" """),
    ],
    solution="""
k get deploy -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{" "}{range .spec.template.spec.initContainers[*]}{.image}{","}{end}{range .spec.template.spec.containers[*]}{.image}{","}{end}{"\\n"}{end}' \\
  | sed 's/,$//' | sort > {WS}/q311/images.txt
k get deploy -A -o jsonpath='{range .items[*]}{range .spec.template.spec.initContainers[*]}{.image}{"\\n"}{end}{range .spec.template.spec.containers[*]}{.image}{"\\n"}{end}{end}' \\
  | grep . | sort -u | wc -l > {WS}/q311/distinct.txt
# Two `range` blocks over the same pod spec keeps init and normal containers in order.
# A trailing comma is easiest to strip afterwards with sed.
""",
)

# ------------------------------------------------------------------ 312 ----
task(
    id=312, domain="WORK", title="A Job that fails, then one that doesn't",
    task="""
In Namespace `project-orion`:

1. Job `flaky` exists and keeps failing. Let it reach its terminal state, then write the name of
   the condition that marks it failed and its reason, as `<Type>/<Reason>`, to `{WS}/q312/why.txt`.
2. Create a **new** Job `steady` that does the same work correctly: image `busybox:1.36`,
   6 completions, 3 in parallel, `backoffLimit: 2`, restartPolicy Never, each Pod printing
   `ok` and exiting 0.
3. `steady` must complete all 6.
""",
    setup=NS + r"""
mkns project-orion
rm -rf "$WS/q312"; mkdir -p "$WS/q312"
$K -n project-orion delete job flaky steady --ignore-not-found >/dev/null
$K apply -f - >/dev/null <<'EOF'
apiVersion: batch/v1
kind: Job
metadata: {name: flaky, namespace: project-orion}
spec:
  completions: 2
  parallelism: 1
  backoffLimit: 1
  template:
    spec:
      restartPolicy: Never
      containers: [{name: c, image: busybox:1.36, command: [sh, -c, "echo starting; exit 7"]}]
EOF
""",
    checks=[
        C("why.txt names the Failed condition and BackoffLimitExceeded", 3, r"""
f="$WS/q312/why.txt"; grep -qi 'Failed' "$f" 2>/dev/null && grep -qi 'BackoffLimitExceeded' "$f" """),
        C("Job steady: 6 completions, parallelism 3, backoffLimit 2, restartPolicy Never", 3, r"""
j1="$K -n project-orion get job steady"
eq "$($j1 -o jsonpath='{.spec.completions}' 2>/dev/null)" 6 && eq "$($j1 -o jsonpath='{.spec.parallelism}')" 3 \
 && eq "$($j1 -o jsonpath='{.spec.backoffLimit}')" 2 && eq "$($j1 -o jsonpath='{.spec.template.spec.restartPolicy}')" Never"""),
        C("steady completed all 6", 4, r"""for i in $(seq 1 20); do [ "$(j -n project-orion job steady '{.status.succeeded}')" = 6 ] && exit 0; sleep 3; done; exit 1"""),
    ],
    solution="""
k -n project-orion get job flaky -o yaml | grep -A6 conditions
#   type: Failed / reason: BackoffLimitExceeded
k -n project-orion describe job flaky
k -n project-orion logs job/flaky        # exit 7
echo Failed/BackoffLimitExceeded > {WS}/q312/why.txt
k -n project-orion create job steady --image=busybox:1.36 $do -- sh -c "echo ok" > steady.yaml
# set spec.completions: 6, spec.parallelism: 3, spec.backoffLimit: 2
k apply -f steady.yaml
k -n project-orion wait --for=condition=complete job/steady --timeout=120s
""",
)

# ------------------------------------------------------------------ 313 ----
task(
    id=313, domain="NET", title="Two hosts, TLS, and a default backend",
    task="""
Namespace `project-r700` has Services `web-classic` and `web-next` (port 80).

Create Ingress `multi` in that Namespace with class `nginx`:

- host `classic.r700.io`, path `/` (Prefix) → `web-classic:80`, TLS with secret `classic-tls`
- host `next.r700.io`, path `/` (Prefix) → `web-next:80`, TLS with the same secret
- a **default backend** (requests matching no rule) → `web-classic:80`

Also create the TLS secret `classic-tls` from a self-signed certificate whose CN is `classic.r700.io`.
""",
    setup=NS + r"""
mkns project-r700
for s in web-classic web-next; do
  $K -n project-r700 create deploy $s --image=nginx:1.27 --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null
  $K -n project-r700 expose deploy $s --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null
done
$K -n project-r700 delete ingress multi --ignore-not-found >/dev/null
$K -n project-r700 delete secret classic-tls --ignore-not-found >/dev/null
""",
    checks=[
        C("Secret classic-tls is a TLS secret with CN classic.r700.io", 3, r"""
eq "$(j -n project-r700 secret classic-tls '{.type}')" kubernetes.io/tls \
 && j -n project-r700 secret classic-tls '{.data.tls\.crt}' | base64 -d | openssl x509 -noout -subject 2>/dev/null | grep -q 'CN *= *classic.r700.io'"""),
        C("both host rules route to the right Services on /", 3, r"""
$K -n project-r700 get ingress multi -o json 2>/dev/null | python3 -c '
import sys,json
rs=json.load(sys.stdin)["spec"]["rules"]
m={r["host"]:r["http"]["paths"][0] for r in rs if r.get("host")}
ok=all(h in m and m[h]["pathType"]=="Prefix" and m[h]["path"]=="/" for h in ("classic.r700.io","next.r700.io"))
ok=ok and m["classic.r700.io"]["backend"]["service"]["name"]=="web-classic" and m["next.r700.io"]["backend"]["service"]["name"]=="web-next"
sys.exit(0 if ok else 1)'"""),
        C("TLS covers both hosts with classic-tls", 2, r"""
$K -n project-r700 get ingress multi -o json 2>/dev/null | python3 -c '
import sys,json
t=json.load(sys.stdin)["spec"].get("tls",[])
hosts={h for e in t for h in e.get("hosts",[])}; secs={e.get("secretName") for e in t}
sys.exit(0 if {"classic.r700.io","next.r700.io"}<=hosts and secs=={"classic-tls"} else 1)'"""),
        C("defaultBackend points at web-classic:80, class is nginx", 3, r"""
i="$K -n project-r700 get ingress multi"
eq "$($i -o jsonpath='{.spec.ingressClassName}' 2>/dev/null)" nginx \
 && eq "$($i -o jsonpath='{.spec.defaultBackend.service.name}')" web-classic && eq "$($i -o jsonpath='{.spec.defaultBackend.service.port.number}')" 80"""),
    ],
    solution="""
openssl req -x509 -newkey rsa:2048 -nodes -days 365 -subj "/CN=classic.r700.io" -keyout t.key -out t.crt
k -n project-r700 create secret tls classic-tls --cert=t.crt --key=t.key
cat <<EOF | k apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata: {name: multi, namespace: project-r700}
spec:
  ingressClassName: nginx
  defaultBackend: {service: {name: web-classic, port: {number: 80}}}
  tls:
    - hosts: [classic.r700.io, next.r700.io]
      secretName: classic-tls
  rules:
    - host: classic.r700.io
      http: {paths: [{path: /, pathType: Prefix, backend: {service: {name: web-classic, port: {number: 80}}}}]}
    - host: next.r700.io
      http: {paths: [{path: /, pathType: Prefix, backend: {service: {name: web-next, port: {number: 80}}}}]}
EOF
# `kubectl create ingress` can do most of this:
#   k -n project-r700 create ingress multi --class=nginx \\
#     --rule="classic.r700.io/*=web-classic:80,tls=classic-tls" \\
#     --rule="next.r700.io/*=web-next:80,tls=classic-tls" \\
#     --default-backend=web-classic:80
""",
)

# ------------------------------------------------------------------ 314 ----
task(
    id=314, domain="TSHOOT", title="A static pod that never starts", breaks="w3-kubelet",
    task="""
Someone added a static Pod manifest on node `{W3}` for a Pod called `edge-cache`, but no such Pod
ever appears in the cluster and the node is otherwise healthy.

Find out why, fix it on the node, and get the mirror Pod `edge-cache-{W3}` Running in Namespace
`default` with image `nginx:1.27`.

Write the one-line reason it wasn't starting to `{WS}/q314/reason.txt`.
""",
    setup=r"""
rm -rf "$WS/q314"; mkdir -p "$WS/q314"
nx "$W3" 'rm -f /etc/kubernetes/manifests/edge-cache.yaml
d=$(grep staticPodPath /var/lib/kubelet/config.yaml | awk "{print \$2}")
mkdir -p "$d"
cat > "$d/edge-cache.yaml" <<EOF
apiVersion: v1
kind: Pod
metadata:
  name: edge-cache
  namespace: default
spec:
  containers:
    - name: cache
      image: nginx:1.27
      volumeMounts:
        - name: conf
          mountPath: /etc/nginx/conf.d
  volumes:
    - name: conf
      hostPath:
        path: /opt/edge-cache/conf
        type: Directory
EOF
rm -rf /opt/edge-cache
systemctl restart kubelet'
""",
    checks=[
        C("mirror pod edge-cache-<node> is Running with nginx:1.27", 5, r"""
for i in $(seq 1 12); do
  [ "$(j -n default pod "edge-cache-$W3" '{.status.phase}')" = Running ] && [ "$(j -n default pod "edge-cache-$W3" '{.spec.containers[0].image}')" = nginx:1.27 ] && exit 0
  sleep 5
done; exit 1"""),
        # -h: with two file operands GNU grep prefixes every line with the file name, which made
        # awk's $2 the literal word "path:" and this check unpassable.
        C("the hostPath the manifest references now exists on the node", 3, r"""
p=$(nx "$W3" 'grep -h -A2 hostPath /etc/kubernetes/manifests/edge-cache.yaml /etc/kubernetes/static/edge-cache.yaml 2>/dev/null | grep "path:" | head -1' | awk '{print $2}')
[ -n "$p" ] && nx "$W3" "test -d $p" """),
        C("reason.txt mentions the missing host directory", 2, r"""grep -Eqi 'director|hostpath|/opt/edge-cache|not exist|missing' "$WS/q314/reason.txt" 2>/dev/null"""),
    ],
    solution="""
k get pod -A | grep edge-cache          # nothing
{SHELL_W3}
journalctl -u kubelet -n 40 --no-pager | grep -i edge-cache
#   ... failed to start ... hostPath type check failed: /opt/edge-cache/conf is not a directory
grep staticPodPath /var/lib/kubelet/config.yaml
cat <staticPodPath>/edge-cache.yaml     # hostPath /opt/edge-cache/conf with type: Directory
mkdir -p /opt/edge-cache/conf           # type: Directory means the kubelet will NOT create it
exit
k get pod edge-cache-{W3}
echo 'hostPath /opt/edge-cache/conf did not exist and type was Directory' > {WS}/q314/reason.txt
# The alternative fix is type: DirectoryOrCreate, but the task asks for the pod to run as written.
""",
)

# ------------------------------------------------------------------ 315 ----
task(
    id=315, domain="STOR", title="Keep the data when the claim goes",
    task="""
In Namespace `project-orbit`, Pod `keeper` writes to PVC `keeper-claim`, which was provisioned
dynamically by the default StorageClass — whose reclaim policy is `Delete`.

The data must survive the claim being deleted:

1. Without recreating anything, make sure the bound PersistentVolume is **retained** when its claim
   goes away.
2. Write the PV's name to `{WS}/q315/pv.txt`.
3. Delete Pod `keeper` and PVC `keeper-claim`.
4. The PV must still exist afterwards, in phase `Released`.
""",
    setup=NS + r"""
mkns project-orbit
rm -rf "$WS/q315"; mkdir -p "$WS/q315"
$K -n project-orbit delete pod keeper --ignore-not-found --wait=true >/dev/null
$K -n project-orbit delete pvc keeper-claim --ignore-not-found >/dev/null
$K apply -f - >/dev/null <<'EOF'
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: keeper-claim, namespace: project-orbit}
spec: {accessModes: [ReadWriteOnce], resources: {requests: {storage: 200Mi}}}
---
apiVersion: v1
kind: Pod
metadata: {name: keeper, namespace: project-orbit}
spec:
  volumes: [{name: d, persistentVolumeClaim: {claimName: keeper-claim}}]
  containers: [{name: c, image: busybox:1.36, command: [sh, -c, "echo important > /data/keep.txt; sleep 86400"], volumeMounts: [{name: d, mountPath: /data}]}]
EOF
for i in $(seq 1 30); do [ "$(j -n project-orbit pod keeper '{.status.phase}')" = Running ] && break; sleep 2; done
""",
    checks=[
        C("pv.txt names the PV that was bound to keeper-claim", 3, r"""
p=$(tr -d '[:space:]' < "$WS/q315/pv.txt" 2>/dev/null); [ -n "$p" ] || exit 1
eq "$(j pv "$p" '{.spec.claimRef.name}')" keeper-claim"""),
        C("that PV has reclaim policy Retain", 4, r"""eq "$(j pv "$(tr -d '[:space:]' < "$WS/q315/pv.txt")" '{.spec.persistentVolumeReclaimPolicy}')" Retain"""),
        C("Pod and PVC are gone", 2, r"""! $K -n project-orbit get pod keeper >/dev/null 2>&1 && ! $K -n project-orbit get pvc keeper-claim >/dev/null 2>&1"""),
        C("the PV still exists and is Released", 3, r"""
p=$(tr -d '[:space:]' < "$WS/q315/pv.txt"); for i in $(seq 1 10); do [ "$(j pv "$p" '{.status.phase}')" = Released ] && exit 0; sleep 3; done; exit 1"""),
    ],
    solution="""
k -n project-orbit get pvc keeper-claim -o jsonpath='{.spec.volumeName}' | tee {WS}/q315/pv.txt
PV=$(cat {WS}/q315/pv.txt)
k patch pv $PV -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}'
k get pv $PV                       # RECLAIM POLICY: Retain
k -n project-orbit delete pod keeper --now
k -n project-orbit delete pvc keeper-claim
k get pv $PV                       # STATUS: Released - data still on disk
# Patch the PV, not the StorageClass: the class only sets the policy for volumes created later.
# To reuse it, clear .spec.claimRef and it becomes Available again.
""",
)

# ------------------------------------------------------------------ 316 ----
task(
    id=316, domain="ARCH", title="Plan a control-plane upgrade",
    task="""
The control-plane node must be upgraded to the next patch release.

1. Write the cluster's current control-plane version to `{WS}/q316/current.txt` (as `kubectl` reports it).
2. Run the kubeadm command that shows what an upgrade would do, and save its output to
   `{WS}/q316/plan.txt`.
3. Write to `{WS}/q316/steps.sh` the ordered commands you would run to upgrade the control-plane
   node — package install of kubeadm, the kubeadm upgrade step, draining, upgrading kubelet and
   kubectl, restarting the kubelet, and uncordoning — one command per line, in the order they must
   be executed. Do **not** execute them.
""",
    setup=r"""rm -rf "$WS/q316"; mkdir -p "$WS/q316" """,
    checks=[
        C("current.txt matches the server version", 2, r"""
v=$($K version -o json 2>/dev/null | python3 -c 'import sys,json;print(json.load(sys.stdin)["serverVersion"]["gitVersion"])')
grep -qF "$v" "$WS/q316/current.txt" 2>/dev/null"""),
        C("plan.txt is real kubeadm upgrade plan output", 3, r"""grep -Eqi 'upgrade|component|version' "$WS/q316/plan.txt" 2>/dev/null && [ "$(wc -l < "$WS/q316/plan.txt")" -ge 5 ]"""),
        C("steps.sh has the commands in the correct order", 5, r"""
f="$WS/q316/steps.sh"; [ -f "$f" ] || exit 1
grep -q 'kubeadm upgrade apply' "$f" || exit 1
grep -Eq '(apt|yum|dnf).*(install).*kubeadm' "$f" || exit 1
grep -q 'drain' "$f" && grep -q 'uncordon' "$f" || exit 1
grep -Eq '(apt|yum|dnf).*(install).*kubelet' "$f" || exit 1
grep -Eq 'systemctl (restart|daemon-reload)' "$f" || exit 1
n=$(grep -n 'kubeadm upgrade apply' "$f" | head -1 | cut -d: -f1)
ka=$(grep -nE '(apt|yum|dnf).*install.*kubeadm' "$f" | head -1 | cut -d: -f1)
kl=$(grep -nE '(apt|yum|dnf).*install.*kubelet' "$f" | head -1 | cut -d: -f1)
dr=$(grep -n 'drain' "$f" | head -1 | cut -d: -f1)
un=$(grep -n 'uncordon' "$f" | head -1 | cut -d: -f1)
[ "$ka" -lt "$n" ] && [ "$n" -lt "$kl" ] && [ "$dr" -lt "$kl" ] && [ "$kl" -lt "$un" ]"""),
    ],
    solution="""
k version -o json | python3 -c 'import sys,json;print(json.load(sys.stdin)["serverVersion"]["gitVersion"])' > {WS}/q316/current.txt
{SHELL_CP}
kubeadm upgrade plan | tee /tmp/plan.txt      # copy into {WS}/q316/plan.txt
cat > {WS}/q316/steps.sh <<'EOF'
apt-get update && apt-get install -y kubeadm=1.34.1-1.1
kubeadm upgrade plan
kubeadm upgrade apply v1.34.1
kubectl drain <control-plane> --ignore-daemonsets
apt-get install -y kubelet=1.34.1-1.1 kubectl=1.34.1-1.1
systemctl daemon-reload
systemctl restart kubelet
kubectl uncordon <control-plane>
EOF
# Order matters and is the point of this task: kubeadm binary first, then `kubeadm upgrade apply`
# (it upgrades the static pod control plane), then drain, then kubelet/kubectl packages, restart,
# uncordon. Workers use `kubeadm upgrade node` instead of `upgrade apply`.
""",
)

# ------------------------------------------------------------------ 317 ----
# The sabotage switches off ONE controller. It used to point --cluster-signing-cert-file at a
# missing file instead, but a controller that fails to start is fatal to kube-controller-manager
# (verified on 1.37: "Error building controllers" -> exit), so the whole KCM crash-looped and every
# other set-B task that needs a controller (Deployments, Jobs, EndpointSlices, PV phases, PDB
# status, token Secrets) silently stopped working until this task - the last one - was fixed.
task(
    id=317, domain="TSHOOT", title="Certificates stop being issued", breaks="controller-manager",
    task="""
Certificate signing requests are approved but never get a certificate: `kubectl get csr` shows
them `Approved` and never `Approved,Issued`. Everything else in the cluster works.

Find the responsible control-plane component, repair it on the control-plane node, and prove it
works: a CertificateSigningRequest named `issue-probe` (CN `probe`, signer
`kubernetes.io/kube-apiserver-client`, usage `client auth`) that you create and approve must
receive a certificate.
""",
    setup=r"""
$K delete csr issue-probe --ignore-not-found >/dev/null 2>&1
nx "$CP" '
f=/etc/kubernetes/manifests/kube-controller-manager.yaml
grep -q -- "-certificatesigningrequest-signing-controller" $f || sed -i "s|^\( *- --controllers=.*\)$|\1,-certificatesigningrequest-signing-controller|" $f
'
""",
    checks=[
        C("kube-controller-manager runs the CSR signing controller again", 4, r"""
c=$(nx "$CP" 'grep -o -- "--controllers=[^ ]*" /etc/kubernetes/manifests/kube-controller-manager.yaml')
[ -n "$c" ] && ! echo "$c" | grep -Eq -- '-(certificatesigningrequest-signing-controller|csrsigning)(,|$)'"""),
        C("kube-controller-manager still Running after your edit", 3, r"""
for i in $(seq 1 12); do [ "$(j -n kube-system pod -l component=kube-controller-manager '{.items[0].status.phase}')" = Running ] && exit 0; sleep 5; done; exit 1"""),
        C("an approved CSR now receives a certificate", 5, r"""
$K get csr issue-probe >/dev/null 2>&1 || exit 1
$K get csr issue-probe -o jsonpath='{.status.conditions[*].type}' | grep -q Approved || exit 1
for i in $(seq 1 15); do [ -n "$(j csr issue-probe '{.status.certificate}')" ] && exit 0; sleep 4; done; exit 1"""),
    ],
    solution="""
k get csr                                     # Approved - but never Approved,Issued
# Approving is the API's job; ISSUING is a controller's job, and controllers live in
# kube-controller-manager. So: is it running, and is the right controller in it running?
k -n kube-system get pod -l component=kube-controller-manager      # Running - so not a crash
k -n kube-system logs kube-controller-manager-{CP} | grep -i disabled
#   "Warning: controller is disabled" controller="certificatesigningrequest-signing-controller"
{SHELL_CP}
grep -- --controllers /etc/kubernetes/manifests/kube-controller-manager.yaml
#   --controllers=*,bootstrapsigner,tokencleaner,-certificatesigningrequest-signing-controller
vi /etc/kubernetes/manifests/kube-controller-manager.yaml    # remove ,-certificatesigningrequest-signing-controller
exit
k -n kube-system get pod -l component=kube-controller-manager -w   # the kubelet restarts the static pod
openssl genrsa -out probe.key 2048
openssl req -new -key probe.key -subj "/CN=probe" -out probe.csr
cat <<EOF | k apply -f -
apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata: {name: issue-probe}
spec: {request: $(base64 -w0 probe.csr), signerName: kubernetes.io/kube-apiserver-client, usages: [client auth]}
EOF
k certificate approve issue-probe
k get csr issue-probe                          # Approved,Issued
# `--controllers`: '*' = every default-on controller, 'name' adds one, '-name' removes one.
# kubeadm itself adds bootstrapsigner and tokencleaner, which are off by default.
""",
)

QUESTIONS = Q