"""Simulator Set D - 17 tasks, fixed set, killer.sh-style.

Original scenarios, not copies of any commercial simulator. They exercise the techniques the
killer.sh CKA environment is known for that sets A-C do not already cover: saying how every
control-plane piece is run, cluster facts to an exact file, readiness gated on a Service,
`--sort-by` scripts, event forensics around kube-proxy with crictl, API-resource and RBAC census,
a kubelet whose systemd unit points at the wrong binary, Secrets mounted read-only and renamed into
env vars, a static Pod on the control plane behind a Service, kubelet client/serving certificate
forensics, control-plane-only scheduling, stopping the scheduler and scheduling by hand, Helm
release management (including a release stuck in pending-install), StatefulSet storage, the
namespaceSelector-AND-podSelector NetworkPolicy trap (enforced here - kindnet enforces), a
dnsPolicy that silently breaks name resolution, and a claim Pending on an access mode.

Every value a grader compares against is derived from the live cluster, never hardcoded, and every
task was seeded, solved and graded on an isolated kind cluster before it went in.

Run with:  ./exam sim d
"""
from tracks.cka.common import C, NS

Q = []
CTX = "Use context: `kubectl config use-context %{ctx}`\n\n"


def task(**kw):
    kw.setdefault("difficulty", "hard")
    kw.setdefault("params", lambda r: {"ctx": "k8s-c1-H"})
    kw["task"] = CTX + kw["task"].lstrip("\n")
    kw["set"] = "d"
    Q.append(kw)


# ------------------------------------------------------------------ 501 ----
task(
    id=501, domain="ARCH", title="How the control plane is run",
    task="""
The operations handbook has to say how each part of this cluster's control plane is run. Write
`{WS}/q501/controlplane.txt` with exactly these six lines, each value one of `process`,
`static-pod`, `pod`, `deployment` or `daemonset`:

```
kubelet: <value>
kube-apiserver: <value>
kube-scheduler: <value>
kube-controller-manager: <value>
etcd: <value>
dns: <value> <name of that object>
```

`dns` is whatever serves cluster DNS, for example `dns: daemonset some-name`.
""",
    setup=r"""rm -rf "$WS/q501"; mkdir -p "$WS/q501" """,
    checks=[
        C("kubelet is a process", 1, r"""
f="$WS/q501/controlplane.txt"; [ -s "$f" ] || exit 1
want=process; [ "$(nx "$CP" 'systemctl is-active kubelet' </dev/null)" = active ] || want=unknown
eq "$(sed -n 's/^ *kubelet *: *//p' "$f" | head -1 | tr -d '[:space:]' | tr A-Z a-z)" "$want" """),
        C("apiserver, scheduler, controller-manager and etcd are static pods", 3, r"""
f="$WS/q501/controlplane.txt"
# a kubeadm control plane: the manifests live in the kubelet's staticPodPath on the node
nx "$CP" 'ls /etc/kubernetes/manifests/kube-apiserver.yaml' </dev/null >/dev/null 2>&1 || exit 1
for c in kube-apiserver kube-scheduler kube-controller-manager etcd; do
  eq "$(sed -n "s/^ *$c *: *//p" "$f" | head -1 | tr -d '[:space:]' | tr A-Z a-z)" static-pod || exit 1
done"""),
        C("dns names the right kind of object and its name", 2, r"""
f="$WS/q501/controlplane.txt"
sel=$(j -n kube-system svc kube-dns '{.spec.selector.k8s-app}')
own=$(j -n kube-system pod -l k8s-app=$sel '{.items[0].metadata.ownerReferences[0].kind}')
name=$(j -n kube-system pod -l k8s-app=$sel '{.items[0].metadata.ownerReferences[0].name}')
if [ "$own" = ReplicaSet ]; then want="deployment $(j -n kube-system rs "$name" '{.metadata.ownerReferences[0].name}')"
else want="$(echo "$own" | tr A-Z a-z) $name"; fi
eq "$(sed -n 's/^ *dns *: *//p' "$f" | head -1 | tr -s ' ' | tr A-Z a-z | sed 's/ *$//')" "$want" """),
    ],
    solution="""
{SHELL_CP}
systemctl status kubelet | head -3          # a systemd service: a plain process on the node
ls /etc/kubernetes/manifests                # etcd, kube-apiserver, kube-controller-manager, kube-scheduler
exit
k -n kube-system get pod -o wide            # static pods carry the node name as a suffix...
k -n kube-system get pod kube-apiserver-{CP} -o jsonpath='{.metadata.ownerReferences[0].kind}{"\\n"}'   # ...and are owned by the Node
k -n kube-system get deploy,ds              # coredns is a Deployment (kube-proxy, kindnet are DaemonSets)
cat > {WS}/q501/controlplane.txt <<'EOF'
kubelet: process
kube-apiserver: static-pod
kube-scheduler: static-pod
kube-controller-manager: static-pod
etcd: static-pod
dns: deployment coredns
EOF
# The Service is called kube-dns for historical reasons; what actually answers is the coredns
# Deployment. Follow the chain svc selector -> pod -> ownerReferences when unsure.
""",
)

# ------------------------------------------------------------------ 502 ----
task(
    id=502, domain="TSHOOT", title="Cluster facts to an exact file",
    task="""
Write `{WS}/q502/cluster-info.txt` — answers only, exactly this format:

```
1: <number of control-plane nodes>
2: <number of worker nodes>
3: <the Service CIDR>
4: <name of the CNI plugin>, <full path of its configuration file on the nodes>
5: <the suffix the names of static Pods running on node {W3} end with>
```
""",
    setup=r"""rm -rf "$WS/q502"; mkdir -p "$WS/q502" """,
    checks=[
        C("node counts (lines 1 and 2)", 2, r"""
f="$WS/q502/cluster-info.txt"; [ -s "$f" ] || exit 1
cp=$($K get nodes -l node-role.kubernetes.io/control-plane --no-headers 2>/dev/null | wc -l); all=$($K get nodes --no-headers | wc -l)
eq "$(sed -n 's/^ *1 *: *//p' "$f" | tr -d '[:space:]')" "$cp" && eq "$(sed -n 's/^ *2 *: *//p' "$f" | tr -d '[:space:]')" "$((all - cp))" """),
        C("Service CIDR (line 3)", 2, r"""
sc=$(j -n kube-system pod -l component=kube-apiserver '{.items[0].spec.containers[0].command}' | tr ',' '\n' | grep -o 'service-cluster-ip-range=[^"]*' | cut -d= -f2)
[ -n "$sc" ] && eq "$(sed -n 's/^ *3 *: *//p' "$WS/q502/cluster-info.txt" | tr -d '[:space:]')" "$sc" """),
        C("CNI plugin and its config file (line 4)", 2, r"""
l=$(sed -n 's/^ *4 *: *//p' "$WS/q502/cluster-info.txt" | tr -d '[:space:]')
cni=$($K -n kube-system get ds -o name | sed 's|.*/||' | grep -Ei 'kindnet|calico|cilium|flannel|weave' | head -1)
cfg=$(nx "$W1" 'ls /etc/cni/net.d/*.conflist /etc/cni/net.d/*.conf 2>/dev/null | head -1' </dev/null)
[ -n "$cni" ] && [ -n "$cfg" ] && echo "$l" | grep -qi "$cni" && echo "$l" | grep -qF "$cfg" """),
        C("static Pod name suffix on {W3} (line 5)", 1, r"""
v=$(sed -n 's/^ *5 *: *//p' "$WS/q502/cluster-info.txt" | tr -d '[:space:]'); [ "$v" = "-$W3" ] || [ "$v" = "$W3" ]"""),
    ],
    solution="""
k get nodes                                   # ROLES shows control-plane; count the rest
k -n kube-system get pod -l component=kube-apiserver -o yaml | grep service-cluster-ip-range
k -n kube-system get ds                       # kindnet
{SHELL_W1}
ls /etc/cni/net.d/                            # 10-kindnet.conflist
exit
cat > {WS}/q502/cluster-info.txt <<'EOF'
1: 1
2: 3
3: 10.96.0.0/16
4: kindnet, /etc/cni/net.d/10-kindnet.conflist
5: -{W3}
EOF
# A static Pod's mirror is named <pod>-<node name>, e.g. kube-apiserver-{CP}. Values come from THIS
# cluster - check each one instead of trusting the ones shown here.
""",
)

# ------------------------------------------------------------------ 503 ----
task(
    id=503, domain="NET", title="Ready only when its dependency answers",
    task="""
In Namespace `oslo-web`:

1. Create Pod `gatekeeper` (image `busybox:1.36`, command `sleep 86400`) with an **exec readiness
   probe** that runs `wget -T2 -qO- http://ledger-api:8080`, initial delay 5 seconds, period 5
   seconds. For now it must stay **not ready** — confirm that it does.
2. Deployment `ledger` serves HTTP on port 80, but nothing exposes it. Create the Service
   `ledger-api` (ClusterIP, port **8080**) for it, so that `gatekeeper` turns Ready without you
   touching the Pod again.
""",
    setup=NS + r"""
mkns oslo-web
$K -n oslo-web delete pod gatekeeper --ignore-not-found --force --grace-period=0 >/dev/null 2>&1
$K -n oslo-web delete svc ledger-api --ignore-not-found >/dev/null
$K -n oslo-web create deploy ledger --image=nginx:1.27 --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null
$K -n oslo-web rollout status deploy ledger --timeout=120s >/dev/null 2>&1
""",
    checks=[
        C("gatekeeper has the exec readiness probe (delay 5, period 5)", 2, r"""
p='{.spec.containers[0].readinessProbe'
j -n oslo-web pod gatekeeper "$p.exec.command}" | grep -q 'ledger-api:8080' \
 && eq "$(j -n oslo-web pod gatekeeper "$p.initialDelaySeconds}")" 5 && eq "$(j -n oslo-web pod gatekeeper "$p.periodSeconds}")" 5"""),
        C("Service ledger-api: 8080 -> 80 with ready endpoints", 2, r"""
eq "$(j -n oslo-web svc ledger-api '{.spec.ports[0].port}')" 8080 && eq "$(j -n oslo-web svc ledger-api '{.spec.ports[0].targetPort}')" 80 \
 && [ -n "$(j -n oslo-web endpointslices -l kubernetes.io/service-name=ledger-api '{.items[*].endpoints[?(@.conditions.ready==true)].addresses[*]}')" ]"""),
        C("gatekeeper is Ready", 3, r"""
for i in $(seq 1 20); do [ "$(j -n oslo-web pod gatekeeper '{.status.conditions[?(@.type=="Ready")].status}')" = True ] && exit 0; sleep 3; done; exit 1"""),
    ],
    solution="""
k -n oslo-web run gatekeeper --image=busybox:1.36 $do -- sleep 86400 > gk.yaml
# add to the container:
#   readinessProbe:
#     exec:
#       command: ["sh", "-c", "wget -T2 -qO- http://ledger-api:8080"]
#     initialDelaySeconds: 5
#     periodSeconds: 5
k apply -f gk.yaml
k -n oslo-web get pod gatekeeper                  # 0/1 Running - the probe cannot resolve ledger-api
k -n oslo-web describe pod gatekeeper | tail -3   # Readiness probe failed: wget: bad address 'ledger-api:8080'
k -n oslo-web expose deploy ledger --name=ledger-api --port=8080 --target-port=80
k -n oslo-web get pod gatekeeper -w               # 1/1 after the next probe
# Readiness removes a Pod from Service endpoints and blocks rollouts; liveness restarts it.
# A dependency check belongs in readiness - failing liveness here would just restart-loop the Pod.
""",
)

# ------------------------------------------------------------------ 504 ----
task(
    id=504, domain="WORK", title="Sorting done by kubectl",
    task="""
Write two executable scripts that call `kubectl` (spelled out — aliases do not exist in a fresh
shell) and let **kubectl itself** do the sorting with `--sort-by`:

1. `{WS}/q504/pods-by-age.sh` — every Pod in every Namespace, oldest first (by creation time).
2. `{WS}/q504/pods-by-uid.sh` — the Pods in Namespace `kube-system`, sorted by `metadata.uid`.
""",
    setup=r"""rm -rf "$WS/q504"; mkdir -p "$WS/q504" """,
    checks=[
        C("pods-by-age.sh: all namespaces, --sort-by creationTimestamp, runs", 2, r"""
f="$WS/q504/pods-by-age.sh"; [ -x "$f" ] || exit 1
grep -Eq 'kubectl\b.*\bget\b.*\b(po|pods?)\b' "$f" && grep -Eq -- '(-A|--all-namespaces)' "$f" \
 && grep -Eq -- "--sort-by[= ]*['\"]?\{?\.?metadata\.creationTimestamp" "$f" || exit 1
out=$(timeout 60 bash "$f" 2>/dev/null); [ "$(echo "$out" | wc -l)" -ge 5 ] && echo "$out" | grep -q kube-system"""),
        C("pods-by-uid.sh: kube-system, --sort-by metadata.uid, runs", 2, r"""
f="$WS/q504/pods-by-uid.sh"; [ -x "$f" ] || exit 1
grep -Eq 'kubectl\b.*\bget\b.*\b(po|pods?)\b' "$f" && grep -Eq -- '(-n|--namespace)[= ]*kube-system' "$f" \
 && grep -Eq -- "--sort-by[= ]*['\"]?\{?\.?metadata\.uid" "$f" || exit 1
timeout 60 bash "$f" 2>/dev/null | grep -q etcd"""),
        C("no aliases in either script", 1, r"""[ -s "$WS/q504/pods-by-age.sh" ] && [ -s "$WS/q504/pods-by-uid.sh" ] && ! grep -Eq '(^|[;|&(] *| )k +' "$WS/q504/pods-by-age.sh" "$WS/q504/pods-by-uid.sh" """),
    ],
    solution="""
cat > {WS}/q504/pods-by-age.sh <<'EOF'
kubectl get pods -A --sort-by=.metadata.creationTimestamp
EOF
cat > {WS}/q504/pods-by-uid.sh <<'EOF'
kubectl -n kube-system get pods --sort-by=.metadata.uid
EOF
chmod +x {WS}/q504/*.sh
bash {WS}/q504/pods-by-age.sh | head -3        # the control-plane pods come first - they are oldest
# --sort-by takes a JSONPath into each item; ascending only, so "newest first" is `| tac` or
# sorting by a field that grows the other way.
""",
)

# ------------------------------------------------------------------ 505 ----
task(
    id=505, domain="TSHOOT", title="Event forensics around kube-proxy", breaks="w2-kube-proxy",
    task="""
1. Write `{WS}/q505/cluster-events.sh`: a script that prints the latest events of the **whole**
   cluster (all Namespaces) sorted by creation time, using `kubectl`. Make it executable.
2. Delete the kube-proxy Pod that runs on node `{W2}`. Write the events this caused to
   `{WS}/q505/pod-deleted.log`.
3. On node `{W2}`, stop **and remove** the kube-proxy **container** with `crictl` — leave the Pod
   object alone. Write the events this caused to `{WS}/q505/container-killed.log`.
4. Which of the two actions made the Pod's `RESTARTS` counter go up? Write `pod` or `container` to
   `{WS}/q505/restarts.txt`.

Do step 2 before step 3.
""",
    setup=r"""
rm -rf "$WS/q505"; mkdir -p "$WS/q505"
$K -n kube-system get pod -l k8s-app=kube-proxy --field-selector spec.nodeName=$W2 -o jsonpath='{.items[0].metadata.uid}' > "$WS/.q505-uid"
""",
    checks=[
        C("cluster-events.sh: all namespaces, sorted by time, runs", 2, r"""
f="$WS/q505/cluster-events.sh"; [ -x "$f" ] || exit 1
grep -Eq 'kubectl\b.*\bget\b.*\b(ev|events?)\b' "$f" && grep -Eq -- '(-A|--all-namespaces)' "$f" \
 && grep -Eq -- "--sort-by[= ]*['\"]?\{?\.?(metadata\.creationTimestamp|lastTimestamp|eventTime|firstTimestamp)" "$f" || exit 1
timeout 60 bash "$f" >/dev/null 2>&1"""),
        C("the kube-proxy Pod on {W2} was deleted and replaced", 2, r"""
old=$(cat "$WS/.q505-uid" 2>/dev/null); [ -n "$old" ] || exit 1
now=$(j -n kube-system pod -l k8s-app=kube-proxy --field-selector spec.nodeName=$W2 '{.items[0].metadata.uid}')
[ -n "$now" ] && [ "$now" != "$old" ] && eq "$(j -n kube-system pod -l k8s-app=kube-proxy --field-selector spec.nodeName=$W2 '{.items[0].status.phase}')" Running"""),
        # counted on the REPLACEMENT pod, which starts at 0 - after a node reboot the original pod
        # often already shows a restart, which would otherwise hand these points out for free
        C("then its container was killed (restart counter >= 1)", 2, r"""
old=$(cat "$WS/.q505-uid" 2>/dev/null); now=$(j -n kube-system pod -l k8s-app=kube-proxy --field-selector spec.nodeName=$W2 '{.items[0].metadata.uid}')
[ -n "$old" ] && [ -n "$now" ] && [ "$now" != "$old" ] || exit 1
[ "$(j -n kube-system pod -l k8s-app=kube-proxy --field-selector spec.nodeName=$W2 '{.items[0].status.containerStatuses[0].restartCount}')" -ge 1 ] 2>/dev/null"""),
        C("pod-deleted.log and container-killed.log hold the events", 2, r"""
for f in pod-deleted.log container-killed.log; do
  grep -q 'kube-proxy' "$WS/q505/$f" 2>/dev/null && grep -Eq 'Killing|SuccessfulCreate|SuccessfulDelete|Scheduled|Pulled|Created|Started|BackOff' "$WS/q505/$f" || exit 1
done"""),
        C("restarts.txt says container", 2, r"""eq "$(tr -d '[:space:]' < "$WS/q505/restarts.txt" 2>/dev/null | tr A-Z a-z)" container"""),
    ],
    solution="""
cat > {WS}/q505/cluster-events.sh <<'EOF'
kubectl get events -A --sort-by=.metadata.creationTimestamp
EOF
chmod +x {WS}/q505/cluster-events.sh

k -n kube-system get pod -l k8s-app=kube-proxy -o wide              # the one on {W2}
k -n kube-system delete pod <kube-proxy-on-{W2}>
k -n kube-system get events --sort-by=.metadata.creationTimestamp | grep kube-proxy | tail -8 > {WS}/q505/pod-deleted.log
#   Killing (old pod), SuccessfulCreate (DaemonSet), Scheduled, Pulled, Created, Started (new pod)

{SHELL_W2}
crictl ps --name kube-proxy                 # container id
crictl stop <id> && crictl rm <id>
exit
k -n kube-system get events --sort-by=.metadata.creationTimestamp | grep kube-proxy | tail -4 > {WS}/q505/container-killed.log
#   Pulled, Created, Started - same Pod, new container
k -n kube-system get pod -l k8s-app=kube-proxy -o wide              # RESTARTS 1 on {W2}
echo container > {WS}/q505/restarts.txt
# Deleting the Pod makes the DaemonSet create a brand-new Pod (RESTARTS 0). Killing only the
# container makes the kubelet start a new container inside the SAME Pod - that is a restart.
""",
)

# ------------------------------------------------------------------ 506 ----
def _census(r):
    n = r.sample(range(2, 10), 4)
    return {"ctx": "k8s-c1-H", "north": n[0], "south": n[1], "east": n[2], "west": n[3]}


task(
    id=506, domain="ARCH", title="API resources and a Role census", params=_census,
    task="""
1. Write the names of every **namespaced** resource type this API server offers to
   `{WS}/q506/namespaced.txt`, one per line — plural names, as `kubectl api-resources` prints them.
2. Of the Namespaces `census-north`, `census-south`, `census-east` and `census-west`, which one holds
   the most Roles? Write `<namespace> <number of Roles>` to `{WS}/q506/most-roles.txt`.
""",
    setup=NS + r"""
rm -rf "$WS/q506"; mkdir -p "$WS/q506"
for pair in north:%{north} south:%{south} east:%{east} west:%{west}; do
  ns=census-${pair%%:*}; n=${pair##*:}
  mkns $ns
  $K -n $ns delete role --all >/dev/null 2>&1
  for i in $(seq 1 $n); do $K -n $ns create role reader-$i --verb=get,list --resource=configmaps >/dev/null; done
done
""",
    checks=[
        C("namespaced.txt lists exactly the namespaced resource types", 3, r"""
f="$WS/q506/namespaced.txt"; [ -s "$f" ] || exit 1
want=$($K api-resources --namespaced=true -o name 2>/dev/null | sed 's/\..*//' | sort -u)
got=$(awk '{print $1}' "$f" | sed 's/\..*//' | grep . | sort -u)
[ -n "$want" ] && [ "$got" = "$want" ]"""),
        C("most-roles.txt names the namespace and its Role count", 3, r"""
best=$(for ns in census-north census-south census-east census-west; do echo "$ns $($K -n $ns get role --no-headers 2>/dev/null | wc -l)"; done | sort -k2 -n | tail -1)
eq "$(tr -s ' ' < "$WS/q506/most-roles.txt" 2>/dev/null | sed 's/^ *//; s/ *$//')" "$best" """),
    ],
    solution="""
k api-resources --namespaced=true -o name > /tmp/r.txt       # deployments.apps, ... (group suffix)
k api-resources --namespaced=true --no-headers | awk '{print $1}' > {WS}/q506/namespaced.txt
for ns in census-north census-south census-east census-west; do echo "$ns $(k -n $ns get role --no-headers | wc -l)"; done
#   pick the largest, e.g.  echo "census-east 8" > {WS}/q506/most-roles.txt
# --namespaced=false gives the cluster-scoped half (nodes, pv, storageclasses, clusterroles...).
# `k get role -A --no-headers | awk '{print $1}' | sort | uniq -c | sort -n` counts every
# namespace at once.
""",
)

# ------------------------------------------------------------------ 507 ----
task(
    id=507, domain="TSHOOT", title="Kubelet unit points at nothing", breaks="w1-kubelet", disruptive=True,
    task="""
Node `{W1}` went `NotReady` after someone "tidied up" its kubelet installation. Bring it back to
`Ready`; the kubelet has to run as the systemd service it always was. Then write the full path of the
file you had to fix to `{WS}/q507/fixed-file.txt`.

Reach the node with `{SHELL_W1}`. Pods on a node that stays NotReady are evicted after five minutes.
""",
    setup=r"""
rm -rf "$WS/q507"; mkdir -p "$WS/q507"
nx "$W1" '
d=/etc/systemd/system/kubelet.service.d/10-kubeadm.conf
grep -q "^ExecStart=/usr/bin/kubelet " $d && sed -i "s|^ExecStart=/usr/bin/kubelet |ExecStart=/usr/local/bin/kubelet |" $d
systemctl daemon-reload; systemctl restart kubelet' </dev/null >/dev/null 2>&1
exit 0
""",
    checks=[
        C("{W1} is Ready", 4, r"""for i in $(seq 1 15); do [ "$(j node "$W1" '{.status.conditions[?(@.type=="Ready")].status}')" = True ] && exit 0; sleep 4; done; exit 1"""),
        C("kubelet active, and its unit runs a kubelet binary that exists", 2, r"""
nx "$W1" 'systemctl is-active --quiet kubelet && b=$(sed -n "s|^ExecStart=\([^ ]*kubelet\) .*|\1|p" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf | tail -1) && [ -x "$b" ]' </dev/null"""),
        C("fixed-file.txt names the drop-in", 2, r"""grep -q '/etc/systemd/system/kubelet.service.d/10-kubeadm.conf' "$WS/q507/fixed-file.txt" 2>/dev/null"""),
    ],
    solution="""
k get nodes                                   # {W1} NotReady
{SHELL_W1}
systemctl status kubelet
#   Process: ... ExecStart=/usr/local/bin/kubelet ... (code=exited, status=203/EXEC)
journalctl -u kubelet -n 20 --no-pager        # Unable to locate executable '/usr/local/bin/kubelet'
which kubelet                                 # /usr/bin/kubelet - the binary was never moved
systemctl cat kubelet                         # shows the unit AND every drop-in, with file names
vi /etc/systemd/system/kubelet.service.d/10-kubeadm.conf     # ExecStart=/usr/bin/kubelet ...
systemctl daemon-reload                       # systemd caches unit files - without this nothing changes
systemctl restart kubelet
systemctl status kubelet                      # active (running)
exit
echo /etc/systemd/system/kubelet.service.d/10-kubeadm.conf > {WS}/q507/fixed-file.txt
k get nodes -w
# status=203/EXEC means systemd could not exec the binary at all - a path problem, not a kubelet
# configuration problem. `ExecStart=` (empty) in a drop-in clears the unit's own ExecStart first.
""",
)

# ------------------------------------------------------------------ 508 ----
task(
    id=508, domain="WORK", title="Secrets as a read-only volume and renamed env vars",
    params=lambda r: {"ctx": "k8s-c1-H", "pw": "ledger-" + "".join(r.choice("abcdefghjkmnpqrstuvwxyz23456789") for _ in range(10))},
    task="""
In Namespace `oslo-secure`:

1. Create Secret `bundle` from the file `{WS}/q508/tls-bundle.pem`; the key must be the file's name.
2. Create Secret `db-login` with `user=svc-ledger` and `pass=%{pw}`.
3. Create Pod `vault-reader` (image `busybox:1.36`, command `sleep 86400`) that
   - mounts Secret `bundle` **read-only** at `/etc/bundle`,
   - receives `db-login`'s keys as environment variables named **`APP_USER`** and **`APP_PASS`**.
""",
    setup=NS + r"""
mkns oslo-secure
rm -rf "$WS/q508"; mkdir -p "$WS/q508"
openssl req -x509 -newkey rsa:2048 -nodes -days 30 -subj "/CN=ledger.oslo.internal" -keyout /dev/null -out "$WS/q508/tls-bundle.pem" >/dev/null 2>&1
$K -n oslo-secure delete pod vault-reader --ignore-not-found --force --grace-period=0 >/dev/null 2>&1
$K -n oslo-secure delete secret bundle db-login --ignore-not-found >/dev/null
""",
    checks=[
        C("Secrets bundle (key tls-bundle.pem, the file's content) and db-login", 2, r"""
[ "$(j -n oslo-secure secret bundle '{.data.tls-bundle\.pem}' | base64 -d 2>/dev/null)" = "$(cat "$WS/q508/tls-bundle.pem")" ] \
 && eq "$(j -n oslo-secure secret db-login '{.data.user}' | base64 -d)" svc-ledger && eq "$(j -n oslo-secure secret db-login '{.data.pass}' | base64 -d)" '%{pw}'"""),
        C("vault-reader Running with bundle mounted read-only at /etc/bundle", 3, r"""
p="$K -n oslo-secure get pod vault-reader"
eq "$($p -o jsonpath='{.status.phase}' 2>/dev/null)" Running || exit 1
v=$($p -o jsonpath='{.spec.volumes[?(@.secret.secretName=="bundle")].name}'); [ -n "$v" ] || exit 1
eq "$($p -o jsonpath="{.spec.containers[0].volumeMounts[?(@.name==\"$v\")].mountPath}")" /etc/bundle \
 && eq "$($p -o jsonpath="{.spec.containers[0].volumeMounts[?(@.name==\"$v\")].readOnly}")" true \
 && $K -n oslo-secure exec vault-reader -- cat /etc/bundle/tls-bundle.pem 2>/dev/null | grep -q 'BEGIN CERTIFICATE'"""),
        C("APP_USER / APP_PASS come from db-login", 3, r"""
e=$(j -n oslo-secure pod vault-reader '{.spec.containers[0].env}')
echo "$e" | grep -q '"name":"APP_USER"' && echo "$e" | grep -q '"name":"APP_PASS"' && echo "$e" | grep -q 'db-login' || exit 1
env=$($K -n oslo-secure exec vault-reader -- env 2>/dev/null)
echo "$env" | grep -qx 'APP_USER=svc-ledger' && echo "$env" | grep -qx 'APP_PASS=%{pw}'"""),
    ],
    solution="""
k -n oslo-secure create secret generic bundle --from-file={WS}/q508/tls-bundle.pem   # key = file name
k -n oslo-secure create secret generic db-login --from-literal=user=svc-ledger --from-literal=pass='%{pw}'
k -n oslo-secure run vault-reader --image=busybox:1.36 $do -- sleep 86400 > vr.yaml
# in vr.yaml, pod level:
#   volumes: [{name: bundle, secret: {secretName: bundle}}]
# in the container:
#   volumeMounts: [{name: bundle, mountPath: /etc/bundle, readOnly: true}]
#   env:
#     - name: APP_USER
#       valueFrom: {secretKeyRef: {name: db-login, key: user}}
#     - name: APP_PASS
#       valueFrom: {secretKeyRef: {name: db-login, key: pass}}
k apply -f vr.yaml
k -n oslo-secure exec vault-reader -- env | grep APP_
k -n oslo-secure exec vault-reader -- ls /etc/bundle
# envFrom would inject the keys under their own names (user, pass); renaming needs env + secretKeyRef.
# A Secret volume updates in place when the Secret changes; env vars never do.
""",
)

# ------------------------------------------------------------------ 509 ----
task(
    id=509, domain="ARCH", title="Static Pod on the control plane behind a Service",
    task="""
1. On the control-plane node (`{SHELL_CP}`) create a **static Pod** `ops-beacon` in Namespace
   `default`: image `nginx:1.27`, requests `cpu: 10m` and `memory: 20Mi`, label `app=ops-beacon`.
2. Expose it with a Service `ops-beacon-svc` of type **NodePort** (port 80 → 80) whose endpoints
   contain that static Pod.
3. Write the node port the Service was given to `{WS}/q509/nodeport.txt`.
""",
    setup=r"""
rm -rf "$WS/q509"; mkdir -p "$WS/q509"
nx "$CP" 'rm -f /etc/kubernetes/manifests/ops-beacon*.yaml' </dev/null
$K -n default delete svc ops-beacon-svc --ignore-not-found >/dev/null
# the kubelet drops the mirror pod on its next sync of the manifest directory
for i in $(seq 1 30); do $K -n default get pod "ops-beacon-$CP" >/dev/null 2>&1 || break; sleep 2; done
""",
    checks=[
        C("manifest in the control-plane node's staticPodPath", 1, r"""
nx "$CP" 'd=$(awk "/^staticPodPath:/{print \$2}" /var/lib/kubelet/config.yaml); grep -ls "ops-beacon" "$d"/*.yaml' </dev/null >/dev/null 2>&1"""),
        C("mirror Pod ops-beacon-{CP} Running with the requests and label", 3, r"""
p="$K -n default get pod ops-beacon-$CP"
eq "$($p -o jsonpath='{.status.phase}' 2>/dev/null)" Running && eq "$($p -o jsonpath='{.metadata.labels.app}')" ops-beacon \
 && eq "$($p -o jsonpath='{.spec.containers[0].resources.requests.cpu}')" 10m && eq "$($p -o jsonpath='{.spec.containers[0].resources.requests.memory}')" 20Mi"""),
        C("NodePort Service ops-beacon-svc 80 -> 80 with the static Pod as endpoint", 3, r"""
s="$K -n default get svc ops-beacon-svc"
eq "$($s -o jsonpath='{.spec.type}' 2>/dev/null)" NodePort && eq "$($s -o jsonpath='{.spec.ports[0].port}')" 80 && eq "$($s -o jsonpath='{.spec.ports[0].targetPort}')" 80 || exit 1
ip=$(j -n default pod "ops-beacon-$CP" '{.status.podIP}'); [ -n "$ip" ] || exit 1
j -n default endpointslices -l kubernetes.io/service-name=ops-beacon-svc '{.items[*].endpoints[*].addresses[*]}' | tr ' ' '\n' | grep -qx "$ip" """),
        C("nodeport.txt holds the Service's node port", 1, r"""np=$(j -n default svc ops-beacon-svc '{.spec.ports[0].nodePort}'); [ -n "$np" ] && eq "$(tr -d '[:space:]' < "$WS/q509/nodeport.txt" 2>/dev/null)" "$np" """),
    ],
    solution="""
k run ops-beacon -n default --image=nginx:1.27 --labels=app=ops-beacon $do > beacon.yaml
# add to the container:  resources: {requests: {cpu: 10m, memory: 20Mi}}
{SHELL_CP}
grep staticPodPath /var/lib/kubelet/config.yaml          # /etc/kubernetes/manifests
vi /etc/kubernetes/manifests/ops-beacon.yaml             # paste beacon.yaml
exit
k -n default get pod ops-beacon-{CP} -o wide             # mirror pod, named <pod>-<node>
k -n default expose pod ops-beacon-{CP} --name=ops-beacon-svc --type=NodePort --port=80 --target-port=80
k -n default get endpointslices -l kubernetes.io/service-name=ops-beacon-svc   # the mirror pod's IP
k -n default get svc ops-beacon-svc -o jsonpath='{.spec.ports[0].nodePort}' > {WS}/q509/nodeport.txt
# The control-plane taint only keeps the SCHEDULER from placing pods there; the kubelet runs its
# static pods regardless. `expose pod` copies the pod's labels into the selector.
""",
)

# ------------------------------------------------------------------ 510 ----
task(
    id=510, domain="ARCH", title="The kubelet's two certificates",
    task="""
On node `{W2}` find the certificate the kubelet uses as a **client** (to talk to the API server)
and the one it **serves** its own API with (port 10250). Write exactly this to
`{WS}/q510/kubelet-certs.txt`:

```
client-issuer: <issuer, as openssl prints it>
client-eku: <the extended key usage>
server-issuer: <issuer>
server-eku: <the extended key usage>
```

Reach the node with `{SHELL_W2}`.
""",
    setup=r"""rm -rf "$WS/q510"; mkdir -p "$WS/q510" """,
    checks=[
        C("client certificate: issuer and extended key usage", 3, r"""
f="$WS/q510/kubelet-certs.txt"; [ -s "$f" ] || exit 1
iss=$(nx "$W2" 'openssl x509 -in /var/lib/kubelet/pki/kubelet-client-current.pem -noout -issuer' </dev/null | sed 's/^issuer= *//' | tr -d ' ' | tr A-Z a-z)
[ -n "$iss" ] && sed -n 's/^ *client-issuer *: *//p' "$f" | tr -d ' ' | tr A-Z a-z | grep -qF "$iss" \
 && sed -n 's/^ *client-eku *: *//p' "$f" | grep -qi 'client auth'"""),
        C("serving certificate: issuer and extended key usage", 3, r"""
f="$WS/q510/kubelet-certs.txt"
iss=$(nx "$W2" 'openssl x509 -in /var/lib/kubelet/pki/kubelet.crt -noout -issuer' </dev/null | sed 's/^issuer= *//' | tr -d ' ' | tr A-Z a-z)
[ -n "$iss" ] && sed -n 's/^ *server-issuer *: *//p' "$f" | tr -d ' ' | tr A-Z a-z | grep -qF "$iss" \
 && sed -n 's/^ *server-eku *: *//p' "$f" | grep -qi 'server auth'"""),
    ],
    solution="""
{SHELL_W2}
grep -A3 users: /etc/kubernetes/kubelet.conf      # client-certificate: /var/lib/kubelet/pki/kubelet-client-current.pem
ls /var/lib/kubelet/pki/                          # kubelet.crt / kubelet.key = the serving pair
openssl x509 -in /var/lib/kubelet/pki/kubelet-client-current.pem -noout -text | grep -E 'Issuer|Extended Key Usage' -A1
#   Issuer: CN = kubernetes          Extended Key Usage: TLS Web Client Authentication
openssl x509 -in /var/lib/kubelet/pki/kubelet.crt -noout -text | grep -E 'Issuer|Extended Key Usage' -A1
#   Issuer: CN = {W2}-ca@<timestamp>   Extended Key Usage: TLS Web Server Authentication
exit
# write the four lines with the values YOUR openssl printed:
#   client-issuer: CN=kubernetes
#   client-eku: TLS Web Client Authentication
#   server-issuer: CN={W2}-ca@<timestamp>
#   server-eku: TLS Web Server Authentication
# The client cert is signed by the cluster CA and rotated by the kubelet through CSRs. The serving
# cert here is self-signed by the kubelet at startup (no serverTLSBootstrap) - which is exactly why
# metrics-server needs --kubelet-insecure-tls on this cluster.
""",
)

# ------------------------------------------------------------------ 511 ----
task(
    id=511, domain="WORK", title="A Pod for the control plane only",
    task="""
Create Pod `cp-watch` in Namespace `default`: image `httpd:2-alpine`, container name `watch`.

It must be able to run **only** on control-plane nodes. Select them by the node label kubeadm puts
on control-plane nodes — not by host name, and not with `nodeName` — and tolerate whatever keeps
ordinary Pods off them. It must be Running.
""",
    setup=r"""$K -n default delete pod cp-watch --ignore-not-found --force --grace-period=0 >/dev/null 2>&1""",
    checks=[
        C("cp-watch Running on the control-plane node, container named watch", 2, r"""
eq "$(j -n default pod cp-watch '{.status.phase}')" Running && eq "$(j -n default pod cp-watch '{.spec.nodeName}')" "$CP" && eq "$(j -n default pod cp-watch '{.spec.containers[0].name}')" watch"""),
        C("selected by the node-role.kubernetes.io/control-plane label", 2, r"""
s=$(j -n default pod cp-watch '{.spec.nodeSelector}{.spec.affinity.nodeAffinity}')
echo "$s" | grep -q 'node-role.kubernetes.io/control-plane' && ! echo "$s" | grep -q 'kubernetes.io/hostname'"""),
        C("tolerates the control-plane taint, no nodeName in the manifest", 2, r"""
$K -n default get pod cp-watch -o json 2>/dev/null | python3 -c '
import sys,json; t=json.load(sys.stdin)["spec"].get("tolerations") or []
sys.exit(0 if any("control-plane" in (x.get("key") or "") or (x.get("operator")=="Exists" and not x.get("key")) for x in t) else 1)' || exit 1
$K -n default get pod cp-watch -o json --show-managed-fields | python3 -c '
import sys,json
mf=json.load(sys.stdin)["metadata"].get("managedFields",[])
sys.exit(1 if any("f:nodeName" in json.dumps(m.get("fieldsV1",{})) for m in mf) else 0)'"""),
    ],
    solution="""
k get node {CP} --show-labels | tr ',' '\\n' | grep node-role     # node-role.kubernetes.io/control-plane=
k describe node {CP} | grep Taints                                 # node-role.kubernetes.io/control-plane:NoSchedule
k -n default run cp-watch --image=httpd:2-alpine $do > cp.yaml
# rename the container to watch, then add at pod level:
#   nodeSelector:
#     node-role.kubernetes.io/control-plane: ""
#   tolerations:
#     - {key: node-role.kubernetes.io/control-plane, operator: Exists, effect: NoSchedule}
k apply -f cp.yaml
k -n default get pod cp-watch -o wide
# The toleration only ALLOWS the control plane; the nodeSelector is what REQUIRES it. You need both.
""",
)

# ------------------------------------------------------------------ 512 ----
task(
    id=512, domain="TSHOOT", title="Stop the scheduler, schedule by hand", breaks="scheduler",
    task="""
1. Stop the kube-scheduler **temporarily** — it runs as a static Pod on the control-plane node — and
   confirm it is gone.
2. Create Pod `manual-1` (image `busybox:1.36`, command `sleep 86400`) in Namespace `default` and
   confirm it stays `Pending`.
3. Schedule `manual-1` **by hand** onto the control-plane node and get it Running.
4. Start the kube-scheduler again, then create Pod `manual-2` (same image and command) in `default`
   and confirm that the scheduler places it.
""",
    setup=r"""
# a pristine copy outside the static pod directory, so the pre-session baseline can put the
# scheduler back if this task is abandoned half-done
nx "$CP" 'mkdir -p /etc/kubernetes/.cka-sim && [ -f /etc/kubernetes/manifests/kube-scheduler.yaml ] && cp /etc/kubernetes/manifests/kube-scheduler.yaml /etc/kubernetes/.cka-sim/kube-scheduler.yaml' </dev/null
$K -n default delete pod manual-1 manual-2 --ignore-not-found --force --grace-period=0 >/dev/null 2>&1
exit 0
""",
    checks=[
        C("kube-scheduler is back and Running", 2, r"""
$K -n default get pod manual-1 >/dev/null 2>&1 || exit 1          # only once the task was actually done
nx "$CP" 'test -f /etc/kubernetes/manifests/kube-scheduler.yaml' </dev/null || exit 1
for i in $(seq 1 15); do [ "$(j -n kube-system pod -l component=kube-scheduler '{.items[0].status.containerStatuses[0].ready}')" = true ] && exit 0; sleep 4; done; exit 1"""),
        C("manual-1 Running on the control-plane node", 3, r"""
eq "$(j -n default pod manual-1 '{.status.phase}')" Running && eq "$(j -n default pod manual-1 '{.spec.nodeName}')" "$CP" """),
        C("manual-2 Running, placed by the scheduler (no nodeName in its manifest)", 3, r"""
eq "$(j -n default pod manual-2 '{.status.phase}')" Running && [ -n "$(j -n default pod manual-2 '{.spec.nodeName}')" ] || exit 1
$K -n default get pod manual-2 -o json --show-managed-fields | python3 -c '
import sys,json
mf=json.load(sys.stdin)["metadata"].get("managedFields",[])
sys.exit(1 if any("f:nodeName" in json.dumps(m.get("fieldsV1",{})) for m in mf) else 0)'"""),
    ],
    solution="""
{SHELL_CP}
mv /etc/kubernetes/manifests/kube-scheduler.yaml /etc/kubernetes/     # the kubelet stops what it no longer sees
exit
k -n kube-system get pod -l component=kube-scheduler                   # gone (give it ~20s)
k -n default run manual-1 --image=busybox:1.36 -- sleep 86400
k -n default get pod manual-1                                          # Pending, no node
# spec.nodeName is only settable on creation, so recreate it with the node filled in:
k -n default get pod manual-1 -o yaml > m1.yaml
#   add under spec:  nodeName: {CP}
k replace --force -f m1.yaml
k -n default get pod manual-1 -o wide                                  # Running on {CP}
{SHELL_CP}
mv /etc/kubernetes/kube-scheduler.yaml /etc/kubernetes/manifests/
exit
k -n kube-system get pod -l component=kube-scheduler -w                # Running again
k -n default run manual-2 --image=busybox:1.36 -- sleep 86400
k -n default get pod manual-2 -o wide                                  # a worker, chosen by the scheduler
# nodeName skips the scheduler entirely - including its taint checks, which is why manual-1 can
# sit on the tainted control plane. (The Binding API does the same without recreating the pod.)
""",
)

# ------------------------------------------------------------------ 513 ----
_CHART = r'''
mkchart() {  # dir name version image replicas
  mkdir -p "$1/templates"
  printf 'apiVersion: v2\nname: %s\nversion: %s\nappVersion: "%s"\n' "$2" "$3" "${4##*:}" > "$1/Chart.yaml"
  printf 'replicaCount: %s\nimage: %s\n' "$5" "$4" > "$1/values.yaml"
  cat > "$1/templates/deployment.yaml" <<'YAML'
apiVersion: apps/v1
kind: Deployment
metadata: {name: {{ .Release.Name }}, labels: {app: {{ .Release.Name }}}}
spec:
  replicas: {{ .Values.replicaCount }}
  selector: {matchLabels: {app: {{ .Release.Name }}}}
  template:
    metadata: {labels: {app: {{ .Release.Name }}}}
    spec:
      containers: [{name: main, image: "{{ .Values.image }}"}]
YAML
}
'''

task(
    id=513, domain="ARCH", title="Helm release housekeeping",
    task="""
Namespace `oslo-helm` holds several Helm releases; local charts are under `{WS}/q513/charts`.

1. Delete the release `web-legacy`.
2. Upgrade the release `web-main` to chart version **1.1.0** (`{WS}/q513/charts/nginx-lite-1.1.0`).
3. Install a new release `cache` from `{WS}/q513/charts/redis-lite` with **2** replicas (the chart's
   `replicaCount` value).
4. One release in the Namespace never finished installing. Find it and remove it.
""",
    setup=NS + _CHART + r"""
mkns oslo-helm
C="$WS/q513/charts"; rm -rf "$WS/q513"; mkdir -p "$C"
mkchart "$C/nginx-lite-1.0.0" nginx-lite 1.0.0 nginx:1.25 1
mkchart "$C/nginx-lite-1.1.0" nginx-lite 1.1.0 nginx:1.27 1
mkchart "$C/redis-lite" redis-lite 0.3.0 redis:7-alpine 1
mkchart /tmp/q513-stuck cache-edge 0.1.0 busybox:0.0-does-not-exist 1
for r in $(helm -n oslo-helm list -q 2>/dev/null); do helm -n oslo-helm uninstall "$r" >/dev/null 2>&1; done
helm -n oslo-helm install web-legacy "$C/nginx-lite-1.0.0" >/dev/null
helm -n oslo-helm install web-main "$C/nginx-lite-1.0.0" >/dev/null
# helm records the release as pending-install BEFORE it starts waiting; kill it mid-wait and it
# stays that way - the classic result of an interrupted install or a CI job that timed out
timeout -s KILL 6 helm -n oslo-helm install cache-broken /tmp/q513-stuck --wait --timeout 120s >/dev/null 2>&1
rm -rf /tmp/q513-stuck
helm -n oslo-helm status cache-broken -o json 2>/dev/null | grep -q '"status":"pending-install"' || echo "warn: cache-broken did not end up pending-install"
""",
    checks=[
        C("web-legacy is gone", 1, r"""! helm -n oslo-helm status web-legacy >/dev/null 2>&1 && helm -n oslo-helm status web-main >/dev/null 2>&1"""),
        C("web-main deployed at chart version 1.1.0", 2, r"""
helm -n oslo-helm list -o json 2>/dev/null | python3 -c '
import sys,json
r={x["name"]:x for x in json.load(sys.stdin)}.get("web-main") or {}
sys.exit(0 if r.get("chart")=="nginx-lite-1.1.0" and r.get("status")=="deployed" and int(r.get("revision",0))>=2 else 1)'"""),
        C("cache deployed from redis-lite with 2 available replicas", 2, r"""
helm -n oslo-helm list -o json 2>/dev/null | python3 -c '
import sys,json
r={x["name"]:x for x in json.load(sys.stdin)}.get("cache") or {}
sys.exit(0 if r.get("chart","").startswith("redis-lite-") and r.get("status")=="deployed" else 1)' || exit 1
for i in $(seq 1 20); do [ "$(j -n oslo-helm deploy cache '{.status.availableReplicas}')" = 2 ] && exit 0; sleep 3; done; exit 1"""),
        C("the stuck release is removed; nothing pending remains", 2, r"""
! helm -n oslo-helm status cache-broken >/dev/null 2>&1 && [ -z "$(helm -n oslo-helm list --pending -q 2>/dev/null)" ] && ! $K -n oslo-helm get deploy cache-broken >/dev/null 2>&1"""),
    ],
    solution="""
helm -n oslo-helm list                         # Helm 4 shows every status; Helm 3 hides pending ones
helm -n oslo-helm list --pending               # works in both: cache-broken   pending-install
helm -n oslo-helm uninstall web-legacy
helm -n oslo-helm upgrade web-main {WS}/q513/charts/nginx-lite-1.1.0
helm -n oslo-helm install cache {WS}/q513/charts/redis-lite --set replicaCount=2
helm -n oslo-helm uninstall cache-broken       # also removes the Deployment it half-created
helm -n oslo-helm list                         # web-main (1.1.0, rev 2) and cache, both deployed
# A pending-install release blocks a plain re-install under the same name ("cannot re-use a name
# that is still in use"). `helm history <rel>` shows when it got stuck; uninstall clears it.
""",
)

# ------------------------------------------------------------------ 514 ----
task(
    id=514, domain="STOR", title="Per-replica storage for a StatefulSet",
    task="""
In Namespace `lima-data`:

1. Create a headless Service `ledger-db` (port 5432) and a StatefulSet `ledger-db` with **3**
   replicas: image `busybox:1.36`, command `sh -c 'hostname > /data/id.txt && sleep 86400'`, and a
   `volumeClaimTemplates` entry `data` — 100Mi, ReadWriteOnce, StorageClass `standard` — mounted at
   `/data`.
2. Once all 3 are Ready, scale the StatefulSet down to **2**.
3. Write the names of the PersistentVolumeClaims that belong to the StatefulSet and still exist
   after the scale-down to `{WS}/q514/pvcs.txt`, one per line, sorted.
""",
    setup=NS + r"""
mkns lima-data
rm -rf "$WS/q514"; mkdir -p "$WS/q514"
$K -n lima-data delete sts ledger-db --ignore-not-found --wait=true >/dev/null 2>&1
$K -n lima-data delete svc ledger-db --ignore-not-found >/dev/null
$K -n lima-data delete pvc -l app=ledger-db --ignore-not-found >/dev/null 2>&1
for i in 0 1 2 3 4; do $K -n lima-data delete pvc data-ledger-db-$i --ignore-not-found --wait=false >/dev/null 2>&1; done
""",
    checks=[
        C("headless Service and a StatefulSet with the right claim template", 2, r"""
eq "$(j -n lima-data svc ledger-db '{.spec.clusterIP}')" None && eq "$(j -n lima-data sts ledger-db '{.spec.serviceName}')" ledger-db || exit 1
t='{.spec.volumeClaimTemplates[?(@.metadata.name=="data")]'
eq "$(j -n lima-data sts ledger-db "$t.spec.resources.requests.storage}")" 100Mi && eq "$(j -n lima-data sts ledger-db "$t.spec.storageClassName}")" standard \
 && j -n lima-data sts ledger-db "$t.spec.accessModes}" | grep -q ReadWriteOnce"""),
        C("scaled to 2, both Ready", 2, r"""eq "$(j -n lima-data sts ledger-db '{.spec.replicas}')" 2 && eq "$(j -n lima-data sts ledger-db '{.status.readyReplicas}')" 2"""),
        C("each replica wrote its own identity to its own volume", 2, r"""
for i in 0 1; do eq "$($K -n lima-data exec ledger-db-$i -- cat /data/id.txt 2>/dev/null | tr -d '[:space:]')" "ledger-db-$i" || exit 1; done
[ "$(j -n lima-data pod ledger-db-0 '{.spec.volumes[?(@.name=="data")].persistentVolumeClaim.claimName}')" = data-ledger-db-0 ]"""),
        C("pvcs.txt lists every claim that survived the scale-down", 2, r"""
want=$($K -n lima-data get pvc --no-headers -o custom-columns=N:.metadata.name 2>/dev/null | grep '^data-ledger-db-' | sort)
[ -n "$want" ] && [ "$(echo "$want" | wc -l)" -ge 3 ] && [ "$(grep . "$WS/q514/pvcs.txt" 2>/dev/null)" = "$want" ]"""),
    ],
    solution="""
cat <<EOF | k apply -f -
apiVersion: v1
kind: Service
metadata: {name: ledger-db, namespace: lima-data}
spec: {clusterIP: None, selector: {app: ledger-db}, ports: [{port: 5432}]}
---
apiVersion: apps/v1
kind: StatefulSet
metadata: {name: ledger-db, namespace: lima-data}
spec:
  serviceName: ledger-db
  replicas: 3
  selector: {matchLabels: {app: ledger-db}}
  template:
    metadata: {labels: {app: ledger-db}}
    spec:
      containers:
        - name: db
          image: busybox:1.36
          command: [sh, -c, "hostname > /data/id.txt && sleep 86400"]
          volumeMounts: [{name: data, mountPath: /data}]
  volumeClaimTemplates:
    - metadata: {name: data}
      spec: {accessModes: [ReadWriteOnce], storageClassName: standard, resources: {requests: {storage: 100Mi}}}
EOF
k -n lima-data rollout status sts ledger-db
k -n lima-data scale sts ledger-db --replicas=2
k -n lima-data get pvc -o custom-columns=N:.metadata.name --no-headers | grep '^data-ledger-db-' | sort > {WS}/q514/pvcs.txt
cat {WS}/q514/pvcs.txt           # all THREE: data-ledger-db-0, -1, -2
# Claims are named <template>-<statefulset>-<ordinal> and outlive their pods: scaling down (or
# deleting the StatefulSet) keeps them unless spec.persistentVolumeClaimRetentionPolicy says
# otherwise, so scaling back up re-attaches ledger-db-2 to its old data.
""",
)

# ------------------------------------------------------------------ 515 ----
task(
    id=515, domain="NET", title="Namespace AND pod selector, enforced",
    task="""
Pods labelled `app=db` in Namespace `oslo-db` must accept connections on TCP port 80 **only** from
Pods labelled `role=backend` that run in Namespaces labelled `env=prod`. Every other source must be
refused. Do not restrict egress.

Create the NetworkPolicy `db-ingress` in `oslo-db`. This cluster's CNI enforces NetworkPolicy, so
test what you built: the `api` Pods in `oslo-api` must get through to Service `db`, the `debug`
Pods in `oslo-api` must not, and neither must the `api` Pods in `lima-api`.
""",
    setup=NS + r"""
mkns oslo-db; mkns oslo-api; mkns lima-api
$K label ns oslo-api env=prod --overwrite >/dev/null; $K label ns lima-api env=dev --overwrite >/dev/null
$K -n oslo-db delete netpol --all >/dev/null 2>&1
$K -n oslo-db create deploy db --image=nginx:1.27 --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null
$K -n oslo-db expose deploy db --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null
mk() {  # ns name role
  cat <<EOF | $K apply -f - >/dev/null
apiVersion: apps/v1
kind: Deployment
metadata: {name: $2, namespace: $1}
spec:
  replicas: 1
  selector: {matchLabels: {app: $2}}
  template:
    metadata: {labels: {app: $2, role: $3}}
    spec: {containers: [{name: c, image: busybox:1.36, command: [sleep, "86400"]}]}
EOF
}
mk oslo-api api backend; mk oslo-api debug debug; mk lima-api api backend
for d in oslo-db/db oslo-api/api oslo-api/debug lima-api/api; do $K -n ${d%%/*} rollout status deploy ${d##*/} --timeout=120s >/dev/null 2>&1; done
""",
    checks=[
        C("selects app=db, Ingress only", 1, r"""
n="$K -n oslo-db get netpol db-ingress"
eq "$($n -o jsonpath='{.spec.podSelector.matchLabels.app}' 2>/dev/null)" db && eq "$($n -o jsonpath='{.spec.policyTypes}')" '["Ingress"]'"""),
        C("one peer holding BOTH namespaceSelector env=prod AND podSelector role=backend, on TCP 80", 3, r"""
$K -n oslo-db get netpol db-ingress -o json 2>/dev/null | python3 -c '
import sys,json
ing=json.load(sys.stdin)["spec"].get("ingress",[])
ok=len(ing)==1 and len(ing[0].get("from",[]))==1
if ok:
    f=ing[0]["from"][0]
    ok=f.get("namespaceSelector",{}).get("matchLabels",{}).get("env")=="prod" and f.get("podSelector",{}).get("matchLabels",{}).get("role")=="backend"
    ok=ok and [(str(p.get("port")),p.get("protocol","TCP")) for p in ing[0].get("ports",[])]==[("80","TCP")]
sys.exit(0 if ok else 1)'"""),
        C("oslo-api/api reaches db", 2, r"""
$K -n oslo-db get netpol db-ingress >/dev/null 2>&1 || exit 1     # no policy yet: nothing earned
for i in 1 2 3 4 5; do $K -n oslo-api exec deploy/api -- wget -qO- -T3 http://db.oslo-db >/dev/null 2>&1 && exit 0; sleep 2; done; exit 1"""),
        C("oslo-api/debug and lima-api/api are refused", 2, r"""
$K -n oslo-db get netpol db-ingress >/dev/null 2>&1 || exit 1
! $K -n oslo-api exec deploy/debug -- wget -qO- -T3 http://db.oslo-db >/dev/null 2>&1 \
 && ! $K -n lima-api exec deploy/api -- wget -qO- -T3 http://db.oslo-db >/dev/null 2>&1"""),
    ],
    solution="""
k get ns --show-labels | grep -E 'oslo-api|lima-api'    # env=prod / env=dev
cat <<EOF | k apply -f -
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: db-ingress, namespace: oslo-db}
spec:
  podSelector: {matchLabels: {app: db}}
  policyTypes: [Ingress]
  ingress:
    - from:
        - namespaceSelector: {matchLabels: {env: prod}}
          podSelector: {matchLabels: {role: backend}}      # SAME list item = AND
      ports: [{protocol: TCP, port: 80}]
EOF
k -n oslo-api exec deploy/api   -- wget -qO- -T3 http://db.oslo-db | head -1     # <!DOCTYPE html>
k -n oslo-api exec deploy/debug -- wget -qO- -T3 http://db.oslo-db               # download timed out
k -n lima-api exec deploy/api   -- wget -qO- -T3 http://db.oslo-db               # download timed out
# The trap is one dash: "- namespaceSelector: ...\\n  - podSelector: ..." (two list items) means OR -
# any pod in a prod namespace, OR a backend pod in oslo-db's own namespace. One item with both
# selectors means AND.
""",
)

# ------------------------------------------------------------------ 516 ----
task(
    id=516, domain="NET", title="Resolves everything except the cluster",
    task="""
Deployment `reporter` in Namespace `lima-client` cannot reach Service `ledger` — its log says
`bad address`. The Service, its endpoints and cluster DNS are all fine: other Pods in the Namespace
resolve `ledger` without trouble.

Fix the `reporter` Deployment (not the Service), without changing its image or command, so that its
requests succeed. Write the field you changed and its new value, as `<field>=<value>`, to
`{WS}/q516/fix.txt`.
""",
    setup=NS + r"""
mkns lima-client
rm -rf "$WS/q516"; mkdir -p "$WS/q516"
$K -n lima-client create deploy ledger --image=nginx:1.27 --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null
$K -n lima-client expose deploy ledger --port=8080 --target-port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null
$K -n lima-client delete deploy reporter --ignore-not-found --wait=true >/dev/null 2>&1
$K apply -f - >/dev/null <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata: {name: reporter, namespace: lima-client}
spec:
  replicas: 1
  selector: {matchLabels: {app: reporter}}
  template:
    metadata: {labels: {app: reporter}}
    spec:
      dnsPolicy: Default
      containers:
        - name: reporter
          image: busybox:1.36
          command: [sh, -c, 'while true; do if wget -qO- -T2 http://ledger:8080/ >/dev/null 2>&1; then echo "$(date +%T) ledger ok"; else wget -qO- -T2 http://ledger:8080/ 2>&1 | head -1; fi; sleep 3; done']
EOF
$K -n lima-client rollout status deploy ledger --timeout=120s >/dev/null 2>&1
""",
    checks=[
        C("reporter uses the cluster DNS policy again", 3, r"""
p=$(j -n lima-client deploy reporter '{.spec.template.spec.dnsPolicy}'); [ -z "$p" ] || [ "$p" = ClusterFirst ] || [ "$p" = ClusterFirstWithHostNet ]"""),
        C("reporter's requests now succeed", 3, r"""
for i in $(seq 1 15); do
  p=$($K -n lima-client get pod -l app=reporter --field-selector status.phase=Running -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
  [ -n "$p" ] && $K -n lima-client logs "$p" --tail=3 2>/dev/null | grep -q 'ledger ok' && exit 0; sleep 3
done; exit 1"""),
        C("fix.txt names dnsPolicy and its new value", 1, r"""grep -Eqi '^ *dnsPolicy *= *ClusterFirst *$' "$WS/q516/fix.txt" 2>/dev/null"""),
    ],
    solution="""
k -n lima-client logs deploy/reporter --tail=3            # wget: bad address 'ledger:8080'
k -n lima-client get endpointslices                       # the Service has endpoints - not the problem
k -n lima-client get deploy reporter -o yaml | grep -n dns
#   dnsPolicy: Default   <- "Default" means: use the NODE's resolver, NOT cluster DNS
k -n lima-client exec deploy/reporter -- cat /etc/resolv.conf   # the podman/host resolver, no cluster search domains
k -n lima-client patch deploy reporter -p '{"spec":{"template":{"spec":{"dnsPolicy":"ClusterFirst"}}}}'
k -n lima-client logs deploy/reporter -f                  # "ledger ok"
echo dnsPolicy=ClusterFirst > {WS}/q516/fix.txt
# The trap is the name: the DEFAULT dnsPolicy is ClusterFirst, while the policy literally called
# "Default" inherits the node's /etc/resolv.conf. ClusterFirstWithHostNet is for hostNetwork pods.
""",
)

# ------------------------------------------------------------------ 517 ----
task(
    id=517, domain="STOR", title="A claim that can never bind",
    task="""
Deployment `uploader` in Namespace `oslo-files` has no running Pod. Find out why and fix it, so its
Pod runs with the claim mounted at `/files`. Keep the claim's name (`shared-files`), its size and its
StorageClass. Write the root cause, in one line, to `{WS}/q517/reason.txt`.
""",
    setup=NS + r"""
mkns oslo-files
rm -rf "$WS/q517"; mkdir -p "$WS/q517"
$K -n oslo-files delete deploy uploader --ignore-not-found --wait=true >/dev/null 2>&1
$K -n oslo-files delete pvc shared-files --ignore-not-found --wait=true >/dev/null 2>&1
$K apply -f - >/dev/null <<'EOF'
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: shared-files, namespace: oslo-files}
spec: {accessModes: [ReadWriteMany], storageClassName: standard, resources: {requests: {storage: 1Gi}}}
---
apiVersion: apps/v1
kind: Deployment
metadata: {name: uploader, namespace: oslo-files}
spec:
  replicas: 1
  selector: {matchLabels: {app: uploader}}
  template:
    metadata: {labels: {app: uploader}}
    spec:
      volumes: [{name: files, persistentVolumeClaim: {claimName: shared-files}}]
      containers: [{name: up, image: busybox:1.36, command: [sleep, "86400"], volumeMounts: [{name: files, mountPath: /files}]}]
EOF
""",
    checks=[
        C("shared-files Bound: 1Gi, class standard, an access mode local-path supports", 3, r"""
eq "$(j -n oslo-files pvc shared-files '{.status.phase}')" Bound && eq "$(j -n oslo-files pvc shared-files '{.spec.resources.requests.storage}')" 1Gi \
 && eq "$(j -n oslo-files pvc shared-files '{.spec.storageClassName}')" standard \
 && ! j -n oslo-files pvc shared-files '{.spec.accessModes}' | grep -q ReadWriteMany"""),
        C("uploader available with the claim at /files", 2, r"""
for i in $(seq 1 20); do [ "$(j -n oslo-files deploy uploader '{.status.availableReplicas}')" = 1 ] && break; sleep 3; done
eq "$(j -n oslo-files deploy uploader '{.status.availableReplicas}')" 1 \
 && eq "$(j -n oslo-files deploy uploader '{.spec.template.spec.volumes[?(@.persistentVolumeClaim.claimName=="shared-files")].name}')" files \
 && j -n oslo-files deploy uploader '{.spec.template.spec.containers[0].volumeMounts[*].mountPath}' | grep -qw /files"""),
        C("reason.txt names the unsupported access mode", 2, r"""grep -Eqi 'ReadWriteMany|RWX|access ?mode' "$WS/q517/reason.txt" 2>/dev/null"""),
    ],
    solution="""
k -n oslo-files get pod                          # Pending
k -n oslo-files describe pod -l app=uploader | tail -4    # unbound immediate PersistentVolumeClaims / waiting
k -n oslo-files describe pvc shared-files | tail -4
#   ProvisioningFailed ... NodePath only supports ReadWriteOnce and ReadWriteOncePod (1.22+) access modes
# accessModes is immutable on a PVC, so recreate it. The Pending pod was never scheduled, so it
# does not hold the claim (pvc-protection only counts scheduled pods) and the delete goes through.
k -n oslo-files get pvc shared-files -o yaml > pvc.yaml     # change ReadWriteMany -> ReadWriteOnce, strip status/uid
k -n oslo-files delete pvc shared-files
k apply -f pvc.yaml
k -n oslo-files rollout restart deploy uploader            # or just wait - the pending pod retries
k -n oslo-files get pvc,pod
echo 'PVC requested ReadWriteMany, which the local-path provisioner cannot provide (RWO/RWOP only)' > {WS}/q517/reason.txt
# RWX needs a filesystem many nodes can mount at once (NFS, CephFS, a cloud file service).
# ReadWriteOnce is per NODE, not per pod - ReadWriteOncePod is the per-pod one.
""",
)

QUESTIONS = Q