"""CKA tasks 103-113: pool tasks for the curriculum objectives the bank covered thinly or not at all.

curriculum.py maps every task to the objectives of the CKA curriculum and turns the bank into a
study path (./ksim objectives). Mapping the first 102 tasks left eight objectives with fewer than
two pool tasks; these close them:

  2.3 Monitor cluster and application resource usage     103 kubectl top
  2.4 Manage and evaluate container output streams       104 logs by selector, of one replica, of an init container
  3.3 Configure workload autoscaling                     105 an HPA on CPU with a scale-down policy
  3.4 Robust, self-healing deployments                   106 probes that check the wrong thing
  4.2 Prepare underlying infrastructure                  107 kernel modules and sysctls   108 the runtime behind a Pod
  4.4 Manage the lifecycle of clusters                   109 drain the control-plane node and bring it back
  4.5 Highly-available control plane                     110 the join of a second control-plane node   111 leaders and etcd members
  4.7 Extension interfaces (CNI, CSI, CRI)               112 what this cluster plugs in   (and 108)
  5.1 Connectivity between Pods                          113 Pod to Pod across nodes, Pod CIDRs, hostNetwork

103, 104 and 106 are the CKAD tasks for the same skill (kubectl top, logs, probes) under CKA ids:
the objective is word for word the same, so is the scenario. Everything else is CKA ground - nodes,
kubeadm, etcd, the runtime. All normal difficulty, all in the random pool; nothing here breaks a
node or a control-plane component (109 cordons the control-plane node while you work on it and has
you uncordon it; the session baseline uncordons it if you do not).
"""
from sim.bank import C, NS, ns_pick, name_pick, cmd, hand
from ckad import bank_extra as _ckad_extra, bank_more as _ckad_more


def _port(bank, src_id, new_id, domain, **over):
    """A CKAD task that serves a CKA objective as it is: same scenario and graders, CKA id and paths."""
    src = next(q for q in bank.QUESTIONS if q["id"] == src_id)
    old, new = f"q{src_id}", f"q{new_id}"
    q = dict(src, id=new_id, domain=domain, **over)
    for k in ("task", "setup", "solution"):
        if k not in over:   # a solution of its own (in parts) is the CKA one already
            q[k] = src[k].replace(old, new)
    q["checks"] = [dict(c, script=c["script"].replace(old, new), name=c["name"].replace(old, new)) for c in src["checks"]]
    return q


# 103's own solution: the CKAD task's, done as in the exam
_SOL103 = [hand(r"""k -n %{ns} top pod""",
                r"""Again until it shows all three: metrics-server needs about 30 seconds after the Pods start. Without it, `Metrics API not available`.""",
                run=r"""for i in $(seq 1 40); do [ "$(kubectl -n %{ns} top pod --no-headers 2>/dev/null | wc -l)" -ge 3 ] && break; sleep 5; done"""),
           cmd(r"""k -n %{ns} top pod --sort-by=cpu
k -n %{ns} top pod --sort-by=memory""", r"""The first line of each is the heaviest."""),
           hand(r"""echo %{cpu} > {WS}/q103/cpu.txt
echo %{mem} > {WS}/q103/memory.txt""",
                r"""With the names on those first lines.""",
                run=r"""kubectl -n %{ns} top pod --sort-by=cpu --no-headers | head -1 | awk '{print $1}' > {WS}/q103/cpu.txt
kubectl -n %{ns} top pod --sort-by=memory --no-headers | head -1 | awk '{print $1}' > {WS}/q103/memory.txt"""),
           cmd(r"""echo 'kubectl top node' > {WS}/q103/nodes.sh
echo 'kubectl top pod -A --sort-by=memory' > {WS}/q103/pods.sh""", r"""`k top` reads the metrics API (`metrics.k8s.io`) that metrics-server serves; `--containers` breaks a Pod down per container.""")]


ETCDCTL = ("etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt "
           "--cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key")

QUESTIONS = [
# ---------------------------------------------------------------- TSHOOT ----
# needs a working metrics API, which Q23 breaks on purpose: never in one session with it
_port(_ckad_extra, 23, 103, "TSHOOT", breaks="metrics-server", solution=_SOL103),
_port(_ckad_more, 64, 104, "TSHOOT"),
# ------------------------------------------------------------------ WORK ----
{
 "id": 105, "domain": "WORK", "title": "Autoscale on CPU, scale down calmly",
 "params": lambda r: {"ns": ns_pick(r, "scale"), "dep": name_pick(r, "render"), "min": r.choice([2, 3]),
                      "max": r.choice([6, 8, 10]), "cpu": r.choice([50, 60, 70]), "win": r.choice([120, 180, 300])},
 "task": """
Deployment `%{dep}` in Namespace `%{ns}` has to scale with its load.

1. An autoscaler that works on CPU utilisation needs a CPU request to measure against. Give the
   container `web` a CPU request of `100m` and leave everything else in the Deployment as it is.
2. Create HorizontalPodAutoscaler `%{dep}` for the Deployment: at least **%{min}** replicas, at
   most **%{max}**, target average CPU utilisation **%{cpu}%**.
3. Scaling down must be calm: a stabilisation window of **%{win}** seconds, and at most **1 Pod
   removed per 60 seconds**. Leave scaling up at its defaults.
""",
 "setup": NS + r"""
mkns %{ns}
$K -n %{ns} delete hpa --all >/dev/null 2>&1
$K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null 2>&1
$K apply -f - >/dev/null <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata: {name: %{dep}, namespace: %{ns}, labels: {app: %{dep}}}
spec:
  replicas: 1
  selector: {matchLabels: {app: %{dep}}}
  template:
    metadata: {labels: {app: %{dep}}}
    spec:
      containers:
        - name: web
          image: nginx:1.27
          ports: [{containerPort: 80}]
          resources: {requests: {memory: 32Mi}}
EOF
$K -n %{ns} rollout status deploy %{dep} --timeout=150s >/dev/null
""",
 "checks": [
  C("container web requests cpu 100m, the rest of it unchanged", 2, r"""
eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].name}/{.spec.template.spec.containers[0].image}/{.spec.template.spec.containers[0].resources.requests.cpu}/{.spec.template.spec.containers[0].resources.requests.memory}')" \
   "web/nginx:1.27/100m/32Mi" """),
  C("HPA %{dep}: Deployment %{dep}, %{min}-%{max} replicas, %{cpu}% average CPU utilisation", 2, r"""
$K -n %{ns} get hpa %{dep} -o json 2>/dev/null | python3 -c '
import sys, json
s = json.load(sys.stdin)["spec"]; t = s["scaleTargetRef"]; m = s.get("metrics") or []
cpu = [x["resource"]["target"] for x in m if x.get("type") == "Resource" and x["resource"]["name"] == "cpu"]
ok = (t["kind"], t["name"]) == ("Deployment", "%{dep}") and (s.get("minReplicas"), s["maxReplicas"]) == (%{min}, %{max}) \
     and len(m) == 1 and len(cpu) == 1 and cpu[0].get("type") == "Utilization" and cpu[0].get("averageUtilization") == %{cpu}
sys.exit(0 if ok else 1)'"""),
  C("scale-down: %{win} s stabilisation window, at most 1 Pod per 60 s", 2, r"""
$K -n %{ns} get hpa %{dep} -o json 2>/dev/null | python3 -c '
import sys, json
d = (json.load(sys.stdin)["spec"].get("behavior") or {}).get("scaleDown") or {}
p = d.get("policies") or []
ok = d.get("stabilizationWindowSeconds") == %{win} and len(p) == 1 \
     and (p[0].get("type"), p[0].get("value"), p[0].get("periodSeconds")) == ("Pods", 1, 60)
sys.exit(0 if ok else 1)'"""),
  C("the autoscaler has taken the Deployment to at least %{min} replicas", 1, r"""
$K -n %{ns} get hpa %{dep} >/dev/null 2>&1 || exit 1
for i in $(seq 1 20); do
  [ "$(j -n %{ns} deploy %{dep} '{.spec.replicas}')" -ge %{min} ] 2>/dev/null && exit 0; sleep 3
done; exit 1"""),
 ],
 "solution": """
k -n %{ns} set resources deploy %{dep} -c web --requests=cpu=100m
k -n %{ns} autoscale deploy %{dep} --min=%{min} --max=%{max} --cpu=%{cpu}%          # kubectl before 1.34: --cpu-percent=%{cpu}
k -n %{ns} edit hpa %{dep}                       # `autoscale` cannot write behaviour - add under spec:
#   behavior:
#     scaleDown:
#       stabilizationWindowSeconds: %{win}
#       policies:
#         - {type: Pods, value: 1, periodSeconds: 60}
k -n %{ns} get hpa %{dep}                        # MINPODS %{min}, MAXPODS %{max}; REPLICAS climbs to %{min}
k -n %{ns} describe hpa %{dep}
# The whole object, for when you write it instead:
#   apiVersion: autoscaling/v2
#   kind: HorizontalPodAutoscaler
#   metadata: {name: %{dep}, namespace: %{ns}}
#   spec:
#     scaleTargetRef: {apiVersion: apps/v1, kind: Deployment, name: %{dep}}
#     minReplicas: %{min}
#     maxReplicas: %{max}
#     metrics:
#       - type: Resource
#         resource: {name: cpu, target: {type: Utilization, averageUtilization: %{cpu}}}
#     behavior: {scaleDown: {stabilizationWindowSeconds: %{win}, policies: [{type: Pods, value: 1, periodSeconds: 60}]}}
# Utilisation is usage divided by the REQUEST, so without a CPU request the HPA shows <unknown>
# and never scales on load. It does enforce minReplicas at once, metrics or not. TARGETS stays
# <unknown> here until metrics-server is installed and has a minute of data.
""",
},
_port(_ckad_more, 62, 106, "WORK"),
# ------------------------------------------------------------------ ARCH ----
{
 "id": 107, "domain": "ARCH", "title": "Kernel prerequisites for a Kubernetes node",
 "params": lambda r: {"name": r.choice(["k8s", "kubernetes", "99-kubernetes-cri"])},
 "task": """
Node `{W3}` is to be prepared the way a machine is before `kubeadm` puts Kubernetes on it. Reach
it with `{SHELL_W3}`.

1. Make the kernel modules `overlay` and `br_netfilter` load at every boot: list them, one per
   line, in `/etc/modules-load.d/%{name}.conf`.
2. Make these kernel parameters permanent in `/etc/sysctl.d/%{name}.conf`:
   `net.bridge.bridge-nf-call-iptables = 1`, `net.bridge.bridge-nf-call-ip6tables = 1` and
   `net.ipv4.ip_forward = 1`.
3. Write what the node reports right now to `{WS}/q107/node.txt`, in exactly this format:

```
ip_forward=<current value of net.ipv4.ip_forward>
swap=<on or off - is any swap device active on the node>
fail-swap-on=<the kubelet's failSwapOn setting>
cgroup-driver=<the kubelet's cgroupDriver>
```

(The nodes of this simulator are containers, so a module cannot really be loaded here: the two
files and the report are what is graded.)
""",
 "setup": r"""
rm -rf "$WS/q107"; mkdir -p "$WS/q107"
nx "$W3" 'rm -f /etc/modules-load.d/%{name}.conf /etc/sysctl.d/%{name}.conf' </dev/null
""",
 "checks": [
  C("/etc/modules-load.d/%{name}.conf lists overlay and br_netfilter", 2, r"""
got=$(nx "$W3" "grep -Ev '^[[:space:]]*(#|\$)' /etc/modules-load.d/%{name}.conf 2>/dev/null" </dev/null | tr -d ' \t\r' | sort | tr '\n' ' ')
eq "$got" "br_netfilter overlay " """),
  C("/etc/sysctl.d/%{name}.conf sets the three parameters to 1", 3, r"""
f=$(nx "$W3" "cat /etc/sysctl.d/%{name}.conf 2>/dev/null" </dev/null | tr -d ' \t\r')
[ -n "$f" ] || exit 1
for kv in net.bridge.bridge-nf-call-iptables=1 net.bridge.bridge-nf-call-ip6tables=1 net.ipv4.ip_forward=1; do
  echo "$f" | grep -qx "$kv" || exit 1
done"""),
  C("node.txt: ip_forward, swap, fail-swap-on and cgroup-driver as the node has them", 2, r"""
fwd=$(nx "$W3" "sysctl -n net.ipv4.ip_forward" </dev/null | tr -d '[:space:]')
swap=off; [ "$(nx "$W3" "sed 1d /proc/swaps | grep -c ." </dev/null | tr -d '[:space:]')" -gt 0 ] 2>/dev/null && swap=on
fso=$(nx "$W3" "awk '/^failSwapOn:/{print \$2}' /var/lib/kubelet/config.yaml" </dev/null | tr -d '[:space:]'); [ -n "$fso" ] || fso=true
cg=$(nx "$W3" "awk '/^cgroupDriver:/{print \$2}' /var/lib/kubelet/config.yaml" </dev/null | tr -d '[:space:]')
[ -n "$fwd" ] && [ -n "$cg" ] || exit 1
want=$(printf 'ip_forward=%s\nswap=%s\nfail-swap-on=%s\ncgroup-driver=%s' "$fwd" "$swap" "$fso" "$cg")
eq "$(tr -d ' \t\r' < "$WS/q107/node.txt" 2>/dev/null | grep .)" "$want" """),
 ],
 "solution": [hand(r"""{SHELL_W3}
sudo -i
vim /etc/modules-load.d/%{name}.conf""",
                   r"""The docs page *Container Runtimes* has both files under *Install and configure prerequisites*. One module per line:
```
overlay
br_netfilter
```""",
                   run=r"""ssh -n {W3} 'printf "overlay\nbr_netfilter\n" | sudo tee /etc/modules-load.d/%{name}.conf >/dev/null'"""),
              hand(r"""vim /etc/sysctl.d/%{name}.conf""",
                   r"""```
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
```
On a real machine they would now be loaded and applied without a reboot: `modprobe overlay`, `modprobe br_netfilter`, `sysctl --system`. `sysctl -w` and `modprobe` alone are gone after a reboot; the files make them stick.""",
                   run=r"""ssh -n {W3} 'printf "net.bridge.bridge-nf-call-iptables = 1\nnet.bridge.bridge-nf-call-ip6tables = 1\nnet.ipv4.ip_forward = 1\n" | sudo tee /etc/sysctl.d/%{name}.conf >/dev/null'"""),
              hand(r"""sysctl -n net.ipv4.ip_forward
cat /proc/swaps
grep -e failSwapOn -e cgroupDriver /var/lib/kubelet/config.yaml
exit
exit""",
                   r"""Any line below the header of `/proc/swaps` means swap is on. No `failSwapOn` line means the default, `true`.""",
                   run=r""""""),
              hand(r"""vim {WS}/q107/node.txt""",
                   r"""What the node printed:
```
ip_forward=<0 or 1>
swap=<on or off>
fail-swap-on=<true or false>
cgroup-driver=<systemd or cgroupfs>
```
kubeadm's preflight refuses a node with swap on unless the kubelet tolerates it (`failSwapOn: false`); and the kubelet's cgroup driver has to match the runtime's (`SystemdCgroup = true` in containerd goes with `cgroupDriver: systemd`).""",
                   run=r"""IPF=$(ssh -n {W3} 'sudo sysctl -n net.ipv4.ip_forward')
SW=off; [ "$(ssh -n {W3} 'sed 1d /proc/swaps | grep -c .')" -gt 0 ] && SW=on
FS=$(ssh -n {W3} "sudo awk '/^failSwapOn:/ {print \$2}' /var/lib/kubelet/config.yaml"); [ -n "$FS" ] || FS=true
CG=$(ssh -n {W3} "sudo awk '/^cgroupDriver:/ {print \$2}' /var/lib/kubelet/config.yaml")
printf 'ip_forward=%s\nswap=%s\nfail-swap-on=%s\ncgroup-driver=%s\n' "$IPF" "$SW" "$FS" "$CG" > {WS}/q107/node.txt""")],
},
{
 "id": 108, "domain": "ARCH", "title": "The container runtime behind a Pod",
 "params": lambda r: {"ns": ns_pick(r, "runtime"), "pod": name_pick(r, "probe")},
 "task": """
Pod `%{pod}` in Namespace `%{ns}` runs on node `{W2}`. Look at it from the node's side, through
the container runtime (`{SHELL_W2}`, then `crictl`), and write `{WS}/q108/runtime.txt` in exactly
this format:

```
runtime=<runtime name> <runtime version>
socket=<the runtime endpoint the kubelet on that node is configured with>
container-id=<the full ID of the Pod's container `app`>
pid=<the process ID of that container on the node>
log=<the full path of that container's log file on the node>
```
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q108"; mkdir -p "$WS/q108"
$K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1
$K apply -f - >/dev/null <<'EOF'
apiVersion: v1
kind: Pod
metadata: {name: %{pod}, namespace: %{ns}}
spec:
  terminationGracePeriodSeconds: 1
  nodeSelector: {kubernetes.io/hostname: "{W2}"}
  containers:
    - name: app
      image: busybox:1.36
      command: [sh, -c, "echo 'runtime probe started'; sleep 86400"]
EOF
$K -n %{ns} wait --for=condition=Ready pod/%{pod} --timeout=150s >/dev/null
""",
 "checks": [
  C("runtime and socket lines", 2, r"""
ver=$(nx "$W2" "crictl version 2>/dev/null" </dev/null)
name=$(echo "$ver" | awk '/^RuntimeName:/{print $2}'); rv=$(echo "$ver" | awk '/^RuntimeVersion:/{print $2}')
sock=$(nx "$W2" "awk '/^containerRuntimeEndpoint:/{print \$2}' /var/lib/kubelet/config.yaml" </dev/null | tr -d '"[:space:]')
[ -n "$name" ] && [ -n "$rv" ] && [ -n "$sock" ] || exit 1
f="$WS/q108/runtime.txt"
tr -d '\r' < "$f" 2>/dev/null | sed 's/[[:space:]]*$//' | grep -qx "runtime=$name $rv" && tr -d ' \t\r' < "$f" | grep -qx "socket=$sock" """),
  C("container-id is the full ID of container app", 2, r"""
cid=$($K -n %{ns} get pod %{pod} -o jsonpath='{.status.containerStatuses[0].containerID}' 2>/dev/null | sed 's#^[a-z-]*://##')
[ "${#cid}" = 64 ] && tr -d ' \t\r' < "$WS/q108/runtime.txt" 2>/dev/null | grep -qx "container-id=$cid" """),
  C("pid is that container's process on the node", 2, r"""
cid=$($K -n %{ns} get pod %{pod} -o jsonpath='{.status.containerStatuses[0].containerID}' 2>/dev/null | sed 's#^[a-z-]*://##')
pid=$(nx "$W2" "crictl inspect -o go-template --template '{{.info.pid}}' $cid 2>/dev/null" </dev/null | tr -d '[:space:]')
[ -n "$cid" ] && [ -n "$pid" ] && tr -d ' \t\r' < "$WS/q108/runtime.txt" 2>/dev/null | grep -qx "pid=$pid" """),
  C("log is the container's log file on the node", 1, r"""
cid=$($K -n %{ns} get pod %{pod} -o jsonpath='{.status.containerStatuses[0].containerID}' 2>/dev/null | sed 's#^[a-z-]*://##')
log=$(nx "$W2" "crictl inspect -o go-template --template '{{.status.logPath}}' $cid 2>/dev/null" </dev/null | tr -d '[:space:]')
[ -n "$cid" ] && [ -n "$log" ] && tr -d ' \t\r' < "$WS/q108/runtime.txt" 2>/dev/null | grep -qx "log=$log" """),
 ],
 "solution": [cmd(r"""k -n %{ns} get pod %{pod} -o wide""", r"""On `{W2}`."""),
              hand(r"""{SHELL_W2}
sudo -i
crictl version
grep containerRuntimeEndpoint /var/lib/kubelet/config.yaml
crictl ps --no-trunc --name app""",
                   r"""`RuntimeName` and `RuntimeVersion`; the CRI socket the kubelet uses - crictl speaks the same protocol, which is why it works even when the API server does not; and the container's full ID (`--no-trunc`), the line whose `POD` is `%{pod}`.""",
                   run=r""""""),
              hand(r"""crictl inspect 0123abcd... | grep -e '"pid"' -e logPath
exit
exit""",
                   r"""With that ID: `.info.pid` and `.status.logPath`. The log is the file `kubectl logs` reads, under `/var/log/pods`.""",
                   run=r""""""),
              hand(r"""vim {WS}/q108/runtime.txt""",
                   r"""The values you read:
```
runtime=containerd <version>
socket=unix:///run/containerd/containerd.sock
container-id=<the 64 hex characters>
pid=<the number>
log=/var/log/pods/%{ns}_%{pod}_<pod uid>/app/0.log
```""",
                   run=r"""V=$(ssh -n {W2} 'sudo crictl version 2>/dev/null')
RN=$(echo "$V" | awk '/^RuntimeName:/ {print $2}'); RV=$(echo "$V" | awk '/^RuntimeVersion:/ {print $2}')
SOCK=$(ssh -n {W2} "sudo awk '/^containerRuntimeEndpoint:/ {print \$2}' /var/lib/kubelet/config.yaml" | tr -d '"')
CID=$(kubectl -n %{ns} get pod %{pod} -o jsonpath='{.status.containerStatuses[0].containerID}' | sed 's#^[a-z-]*://##')
PID=$(ssh -n {W2} "sudo crictl inspect -o go-template --template '{{.info.pid}}' $CID 2>/dev/null")
LOG=$(ssh -n {W2} "sudo crictl inspect -o go-template --template '{{.status.logPath}}' $CID 2>/dev/null")
printf 'runtime=%s %s\nsocket=%s\ncontainer-id=%s\npid=%s\nlog=%s\n' "$RN" "$RV" "$SOCK" "$CID" "$PID" "$LOG" > {WS}/q108/runtime.txt""")],
},
{
 # drains the control-plane node: CoreDNS moves off it, so never beside the tasks that break DNS
 "id": 109, "domain": "ARCH", "title": "Drain the control-plane node and bring it back", "breaks": "coredns",
 "params": lambda r: {"ns": ns_pick(r, "maint"), "dep": name_pick(r, "beacon")},
 "task": """
The control-plane node `{CP}` is about to be upgraded and has to be emptied first.

1. Drain `{CP}`: it must be unschedulable, and the only Pods left on it are static Pods and Pods
   of DaemonSets. Deployment `%{dep}` in Namespace `%{ns}` has a Pod with an `emptyDir` volume on
   that node - losing that volume is accepted.
2. While the node is drained, save the output of `kubectl get nodes` to `{WS}/q109/drained.txt`,
   and write the phase the Pod of `%{dep}` is in now to `{WS}/q109/phase.txt`.
3. The upgrade is over: make the node schedulable again and wait until `%{dep}` has its Pod back.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q109"; mkdir -p "$WS/q109"
$K uncordon "$CP" >/dev/null 2>&1
$K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null 2>&1
$K -n %{ns} delete pod -l app=%{dep} --wait=true --grace-period=1 >/dev/null 2>&1
$K apply -f - >/dev/null <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata: {name: %{dep}, namespace: %{ns}, labels: {app: %{dep}}}
spec:
  replicas: 1
  selector: {matchLabels: {app: %{dep}}}
  template:
    metadata: {labels: {app: %{dep}}}
    spec:
      terminationGracePeriodSeconds: 2
      nodeSelector: {kubernetes.io/hostname: "{CP}"}
      tolerations:
        - {key: node-role.kubernetes.io/control-plane, operator: Exists, effect: NoSchedule}
      volumes: [{name: scratch, emptyDir: {}}]
      containers:
        - name: beacon
          image: busybox:1.36
          command: [sh, -c, "date > /scratch/started; sleep 86400"]
          volumeMounts: [{name: scratch, mountPath: /scratch}]
EOF
$K -n %{ns} rollout status deploy %{dep} --timeout=150s >/dev/null
$K -n %{ns} get pod -l app=%{dep} -o jsonpath='{.items[0].metadata.uid}' > "$WS/.q109-uid"
""",
 "checks": [
  C("drained.txt shows {CP} as SchedulingDisabled", 2, r"""
grep -Eq "^$CP[[:space:]]+Ready,SchedulingDisabled([[:space:]]|\$)" "$WS/q109/drained.txt" 2>/dev/null"""),
  C("phase.txt holds Pending", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q109/phase.txt" 2>/dev/null)" Pending"""),
  C("the Pod of %{dep} was evicted, and its replacement runs on {CP} again", 2, r"""
old=$(cat "$WS/.q109-uid" 2>/dev/null); [ -n "$old" ] || exit 1
for i in $(seq 1 20); do
  new=$($K -n %{ns} get pod -l app=%{dep} -o jsonpath='{range .items[?(@.status.phase=="Running")]}{.metadata.uid} {.spec.nodeName}{"\n"}{end}' 2>/dev/null | head -1)
  [ -n "$new" ] && break; sleep 3
done
[ -n "$new" ] && [ "${new%% *}" != "$old" ] && [ "${new##* }" = "$CP" ]"""),
  C("{CP} is schedulable again and CoreDNS is fully available", 2, r"""
old=$(cat "$WS/.q109-uid" 2>/dev/null); [ -n "$old" ] || exit 1
$K -n %{ns} get pod -l app=%{dep} -o jsonpath='{.items[*].metadata.uid}' 2>/dev/null | grep -q "$old" && exit 1
[ "$(j node "$CP" '{.spec.unschedulable}')" != true ] || exit 1
for i in $(seq 1 20); do
  [ "$(j -n kube-system deploy coredns '{.status.availableReplicas}')" = "$(j -n kube-system deploy coredns '{.spec.replicas}')" ] && exit 0; sleep 3
done; exit 1"""),
 ],
 "solution": """
k drain {CP} --ignore-daemonsets --delete-emptydir-data
#   without the flags it stops at:  cannot delete DaemonSet-managed Pods ... / cannot delete Pods with local storage ...
k get nodes > {WS}/q109/drained.txt                 # {CP}   Ready,SchedulingDisabled   control-plane
k -n %{ns} get pod -l app=%{dep}                    # Pending: it may only run on {CP}, and that node is cordoned
echo Pending > {WS}/q109/phase.txt
k uncordon {CP}
k -n %{ns} rollout status deploy %{dep}
k get nodes; k -n kube-system get deploy coredns    # CoreDNS moved to the workers during the drain and stays there
# drain = cordon + evict. It leaves DaemonSet Pods (they would come straight back) and static Pods,
# refuses Pods with emptyDir data until you accept the loss, and refuses bare Pods until --force -
# which deletes them for good. The node stays cordoned until you uncordon it: a real upgrade
# (kubeadm upgrade apply, then the kubelet package) happens between those two commands.
""",
},
{
 "id": 110, "domain": "ARCH", "title": "Prepare the join of a second control-plane node",
 "task": """
The cluster is getting a second control-plane node. Prepare what the new machine needs - on the
control-plane node, `{SHELL_CP}` - without joining anything.

1. Write the address every node uses to reach the API server - the cluster's control-plane
   endpoint, as kubeadm has it configured - to `{WS}/q110/endpoint.txt`.
2. Upload the control-plane certificates to the cluster, so that a joining control-plane node can
   fetch them.
3. Save the complete `kubeadm join` command a new **control-plane** node has to run - with a new
   token, the CA certificate hash and the certificate key from step 2 - to
   `{WS}/q110/join-control-plane.sh`, on one line.
""",
 "setup": r"""
rm -rf "$WS/q110"; mkdir -p "$WS/q110"
$K -n kube-system delete secret kubeadm-certs --ignore-not-found >/dev/null 2>&1
date +%s > "$WS/.q110-t0"
""",
 "checks": [
  C("endpoint.txt holds the control-plane endpoint", 1, r"""
want=$($K -n kube-system get cm kubeadm-config -o jsonpath='{.data.ClusterConfiguration}' 2>/dev/null | awk '/^controlPlaneEndpoint:/{print $2}' | tr -d '"')
[ -n "$want" ] && eq "$(tr -d '[:space:]' < "$WS/q110/endpoint.txt" 2>/dev/null)" "$want" """),
  C("the control-plane certificates were uploaded (Secret kubeadm-certs)", 2, r"""
ct=$(j -n kube-system secret kubeadm-certs '{.metadata.creationTimestamp}'); [ -n "$ct" ] || exit 1
[ "$(date -d "$ct" +%s)" -ge "$(( $(cat "$WS/.q110-t0" 2>/dev/null || echo 9999999999) - 5 ))" ]"""),
  C("join command for a control-plane node: endpoint, CA hash, --control-plane, a certificate key", 2, r"""
f="$WS/q110/join-control-plane.sh"
ep=$($K -n kube-system get cm kubeadm-config -o jsonpath='{.data.ClusterConfiguration}' 2>/dev/null | awk '/^controlPlaneEndpoint:/{print $2}' | tr -d '"')
hash=$(nx "$CP" "openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'" </dev/null | tr -d '[:space:]')
[ -n "$ep" ] && [ -n "$hash" ] || exit 1
l=$(grep 'kubeadm join' "$f" 2>/dev/null | head -1)
echo "$l" | grep -q "kubeadm join $ep " && echo "$l" | grep -q -- "--discovery-token-ca-cert-hash sha256:$hash" \
 && echo "$l" | grep -Eq -- '--control-plane( |$)' && echo "$l" | grep -Eq -- '--certificate-key [0-9a-f]{64}( |$)' """),
  C("its token is a real, unexpired bootstrap token", 2, r"""
tok=$(grep -oE -- '--token [a-z0-9]{6}\.[a-z0-9]{16}' "$WS/q110/join-control-plane.sh" 2>/dev/null | head -1 | awk '{print $2}'); [ -n "$tok" ] || exit 1
exp=$(j -n kube-system secret bootstrap-token-${tok%%.*} '{.data.expiration}' | base64 -d 2>/dev/null); [ -n "$exp" ] || exit 1
[ "$(date -d "$exp" +%s)" -gt "$(date +%s)" ]"""),
 ],
 "solution": """
k -n kube-system get cm kubeadm-config -o yaml | grep controlPlaneEndpoint       # {CP}:6443
echo {CP}:6443 > {WS}/q110/endpoint.txt
sudo -i                # the task's host is the control-plane node: no hop needed
kubeadm init phase upload-certs --upload-certs
#   [upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
#   [upload-certs] Using certificate key:
#   <64 hex characters>
kubeadm token create --print-join-command --certificate-key <that key>
#   kubeadm join {CP}:6443 --token ... --discovery-token-ca-cert-hash sha256:... --control-plane --certificate-key ...
exit
# paste that line into {WS}/q110/join-control-plane.sh
# A control-plane node needs the cluster's CA, front-proxy CA, etcd CA and service-account keys.
# upload-certs puts them, encrypted with the certificate key, into the Secret kubeadm-certs for
# two hours; `kubeadm join --control-plane --certificate-key` downloads and decrypts them. All of
# it only works because the cluster was initialised with a controlPlaneEndpoint - the one stable
# address (a load balancer or VIP in a real HA cluster) that every node and kubeconfig points at.
""",
},
{
 "id": 111, "domain": "ARCH", "title": "Who leads, and who is in etcd",
 "task": """
Before a second control-plane node is added, record how the control plane is laid out today.
Write `{WS}/q111/ha.txt` in exactly this format:

```
control-plane-nodes=<number of control-plane nodes>
etcd-topology=<stacked or external>
etcd-members=<number of members of the etcd cluster>
etcd-leader=<name of the etcd member that is the leader>
scheduler-leader=<name of the node whose kube-scheduler holds the leader lease>
controller-manager-leader=<name of the node whose kube-controller-manager holds the leader lease>
```

`stacked` means etcd runs on the control-plane nodes themselves, `external` that the API server
uses etcd hosts of their own. `etcdctl` is on the control-plane node (`{SHELL_CP}`); etcd's
certificates are under `/etc/kubernetes/pki/etcd`.
""",
 "setup": r"""
rm -rf "$WS/q111"; mkdir -p "$WS/q111"
ensure_etcd_tools >/dev/null 2>&1 || { echo "no etcdctl on $CP - rebuild the exam host image: ./ksim build"; exit 1; }
""",
 "checks": [
  C("control-plane-nodes and etcd-topology", 2, r"""
n=$($K get nodes -l node-role.kubernetes.io/control-plane --no-headers 2>/dev/null | grep -c .)
topo=external; [ "$($K -n kube-system get pods -l component=etcd,tier=control-plane --no-headers 2>/dev/null | grep -c .)" -ge 1 ] && topo=stacked
f=$(tr -d ' \t\r' < "$WS/q111/ha.txt" 2>/dev/null)
[ "$n" -ge 1 ] && echo "$f" | grep -qx "control-plane-nodes=$n" && echo "$f" | grep -qx "etcd-topology=$topo" """),
  C("etcd-members and etcd-leader", 3, r"""
E='""" + ETCDCTL + r"""'
out=$( { nx "$CP" "$E endpoint status -w json 2>/dev/null" </dev/null; echo; nx "$CP" "$E member list -w json 2>/dev/null" </dev/null; } | python3 -c '
import sys, json
docs = [json.loads(l) for l in sys.stdin.read().splitlines() if l.strip()[:1] in "[{" and l.strip()]
status, members = docs[0], docs[1]["members"]
leader = status[0]["Status"]["leader"]
print(len(members), next(m["name"] for m in members if m["ID"] == leader))' 2>/dev/null)
[ -n "$out" ] || exit 1
f=$(tr -d ' \t\r' < "$WS/q111/ha.txt" 2>/dev/null)
echo "$f" | grep -qx "etcd-members=${out%% *}" && echo "$f" | grep -qx "etcd-leader=${out##* }" """),
  C("scheduler-leader and controller-manager-leader", 2, r"""
s=$(j -n kube-system lease kube-scheduler '{.spec.holderIdentity}' | cut -d_ -f1)
c=$(j -n kube-system lease kube-controller-manager '{.spec.holderIdentity}' | cut -d_ -f1)
f=$(tr -d ' \t\r' < "$WS/q111/ha.txt" 2>/dev/null)
[ -n "$s" ] && [ -n "$c" ] && echo "$f" | grep -qx "scheduler-leader=$s" && echo "$f" | grep -qx "controller-manager-leader=$c" """),
 ],
 "solution": [cmd(r"""k get nodes -l node-role.kubernetes.io/control-plane
k -n kube-system get pods -l component=etcd -o wide
k -n kube-system get lease kube-scheduler kube-controller-manager""", r"""An etcd static Pod on the control-plane node: stacked. A Lease's `HOLDER` is `<node>_<uuid>`."""),
              hand(r"""sudo -i
export ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCDCTL_CACERT=/etc/kubernetes/pki/etcd/ca.crt ETCDCTL_CERT=/etc/kubernetes/pki/etcd/server.crt ETCDCTL_KEY=/etc/kubernetes/pki/etcd/server.key
etcdctl member list -w table
etcdctl endpoint status -w table
exit""",
                   r"""The host is the control-plane node. One line per member; `IS LEADER` in the status (`--cluster` asks every member).""",
                   run=r""""""),
              hand(r"""vim {WS}/q111/ha.txt""",
                   r"""```
control-plane-nodes=1
etcd-topology=stacked
etcd-members=1
etcd-leader={CP}
scheduler-leader={CP}
controller-manager-leader={CP}
```
In an HA control plane every API server serves at once, but one scheduler and one controller manager act, through the leader Lease. etcd elects its own leader and needs a majority to stay writable: three members survive the loss of one, five of two.""",
                   run=r"""E='sudo etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key'
N=$(kubectl get nodes -l node-role.kubernetes.io/control-plane --no-headers | wc -l)
M=$($E member list | wc -l)
L=$($E member list | awk -F', ' 'NR==1 {print $3}')
S=$(kubectl -n kube-system get lease kube-scheduler -o jsonpath='{.spec.holderIdentity}' | cut -d_ -f1)
C=$(kubectl -n kube-system get lease kube-controller-manager -o jsonpath='{.spec.holderIdentity}' | cut -d_ -f1)
printf 'control-plane-nodes=%s\netcd-topology=stacked\netcd-members=%s\netcd-leader=%s\nscheduler-leader=%s\ncontroller-manager-leader=%s\n' "$N" "$M" "$L" "$S" "$C" > {WS}/q111/ha.txt""")],
},
{
 "id": 112, "domain": "ARCH", "title": "CNI, CSI and CRI: what this cluster plugs in",
 "task": """
Kubernetes leaves networking, storage and running containers to plugins. Record which ones this
cluster uses in `{WS}/q112/interfaces.txt`, in exactly this format:

```
cni-config=<full path of the CNI configuration file on node {W1}>
cni-plugins=<the type of every plugin in that file's chain, in order, separated by commas>
cni-ipam=<the type of the IPAM plugin it uses>
cri-socket=<the runtime endpoint the kubelet on {W1} is configured with>
csi-drivers=<number of CSIDriver objects in the cluster>
default-provisioner=<provisioner of the default StorageClass>
```

Reach the node with `{SHELL_W1}`.
""",
 "setup": r"""
rm -rf "$WS/q112"; mkdir -p "$WS/q112"
""",
 "checks": [
  C("cni-config, cni-plugins and cni-ipam", 3, r"""
conf=$(nx "$W1" "ls /etc/cni/net.d/*.conflist /etc/cni/net.d/*.conf 2>/dev/null | sort | head -1" </dev/null | tr -d '[:space:]'); [ -n "$conf" ] || exit 1
out=$(nx "$W1" "cat $conf" </dev/null | python3 -c '
import sys, json
d = json.load(sys.stdin); p = d.get("plugins", [d])
print(",".join(x["type"] for x in p), next((x["ipam"]["type"] for x in p if "ipam" in x), ""))' 2>/dev/null)
[ -n "$out" ] || exit 1
f=$(tr -d ' \t\r' < "$WS/q112/interfaces.txt" 2>/dev/null)
echo "$f" | grep -qx "cni-config=$conf" && echo "$f" | grep -qx "cni-plugins=${out%% *}" && echo "$f" | grep -qx "cni-ipam=${out##* }" """),
  C("cri-socket", 2, r"""
sock=$(nx "$W1" "awk '/^containerRuntimeEndpoint:/{print \$2}' /var/lib/kubelet/config.yaml" </dev/null | tr -d '"[:space:]')
[ -n "$sock" ] && tr -d ' \t\r' < "$WS/q112/interfaces.txt" 2>/dev/null | grep -qx "cri-socket=$sock" """),
  C("csi-drivers and default-provisioner", 2, r"""
n=$($K get csidrivers --no-headers 2>/dev/null | grep -c .)
prov=$($K get sc -o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.provisioner}{"\n"}{end}' 2>/dev/null | head -1)
[ -n "$prov" ] || exit 1
f=$(tr -d ' \t\r' < "$WS/q112/interfaces.txt" 2>/dev/null)
echo "$f" | grep -qx "csi-drivers=$n" && echo "$f" | grep -qx "default-provisioner=$prov" """),
 ],
 "solution": [hand(r"""{SHELL_W1}
sudo -i
ls /etc/cni/net.d
cat /etc/cni/net.d/10-kindnet.conflist
grep containerRuntimeEndpoint /var/lib/kubelet/config.yaml
exit
exit""",
                   r"""The runtime takes the first file of `/etc/cni/net.d`. Its `plugins` chain, in order - each `type` names a binary in `/opt/cni/bin` - and the `ipam` type inside one of them.""",
                   run=r""""""),
              cmd(r"""k get csidrivers
k get sc""", r"""`No resources found`: no CSI driver - the local-path provisioner is not one. The `(default)` class's `PROVISIONER`."""),
              hand(r"""vim {WS}/q112/interfaces.txt""",
                   r"""The values you read - here:
```
cni-config=/etc/cni/net.d/10-kindnet.conflist
cni-plugins=ptp,portmap
cni-ipam=host-local
cri-socket=unix:///run/containerd/containerd.sock
csi-drivers=0
default-provisioner=rancher.io/local-path
```
`ptp` wires each Pod's veth pair, `host-local` hands out addresses from the node's Pod CIDR, `portmap` serves hostPorts.""",
                   run=r"""CONF=$(ssh -n {W1} "sudo sh -c 'ls /etc/cni/net.d/*.conflist | sort | head -1'")
PL=$(ssh -n {W1} "sudo cat $CONF" | jq -r '[.plugins[].type] | join(",")')
IPAM=$(ssh -n {W1} "sudo cat $CONF" | jq -r '[.plugins[] | select(.ipam) | .ipam.type][0]')
SOCK=$(ssh -n {W1} "sudo awk '/^containerRuntimeEndpoint:/ {print \$2}' /var/lib/kubelet/config.yaml" | tr -d '"')
N=$(kubectl get csidrivers --no-headers 2>/dev/null | wc -l)
PROV=$(kubectl get sc -o json | jq -r '.items[] | select(.metadata.annotations["storageclass.kubernetes.io/is-default-class"] == "true") | .provisioner')
printf 'cni-config=%s\ncni-plugins=%s\ncni-ipam=%s\ncri-socket=%s\ncsi-drivers=%s\ndefault-provisioner=%s\n' "$CONF" "$PL" "$IPAM" "$SOCK" "$N" "$PROV" > {WS}/q112/interfaces.txt""")],
},
# ------------------------------------------------------------------- NET ----
{
 "id": 113, "domain": "NET", "title": "Pod to Pod, across nodes",
 "params": lambda r: {"ns": ns_pick(r, "wire"), "web": name_pick(r, "web"), "client": name_pick(r, "client"), "edge": name_pick(r, "edge")},
 "task": """
Namespace `%{ns}` runs a few Pods on different nodes. No Service is involved in this task.

1. Write the IP address of Pod `%{web}` and the node it runs on to `{WS}/q113/web.txt`, as
   `<ip> <node>`.
2. From Pod `%{client}`, which runs on another node, request `http://<that IP>` directly and save
   the response to `{WS}/q113/direct.html`.
3. Write the Pod CIDR of every node of the cluster to `{WS}/q113/pod-cidrs.txt`, one node per
   line as `<node> <cidr>`.
4. One Pod in `%{ns}` has an address that lies in none of those ranges. Write its name to
   `{WS}/q113/outside.txt` and the Pod spec field that causes it to `{WS}/q113/why.txt`.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q113"; mkdir -p "$WS/q113"
$K -n %{ns} delete pod --all --wait=true --grace-period=1 >/dev/null 2>&1
$K apply -f - >/dev/null <<'EOF'
apiVersion: v1
kind: Pod
metadata: {name: %{web}, namespace: %{ns}, labels: {app: web}}
spec:
  nodeSelector: {kubernetes.io/hostname: "{W1}"}
  containers: [{name: web, image: "nginx:1.27", ports: [{containerPort: 80}]}]
---
apiVersion: v1
kind: Pod
metadata: {name: %{client}, namespace: %{ns}, labels: {app: client}}
spec:
  terminationGracePeriodSeconds: 1
  nodeSelector: {kubernetes.io/hostname: "{W2}"}
  containers: [{name: client, image: "busybox:1.36", command: [sleep, "86400"]}]
---
apiVersion: v1
kind: Pod
metadata: {name: %{edge}, namespace: %{ns}, labels: {app: edge}}
spec:
  terminationGracePeriodSeconds: 1
  hostNetwork: true
  nodeSelector: {kubernetes.io/hostname: "{W3}"}
  containers: [{name: edge, image: "busybox:1.36", command: [sleep, "86400"]}]
---
apiVersion: v1
kind: Pod
metadata: {name: spare-%{edge}, namespace: %{ns}, labels: {app: spare}}
spec:
  terminationGracePeriodSeconds: 1
  nodeSelector: {kubernetes.io/hostname: "{W3}"}
  containers: [{name: spare, image: "busybox:1.36", command: [sleep, "86400"]}]
EOF
$K -n %{ns} wait --for=condition=Ready pod --all --timeout=150s >/dev/null
""",
 "checks": [
  C("web.txt holds the Pod's IP and node", 1, r"""
want=$(j -n %{ns} pod %{web} '{.status.podIP} {.spec.nodeName}')
[ -n "$want" ] && eq "$(tr -s ' \t' ' ' < "$WS/q113/web.txt" 2>/dev/null | tr -d '\r' | sed 's/^ *//; s/ *$//' | grep .)" "$want" """),
  C("direct.html holds the page fetched from the Pod's IP", 2, r"""grep -q 'Welcome to nginx' "$WS/q113/direct.html" 2>/dev/null"""),
  C("pod-cidrs.txt lists every node with its Pod CIDR", 2, r"""
want=$($K get nodes -o jsonpath='{range .items[*]}{.metadata.name} {.spec.podCIDR}{"\n"}{end}' 2>/dev/null | sort)
[ "$(echo "$want" | grep -c .)" -ge 2 ] || exit 1
eq "$(tr -s ' \t' ' ' < "$WS/q113/pod-cidrs.txt" 2>/dev/null | tr -d '\r' | sed 's/^ *//; s/ *$//' | grep . | sort)" "$want" """),
  C("outside.txt names %{edge}, why.txt names hostNetwork", 2, r"""
eq "$(tr -d '[:space:]' < "$WS/q113/outside.txt" 2>/dev/null | sed 's#^pod/##')" %{edge} && grep -qi 'hostNetwork' "$WS/q113/why.txt" 2>/dev/null"""),
 ],
 "solution": [cmd(r"""k -n %{ns} get pods -o wide
k get nodes -o wide""", r"""The `IP` and `NODE` of every Pod. `%{edge}`'s IP is its node's own `INTERNAL-IP`, not one from a Pod CIDR."""),
              hand(r"""echo "10.244.1.5 {W1}" > {WS}/q113/web.txt""",
                   r"""With `%{web}`'s IP and node.""",
                   run=r"""kubectl -n %{ns} get pod %{web} -o jsonpath='{.status.podIP} {.spec.nodeName}' > {WS}/q113/web.txt"""),
              hand(r"""k -n %{ns} exec %{client} -- wget -qO- -T3 http://10.244.1.5 > {WS}/q113/direct.html""",
                   r"""With that IP: Pod to Pod, across nodes, no Service and no NAT - the network model the CNI makes true.""",
                   run=r"""kubectl -n %{ns} exec %{client} -- wget -qO- -T3 http://$(kubectl -n %{ns} get pod %{web} -o jsonpath='{.status.podIP}') > {WS}/q113/direct.html"""),
              cmd(r"""k get nodes -o custom-columns=NAME:.metadata.name,CIDR:.spec.podCIDR --no-headers > {WS}/q113/pod-cidrs.txt
cat {WS}/q113/pod-cidrs.txt
k -n %{ns} get pod %{edge} -o yaml | grep hostNetwork
echo %{edge} > {WS}/q113/outside.txt
echo hostNetwork > {WS}/q113/why.txt""", r"""A `hostNetwork` Pod lives in its node's network namespace and shares the node's address and ports.""")],
},
]