"""CKA tasks 103-113: pool tasks for the curriculum objectives the bank covered thinly or not at all.

curriculum.py maps every task to the objectives of the CKA curriculum and turns the bank into a
study path (./ksim objectives). Mapping the first 102 tasks left eight objectives with fewer than
two pool tasks; these close them:

  2.3 Monitor cluster and application resource usage     103 kubectl top
  2.4 Manage and evaluate container output streams       104 logs by selector, of one replica, of an init container
  3.3 Configure workload autoscaling                     105 an HPA on CPU with a scale-down policy
  3.4 Robust, self-healing deployments                   106 probes that check the wrong thing
  4.2 Prepare underlying infrastructure                  107 kernel modules and sysctls   108 the runtime behind a Pod
  4.4 Manage the lifecycle of clusters                   109 drain the control-plane node and bring it back
  4.5 Highly-available control plane                     110 the join of a second control-plane node   111 leaders and etcd members
  4.7 Extension interfaces (CNI, CSI, CRI)               112 what this cluster plugs in   (and 108)
  5.1 Connectivity between Pods                          113 Pod to Pod across nodes, Pod CIDRs, hostNetwork

103, 104 and 106 are the CKAD tasks for the same skill (kubectl top, logs, probes) under CKA ids:
the objective is word for word the same, so is the scenario. Everything else is CKA ground - nodes,
kubeadm, etcd, the runtime. All normal difficulty, all in the random pool; nothing here breaks a
node or a control-plane component (109 cordons the control-plane node while you work on it and has
you uncordon it; the session baseline uncordons it if you do not).
"""
from sim.bank import C, NS, ns_pick, name_pick
from ckad import bank_extra as _ckad_extra, bank_more as _ckad_more


def _port(bank, src_id, new_id, domain, **over):
    """A CKAD task that serves a CKA objective as it is: same scenario and graders, CKA id and paths."""
    src = next(q for q in bank.QUESTIONS if q["id"] == src_id)
    old, new = f"q{src_id}", f"q{new_id}"
    q = dict(src, id=new_id, domain=domain, **over)
    for k in ("task", "setup", "solution"):
        q[k] = src[k].replace(old, new)
    q["checks"] = [dict(c, script=c["script"].replace(old, new), name=c["name"].replace(old, new)) for c in src["checks"]]
    return q


ETCDCTL = ("etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt "
           "--cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key")

QUESTIONS = [
# ---------------------------------------------------------------- TSHOOT ----
# needs a working metrics API, which Q23 breaks on purpose: never in one session with it
_port(_ckad_extra, 23, 103, "TSHOOT", breaks="metrics-server"),
_port(_ckad_more, 64, 104, "TSHOOT"),
# ------------------------------------------------------------------ WORK ----
{
 "id": 105, "domain": "WORK", "title": "Autoscale on CPU, scale down calmly",
 "params": lambda r: {"ns": ns_pick(r, "scale"), "dep": name_pick(r, "render"), "min": r.choice([2, 3]),
                      "max": r.choice([6, 8, 10]), "cpu": r.choice([50, 60, 70]), "win": r.choice([120, 180, 300])},
 "task": """
Deployment `%{dep}` in Namespace `%{ns}` has to scale with its load.

1. An autoscaler that works on CPU utilisation needs a CPU request to measure against. Give the
   container `web` a CPU request of `100m` and leave everything else in the Deployment as it is.
2. Create HorizontalPodAutoscaler `%{dep}` for the Deployment: at least **%{min}** replicas, at
   most **%{max}**, target average CPU utilisation **%{cpu}%**.
3. Scaling down must be calm: a stabilisation window of **%{win}** seconds, and at most **1 Pod
   removed per 60 seconds**. Leave scaling up at its defaults.
""",
 "setup": NS + r"""
mkns %{ns}
$K -n %{ns} delete hpa --all >/dev/null 2>&1
$K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null 2>&1
$K apply -f - >/dev/null <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata: {name: %{dep}, namespace: %{ns}, labels: {app: %{dep}}}
spec:
  replicas: 1
  selector: {matchLabels: {app: %{dep}}}
  template:
    metadata: {labels: {app: %{dep}}}
    spec:
      containers:
        - name: web
          image: nginx:1.27
          ports: [{containerPort: 80}]
          resources: {requests: {memory: 32Mi}}
EOF
$K -n %{ns} rollout status deploy %{dep} --timeout=150s >/dev/null
""",
 "checks": [
  C("container web requests cpu 100m, the rest of it unchanged", 2, r"""
eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].name}/{.spec.template.spec.containers[0].image}/{.spec.template.spec.containers[0].resources.requests.cpu}/{.spec.template.spec.containers[0].resources.requests.memory}')" \
   "web/nginx:1.27/100m/32Mi" """),
  C("HPA %{dep}: Deployment %{dep}, %{min}-%{max} replicas, %{cpu}% average CPU utilisation", 2, r"""
$K -n %{ns} get hpa %{dep} -o json 2>/dev/null | python3 -c '
import sys, json
s = json.load(sys.stdin)["spec"]; t = s["scaleTargetRef"]; m = s.get("metrics") or []
cpu = [x["resource"]["target"] for x in m if x.get("type") == "Resource" and x["resource"]["name"] == "cpu"]
ok = (t["kind"], t["name"]) == ("Deployment", "%{dep}") and (s.get("minReplicas"), s["maxReplicas"]) == (%{min}, %{max}) \
     and len(m) == 1 and len(cpu) == 1 and cpu[0].get("type") == "Utilization" and cpu[0].get("averageUtilization") == %{cpu}
sys.exit(0 if ok else 1)'"""),
  C("scale-down: %{win} s stabilisation window, at most 1 Pod per 60 s", 2, r"""
$K -n %{ns} get hpa %{dep} -o json 2>/dev/null | python3 -c '
import sys, json
d = (json.load(sys.stdin)["spec"].get("behavior") or {}).get("scaleDown") or {}
p = d.get("policies") or []
ok = d.get("stabilizationWindowSeconds") == %{win} and len(p) == 1 \
     and (p[0].get("type"), p[0].get("value"), p[0].get("periodSeconds")) == ("Pods", 1, 60)
sys.exit(0 if ok else 1)'"""),
  C("the autoscaler has taken the Deployment to at least %{min} replicas", 1, r"""
$K -n %{ns} get hpa %{dep} >/dev/null 2>&1 || exit 1
for i in $(seq 1 20); do
  [ "$(j -n %{ns} deploy %{dep} '{.spec.replicas}')" -ge %{min} ] 2>/dev/null && exit 0; sleep 3
done; exit 1"""),
 ],
 "solution": """
k -n %{ns} set resources deploy %{dep} -c web --requests=cpu=100m
k -n %{ns} autoscale deploy %{dep} --min=%{min} --max=%{max} --cpu=%{cpu}%          # kubectl before 1.34: --cpu-percent=%{cpu}
k -n %{ns} edit hpa %{dep}                       # `autoscale` cannot write behaviour - add under spec:
#   behavior:
#     scaleDown:
#       stabilizationWindowSeconds: %{win}
#       policies:
#         - {type: Pods, value: 1, periodSeconds: 60}
k -n %{ns} get hpa %{dep}                        # MINPODS %{min}, MAXPODS %{max}; REPLICAS climbs to %{min}
k -n %{ns} describe hpa %{dep}
# The whole object, for when you write it instead:
#   apiVersion: autoscaling/v2
#   kind: HorizontalPodAutoscaler
#   metadata: {name: %{dep}, namespace: %{ns}}
#   spec:
#     scaleTargetRef: {apiVersion: apps/v1, kind: Deployment, name: %{dep}}
#     minReplicas: %{min}
#     maxReplicas: %{max}
#     metrics:
#       - type: Resource
#         resource: {name: cpu, target: {type: Utilization, averageUtilization: %{cpu}}}
#     behavior: {scaleDown: {stabilizationWindowSeconds: %{win}, policies: [{type: Pods, value: 1, periodSeconds: 60}]}}
# Utilisation is usage divided by the REQUEST, so without a CPU request the HPA shows <unknown>
# and never scales on load. It does enforce minReplicas at once, metrics or not. TARGETS stays
# <unknown> here until metrics-server is installed and has a minute of data.
""",
},
_port(_ckad_more, 62, 106, "WORK"),
# ------------------------------------------------------------------ ARCH ----
{
 "id": 107, "domain": "ARCH", "title": "Kernel prerequisites for a Kubernetes node",
 "params": lambda r: {"name": r.choice(["k8s", "kubernetes", "99-kubernetes-cri"])},
 "task": """
Node `{W3}` is to be prepared the way a machine is before `kubeadm` puts Kubernetes on it. Reach
it with `{SHELL_W3}`.

1. Make the kernel modules `overlay` and `br_netfilter` load at every boot: list them, one per
   line, in `/etc/modules-load.d/%{name}.conf`.
2. Make these kernel parameters permanent in `/etc/sysctl.d/%{name}.conf`:
   `net.bridge.bridge-nf-call-iptables = 1`, `net.bridge.bridge-nf-call-ip6tables = 1` and
   `net.ipv4.ip_forward = 1`.
3. Write what the node reports right now to `{WS}/q107/node.txt`, in exactly this format:

```
ip_forward=<current value of net.ipv4.ip_forward>
swap=<on or off - is any swap device active on the node>
fail-swap-on=<the kubelet's failSwapOn setting>
cgroup-driver=<the kubelet's cgroupDriver>
```

(The nodes of this simulator are containers, so a module cannot really be loaded here: the two
files and the report are what is graded.)
""",
 "setup": r"""
rm -rf "$WS/q107"; mkdir -p "$WS/q107"
nx "$W3" 'rm -f /etc/modules-load.d/%{name}.conf /etc/sysctl.d/%{name}.conf' </dev/null
""",
 "checks": [
  C("/etc/modules-load.d/%{name}.conf lists overlay and br_netfilter", 2, r"""
got=$(nx "$W3" "grep -Ev '^[[:space:]]*(#|\$)' /etc/modules-load.d/%{name}.conf 2>/dev/null" </dev/null | tr -d ' \t\r' | sort | tr '\n' ' ')
eq "$got" "br_netfilter overlay " """),
  C("/etc/sysctl.d/%{name}.conf sets the three parameters to 1", 3, r"""
f=$(nx "$W3" "cat /etc/sysctl.d/%{name}.conf 2>/dev/null" </dev/null | tr -d ' \t\r')
[ -n "$f" ] || exit 1
for kv in net.bridge.bridge-nf-call-iptables=1 net.bridge.bridge-nf-call-ip6tables=1 net.ipv4.ip_forward=1; do
  echo "$f" | grep -qx "$kv" || exit 1
done"""),
  C("node.txt: ip_forward, swap, fail-swap-on and cgroup-driver as the node has them", 2, r"""
fwd=$(nx "$W3" "sysctl -n net.ipv4.ip_forward" </dev/null | tr -d '[:space:]')
swap=off; [ "$(nx "$W3" "sed 1d /proc/swaps | grep -c ." </dev/null | tr -d '[:space:]')" -gt 0 ] 2>/dev/null && swap=on
fso=$(nx "$W3" "awk '/^failSwapOn:/{print \$2}' /var/lib/kubelet/config.yaml" </dev/null | tr -d '[:space:]'); [ -n "$fso" ] || fso=true
cg=$(nx "$W3" "awk '/^cgroupDriver:/{print \$2}' /var/lib/kubelet/config.yaml" </dev/null | tr -d '[:space:]')
[ -n "$fwd" ] && [ -n "$cg" ] || exit 1
want=$(printf 'ip_forward=%s\nswap=%s\nfail-swap-on=%s\ncgroup-driver=%s' "$fwd" "$swap" "$fso" "$cg")
eq "$(tr -d ' \t\r' < "$WS/q107/node.txt" 2>/dev/null | grep .)" "$want" """),
 ],
 "solution": """
{SHELL_W3}
sudo -i
printf 'overlay\\nbr_netfilter\\n' > /etc/modules-load.d/%{name}.conf
cat > /etc/sysctl.d/%{name}.conf <<EOF
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF
# on a real machine you now load and apply them without a reboot:
#   modprobe overlay && modprobe br_netfilter && sysctl --system
sysctl -n net.ipv4.ip_forward                              # 1
cat /proc/swaps                                            # any line under the header = swap is on  (swapon --show)
grep -E 'failSwapOn|cgroupDriver' /var/lib/kubelet/config.yaml
exit; exit
cat > {WS}/q107/node.txt <<EOF
ip_forward=1
swap=<on|off>
fail-swap-on=<true|false>
cgroup-driver=<systemd|cgroupfs>
EOF
# `sysctl -w` and `modprobe` alone are gone after a reboot - the files are what makes them stick.
# kubeadm's preflight refuses a node with swap on unless the kubelet is told to tolerate it
# (failSwapOn: false), and the kubelet's cgroup driver has to match the container runtime's
# (SystemdCgroup = true in /etc/containerd/config.toml goes with cgroupDriver: systemd).
""",
},
{
 "id": 108, "domain": "ARCH", "title": "The container runtime behind a Pod",
 "params": lambda r: {"ns": ns_pick(r, "runtime"), "pod": name_pick(r, "probe")},
 "task": """
Pod `%{pod}` in Namespace `%{ns}` runs on node `{W2}`. Look at it from the node's side, through
the container runtime (`{SHELL_W2}`, then `crictl`), and write `{WS}/q108/runtime.txt` in exactly
this format:

```
runtime=<runtime name> <runtime version>
socket=<the runtime endpoint the kubelet on that node is configured with>
container-id=<the full ID of the Pod's container `app`>
pid=<the process ID of that container on the node>
log=<the full path of that container's log file on the node>
```
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q108"; mkdir -p "$WS/q108"
$K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1
$K apply -f - >/dev/null <<'EOF'
apiVersion: v1
kind: Pod
metadata: {name: %{pod}, namespace: %{ns}}
spec:
  terminationGracePeriodSeconds: 1
  nodeSelector: {kubernetes.io/hostname: "{W2}"}
  containers:
    - name: app
      image: busybox:1.36
      command: [sh, -c, "echo 'runtime probe started'; sleep 86400"]
EOF
$K -n %{ns} wait --for=condition=Ready pod/%{pod} --timeout=150s >/dev/null
""",
 "checks": [
  C("runtime and socket lines", 2, r"""
ver=$(nx "$W2" "crictl version 2>/dev/null" </dev/null)
name=$(echo "$ver" | awk '/^RuntimeName:/{print $2}'); rv=$(echo "$ver" | awk '/^RuntimeVersion:/{print $2}')
sock=$(nx "$W2" "awk '/^containerRuntimeEndpoint:/{print \$2}' /var/lib/kubelet/config.yaml" </dev/null | tr -d '"[:space:]')
[ -n "$name" ] && [ -n "$rv" ] && [ -n "$sock" ] || exit 1
f="$WS/q108/runtime.txt"
tr -d '\r' < "$f" 2>/dev/null | sed 's/[[:space:]]*$//' | grep -qx "runtime=$name $rv" && tr -d ' \t\r' < "$f" | grep -qx "socket=$sock" """),
  C("container-id is the full ID of container app", 2, r"""
cid=$($K -n %{ns} get pod %{pod} -o jsonpath='{.status.containerStatuses[0].containerID}' 2>/dev/null | sed 's#^[a-z-]*://##')
[ "${#cid}" = 64 ] && tr -d ' \t\r' < "$WS/q108/runtime.txt" 2>/dev/null | grep -qx "container-id=$cid" """),
  C("pid is that container's process on the node", 2, r"""
cid=$($K -n %{ns} get pod %{pod} -o jsonpath='{.status.containerStatuses[0].containerID}' 2>/dev/null | sed 's#^[a-z-]*://##')
pid=$(nx "$W2" "crictl inspect -o go-template --template '{{.info.pid}}' $cid 2>/dev/null" </dev/null | tr -d '[:space:]')
[ -n "$cid" ] && [ -n "$pid" ] && tr -d ' \t\r' < "$WS/q108/runtime.txt" 2>/dev/null | grep -qx "pid=$pid" """),
  C("log is the container's log file on the node", 1, r"""
cid=$($K -n %{ns} get pod %{pod} -o jsonpath='{.status.containerStatuses[0].containerID}' 2>/dev/null | sed 's#^[a-z-]*://##')
log=$(nx "$W2" "crictl inspect -o go-template --template '{{.status.logPath}}' $cid 2>/dev/null" </dev/null | tr -d '[:space:]')
[ -n "$cid" ] && [ -n "$log" ] && tr -d ' \t\r' < "$WS/q108/runtime.txt" 2>/dev/null | grep -qx "log=$log" """),
 ],
 "solution": """
k -n %{ns} get pod %{pod} -o wide                       # on {W2}
{SHELL_W2}
sudo -i
crictl version                                          # RuntimeName: containerd   RuntimeVersion: v2.x.y
grep containerRuntimeEndpoint /var/lib/kubelet/config.yaml       # unix:///run/containerd/containerd.sock
crictl pods --name %{pod}                               # the Pod sandbox
crictl ps --no-trunc --name app --pod $(crictl pods --name %{pod} -q)     # full container ID
crictl inspect <id> | grep -E '"pid"|logPath'           # .info.pid and .status.logPath
#   or:  crictl inspect -o go-template --template '{{.info.pid}} {{.status.logPath}}' <id>
exit; exit
cat > {WS}/q108/runtime.txt <<EOF
runtime=containerd <version>
socket=unix:///run/containerd/containerd.sock
container-id=<64 hex characters>
pid=<number>
log=/var/log/pods/%{ns}_%{pod}_<pod uid>/app/0.log
EOF
# The kubelet drives the runtime over the CRI socket, and crictl speaks the same protocol - which
# is why it still works when the API server does not. `kubectl get pod -o
# jsonpath='{.status.containerStatuses[0].containerID}'` shows the same ID with a containerd://
# prefix; /var/log/pods/... is the file `kubectl logs` reads.
""",
},
{
 # drains the control-plane node: CoreDNS moves off it, so never beside the tasks that break DNS
 "id": 109, "domain": "ARCH", "title": "Drain the control-plane node and bring it back", "breaks": "coredns",
 "params": lambda r: {"ns": ns_pick(r, "maint"), "dep": name_pick(r, "beacon")},
 "task": """
The control-plane node `{CP}` is about to be upgraded and has to be emptied first.

1. Drain `{CP}`: it must be unschedulable, and the only Pods left on it are static Pods and Pods
   of DaemonSets. Deployment `%{dep}` in Namespace `%{ns}` has a Pod with an `emptyDir` volume on
   that node - losing that volume is accepted.
2. While the node is drained, save the output of `kubectl get nodes` to `{WS}/q109/drained.txt`,
   and write the phase the Pod of `%{dep}` is in now to `{WS}/q109/phase.txt`.
3. The upgrade is over: make the node schedulable again and wait until `%{dep}` has its Pod back.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q109"; mkdir -p "$WS/q109"
$K uncordon "$CP" >/dev/null 2>&1
$K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null 2>&1
$K -n %{ns} delete pod -l app=%{dep} --wait=true --grace-period=1 >/dev/null 2>&1
$K apply -f - >/dev/null <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata: {name: %{dep}, namespace: %{ns}, labels: {app: %{dep}}}
spec:
  replicas: 1
  selector: {matchLabels: {app: %{dep}}}
  template:
    metadata: {labels: {app: %{dep}}}
    spec:
      terminationGracePeriodSeconds: 2
      nodeSelector: {kubernetes.io/hostname: "{CP}"}
      tolerations:
        - {key: node-role.kubernetes.io/control-plane, operator: Exists, effect: NoSchedule}
      volumes: [{name: scratch, emptyDir: {}}]
      containers:
        - name: beacon
          image: busybox:1.36
          command: [sh, -c, "date > /scratch/started; sleep 86400"]
          volumeMounts: [{name: scratch, mountPath: /scratch}]
EOF
$K -n %{ns} rollout status deploy %{dep} --timeout=150s >/dev/null
$K -n %{ns} get pod -l app=%{dep} -o jsonpath='{.items[0].metadata.uid}' > "$WS/.q109-uid"
""",
 "checks": [
  C("drained.txt shows {CP} as SchedulingDisabled", 2, r"""
grep -Eq "^$CP[[:space:]]+Ready,SchedulingDisabled([[:space:]]|\$)" "$WS/q109/drained.txt" 2>/dev/null"""),
  C("phase.txt holds Pending", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q109/phase.txt" 2>/dev/null)" Pending"""),
  C("the Pod of %{dep} was evicted, and its replacement runs on {CP} again", 2, r"""
old=$(cat "$WS/.q109-uid" 2>/dev/null); [ -n "$old" ] || exit 1
for i in $(seq 1 20); do
  new=$($K -n %{ns} get pod -l app=%{dep} -o jsonpath='{range .items[?(@.status.phase=="Running")]}{.metadata.uid} {.spec.nodeName}{"\n"}{end}' 2>/dev/null | head -1)
  [ -n "$new" ] && break; sleep 3
done
[ -n "$new" ] && [ "${new%% *}" != "$old" ] && [ "${new##* }" = "$CP" ]"""),
  C("{CP} is schedulable again and CoreDNS is fully available", 2, r"""
old=$(cat "$WS/.q109-uid" 2>/dev/null); [ -n "$old" ] || exit 1
$K -n %{ns} get pod -l app=%{dep} -o jsonpath='{.items[*].metadata.uid}' 2>/dev/null | grep -q "$old" && exit 1
[ "$(j node "$CP" '{.spec.unschedulable}')" != true ] || exit 1
for i in $(seq 1 20); do
  [ "$(j -n kube-system deploy coredns '{.status.availableReplicas}')" = "$(j -n kube-system deploy coredns '{.spec.replicas}')" ] && exit 0; sleep 3
done; exit 1"""),
 ],
 "solution": """
k drain {CP} --ignore-daemonsets --delete-emptydir-data
#   without the flags it stops at:  cannot delete DaemonSet-managed Pods ... / cannot delete Pods with local storage ...
k get nodes > {WS}/q109/drained.txt                 # {CP}   Ready,SchedulingDisabled   control-plane
k -n %{ns} get pod -l app=%{dep}                    # Pending: it may only run on {CP}, and that node is cordoned
echo Pending > {WS}/q109/phase.txt
k uncordon {CP}
k -n %{ns} rollout status deploy %{dep}
k get nodes; k -n kube-system get deploy coredns    # CoreDNS moved to the workers during the drain and stays there
# drain = cordon + evict. It leaves DaemonSet Pods (they would come straight back) and static Pods,
# refuses Pods with emptyDir data until you accept the loss, and refuses bare Pods until --force -
# which deletes them for good. The node stays cordoned until you uncordon it: a real upgrade
# (kubeadm upgrade apply, then the kubelet package) happens between those two commands.
""",
},
{
 "id": 110, "domain": "ARCH", "title": "Prepare the join of a second control-plane node",
 "task": """
The cluster is getting a second control-plane node. Prepare what the new machine needs - on the
control-plane node, `{SHELL_CP}` - without joining anything.

1. Write the address every node uses to reach the API server - the cluster's control-plane
   endpoint, as kubeadm has it configured - to `{WS}/q110/endpoint.txt`.
2. Upload the control-plane certificates to the cluster, so that a joining control-plane node can
   fetch them.
3. Save the complete `kubeadm join` command a new **control-plane** node has to run - with a new
   token, the CA certificate hash and the certificate key from step 2 - to
   `{WS}/q110/join-control-plane.sh`, on one line.
""",
 "setup": r"""
rm -rf "$WS/q110"; mkdir -p "$WS/q110"
$K -n kube-system delete secret kubeadm-certs --ignore-not-found >/dev/null 2>&1
date +%s > "$WS/.q110-t0"
""",
 "checks": [
  C("endpoint.txt holds the control-plane endpoint", 1, r"""
want=$($K -n kube-system get cm kubeadm-config -o jsonpath='{.data.ClusterConfiguration}' 2>/dev/null | awk '/^controlPlaneEndpoint:/{print $2}' | tr -d '"')
[ -n "$want" ] && eq "$(tr -d '[:space:]' < "$WS/q110/endpoint.txt" 2>/dev/null)" "$want" """),
  C("the control-plane certificates were uploaded (Secret kubeadm-certs)", 2, r"""
ct=$(j -n kube-system secret kubeadm-certs '{.metadata.creationTimestamp}'); [ -n "$ct" ] || exit 1
[ "$(date -d "$ct" +%s)" -ge "$(( $(cat "$WS/.q110-t0" 2>/dev/null || echo 9999999999) - 5 ))" ]"""),
  C("join command for a control-plane node: endpoint, CA hash, --control-plane, a certificate key", 2, r"""
f="$WS/q110/join-control-plane.sh"
ep=$($K -n kube-system get cm kubeadm-config -o jsonpath='{.data.ClusterConfiguration}' 2>/dev/null | awk '/^controlPlaneEndpoint:/{print $2}' | tr -d '"')
hash=$(nx "$CP" "openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'" </dev/null | tr -d '[:space:]')
[ -n "$ep" ] && [ -n "$hash" ] || exit 1
l=$(grep 'kubeadm join' "$f" 2>/dev/null | head -1)
echo "$l" | grep -q "kubeadm join $ep " && echo "$l" | grep -q -- "--discovery-token-ca-cert-hash sha256:$hash" \
 && echo "$l" | grep -Eq -- '--control-plane( |$)' && echo "$l" | grep -Eq -- '--certificate-key [0-9a-f]{64}( |$)' """),
  C("its token is a real, unexpired bootstrap token", 2, r"""
tok=$(grep -oE -- '--token [a-z0-9]{6}\.[a-z0-9]{16}' "$WS/q110/join-control-plane.sh" 2>/dev/null | head -1 | awk '{print $2}'); [ -n "$tok" ] || exit 1
exp=$(j -n kube-system secret bootstrap-token-${tok%%.*} '{.data.expiration}' | base64 -d 2>/dev/null); [ -n "$exp" ] || exit 1
[ "$(date -d "$exp" +%s)" -gt "$(date +%s)" ]"""),
 ],
 "solution": """
k -n kube-system get cm kubeadm-config -o yaml | grep controlPlaneEndpoint       # {CP}:6443
echo {CP}:6443 > {WS}/q110/endpoint.txt
sudo -i                # the task's host is the control-plane node: no hop needed
kubeadm init phase upload-certs --upload-certs
#   [upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
#   [upload-certs] Using certificate key:
#   <64 hex characters>
kubeadm token create --print-join-command --certificate-key <that key>
#   kubeadm join {CP}:6443 --token ... --discovery-token-ca-cert-hash sha256:... --control-plane --certificate-key ...
exit
# paste that line into {WS}/q110/join-control-plane.sh
# A control-plane node needs the cluster's CA, front-proxy CA, etcd CA and service-account keys.
# upload-certs puts them, encrypted with the certificate key, into the Secret kubeadm-certs for
# two hours; `kubeadm join --control-plane --certificate-key` downloads and decrypts them. All of
# it only works because the cluster was initialised with a controlPlaneEndpoint - the one stable
# address (a load balancer or VIP in a real HA cluster) that every node and kubeconfig points at.
""",
},
{
 "id": 111, "domain": "ARCH", "title": "Who leads, and who is in etcd",
 "task": """
Before a second control-plane node is added, record how the control plane is laid out today.
Write `{WS}/q111/ha.txt` in exactly this format:

```
control-plane-nodes=<number of control-plane nodes>
etcd-topology=<stacked or external>
etcd-members=<number of members of the etcd cluster>
etcd-leader=<name of the etcd member that is the leader>
scheduler-leader=<name of the node whose kube-scheduler holds the leader lease>
controller-manager-leader=<name of the node whose kube-controller-manager holds the leader lease>
```

`stacked` means etcd runs on the control-plane nodes themselves, `external` that the API server
uses etcd hosts of their own. `etcdctl` is on the control-plane node (`{SHELL_CP}`); etcd's
certificates are under `/etc/kubernetes/pki/etcd`.
""",
 "setup": r"""
rm -rf "$WS/q111"; mkdir -p "$WS/q111"
ensure_etcd_tools >/dev/null 2>&1 || { echo "no etcdctl on $CP - rebuild the exam host image: ./ksim build"; exit 1; }
""",
 "checks": [
  C("control-plane-nodes and etcd-topology", 2, r"""
n=$($K get nodes -l node-role.kubernetes.io/control-plane --no-headers 2>/dev/null | grep -c .)
topo=external; [ "$($K -n kube-system get pods -l component=etcd,tier=control-plane --no-headers 2>/dev/null | grep -c .)" -ge 1 ] && topo=stacked
f=$(tr -d ' \t\r' < "$WS/q111/ha.txt" 2>/dev/null)
[ "$n" -ge 1 ] && echo "$f" | grep -qx "control-plane-nodes=$n" && echo "$f" | grep -qx "etcd-topology=$topo" """),
  C("etcd-members and etcd-leader", 3, r"""
E='""" + ETCDCTL + r"""'
out=$( { nx "$CP" "$E endpoint status -w json 2>/dev/null" </dev/null; echo; nx "$CP" "$E member list -w json 2>/dev/null" </dev/null; } | python3 -c '
import sys, json
docs = [json.loads(l) for l in sys.stdin.read().splitlines() if l.strip()[:1] in "[{" and l.strip()]
status, members = docs[0], docs[1]["members"]
leader = status[0]["Status"]["leader"]
print(len(members), next(m["name"] for m in members if m["ID"] == leader))' 2>/dev/null)
[ -n "$out" ] || exit 1
f=$(tr -d ' \t\r' < "$WS/q111/ha.txt" 2>/dev/null)
echo "$f" | grep -qx "etcd-members=${out%% *}" && echo "$f" | grep -qx "etcd-leader=${out##* }" """),
  C("scheduler-leader and controller-manager-leader", 2, r"""
s=$(j -n kube-system lease kube-scheduler '{.spec.holderIdentity}' | cut -d_ -f1)
c=$(j -n kube-system lease kube-controller-manager '{.spec.holderIdentity}' | cut -d_ -f1)
f=$(tr -d ' \t\r' < "$WS/q111/ha.txt" 2>/dev/null)
[ -n "$s" ] && [ -n "$c" ] && echo "$f" | grep -qx "scheduler-leader=$s" && echo "$f" | grep -qx "controller-manager-leader=$c" """),
 ],
 "solution": """
k get nodes -l node-role.kubernetes.io/control-plane              # how many
k -n kube-system get pods -l component=etcd -o wide               # an etcd static Pod on the control-plane node = stacked
k -n kube-system get lease kube-scheduler kube-controller-manager # HOLDER is <node>_<uuid>
sudo -i                # the task's host is the control-plane node: no hop needed
export ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCDCTL_CACERT=/etc/kubernetes/pki/etcd/ca.crt \\
       ETCDCTL_CERT=/etc/kubernetes/pki/etcd/server.crt ETCDCTL_KEY=/etc/kubernetes/pki/etcd/server.key
etcdctl member list -w table                                      # one line per member: ID, NAME, peer and client URLs
etcdctl endpoint status -w table                                  # IS LEADER (add --cluster to ask every member)
exit
cat > {WS}/q111/ha.txt <<EOF
control-plane-nodes=1
etcd-topology=stacked
etcd-members=1
etcd-leader={CP}
scheduler-leader={CP}
controller-manager-leader={CP}
EOF
# In an HA control plane every kube-apiserver serves at once, but only one kube-scheduler and one
# kube-controller-manager act: they elect a leader through a Lease in kube-system, and the others
# wait. etcd elects its own leader and needs a majority of members to stay writable - three
# members survive the loss of one, five of two, and an even number adds nothing.
""",
},
{
 "id": 112, "domain": "ARCH", "title": "CNI, CSI and CRI: what this cluster plugs in",
 "task": """
Kubernetes leaves networking, storage and running containers to plugins. Record which ones this
cluster uses in `{WS}/q112/interfaces.txt`, in exactly this format:

```
cni-config=<full path of the CNI configuration file on node {W1}>
cni-plugins=<the type of every plugin in that file's chain, in order, separated by commas>
cni-ipam=<the type of the IPAM plugin it uses>
cri-socket=<the runtime endpoint the kubelet on {W1} is configured with>
csi-drivers=<number of CSIDriver objects in the cluster>
default-provisioner=<provisioner of the default StorageClass>
```

Reach the node with `{SHELL_W1}`.
""",
 "setup": r"""
rm -rf "$WS/q112"; mkdir -p "$WS/q112"
""",
 "checks": [
  C("cni-config, cni-plugins and cni-ipam", 3, r"""
conf=$(nx "$W1" "ls /etc/cni/net.d/*.conflist /etc/cni/net.d/*.conf 2>/dev/null | sort | head -1" </dev/null | tr -d '[:space:]'); [ -n "$conf" ] || exit 1
out=$(nx "$W1" "cat $conf" </dev/null | python3 -c '
import sys, json
d = json.load(sys.stdin); p = d.get("plugins", [d])
print(",".join(x["type"] for x in p), next((x["ipam"]["type"] for x in p if "ipam" in x), ""))' 2>/dev/null)
[ -n "$out" ] || exit 1
f=$(tr -d ' \t\r' < "$WS/q112/interfaces.txt" 2>/dev/null)
echo "$f" | grep -qx "cni-config=$conf" && echo "$f" | grep -qx "cni-plugins=${out%% *}" && echo "$f" | grep -qx "cni-ipam=${out##* }" """),
  C("cri-socket", 2, r"""
sock=$(nx "$W1" "awk '/^containerRuntimeEndpoint:/{print \$2}' /var/lib/kubelet/config.yaml" </dev/null | tr -d '"[:space:]')
[ -n "$sock" ] && tr -d ' \t\r' < "$WS/q112/interfaces.txt" 2>/dev/null | grep -qx "cri-socket=$sock" """),
  C("csi-drivers and default-provisioner", 2, r"""
n=$($K get csidrivers --no-headers 2>/dev/null | grep -c .)
prov=$($K get sc -o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.provisioner}{"\n"}{end}' 2>/dev/null | head -1)
[ -n "$prov" ] || exit 1
f=$(tr -d ' \t\r' < "$WS/q112/interfaces.txt" 2>/dev/null)
echo "$f" | grep -qx "csi-drivers=$n" && echo "$f" | grep -qx "default-provisioner=$prov" """),
 ],
 "solution": """
{SHELL_W1}
sudo -i
ls /etc/cni/net.d/                                    # 10-kindnet.conflist - the kubelet's runtime takes the first file
cat /etc/cni/net.d/10-kindnet.conflist                # "plugins": [{"type": "ptp", "ipam": {"type": "host-local"}}, {"type": "portmap"}]
ls /opt/cni/bin                                       # the plugin binaries those types name
grep containerRuntimeEndpoint /var/lib/kubelet/config.yaml
exit; exit
k get csidrivers                                      # No resources found = 0
k get sc                                              # the (default) class and its PROVISIONER
cat > {WS}/q112/interfaces.txt <<EOF
cni-config=/etc/cni/net.d/10-kindnet.conflist
cni-plugins=ptp,portmap
cni-ipam=host-local
cri-socket=unix:///run/containerd/containerd.sock
csi-drivers=0
default-provisioner=rancher.io/local-path
EOF
# CRI: the kubelet talks to the runtime over that socket (crictl uses the same one). CNI: the
# runtime calls the plugins of the chain in order for every Pod sandbox - here ptp wires the veth
# pair, host-local hands out an address from the node's Pod CIDR, portmap serves hostPorts.
# CSI: a storage driver registers a CSIDriver object and a StorageClass names it as provisioner;
# this cluster's local-path provisioner is not a CSI driver, hence none.
""",
},
# ------------------------------------------------------------------- NET ----
{
 "id": 113, "domain": "NET", "title": "Pod to Pod, across nodes",
 "params": lambda r: {"ns": ns_pick(r, "wire"), "web": name_pick(r, "web"), "client": name_pick(r, "client"), "edge": name_pick(r, "edge")},
 "task": """
Namespace `%{ns}` runs a few Pods on different nodes. No Service is involved in this task.

1. Write the IP address of Pod `%{web}` and the node it runs on to `{WS}/q113/web.txt`, as
   `<ip> <node>`.
2. From Pod `%{client}`, which runs on another node, request `http://<that IP>` directly and save
   the response to `{WS}/q113/direct.html`.
3. Write the Pod CIDR of every node of the cluster to `{WS}/q113/pod-cidrs.txt`, one node per
   line as `<node> <cidr>`.
4. One Pod in `%{ns}` has an address that lies in none of those ranges. Write its name to
   `{WS}/q113/outside.txt` and the Pod spec field that causes it to `{WS}/q113/why.txt`.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q113"; mkdir -p "$WS/q113"
$K -n %{ns} delete pod --all --wait=true --grace-period=1 >/dev/null 2>&1
$K apply -f - >/dev/null <<'EOF'
apiVersion: v1
kind: Pod
metadata: {name: %{web}, namespace: %{ns}, labels: {app: web}}
spec:
  nodeSelector: {kubernetes.io/hostname: "{W1}"}
  containers: [{name: web, image: "nginx:1.27", ports: [{containerPort: 80}]}]
---
apiVersion: v1
kind: Pod
metadata: {name: %{client}, namespace: %{ns}, labels: {app: client}}
spec:
  terminationGracePeriodSeconds: 1
  nodeSelector: {kubernetes.io/hostname: "{W2}"}
  containers: [{name: client, image: "busybox:1.36", command: [sleep, "86400"]}]
---
apiVersion: v1
kind: Pod
metadata: {name: %{edge}, namespace: %{ns}, labels: {app: edge}}
spec:
  terminationGracePeriodSeconds: 1
  hostNetwork: true
  nodeSelector: {kubernetes.io/hostname: "{W3}"}
  containers: [{name: edge, image: "busybox:1.36", command: [sleep, "86400"]}]
---
apiVersion: v1
kind: Pod
metadata: {name: spare-%{edge}, namespace: %{ns}, labels: {app: spare}}
spec:
  terminationGracePeriodSeconds: 1
  nodeSelector: {kubernetes.io/hostname: "{W3}"}
  containers: [{name: spare, image: "busybox:1.36", command: [sleep, "86400"]}]
EOF
$K -n %{ns} wait --for=condition=Ready pod --all --timeout=150s >/dev/null
""",
 "checks": [
  C("web.txt holds the Pod's IP and node", 1, r"""
want=$(j -n %{ns} pod %{web} '{.status.podIP} {.spec.nodeName}')
[ -n "$want" ] && eq "$(tr -s ' \t' ' ' < "$WS/q113/web.txt" 2>/dev/null | tr -d '\r' | sed 's/^ *//; s/ *$//' | grep .)" "$want" """),
  C("direct.html holds the page fetched from the Pod's IP", 2, r"""grep -q 'Welcome to nginx' "$WS/q113/direct.html" 2>/dev/null"""),
  C("pod-cidrs.txt lists every node with its Pod CIDR", 2, r"""
want=$($K get nodes -o jsonpath='{range .items[*]}{.metadata.name} {.spec.podCIDR}{"\n"}{end}' 2>/dev/null | sort)
[ "$(echo "$want" | grep -c .)" -ge 2 ] || exit 1
eq "$(tr -s ' \t' ' ' < "$WS/q113/pod-cidrs.txt" 2>/dev/null | tr -d '\r' | sed 's/^ *//; s/ *$//' | grep . | sort)" "$want" """),
  C("outside.txt names %{edge}, why.txt names hostNetwork", 2, r"""
eq "$(tr -d '[:space:]' < "$WS/q113/outside.txt" 2>/dev/null | sed 's#^pod/##')" %{edge} && grep -qi 'hostNetwork' "$WS/q113/why.txt" 2>/dev/null"""),
 ],
 "solution": """
k -n %{ns} get pods -o wide                           # IP and NODE of every Pod
k -n %{ns} get pod %{web} -o jsonpath='{.status.podIP} {.spec.nodeName}' > {WS}/q113/web.txt
k -n %{ns} exec %{client} -- wget -qO- -T3 http://$(cut -d' ' -f1 {WS}/q113/web.txt) > {WS}/q113/direct.html
k get nodes -o jsonpath='{range .items[*]}{.metadata.name} {.spec.podCIDR}{"\\n"}{end}' > {WS}/q113/pod-cidrs.txt
#   %{edge}'s IP is its node's own address (compare with `k get nodes -o wide`), not one from 10.244.x.0/24
echo %{edge} > {WS}/q113/outside.txt
k -n %{ns} get pod %{edge} -o yaml | grep hostNetwork
echo hostNetwork > {WS}/q113/why.txt
# Every Pod gets its own address from its node's Pod CIDR, and any Pod reaches any other Pod on
# any node at that address without NAT - that is the whole network model, and the CNI plugin is
# what makes it true. A hostNetwork Pod opts out: it lives in the node's network namespace and
# shares its address and ports. Pod addresses change with every new Pod, which is why Services exist.
""",
},
]