diff --git a/CHANGELOG.md b/CHANGELOG.md index f680a36..9db585e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,24 @@ Notable changes to kubestronaut-sim, newest first. A version is the `VERSION` st (year.month.day.n); `./ksim help` and the top bar of the web page show the one you are running. Every entry names the commit it came in. +## Unreleased + +### Changed + +- **Every CKA reference solution shows the exam way**, as the lessons do: a generator and the few + lines to add in `vim`, a manifest pasted from the docs page it names, `kubectl edit`, `ssh NODE`, + or a value you read and `echo` - no heredocs, `sed`, JSON patches, `kubectl wait` or loops. 105 + of the 147 were rewritten; the other 42 were commands you type already. The page and `./ksim + solution N` show each thing to type as a block of its own, which a click copies. +- A solution in parts (`cmd`, `hand`) is its own audit script: `./ksim audit` runs what it shows, + and for each edit by hand the script beside it, so the solution you read is the one proven. Their + separate scripts in `cka/solve.py` are gone. + +### Added + +- `./ksim solution lint [N|DOMAIN..]`: what the reference solutions have you type, against the + list `./ksim lesson lint` uses. + ## 2026.10.26.4 — 2026-10-26 Everything below came in `226ef19`. diff --git a/README.md b/README.md index a207827..564a120 100644 --- a/README.md +++ b/README.md @@ -200,7 +200,7 @@ Each track is a directory: `cka/`, `ckad/`, `cks/`. | `bank_*.py` | more tasks; `bank_sim_*` are the fixed simulator sets | | `curriculum.py` | the exam's objectives in order, each with study notes and the tasks that practise it | | `solutions.py` | reference solutions (CKA keeps them apart; the others inline them) | -| `solve.py` | scripted solutions that `./ksim audit` runs on the task's host to prove a task can be solved | +| `solve.py` | scripted solutions that `./ksim audit` runs on the task's host to prove a task can be solved - for the tasks whose reference solution is not in parts | | `lessons.py` | guided labs, one per objective (CKA: all, one `lessons_*.py` per domain · CKAD: 2.1–2.4): introduction, steps with their check, hint and commands, wrap-up | | Track | Kubernetes | Bank today | Target (17–20 pool tasks per domain) | @@ -242,7 +242,8 @@ v1.36), CKAD Q1 and CKS Q60, Q61 and Q64 passed again on the rebuilt images. "title": "...", "task": "...markdown; %{ns} and %{port} are substituted; `code` copies on click...", "setup": "bash that seeds the scenario on the task's own cluster", "checks": [C("what this check proves", 2, "bash; exit 0 = pass"), ...], - "solution": "...", + "solution": [cmd("k ... --dry-run=client -o yaml > f.yaml"), # or one string, a script as it stands + hand("vim f.yaml", "what to add, Markdown", run="the same, scripted: yq, a heredoc")], "workers": 2, # optional: worker nodes of its cluster "k8s": "v1.36"} # optional: a minor of its own (an upgrade task) ``` @@ -287,9 +288,16 @@ and the untouched grade, which catches a check that only passes once the seeded until a stopped kubelet shows as NotReady; `--keep` leaves the host up to look at (`KUBECONFIG=.state/envs//kubeconfig`, `./ksim down` removes it). -In `cka/solve.py`, `AS_WRITTEN` lists the tasks whose reference solution is a script as it stands: -for those the audit runs exactly the text the candidate is shown, so a solution that stops working -with a new Kubernetes release fails the next audit instead of misleading you after a drill. +A reference solution in parts - `cmd()` and `hand()`, as a lesson step's `show` - is its own +script: the audit runs what is typed as it stands, and for each part done by hand its `run`, so the +solution the candidate reads after a drill is the one that is proven, and one that stops working +with a new Kubernetes release fails the next audit. 105 of the 147 CKA solutions are written so +(the page and `./ksim solution N` show each thing to type as a block of its own, which a click +copies, with the edit to make in `vim` beneath it); the other 42 are commands typed as they stand, +each with its script in `solve.py`. `./ksim solution lint` reads what they have you type, against +the same list of things you would not type in the exam as `./ksim lesson lint`. In `cka/solve.py`, +`AS_WRITTEN` lists the few tasks whose reference solution is still one script, which the audit +runs as it stands. **Writing a lesson.** A lesson in `/lessons.py` is a dict with `id` (9000 plus the objective, `9201` for 2.1 — its host is `ckad9201`, which no task's can be), `objective`, `title`, `workers`, diff --git a/cka/bank_course.py b/cka/bank_course.py index 3ccba9c..edad779 100644 --- a/cka/bank_course.py +++ b/cka/bank_course.py @@ -12,6 +12,8 @@ operator with broken RBAC do its job, and 146-147 grow, snapshot, restore and cl the CSI hostpath driver's class `csi-hostpath-sc`, as the real exam's clusters have it (ensure_csi_hostpath). """ + +from sim.bank import cmd, hand from cka.common import C, CAN, NET, NS, ONE, backup_operator, ns_pick, name_pick _ETCD = ("ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt " @@ -59,38 +61,65 @@ for n in "$CP" "$W1"; do done eq "$(j node "$W1" '{.status.nodeInfo.kubeletVersion}')" v1.37.1 && eq "$(j -n %{ns} deploy web '{.status.availableReplicas}')" 2"""), ], - "solution": """ -# the control plane - this host -sudo sed -i 's|/v1.36/|/v1.37/|' /etc/apt/sources.list.d/kubernetes.list # one repository per minor -sudo apt-get update + "solution": [hand(r"""sudo vim /etc/apt/sources.list.d/kubernetes.list""", + r"""The docs page *Upgrading kubeadm clusters* starts here: pkgs.k8s.io has one repository per minor, so 1.37 packages exist for apt only once the line says `/v1.37/` instead of `/v1.36/`.""", + run=r"""apt_get() { sudo DEBIAN_FRONTEND=noninteractive apt-get -qq -y -o Dpkg::Options::=--force-confold "$@" /dev/null; } +sudo sed -i 's|/v1.36/|/v1.37/|' /etc/apt/sources.list.d/kubernetes.list"""), + hand(r"""sudo apt-get update apt-cache madison kubeadm | head -3 -sudo apt-mark unhold kubeadm && sudo apt-get install -y kubeadm=1.37.1-1.1 && sudo apt-mark hold kubeadm -sudo kubeadm upgrade plan -sudo kubeadm upgrade apply v1.37.1 -y # a few minutes -k drain {CP} --ignore-daemonsets --delete-emptydir-data -sudo apt-mark unhold kubelet kubectl && sudo apt-get install -y kubelet=1.37.1-1.1 kubectl=1.37.1-1.1 && sudo apt-mark hold kubelet kubectl -sudo systemctl daemon-reload && sudo systemctl restart kubelet -k uncordon {CP} -# the worker -k drain {W1} --ignore-daemonsets --delete-emptydir-data -{SHELL_W1} +sudo apt-mark unhold kubeadm +sudo apt-get install -y kubeadm=1.37.1-1.1 +sudo apt-mark hold kubeadm""", + r"""The control plane - this host - first, and kubeadm first: it drives the upgrade.""", + run=r"""apt_get update +sudo apt-mark unhold kubeadm >/dev/null && apt_get install kubeadm=1.37.1-1.1 && sudo apt-mark hold kubeadm >/dev/null"""), + hand(r"""sudo kubeadm upgrade plan +sudo kubeadm upgrade apply v1.37.1""", + r"""Answer `y`; it takes a few minutes, and the API server restarts on the new version.""", + run=r"""sudo kubeadm upgrade apply v1.37.1 -y /dev/null && apt_get install kubelet=1.37.1-1.1 kubectl=1.37.1-1.1 && sudo apt-mark hold kubelet kubectl >/dev/null +sudo systemctl daemon-reload && sudo systemctl restart kubelet"""), + hand(r"""k uncordon {CP} +k get nodes""", + r"""Again until `k` answers - the kubelet is restarting. `{CP}` shows `v1.37.1`.""", + run=r"""for i in $(seq 1 30); do kubectl uncordon {CP} >/dev/null 2>&1 && break; sleep 3; done"""), + cmd(r"""k drain {W1} --ignore-daemonsets --delete-emptydir-data"""), + hand(r"""{SHELL_W1} sudo -i -sed -i 's|/v1.36/|/v1.37/|' /etc/apt/sources.list.d/kubernetes.list -apt-get update +vim /etc/apt/sources.list.d/kubernetes.list""", + r"""`/v1.36/` to `/v1.37/` here too.""", + run=r""""""), + hand(r"""apt-get update apt-mark unhold kubeadm kubelet kubectl apt-get install -y kubeadm=1.37.1-1.1 -kubeadm upgrade node # a worker: node, not apply +kubeadm upgrade node apt-get install -y kubelet=1.37.1-1.1 kubectl=1.37.1-1.1 apt-mark hold kubeadm kubelet kubectl +systemctl daemon-reload +systemctl restart kubelet +exit +exit""", + r"""A worker runs `kubeadm upgrade node`, not `apply`: it hosts no control plane, only its kubelet's configuration to update.""", + run=r"""ssh {W1} sudo bash <<'EOF' +export DEBIAN_FRONTEND=noninteractive +sed -i 's|/v1.36/|/v1.37/|' /etc/apt/sources.list.d/kubernetes.list +apt-get update -qq /dev/null +apt-get install -y -qq kubeadm=1.37.1-1.1 /dev/null +kubeadm upgrade node /dev/null +apt-mark hold kubeadm kubelet kubectl >/dev/null systemctl daemon-reload && systemctl restart kubelet -exit; exit -k uncordon {W1} -k get nodes -# docs: "Upgrading kubeadm clusters". pkgs.k8s.io has one repository per minor, so 1.37 packages -# appear only once the line names v1.37. kubeadm first - it drives the upgrade and rewrites the -# kubelet's configuration; kubelet and kubectl after `upgrade apply`, then the restart: `k get -# nodes` shows each kubelet's running version. Workers run `kubeadm upgrade node`. -""", +EOF"""), + cmd(r"""k uncordon {W1} +k get nodes""", r"""Both `Ready`, both `v1.37.1` - `VERSION` is each kubelet's.""")], }, # ------------------------------------------------------------------ 133 ---- { @@ -196,19 +225,16 @@ eq "$(j -n %{ns} pod %{pod} '{.spec.ephemeralContainers[?(@.name=="debugger")].t && eq "$(j -n %{ns} pod %{pod} '{.metadata.uid} {.status.containerStatuses[0].restartCount}')" "$(cat "$PRIV/q134.uid") 0" """), C("mode.txt holds the value the process runs with", 3, ONE + r"""eq "$(one "$WS/q134/mode.txt")" %{mode}"""), ], - "solution": """ -k -n %{ns} exec %{pod} -- sh # fails: no shell in the image -k -n %{ns} get cm %{pod}-env -o yaml # MODE=open - but that is the ConfigMap NOW -k -n %{ns} debug %{pod} --image=busybox:1.36 --target=app -c debugger --profile=general -- sleep 3600 -for i in $(seq 1 30); do k -n %{ns} exec %{pod} -c debugger -- true 2>/dev/null && break; sleep 2; done -k -n %{ns} exec %{pod} -c debugger -- ps # PID 1 is /pause, the app's process -k -n %{ns} exec %{pod} -c debugger -- sh -c 'tr "\\0" "\\n" < /proc/1/environ' | grep '^MODE=' -k -n %{ns} exec %{pod} -c debugger -- sh -c 'tr "\\0" "\\n" < /proc/1/environ' | sed -n 's/^MODE=//p' > {WS}/q134/mode.txt -# Environment variables from a ConfigMap are read once, when the container starts: editing the -# ConfigMap changes nothing for a running process. --target puts the debug container into app's -# process namespace, so /proc/1 is app's process; the `general` profile gives it SYS_PTRACE, -# which reading another user's /proc/PID/environ needs. Interactively: `k debug -it ... -- sh`. -""", + "solution": [cmd(r"""k -n %{ns} exec %{pod} -- sh +k -n %{ns} get cm %{pod}-env -o yaml""", r"""No shell in the image. And the ConfigMap says `MODE=open` - now; the environment was read once, when the container started."""), + cmd(r"""k -n %{ns} debug %{pod} --image=busybox:1.36 --target=app -c debugger --profile=general -- sleep 3600""", r"""`--target` puts the debug container into `app`'s process namespace; the `general` profile gives it `SYS_PTRACE`, which reading another process's environment needs. Interactively: `k debug -it ... -- sh`."""), + hand(r"""k -n %{ns} exec %{pod} -c debugger -- ps""", + r"""Again until it answers: PID 1 is `/pause`, `app`'s process.""", + run=r"""for i in $(seq 1 30); do kubectl -n %{ns} exec %{pod} -c debugger -- true 2>/dev/null && break; sleep 2; done"""), + cmd(r"""k -n %{ns} exec %{pod} -c debugger -- cat /proc/1/environ | tr '\0' '\n'""", r"""`MODE=%{mode}`: what the process runs with."""), + hand(r"""echo %{mode} > {WS}/q134/mode.txt""", + r"""With the value after `MODE=`.""", + run=r"""kubectl -n %{ns} exec %{pod} -c debugger -- sh -c 'tr "\0" "\n" < /proc/1/environ' | sed -n 's/^MODE=//p' > {WS}/q134/mode.txt""")], }, # ------------------------------------------------------------------ 135 ---- { @@ -253,19 +279,19 @@ tok = [v for v in s.get("volumes", []) if any("serviceAccountToken" in src for s sys.exit(1 if tok else 0)' || exit 1 done"""), ], - "solution": """ -k -n %{ns} create serviceaccount pipeline + "solution": [cmd(r"""k -n %{ns} create serviceaccount pipeline k -n %{ns} create role pipeline --verb=get,list,create,update,patch --resource=deployments.apps k -n %{ns} create rolebinding pipeline --role=pipeline --serviceaccount=%{ns}:pipeline -k auth can-i delete deployments -n %{ns} --as=system:serviceaccount:%{ns}:pipeline # no -k -n %{ns} create token pipeline --duration=%{h}h > {WS}/q135/pipeline.token -k -n %{ns} patch deploy frontend -p '{"spec":{"template":{"spec":{"automountServiceAccountToken":false}}}}' -k -n %{ns} rollout status deploy frontend -# `create token` asks the TokenRequest API for a short-lived, bound token - nothing is stored in a -# Secret. automountServiceAccountToken: false on the Pod template (or on the ServiceAccount; the -# Pod's setting wins) leaves out the projected volume kube-api-access-*. To try a token alone, use -# no kubeconfig: a client certificate in yours would be checked first. -""", +k auth can-i delete deployments -n %{ns} --as=system:serviceaccount:%{ns}:pipeline""", r"""`no`."""), + cmd(r"""k -n %{ns} create token pipeline --duration=%{h}h > {WS}/q135/pipeline.token""", r"""The TokenRequest API issues a short-lived, bound token; nothing is stored in a Secret."""), + hand(r"""k -n %{ns} edit deploy frontend""", + r"""In the Pod template's `spec:`, beside `containers:` and indented like it, then `:wq`: +``` + automountServiceAccountToken: false +``` +It leaves out the projected volume `kube-api-access-*`. (On the ServiceAccount it works too; the Pod's setting wins.)""", + run=r"""kubectl -n %{ns} patch deploy frontend -p '{"spec":{"template":{"spec":{"automountServiceAccountToken":false}}}}'"""), + cmd(r"""k -n %{ns} rollout status deploy frontend""")], }, # ------------------------------------------------------------------ 136 ---- { @@ -298,22 +324,20 @@ eq "$(x theme)" dark && eq "$(x locale)" en_GB && eq "$(x db/password)" pw-%{mem x() { $K -n %{ns} exec %{pod} -- cat "/etc/portal/$1" 2>/dev/null; } x labels | grep -qx 'app="%{pod}"' && eq "$(x mem-limit)" %{mem}"""), ], - "solution": """ -k -n %{ns} apply -f - <<'EOF' -apiVersion: v1 -kind: Pod -metadata: - name: %{pod} - labels: {app: %{pod}} -spec: - containers: - - name: portal - image: busybox:1.36 - command: [sleep, '86400'] + "solution": [cmd(r"""k -n %{ns} run %{pod} --image=busybox:1.36 --labels=app=%{pod} --dry-run=client -o yaml -- sleep 86400 > pod.yaml"""), + hand(r"""vim pod.yaml""", + r"""Replace the container's `resources: {}` with the limit and the mount: +``` resources: - limits: {memory: %{mem}Mi} + limits: + memory: %{mem}Mi volumeMounts: - - {name: portal, mountPath: /etc/portal, readOnly: true} + - name: portal + mountPath: /etc/portal + readOnly: true +``` +and add to the Pod's `spec:`, right below `spec:` (the docs page *Projected Volumes* has an example with all three kinds): +``` volumes: - name: portal projected: @@ -323,20 +347,33 @@ spec: - secret: name: portal-db items: - - {key: password, path: db/password} + - key: password + path: db/password - downwardAPI: items: - path: labels - fieldRef: {fieldPath: metadata.labels} + fieldRef: + fieldPath: metadata.labels - path: mem-limit - resourceFieldRef: {containerName: portal, resource: limits.memory, divisor: 1Mi} -EOF -k -n %{ns} wait --for=condition=Ready pod/%{pod} -k -n %{ns} exec %{pod} -- sh -c 'ls -R /etc/portal; cat /etc/portal/labels /etc/portal/mem-limit' -# `items` picks keys and renames them: only `password` from the Secret, as db/password. A -# resourceFieldRef needs the container's name in a volume (in an env var it is implied), and the -# divisor turns bytes into MiB. -""", + resourceFieldRef: + containerName: %{pod} + resource: limits.memory + divisor: 1Mi +``` +`items` picks keys and renames them; in a volume a `resourceFieldRef` needs the container's name, and the divisor turns bytes into MiB.""", + run=r"""yq -i '.spec.containers[0].resources = {"limits": {"memory": "%{mem}Mi"}} + | .spec.containers[0].volumeMounts = [{"name": "portal", "mountPath": "/etc/portal", "readOnly": true}] + | .spec.volumes = [{"name": "portal", "projected": {"sources": [ + {"configMap": {"name": "portal-config"}}, + {"secret": {"name": "portal-db", "items": [{"key": "password", "path": "db/password"}]}}, + {"downwardAPI": {"items": [{"path": "labels", "fieldRef": {"fieldPath": "metadata.labels"}}, + {"path": "mem-limit", "resourceFieldRef": {"containerName": "%{pod}", "resource": "limits.memory", "divisor": "1Mi"}}]}}]}}]' pod.yaml"""), + cmd(r"""k apply -f pod.yaml"""), + hand(r"""k -n %{ns} get pod %{pod}""", + r"""Again until it is `Running`.""", + run=r"""kubectl -n %{ns} wait --for=condition=Ready pod/%{pod} --timeout=120s"""), + cmd(r"""k -n %{ns} exec %{pod} -- ls -R /etc/portal +k -n %{ns} exec %{pod} -- cat /etc/portal/labels /etc/portal/mem-limit""")], }, # ------------------------------------------------------------------ 137 ---- { @@ -404,18 +441,19 @@ eq "$(j -n %{ns} deploy critical '{.status.availableReplicas} {.status.updatedRe eq "$(j -n %{ns} deploy critical '{.spec.template.spec.containers[0].resources.requests.cpu} {.spec.template.spec.containers[0].resources.limits.cpu} {.spec.template.spec.containers[0].resources.requests.memory} {.spec.template.spec.containers[0].resources.limits.memory}')" "200m 200m 128Mi 128Mi" || exit 1 for q in $($K -n %{ns} get pods -l app=critical -o jsonpath='{.items[*].status.qosClass}'); do [ "$q" = Guaranteed ] || exit 1; done"""), ], - "solution": """ -k -n %{ns} get pods -o custom-columns=NAME:.metadata.name,QOS:.status.qosClass -k -n %{ns} get pods alpha beta delta gamma -o jsonpath='{range .items[*]}{.metadata.name} {.status.qosClass}{"\\n"}{end}' > {WS}/q137/qos.txt -echo gamma > {WS}/q137/first.txt -k -n %{ns} set resources deploy critical --requests=cpu=200m,memory=128Mi --limits=cpu=200m,memory=128Mi + "solution": [cmd(r"""k -n %{ns} get pods -o custom-columns=NAME:.metadata.name,QOS:.status.qosClass""", r"""Guaranteed: every container has requests equal to limits, CPU and memory. BestEffort: no container has any. Everything else is Burstable - `delta` too, as one of its containers sets nothing."""), + hand(r"""vim {WS}/q137/qos.txt""", + r"""``` +alpha Guaranteed +beta Burstable +gamma BestEffort +delta Burstable +```""", + run=r"""printf 'alpha Guaranteed\nbeta Burstable\ngamma BestEffort\ndelta Burstable\n' > {WS}/q137/qos.txt"""), + cmd(r"""echo gamma > {WS}/q137/first.txt""", r"""Under memory pressure BestEffort goes first, then Burstable Pods using more than they requested, Guaranteed last."""), + cmd(r"""k -n %{ns} set resources deploy critical --requests=cpu=200m,memory=128Mi --limits=cpu=200m,memory=128Mi k -n %{ns} rollout status deploy critical -k -n %{ns} get pods -l app=critical -o jsonpath='{.items[*].status.qosClass}'; echo -# Guaranteed: every container has requests equal to limits for both CPU and memory. BestEffort: no -# container has any. Everything else is Burstable - delta too, because one container sets nothing. -# Under memory pressure the kubelet evicts BestEffort first, then Burstable Pods using more than -# they requested; Guaranteed Pods last. -""", +k -n %{ns} get pods -l app=critical -o custom-columns=NAME:.metadata.name,QOS:.status.qosClass""")], }, # ------------------------------------------------------------------ 138 ---- { @@ -452,15 +490,21 @@ eq "$(j -n %{ns} pod %{pod} '{.status.containerStatuses[0].resources.requests.cp eq "$(j -n %{ns} pod %{pod} '{.spec.containers[0].resources.limits.cpu}')" %{lim} \ && eq "$(j -n %{ns} pod %{pod} '{.metadata.uid} {.status.containerStatuses[0].restartCount}')" "$(cat "$PRIV/q138.uid") 0" """), ], - "solution": """ -k -n %{ns} get pod %{pod} -o jsonpath='{.spec.containers[0].resources}'; echo -k -n %{ns} patch pod %{pod} --subresource resize --patch '{"spec":{"containers":[{"name":"encoder","resources":{"requests":{"cpu":"%{req}"},"limits":{"cpu":"%{lim}"}}}]}}' -k -n %{ns} get pod %{pod} -o jsonpath='{.status.containerStatuses[0].resources}{"\\n"}{.status.containerStatuses[0].restartCount}{"\\n"}' -# A running Pod's CPU and memory can be changed in place, through the `resize` subresource - a -# plain `k edit` or `k apply` of a Pod's resources is refused. The spec -# says what you asked for; status.containerStatuses[].resources what the kubelet has applied. -# A container's resizePolicy decides whether a resource's change needs a restart (CPU: no). -""", + "solution": [cmd(r"""k -n %{ns} get pod %{pod} -o yaml | grep -A6 resources:"""), + hand(r"""k -n %{ns} edit pod %{pod} --subresource resize""", + r"""Through the `resize` subresource - a plain `k edit` of a Pod's resources is refused. Set the CPU, leave the memory, then `:wq`: +``` + resources: + limits: + cpu: %{lim} + memory: 48Mi + requests: + cpu: %{req} + memory: 48Mi +```""", + run=r"""kubectl -n %{ns} patch pod %{pod} --subresource resize --patch '{"spec":{"containers":[{"name":"encoder","resources":{"requests":{"cpu":"%{req}"},"limits":{"cpu":"%{lim}"}}}]}}'"""), + cmd(r"""k -n %{ns} get pod %{pod} -o yaml | grep -A6 resources: +k -n %{ns} get pod %{pod}""", r"""The spec says what you asked for, `status.containerStatuses[].resources` what the kubelet applied; `RESTARTS 0` - a container's `resizePolicy` decides, and CPU needs no restart.""")], }, # ------------------------------------------------------------------ 139 ---- { @@ -495,24 +539,33 @@ sys.exit(0 if s == "%{soft}Mi" and secs == 90 else 1)'"""), nx "$W1" 'grep -q "%{soft}Mi" /var/lib/kubelet/config.yaml && grep -q "%{mem}Mi" /var/lib/kubelet/config.yaml && grep -q "^staticPodPath:" /var/lib/kubelet/config.yaml && systemctl is-active -q kubelet' {WS}/q140/mode.txt -ip=$(k -n %{ns} get svc %{svc} -o jsonpath='{.spec.clusterIP}') -sudo iptables-save -t nat | grep -- "-d $ip/32" # KUBE-SERVICES ... -j KUBE-SVC-XXXX -chain=$(sudo iptables-save -t nat | grep -- "-d $ip/32" | grep -o 'KUBE-SVC-[A-Z0-9]*' | head -1) -echo $chain > {WS}/q140/chain.txt -sudo iptables-save -t nat | grep -- "^-A $chain " # one -j KUBE-SEP-... per endpoint -sudo iptables-save -t nat | grep -- "^-A $chain " | grep -c -- '-j KUBE-SEP-' > {WS}/q140/endpoints.txt -# KUBE-SERVICES matches ClusterIP:port and jumps to the Service's KUBE-SVC chain; that picks an -# endpoint chain at random (statistic mode random), and each KUBE-SEP chain DNATs to one Pod. -# One KUBE-SEP per ready endpoint. In nftables mode: `sudo nft list ruleset | grep -A5 %{svc}`. -""", + "solution": [cmd(r"""k -n kube-system get cm kube-proxy -o yaml | grep mode""", r"""`mode: iptables` - an empty mode means the default, iptables too."""), + hand(r"""echo iptables > {WS}/q140/mode.txt""", + r"""The mode it printed.""", + run=r"""m=$(kubectl -n kube-system get cm kube-proxy -o jsonpath='{.data.config\.conf}' | grep '^mode:' | awk '{print $2}' | tr -d '"') +echo ${m:-iptables} > {WS}/q140/mode.txt"""), + cmd(r"""k -n %{ns} get svc %{svc}""", r"""Note its `CLUSTER-IP`."""), + hand(r"""sudo iptables-save -t nat | grep 10.96.12.34""", + r"""With that IP: `KUBE-SERVICES` matches the ClusterIP and port and jumps (`-j`) to the Service's own `KUBE-SVC-...` chain.""", + run=r""""""), + hand(r"""echo KUBE-SVC-ABCDEFGHIJKLMNOP > {WS}/q140/chain.txt +sudo iptables-save -t nat | grep -- '-A KUBE-SVC-ABCDEFGHIJKLMNOP '""", + r"""With the chain's name. Its rules pick one endpoint chain at random (`statistic mode random`); each `KUBE-SEP-...` DNATs to one Pod - count them. In nftables mode: `sudo nft list ruleset | grep -A5 %{svc}`.""", + run=r"""ip=$(kubectl -n %{ns} get svc %{svc} -o jsonpath='{.spec.clusterIP}') +sudo iptables-save -t nat | grep -- "-d $ip/32" | grep -o 'KUBE-SVC-[A-Z0-9]*' | head -1 > {WS}/q140/chain.txt"""), + hand(r"""echo 3 > {WS}/q140/endpoints.txt""", + r"""With the number of `-j KUBE-SEP-` rules: one per ready endpoint.""", + run=r"""chain=$(cat {WS}/q140/chain.txt) +sudo iptables-save -t nat | grep -- "^-A $chain " | grep -c -- '-j KUBE-SEP-' > {WS}/q140/endpoints.txt""")], }, # ------------------------------------------------------------------ 141 ---- { @@ -593,36 +651,65 @@ eq "$(j -n %{ns} endpointslice billing-1 '{.metadata.labels.kubernetes\.io/servi C("wget billing from client returns the API's answer", 3, r""" for i in 1 2 3 4 5; do [ "$($K -n %{ns} exec client -- wget -qO- -T 3 billing 2>/dev/null)" = billing-api-%{port} ] && exit 0; sleep 3; done; exit 1"""), ], - "solution": """ -cat {WS}/q141/billing.txt # ADDRESS:PORT -addr=$(cut -d: -f1 {WS}/q141/billing.txt); port=$(cut -d: -f2 {WS}/q141/billing.txt) -k -n %{ns} apply -f - < billing.yaml </dev/null 2>&1 nx "$CP" '""" + _ETCD + r""" get /registry/secrets/%{ns}/ksim-probe --print-value-only 2>/dev/null | head -c 64' /dev/null 2>&1; exit $r"""), ], - "solution": """ -sudo mkdir -p /etc/kubernetes/enc + "solution": [cmd(r"""head -c 32 /dev/urandom | base64""", r"""The host is the control-plane node. Copy the output: that is the key."""), + hand(r"""sudo mkdir -p /etc/kubernetes/enc +sudo vim /etc/kubernetes/enc/enc.yaml""", + r"""Paste the example from the docs page *Encrypting Confidential Data at Rest* (search for `aescbc`) and cut it down to: +``` +apiVersion: apiserver.config.k8s.io/v1 +kind: EncryptionConfiguration +resources: +- resources: [secrets] + providers: + - aescbc: + keys: + - name: key1 + secret: + - identity: {} +``` +The first provider encrypts; `identity` after it still reads what was stored before.""", + run=r"""sudo mkdir -p /etc/kubernetes/enc sudo tee /etc/kubernetes/enc/enc.yaml >/dev/null </dev/null 2>&1 && break; sleep 3; done -k -n %{ns} get secret %{sec} -o json | k replace -f - # stored again: encrypted now -sudo ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt \\ - --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key \\ - get /registry/secrets/%{ns}/%{sec} | od -c | head -3 # k 8 s : e n c : a e s c b c : v 1 : k e y 1 -# By hand: in kube-apiserver.yaml add the flag to the command, a volumeMount for /etc/kubernetes/enc -# and a hostPath volume of the same path - the API server runs in a container and sees only what is -# mounted. The FIRST provider encrypts; `identity` after it reads what was stored before. Every -# Secret: `k get secrets -A -o json | k replace -f -`. -""", + | .spec.volumes += [{"name": "enc", "hostPath": {"path": "/etc/kubernetes/enc", "type": "DirectoryOrCreate"}}]' ~/kube-apiserver.yaml"""), + cmd(r"""sudo cp ~/kube-apiserver.yaml /etc/kubernetes/manifests/"""), + hand(r"""k -n kube-system get pods""", + r"""Again until it answers: the API server is back in a minute or so.""", + run=r"""sleep 20; for i in $(seq 1 40); do kubectl get --raw /readyz >/dev/null 2>&1 && break; sleep 3; done"""), + cmd(r"""k -n %{ns} get secret %{sec} -o yaml | k replace -f -""", r"""Written again, the Secret is stored encrypted; every Secret at once: `k get secrets -A -o json | k replace -f -`."""), + cmd(r"""sudo etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key get /registry/secrets/%{ns}/%{sec} | head -c 80""", r"""`k8s:enc:aescbc:v1:key1:` and no plain text.""")], }, # ------------------------------------------------------------------ 143 ---- { @@ -730,20 +841,39 @@ sys.exit(0 if ok else 1)'"""), eq "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas} {.status.updatedReplicas} {.status.replicas}')" "3 3 3" || exit 1 [ "$($K -n %{ns} get pods -l app=%{dep} --field-selector=status.phase=Running -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' | sort -u | grep -c .)" = 3 ]"""), ], - "solution": """ -k label node {W2} {W3} tier=edge -k -n %{ns} patch deploy %{dep} --type merge -p '{"spec":{"strategy":{"rollingUpdate":{"maxSurge":0,"maxUnavailable":1}},"template":{"spec":{"affinity":{ + "solution": [cmd(r"""k label node {W2} {W3} tier=edge"""), + hand(r"""k -n %{ns} edit deploy %{dep}""", + r"""In the Pod template's `spec:`, above `containers:` (the docs page *Assigning Pods to Nodes* has both kinds): +``` + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: %{dep} + topologyKey: kubernetes.io/hostname + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: tier + operator: In + values: + - edge +``` +and in the same edit, under `strategy:`, let the rollout replace one Pod at a time: +``` + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 +``` +With 3 workers and one Pod each, a surge Pod has nowhere to go, and 25% of 3 unavailable rounds down to 0: the default rollout would wait for ever. Then `:wq`.""", + run=r"""kubectl -n %{ns} patch deploy %{dep} --type merge -p '{"spec":{"strategy":{"rollingUpdate":{"maxSurge":0,"maxUnavailable":1}},"template":{"spec":{"affinity":{ "podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"app":"%{dep}"}},"topologyKey":"kubernetes.io/hostname"}]}, - "nodeAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"weight":100,"preference":{"matchExpressions":[{"key":"tier","operator":"In","values":["edge"]}]}}]}}}}}}' -k -n %{ns} rollout status deploy %{dep} -k -n %{ns} get pods -o wide -# The anti-affinity's labelSelector picks the Pods to stay away from - its own, by its template's -# labels - and topologyKey says per what: one per hostname = one per node. With 3 workers a 4th -# replica would stay Pending - and so does the rolling update's surge Pod, since every node already -# has one of the old Pods (same labels). The default maxUnavailable of 25% is 0 Pods for 3 -# replicas, so the rollout would wait for ever: let it replace one Pod at a time instead -# (maxSurge 0, maxUnavailable 1), in the same change. -""", + "nodeAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"weight":100,"preference":{"matchExpressions":[{"key":"tier","operator":"In","values":["edge"]}]}}]}}}}}}'"""), + cmd(r"""k -n %{ns} rollout status deploy %{dep} +k -n %{ns} get pods -o wide""", r"""Three nodes, one Pod each. The anti-affinity's selector names the Pods to keep away from - its own - and `topologyKey` per what: one per hostname, one per node.""")], }, # ------------------------------------------------------------------ 144 ---- { @@ -787,15 +917,18 @@ eq "$(j -n %{ns} pod %{sts}-0 '{.spec.containers[0].image}') $(j -n %{ns} pod %{ eq "$(for i in 0 1; do j -n %{ns} pod %{sts}-$i '{.metadata.uid}'; echo; done)" "$(cat "$PRIV/q144.uids")" \ && eq "$(j -n %{ns} sts %{sts} '{.spec.template.spec.containers[0].image}')" nginx:1.27"""), ], - "solution": """ -k -n %{ns} patch sts %{sts} -p '{"spec":{"updateStrategy":{"type":"RollingUpdate","rollingUpdate":{"partition":2}}}}' -k -n %{ns} set image sts/%{sts} nginx=nginx:1.27 + "solution": [hand(r"""k -n %{ns} edit sts %{sts}""", + r"""The partition first - a StatefulSet updates from the highest ordinal down, and without one the new image would reach all three within moments. Under `updateStrategy:`, then `:wq`: +``` + updateStrategy: + rollingUpdate: + partition: 2 + type: RollingUpdate +```""", + run=r"""kubectl -n %{ns} patch sts %{sts} -p '{"spec":{"updateStrategy":{"type":"RollingUpdate","rollingUpdate":{"partition":2}}}}'"""), + cmd(r"""k -n %{ns} set image sts %{sts} nginx=nginx:1.27 k -n %{ns} rollout status sts %{sts} -k -n %{ns} get pods -l app=%{sts} -o custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image -# The partition FIRST: a StatefulSet updates from the highest ordinal down, and with no partition -# the new image would reach -1 and -0 within moments. Pods with an ordinal >= partition get the new -# template; the others keep theirs. Next step of the canary: partition 1, then 0. -""", +k -n %{ns} get pods -l app=%{sts} -o custom-columns=NAME:.metadata.name,IMAGE:.spec.containers[0].image""", r"""Ordinals at or above the partition get the new template: `%{sts}-2` alone. The next steps of the canary: partition 1, then 0.""")], }, # ------------------------------------------------------------------ 145 ---- { @@ -825,32 +958,53 @@ eq "$(j crd backups.data.example.com '{.status.conditions[?(@.type=="Established C("Backup %{name} is Scheduled: the controller may update its status", 3, r""" eq "$(j -n %{ns} backup %{name} '{.status.phase}')" Scheduled && eq "$(j clusterrolebinding backup-operator '{.subjects[0].name}')" controller"""), ], - "solution": """ -grep -E '^kind:|^ name:' {WS}/q145/operator.yaml -k apply -f {WS}/q145/operator.yaml -k wait --for condition=established crd/backups.data.example.com -k -n backup-operator rollout status deploy controller -k create ns %{ns} -k -n %{ns} apply -f - <<'EOF' + "solution": [cmd(r"""grep -e ^kind: -e '^ name:' {WS}/q145/operator.yaml +k apply -f {WS}/q145/operator.yaml"""), + hand(r"""k get crd backups.data.example.com +k -n backup-operator get pods""", + r"""Again until the controller is `Running`.""", + run=r"""kubectl wait --for condition=established crd/backups.data.example.com --timeout=60s +kubectl -n backup-operator rollout status deploy controller --timeout=180s"""), + cmd(r"""k create ns %{ns}"""), + hand(r"""vim backup.yaml""", + r"""``` apiVersion: data.example.com/v1 kind: Backup metadata: name: %{name} + namespace: %{ns} spec: schedule: "30 1 * * *" -EOF -sleep 10 -k -n %{ns} get cm backup-%{name} # made - but: -k -n %{ns} get backup %{name} -o jsonpath='{.status}'; echo # no phase -k -n backup-operator logs deploy/controller | tail -3 # ... cannot patch resource "backups/status" ... -k auth can-i patch backups.data.example.com --subresource=status --as=system:serviceaccount:backup-operator:controller -A -k patch clusterrole backup-operator --type json -p '[{"op":"add","path":"/rules/0/resources/-","value":"backups/status"}]' -sleep 10 -k -n %{ns} get backup %{name} -o jsonpath='{.status.phase}'; echo -# The CRD has a status subresource, and a subresource is a resource of its own for RBAC: -# `backups/status`. The controller could read Backups and write ConfigMaps, but not set a phase. -# The log of an operator's controller is where such a gap shows - here as `forbidden`. -""", +```""", + run=r"""cat > backup.yaml <<'EOF' +apiVersion: data.example.com/v1 +kind: Backup +metadata: + name: %{name} + namespace: %{ns} +spec: + schedule: "30 1 * * *" +EOF"""), + cmd(r"""k apply -f backup.yaml"""), + hand(r"""k -n %{ns} get cm backup-%{name} +k -n %{ns} get backup %{name} -o yaml | grep -A1 status +k -n backup-operator logs deploy/controller | tail -3""", + r"""The ConfigMap is made, but no `phase`: the log says `cannot patch resource "backups/status"`. The CRD has a status subresource, and for RBAC a subresource is a resource of its own.""", + run=r"""sleep 10"""), + cmd(r"""k auth can-i patch backups.data.example.com --subresource=status --as=system:serviceaccount:backup-operator:controller -A""", r"""`no`."""), + hand(r"""k edit clusterrole backup-operator""", + r"""Add `backups/status` to the first rule's resources, then `:wq` - the one thing wrong: +``` +- apiGroups: + - data.example.com + resources: + - backups + - backups/status +```""", + run=r"""kubectl patch clusterrole backup-operator --type json -p '[{"op":"add","path":"/rules/0/resources/-","value":"backups/status"}]'"""), + hand(r"""k -n %{ns} get backup %{name} -o yaml | grep -A1 status""", + r"""Again until `phase: Scheduled` - the controller tries every 5 seconds.""", + run=r"""for i in $(seq 1 20); do [ "$(kubectl -n %{ns} get backup %{name} -o jsonpath='{.status.phase}')" = Scheduled ] && break; sleep 3; done""")], }, # ------------------------------------------------------------------ 146 ---- { @@ -881,41 +1035,66 @@ eq "$(j -n %{ns} pod web-server "{.spec.volumes[?(@.name==\"$v\")].persistentVol eq "$(j -n %{ns} pvc pv-volume '{.spec.resources.requests.storage} {.status.capacity.storage}')" "%{size} %{size}" || exit 1 eq "$(j -n %{ns} pod web-server '{.status.phase} {.status.containerStatuses[0].restartCount}')" "Running 0" """), ], - "solution": """ -k get sc csi-hostpath-sc # ALLOWVOLUMEEXPANSION: true - a claim of this class may grow -k -n %{ns} apply -f - <<'EOF' + "solution": [cmd(r"""k get sc csi-hostpath-sc""", r"""`ALLOWVOLUMEEXPANSION true`: a claim of this class may grow."""), + hand(r"""vim pvc.yaml""", + r"""``` apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pv-volume + namespace: %{ns} spec: storageClassName: csi-hostpath-sc - accessModes: [ReadWriteOnce] + accessModes: + - ReadWriteOnce resources: - requests: {storage: 10Mi} ---- + requests: + storage: 10Mi +```""", + run=r"""cat > pvc.yaml <<'EOF' apiVersion: v1 -kind: Pod +kind: PersistentVolumeClaim metadata: - name: web-server + name: pv-volume + namespace: %{ns} spec: - containers: - - name: web-server - image: nginx:1.27 + storageClassName: csi-hostpath-sc + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Mi +EOF"""), + cmd(r"""k apply -f pvc.yaml +k -n %{ns} run web-server --image=nginx:1.27 --dry-run=client -o yaml > pod.yaml"""), + hand(r"""vim pod.yaml""", + r"""Add to the container, below `name: web-server`: +``` volumeMounts: - - {name: html, mountPath: /usr/share/nginx/html} + - name: html + mountPath: /usr/share/nginx/html +``` +and to the Pod's `spec:`, right below `spec:`: +``` volumes: - name: html - persistentVolumeClaim: {claimName: pv-volume} -EOF -k -n %{ns} wait --for=condition=Ready pod/web-server -k -n %{ns} patch pvc pv-volume -p '{"spec":{"resources":{"requests":{"storage":"%{size}"}}}}' -k -n %{ns} get pvc pv-volume # CAPACITY follows once the driver has resized it -# `k edit pvc pv-volume` works just as well. A claim only grows, and only when its class has -# allowVolumeExpansion: true and its driver can expand; spec.resources.requests is what you ask -# for, status.capacity what you have. Filesystem volumes of this driver grow online - the Pod -# keeps running. -""", + persistentVolumeClaim: + claimName: pv-volume +```""", + run=r"""yq -i '.spec.containers[0].volumeMounts = [{"name": "html", "mountPath": "/usr/share/nginx/html"}] + | .spec.volumes = [{"name": "html", "persistentVolumeClaim": {"claimName": "pv-volume"}}]' pod.yaml"""), + cmd(r"""k apply -f pod.yaml"""), + hand(r"""k -n %{ns} get pod web-server""", + r"""Again until it is `Running`.""", + run=r"""kubectl -n %{ns} wait --for=condition=Ready pod/web-server --timeout=180s"""), + hand(r"""k -n %{ns} edit pvc pv-volume""", + r"""Under `spec.resources.requests`, then `:wq` - a claim only grows: +``` + storage: %{size} +```""", + run=r"""kubectl -n %{ns} patch pvc pv-volume -p '{"spec":{"resources":{"requests":{"storage":"%{size}"}}}}'"""), + cmd(r"""k -n %{ns} get pvc pv-volume +k -n %{ns} get pod web-server""", r"""Again until `CAPACITY %{size}`: `spec.resources.requests` is what you asked for, `status.capacity` what the driver has done. This driver grows filesystems online - the Pod keeps running.""")], }, # ------------------------------------------------------------------ 147 ---- { @@ -963,71 +1142,132 @@ x() { $K -n %{ns} exec auditor -- cat "$1/ledger.txt" 2>/dev/null; } eq "$(x /restore)" entries-%{n} && eq "$(x /copy)" entries-%{n} \ && eq "$(j -n %{ns} pod auditor '{.spec.volumes[*].persistentVolumeClaim.claimName}' | tr ' ' '\n' | sort | tr '\n' ' ')" "ledger-copy ledger-restore " """), ], - "solution": """ -k get volumesnapshotclass # csi-hostpath-snapclass, driver hostpath.csi.k8s.io -k -n %{ns} apply -f - <<'EOF' + "solution": [cmd(r"""k get volumesnapshotclass""", r"""`csi-hostpath-snapclass`, driver `hostpath.csi.k8s.io`."""), + hand(r"""vim snap.yaml""", + r"""The docs page *Volume Snapshots* has the VolumeSnapshot: +``` apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshot metadata: name: ledger-snap + namespace: %{ns} spec: volumeSnapshotClassName: csi-hostpath-snapclass source: persistentVolumeClaimName: ledger-data -EOF -k -n %{ns} wait --for=jsonpath='{.status.readyToUse}'=true volumesnapshot/ledger-snap --timeout=120s -k -n %{ns} apply -f - <<'EOF' +```""", + run=r"""cat > snap.yaml <<'EOF' +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshot +metadata: + name: ledger-snap + namespace: %{ns} +spec: + volumeSnapshotClassName: csi-hostpath-snapclass + source: + persistentVolumeClaimName: ledger-data +EOF"""), + cmd(r"""k apply -f snap.yaml"""), + hand(r"""k -n %{ns} get volumesnapshot ledger-snap""", + r"""Again until `READYTOUSE true`.""", + run=r"""kubectl -n %{ns} wait --for=jsonpath='{.status.readyToUse}'=true volumesnapshot/ledger-snap --timeout=120s"""), + hand(r"""vim claims.yaml""", + r"""`dataSource` makes a claim from a snapshot (*Volume Snapshots*) or from another claim of the same class and namespace - a clone (*CSI Volume Cloning*; no `apiGroup`, claims are core). Both at least as big as the source: +``` apiVersion: v1 kind: PersistentVolumeClaim metadata: name: ledger-restore + namespace: %{ns} spec: storageClassName: csi-hostpath-sc - accessModes: [ReadWriteOnce] - resources: - requests: {storage: 50Mi} dataSource: apiGroup: snapshot.storage.k8s.io kind: VolumeSnapshot name: ledger-snap + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Mi --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: ledger-copy + namespace: %{ns} spec: storageClassName: csi-hostpath-sc - accessModes: [ReadWriteOnce] - resources: - requests: {storage: 50Mi} dataSource: kind: PersistentVolumeClaim name: ledger-data + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Mi +```""", + run=r"""cat > claims.yaml <<'EOF' +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ledger-restore + namespace: %{ns} +spec: + storageClassName: csi-hostpath-sc + dataSource: + apiGroup: snapshot.storage.k8s.io + kind: VolumeSnapshot + name: ledger-snap + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Mi --- apiVersion: v1 -kind: Pod +kind: PersistentVolumeClaim metadata: - name: auditor + name: ledger-copy + namespace: %{ns} spec: - containers: - - name: auditor - image: busybox:1.36 - command: [sleep, '86400'] + storageClassName: csi-hostpath-sc + dataSource: + kind: PersistentVolumeClaim + name: ledger-data + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Mi +EOF"""), + cmd(r"""k apply -f claims.yaml +k -n %{ns} run auditor --image=busybox:1.36 --dry-run=client -o yaml -- sleep 86400 > pod.yaml"""), + hand(r"""vim pod.yaml""", + r"""Add to the container, below `name: auditor`: +``` volumeMounts: - - {name: restore, mountPath: /restore} - - {name: copy, mountPath: /copy} + - name: restore + mountPath: /restore + - name: copy + mountPath: /copy +``` +and to the Pod's `spec:`, right below `spec:`: +``` volumes: - name: restore - persistentVolumeClaim: {claimName: ledger-restore} + persistentVolumeClaim: + claimName: ledger-restore - name: copy - persistentVolumeClaim: {claimName: ledger-copy} -EOF -k -n %{ns} wait --for=condition=Ready pod/auditor --timeout=180s -k -n %{ns} exec auditor -- cat /restore/ledger.txt /copy/ledger.txt -# A snapshot is taken by the CSI driver (VolumeSnapshotClass -> driver); `dataSource` makes a new -# claim from it - apiGroup snapshot.storage.k8s.io - or from another claim of the same class and -# Namespace, a clone (no apiGroup: claims are core). Both need a CSI driver that can do it, and -# the new claim must be at least as big as the source. -""", + persistentVolumeClaim: + claimName: ledger-copy +```""", + run=r"""yq -i '.spec.containers[0].volumeMounts = [{"name": "restore", "mountPath": "/restore"}, {"name": "copy", "mountPath": "/copy"}] + | .spec.volumes = [{"name": "restore", "persistentVolumeClaim": {"claimName": "ledger-restore"}}, {"name": "copy", "persistentVolumeClaim": {"claimName": "ledger-copy"}}]' pod.yaml"""), + cmd(r"""k apply -f pod.yaml"""), + hand(r"""k -n %{ns} get pvc,pod""", + r"""Again until both claims are `Bound` and `auditor` `Running`.""", + run=r"""kubectl -n %{ns} wait --for=condition=Ready pod/auditor --timeout=180s"""), + cmd(r"""k -n %{ns} exec auditor -- cat /restore/ledger.txt /copy/ledger.txt""", r"""`entries-%{n}`, twice.""")], }, ] diff --git a/cka/bank_hard.py b/cka/bank_hard.py index fbc96cc..f40e4f4 100644 --- a/cka/bank_hard.py +++ b/cka/bank_hard.py @@ -186,7 +186,9 @@ $K delete ns %{ns} --ignore-not-found --wait=false >/dev/null }, { "id": 49, "domain": "ARCH", "difficulty": "hard", "title": "CRD with validation and defaults", - "params": lambda r: {"ctx": ctx_pick(r), "kind": r.choice(["Rollout", "Canary", "Release"]), "env": r.choice(["prod", "stage"])}, + # low, short: the plural's stem and the shortName, which the solution spells out + "params": lambda r: (lambda p: dict(p, low=p["kind"].lower(), short=p["kind"][:2].lower()))( + {"ctx": ctx_pick(r), "kind": r.choice(["Rollout", "Canary", "Release"]), "env": r.choice(["prod", "stage"])}), "task": CTX + """ Create CRD `%{kind}` (group `deploy.example.com`, v1, Namespaced, plural = the kind in lowercase plus `s`, shortName = the first two letters of the kind in lowercase) with schema: diff --git a/cka/bank_net.py b/cka/bank_net.py index 7358c0a..83775e5 100644 --- a/cka/bank_net.py +++ b/cka/bank_net.py @@ -5,6 +5,8 @@ Written for the per-task hosts: a task may rewrite the cluster's CoreDNS configu resolution and hostPorts are tested for real; Ingress and Gateway objects are graded as objects, as no controller for them runs on the hosts. """ + +from sim.bank import cmd, hand from cka.common import C, DNS, NS, ONE, PODS, PY, ns_pick, name_pick QUESTIONS = [ @@ -153,43 +155,102 @@ parents == [("%{gw}", "%{ns}")] and s.get("hostnames") == ["%{host}.example.com" out=$($K -n %{ns} get ingress %{host} 2>&1); case "$out" in *NotFound*) ;; *) exit 1 ;; esac eq "$(j -n %{ns} secret %{host}-tls '{.metadata.uid}')" "$(cat "$PRIV/q126.uid")" """), ], - "solution": """ -k -n %{ns} get ingress %{host} -o yaml # host, the TLS Secret (%{host}-tls), /api -> api:8080, / -> web:80 -cat < gw.yaml <<'EOF' +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: %{gw} + namespace: %{ns} +spec: + gatewayClassName: %{gc} + listeners: + - name: https + protocol: HTTPS + port: 443 + hostname: %{host}.example.com + tls: + mode: Terminate + certificateRefs: + - kind: Secret + name: %{host}-tls +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: %{host} + namespace: %{ns} +spec: + parentRefs: + - name: %{gw} + hostnames: + - %{host}.example.com + rules: + - matches: + - path: + type: PathPrefix + value: /api + backendRefs: + - name: api + port: 8080 + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: web + port: 80 +EOF"""), + cmd(r"""k apply -f gw.yaml k -n %{ns} get gateway,httproute -k -n %{ns} delete ingress %{host} -# What an Ingress keeps in one object, the Gateway API splits by role: the Gateway is the -# listener - port, hostname, certificate - and belongs to whoever runs the edge; the HTTPRoute is -# the routing and belongs to the application. spec.tls of the Ingress becomes tls.certificateRefs -# on the listener, each path becomes a rule with a PathPrefix match, and a Service backend becomes -# a backendRef. `k explain gateway.spec.listeners.tls` and the Gateway API docs are allowed. -""", +k -n %{ns} delete ingress %{host}""")], }, # ------------------------------------------------------------------ 127 ---- { @@ -399,23 +460,37 @@ eq "$(j -n %{ns} pod %{pod} '{.spec.hostNetwork}')" true || exit 1 pip=$(j -n %{ns} pod %{pod} '{.status.podIP}'); hip=$(j -n %{ns} pod %{pod} '{.status.hostIP}') [ -n "$pip" ] && eq "$pip" "$hip" && eq "$(tr -s ' \t\r\n' ' ' < "$WS/q128/ips.txt" 2>/dev/null | sed 's/^ //; s/ $//')" "$pip $hip" """), ], - "solution": """ -k -n %{ns} edit ds %{ds} # under the container: -# ports: [{containerPort: 8080, hostPort: %{hp}}] -k -n %{ns} rollout status ds %{ds} -k get nodes -o wide # INTERNAL-IP -curl -s http://:%{hp}/ # agent on - -k -n %{ns} logs %{pod} --tail=2 # bad address %{svc} -k -n %{ns} get pod %{pod} -o jsonpath='{.spec.dnsPolicy}'; echo # ClusterFirst - which a hostNetwork Pod does not get -vi {WS}/q128/collector.yaml # spec.dnsPolicy: ClusterFirstWithHostNet -k -n %{ns} delete pod %{pod}; k apply -f {WS}/q128/collector.yaml -k -n %{ns} logs %{pod} --tail=2 # resolved %{svc} -k -n %{ns} get pod %{pod} -o jsonpath='{.status.podIP} {.status.hostIP}' > {WS}/q128/ips.txt -# hostPort maps one port of the node to one Pod on it (the CNI's portmap plugin writes the rule); -# hostNetwork puts the whole Pod into the node's network namespace - its IP IS the node's IP, and -# it uses the node's /etc/resolv.conf, which knows nothing of cluster.local, unless dnsPolicy is -# ClusterFirstWithHostNet. dnsPolicy cannot be changed on a running Pod: recreate it. -""", + "solution": [hand(r"""k -n %{ns} edit ds %{ds}""", + r"""Give the container's port a `hostPort`, then `:wq`: +``` + ports: + - containerPort: 8080 + hostPort: %{hp} + protocol: TCP +``` +`hostPort` maps one port of the node to the Pod on it (the CNI's `portmap` plugin writes the rule); `hostNetwork` would put the whole Pod in the node's network.""", + run=r"""kubectl -n %{ns} patch ds %{ds} --type=json -p '[{"op": "add", "path": "/spec/template/spec/containers/0/ports/0/hostPort", "value": %{hp}}]'"""), + cmd(r"""k -n %{ns} rollout status ds %{ds} +k get nodes -o wide""", r"""Note a worker's `INTERNAL-IP`."""), + hand(r"""curl -s http://172.18.0.3:%{hp}/""", + r"""With that IP: `agent on `.""", + run=r""""""), + cmd(r"""k -n %{ns} logs %{pod} --tail=2 +k -n %{ns} get pod %{pod} -o yaml | grep -e hostNetwork -e dnsPolicy""", r"""`bad address`: a `hostNetwork` Pod uses the node's `/etc/resolv.conf`, which knows nothing of `cluster.local` - unless its `dnsPolicy` is `ClusterFirstWithHostNet`."""), + hand(r"""vim {WS}/q128/collector.yaml""", + r"""In the Pod's `spec:`, below `hostNetwork: true`: +``` + dnsPolicy: ClusterFirstWithHostNet +```""", + run=r"""yq -i '.spec.dnsPolicy = "ClusterFirstWithHostNet"' {WS}/q128/collector.yaml"""), + cmd(r"""k replace --force -f {WS}/q128/collector.yaml""", r"""A running Pod's `dnsPolicy` cannot change: `--force` deletes it and creates it again."""), + hand(r"""k -n %{ns} logs %{pod} --tail=2 +k -n %{ns} get pod %{pod} -o wide""", + r"""Again until `resolved %{svc}`. Its IP is its node's: note it.""", + run=r"""kubectl -n %{ns} wait --for=condition=Ready pod/%{pod} --timeout=180s +sleep 8"""), + hand(r"""echo "172.18.0.3 172.18.0.3" > {WS}/q128/ips.txt""", + r"""The Pod's IP and its node's - the same address.""", + run=r"""kubectl -n %{ns} get pod %{pod} -o jsonpath='{.status.podIP} {.status.hostIP}' > {WS}/q128/ips.txt""")], }, ] diff --git a/cka/bank_objectives.py b/cka/bank_objectives.py index 3c7a927..f00fec4 100644 --- a/cka/bank_objectives.py +++ b/cka/bank_objectives.py @@ -20,7 +20,7 @@ kubeadm, etcd, the runtime. All normal difficulty, all in the random pool; nothi node or a control-plane component (109 cordons the control-plane node while you work on it and has you uncordon it; the session baseline uncordons it if you do not). """ -from sim.bank import C, NS, ns_pick, name_pick +from sim.bank import C, NS, ns_pick, name_pick, cmd, hand from ckad import bank_extra as _ckad_extra, bank_more as _ckad_more @@ -30,18 +30,34 @@ def _port(bank, src_id, new_id, domain, **over): old, new = f"q{src_id}", f"q{new_id}" q = dict(src, id=new_id, domain=domain, **over) for k in ("task", "setup", "solution"): - q[k] = src[k].replace(old, new) + if k not in over: # a solution of its own (in parts) is the CKA one already + q[k] = src[k].replace(old, new) q["checks"] = [dict(c, script=c["script"].replace(old, new), name=c["name"].replace(old, new)) for c in src["checks"]] return q +# 103's own solution: the CKAD task's, done as in the exam +_SOL103 = [hand(r"""k -n %{ns} top pod""", + r"""Again until it shows all three: metrics-server needs about 30 seconds after the Pods start. Without it, `Metrics API not available`.""", + run=r"""for i in $(seq 1 40); do [ "$(kubectl -n %{ns} top pod --no-headers 2>/dev/null | wc -l)" -ge 3 ] && break; sleep 5; done"""), + cmd(r"""k -n %{ns} top pod --sort-by=cpu +k -n %{ns} top pod --sort-by=memory""", r"""The first line of each is the heaviest."""), + hand(r"""echo %{cpu} > {WS}/q103/cpu.txt +echo %{mem} > {WS}/q103/memory.txt""", + r"""With the names on those first lines.""", + run=r"""kubectl -n %{ns} top pod --sort-by=cpu --no-headers | head -1 | awk '{print $1}' > {WS}/q103/cpu.txt +kubectl -n %{ns} top pod --sort-by=memory --no-headers | head -1 | awk '{print $1}' > {WS}/q103/memory.txt"""), + cmd(r"""echo 'kubectl top node' > {WS}/q103/nodes.sh +echo 'kubectl top pod -A --sort-by=memory' > {WS}/q103/pods.sh""", r"""`k top` reads the metrics API (`metrics.k8s.io`) that metrics-server serves; `--containers` breaks a Pod down per container.""")] + + ETCDCTL = ("etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt " "--cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key") QUESTIONS = [ # ---------------------------------------------------------------- TSHOOT ---- # needs a working metrics API, which Q23 breaks on purpose: never in one session with it -_port(_ckad_extra, 23, 103, "TSHOOT", breaks="metrics-server"), +_port(_ckad_extra, 23, 103, "TSHOOT", breaks="metrics-server", solution=_SOL103), _port(_ckad_more, 64, 104, "TSHOOT"), # ------------------------------------------------------------------ WORK ---- { @@ -183,32 +199,44 @@ cg=$(nx "$W3" "awk '/^cgroupDriver:/{print \$2}' /var/lib/kubelet/config.yaml" < want=$(printf 'ip_forward=%s\nswap=%s\nfail-swap-on=%s\ncgroup-driver=%s' "$fwd" "$swap" "$fso" "$cg") eq "$(tr -d ' \t\r' < "$WS/q107/node.txt" 2>/dev/null | grep .)" "$want" """), ], - "solution": """ -{SHELL_W3} + "solution": [hand(r"""{SHELL_W3} sudo -i -printf 'overlay\\nbr_netfilter\\n' > /etc/modules-load.d/%{name}.conf -cat > /etc/sysctl.d/%{name}.conf </dev/null'"""), + hand(r"""vim /etc/sysctl.d/%{name}.conf""", + r"""``` net.bridge.bridge-nf-call-iptables = 1 net.bridge.bridge-nf-call-ip6tables = 1 net.ipv4.ip_forward = 1 -EOF -# on a real machine you now load and apply them without a reboot: -# modprobe overlay && modprobe br_netfilter && sysctl --system -sysctl -n net.ipv4.ip_forward # 1 -cat /proc/swaps # any line under the header = swap is on (swapon --show) -grep -E 'failSwapOn|cgroupDriver' /var/lib/kubelet/config.yaml -exit; exit -cat > {WS}/q107/node.txt < -fail-swap-on= -cgroup-driver= -EOF -# `sysctl -w` and `modprobe` alone are gone after a reboot - the files are what makes them stick. -# kubeadm's preflight refuses a node with swap on unless the kubelet is told to tolerate it -# (failSwapOn: false), and the kubelet's cgroup driver has to match the container runtime's -# (SystemdCgroup = true in /etc/containerd/config.toml goes with cgroupDriver: systemd). -""", +``` +On a real machine they would now be loaded and applied without a reboot: `modprobe overlay`, `modprobe br_netfilter`, `sysctl --system`. `sysctl -w` and `modprobe` alone are gone after a reboot; the files make them stick.""", + run=r"""ssh -n {W3} 'printf "net.bridge.bridge-nf-call-iptables = 1\nnet.bridge.bridge-nf-call-ip6tables = 1\nnet.ipv4.ip_forward = 1\n" | sudo tee /etc/sysctl.d/%{name}.conf >/dev/null'"""), + hand(r"""sysctl -n net.ipv4.ip_forward +cat /proc/swaps +grep -e failSwapOn -e cgroupDriver /var/lib/kubelet/config.yaml +exit +exit""", + r"""Any line below the header of `/proc/swaps` means swap is on. No `failSwapOn` line means the default, `true`.""", + run=r""""""), + hand(r"""vim {WS}/q107/node.txt""", + r"""What the node printed: +``` +ip_forward=<0 or 1> +swap= +fail-swap-on= +cgroup-driver= +``` +kubeadm's preflight refuses a node with swap on unless the kubelet tolerates it (`failSwapOn: false`); and the kubelet's cgroup driver has to match the runtime's (`SystemdCgroup = true` in containerd goes with `cgroupDriver: systemd`).""", + run=r"""IPF=$(ssh -n {W3} 'sudo sysctl -n net.ipv4.ip_forward') +SW=off; [ "$(ssh -n {W3} 'sed 1d /proc/swaps | grep -c .')" -gt 0 ] && SW=on +FS=$(ssh -n {W3} "sudo awk '/^failSwapOn:/ {print \$2}' /var/lib/kubelet/config.yaml"); [ -n "$FS" ] || FS=true +CG=$(ssh -n {W3} "sudo awk '/^cgroupDriver:/ {print \$2}' /var/lib/kubelet/config.yaml") +printf 'ip_forward=%s\nswap=%s\nfail-swap-on=%s\ncgroup-driver=%s\n' "$IPF" "$SW" "$FS" "$CG" > {WS}/q107/node.txt""")], }, { "id": 108, "domain": "ARCH", "title": "The container runtime behind a Pod", @@ -263,29 +291,35 @@ cid=$($K -n %{ns} get pod %{pod} -o jsonpath='{.status.containerStatuses[0].cont log=$(nx "$W2" "crictl inspect -o go-template --template '{{.status.logPath}}' $cid 2>/dev/null" /dev/null | grep -qx "log=$log" """), ], - "solution": """ -k -n %{ns} get pod %{pod} -o wide # on {W2} -{SHELL_W2} + "solution": [cmd(r"""k -n %{ns} get pod %{pod} -o wide""", r"""On `{W2}`."""), + hand(r"""{SHELL_W2} sudo -i -crictl version # RuntimeName: containerd RuntimeVersion: v2.x.y -grep containerRuntimeEndpoint /var/lib/kubelet/config.yaml # unix:///run/containerd/containerd.sock -crictl pods --name %{pod} # the Pod sandbox -crictl ps --no-trunc --name app --pod $(crictl pods --name %{pod} -q) # full container ID -crictl inspect | grep -E '"pid"|logPath' # .info.pid and .status.logPath -# or: crictl inspect -o go-template --template '{{.info.pid}} {{.status.logPath}}' -exit; exit -cat > {WS}/q108/runtime.txt < socket=unix:///run/containerd/containerd.sock -container-id=<64 hex characters> -pid= +container-id= +pid= log=/var/log/pods/%{ns}_%{pod}_/app/0.log -EOF -# The kubelet drives the runtime over the CRI socket, and crictl speaks the same protocol - which -# is why it still works when the API server does not. `kubectl get pod -o -# jsonpath='{.status.containerStatuses[0].containerID}'` shows the same ID with a containerd:// -# prefix; /var/log/pods/... is the file `kubectl logs` reads. -""", +```""", + run=r"""V=$(ssh -n {W2} 'sudo crictl version 2>/dev/null') +RN=$(echo "$V" | awk '/^RuntimeName:/ {print $2}'); RV=$(echo "$V" | awk '/^RuntimeVersion:/ {print $2}') +SOCK=$(ssh -n {W2} "sudo awk '/^containerRuntimeEndpoint:/ {print \$2}' /var/lib/kubelet/config.yaml" | tr -d '"') +CID=$(kubectl -n %{ns} get pod %{pod} -o jsonpath='{.status.containerStatuses[0].containerID}' | sed 's#^[a-z-]*://##') +PID=$(ssh -n {W2} "sudo crictl inspect -o go-template --template '{{.info.pid}}' $CID 2>/dev/null") +LOG=$(ssh -n {W2} "sudo crictl inspect -o go-template --template '{{.status.logPath}}' $CID 2>/dev/null") +printf 'runtime=%s %s\nsocket=%s\ncontainer-id=%s\npid=%s\nlog=%s\n' "$RN" "$RV" "$SOCK" "$CID" "$PID" "$LOG" > {WS}/q108/runtime.txt""")], }, { # drains the control-plane node: CoreDNS moves off it, so never beside the tasks that break DNS @@ -468,29 +502,33 @@ c=$(j -n kube-system lease kube-controller-manager '{.spec.holderIdentity}' | cu f=$(tr -d ' \t\r' < "$WS/q111/ha.txt" 2>/dev/null) [ -n "$s" ] && [ -n "$c" ] && echo "$f" | grep -qx "scheduler-leader=$s" && echo "$f" | grep -qx "controller-manager-leader=$c" """), ], - "solution": """ -k get nodes -l node-role.kubernetes.io/control-plane # how many -k -n kube-system get pods -l component=etcd -o wide # an etcd static Pod on the control-plane node = stacked -k -n kube-system get lease kube-scheduler kube-controller-manager # HOLDER is _ -sudo -i # the task's host is the control-plane node: no hop needed -export ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCDCTL_CACERT=/etc/kubernetes/pki/etcd/ca.crt \\ - ETCDCTL_CERT=/etc/kubernetes/pki/etcd/server.crt ETCDCTL_KEY=/etc/kubernetes/pki/etcd/server.key -etcdctl member list -w table # one line per member: ID, NAME, peer and client URLs -etcdctl endpoint status -w table # IS LEADER (add --cluster to ask every member) -exit -cat > {WS}/q111/ha.txt <_`."""), + hand(r"""sudo -i +export ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCDCTL_CACERT=/etc/kubernetes/pki/etcd/ca.crt ETCDCTL_CERT=/etc/kubernetes/pki/etcd/server.crt ETCDCTL_KEY=/etc/kubernetes/pki/etcd/server.key +etcdctl member list -w table +etcdctl endpoint status -w table +exit""", + r"""The host is the control-plane node. One line per member; `IS LEADER` in the status (`--cluster` asks every member).""", + run=r""""""), + hand(r"""vim {WS}/q111/ha.txt""", + r"""``` control-plane-nodes=1 etcd-topology=stacked etcd-members=1 etcd-leader={CP} scheduler-leader={CP} controller-manager-leader={CP} -EOF -# In an HA control plane every kube-apiserver serves at once, but only one kube-scheduler and one -# kube-controller-manager act: they elect a leader through a Lease in kube-system, and the others -# wait. etcd elects its own leader and needs a majority of members to stay writable - three -# members survive the loss of one, five of two, and an even number adds nothing. -""", +``` +In an HA control plane every API server serves at once, but one scheduler and one controller manager act, through the leader Lease. etcd elects its own leader and needs a majority to stay writable: three members survive the loss of one, five of two.""", + run=r"""E='sudo etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key' +N=$(kubectl get nodes -l node-role.kubernetes.io/control-plane --no-headers | wc -l) +M=$($E member list | wc -l) +L=$($E member list | awk -F', ' 'NR==1 {print $3}') +S=$(kubectl -n kube-system get lease kube-scheduler -o jsonpath='{.spec.holderIdentity}' | cut -d_ -f1) +C=$(kubectl -n kube-system get lease kube-controller-manager -o jsonpath='{.spec.holderIdentity}' | cut -d_ -f1) +printf 'control-plane-nodes=%s\netcd-topology=stacked\netcd-members=%s\netcd-leader=%s\nscheduler-leader=%s\ncontroller-manager-leader=%s\n' "$N" "$M" "$L" "$S" "$C" > {WS}/q111/ha.txt""")], }, { "id": 112, "domain": "ARCH", "title": "CNI, CSI and CRI: what this cluster plugs in", @@ -532,30 +570,35 @@ prov=$($K get sc -o jsonpath='{range .items[?(@.metadata.annotations.storageclas f=$(tr -d ' \t\r' < "$WS/q112/interfaces.txt" 2>/dev/null) echo "$f" | grep -qx "csi-drivers=$n" && echo "$f" | grep -qx "default-provisioner=$prov" """), ], - "solution": """ -{SHELL_W1} + "solution": [hand(r"""{SHELL_W1} sudo -i -ls /etc/cni/net.d/ # 10-kindnet.conflist - the kubelet's runtime takes the first file -cat /etc/cni/net.d/10-kindnet.conflist # "plugins": [{"type": "ptp", "ipam": {"type": "host-local"}}, {"type": "portmap"}] -ls /opt/cni/bin # the plugin binaries those types name +ls /etc/cni/net.d +cat /etc/cni/net.d/10-kindnet.conflist grep containerRuntimeEndpoint /var/lib/kubelet/config.yaml -exit; exit -k get csidrivers # No resources found = 0 -k get sc # the (default) class and its PROVISIONER -cat > {WS}/q112/interfaces.txt </dev/null | wc -l) +PROV=$(kubectl get sc -o json | jq -r '.items[] | select(.metadata.annotations["storageclass.kubernetes.io/is-default-class"] == "true") | .provisioner') +printf 'cni-config=%s\ncni-plugins=%s\ncni-ipam=%s\ncri-socket=%s\ncsi-drivers=%s\ndefault-provisioner=%s\n' "$CONF" "$PL" "$IPAM" "$SOCK" "$N" "$PROV" > {WS}/q112/interfaces.txt""")], }, # ------------------------------------------------------------------- NET ---- { @@ -623,19 +666,18 @@ eq "$(tr -s ' \t' ' ' < "$WS/q113/pod-cidrs.txt" 2>/dev/null | tr -d '\r' | sed C("outside.txt names %{edge}, why.txt names hostNetwork", 2, r""" eq "$(tr -d '[:space:]' < "$WS/q113/outside.txt" 2>/dev/null | sed 's#^pod/##')" %{edge} && grep -qi 'hostNetwork' "$WS/q113/why.txt" 2>/dev/null"""), ], - "solution": """ -k -n %{ns} get pods -o wide # IP and NODE of every Pod -k -n %{ns} get pod %{web} -o jsonpath='{.status.podIP} {.spec.nodeName}' > {WS}/q113/web.txt -k -n %{ns} exec %{client} -- wget -qO- -T3 http://$(cut -d' ' -f1 {WS}/q113/web.txt) > {WS}/q113/direct.html -k get nodes -o jsonpath='{range .items[*]}{.metadata.name} {.spec.podCIDR}{"\\n"}{end}' > {WS}/q113/pod-cidrs.txt -# %{edge}'s IP is its node's own address (compare with `k get nodes -o wide`), not one from 10.244.x.0/24 -echo %{edge} > {WS}/q113/outside.txt + "solution": [cmd(r"""k -n %{ns} get pods -o wide +k get nodes -o wide""", r"""The `IP` and `NODE` of every Pod. `%{edge}`'s IP is its node's own `INTERNAL-IP`, not one from a Pod CIDR."""), + hand(r"""echo "10.244.1.5 {W1}" > {WS}/q113/web.txt""", + r"""With `%{web}`'s IP and node.""", + run=r"""kubectl -n %{ns} get pod %{web} -o jsonpath='{.status.podIP} {.spec.nodeName}' > {WS}/q113/web.txt"""), + hand(r"""k -n %{ns} exec %{client} -- wget -qO- -T3 http://10.244.1.5 > {WS}/q113/direct.html""", + r"""With that IP: Pod to Pod, across nodes, no Service and no NAT - the network model the CNI makes true.""", + run=r"""kubectl -n %{ns} exec %{client} -- wget -qO- -T3 http://$(kubectl -n %{ns} get pod %{web} -o jsonpath='{.status.podIP}') > {WS}/q113/direct.html"""), + cmd(r"""k get nodes -o custom-columns=NAME:.metadata.name,CIDR:.spec.podCIDR --no-headers > {WS}/q113/pod-cidrs.txt +cat {WS}/q113/pod-cidrs.txt k -n %{ns} get pod %{edge} -o yaml | grep hostNetwork -echo hostNetwork > {WS}/q113/why.txt -# Every Pod gets its own address from its node's Pod CIDR, and any Pod reaches any other Pod on -# any node at that address without NAT - that is the whole network model, and the CNI plugin is -# what makes it true. A hostNetwork Pod opts out: it lives in the node's network namespace and -# shares its address and ports. Pod addresses change with every new Pod, which is why Services exist. -""", +echo %{edge} > {WS}/q113/outside.txt +echo hostNetwork > {WS}/q113/why.txt""", r"""A `hostNetwork` Pod lives in its node's network namespace and shares the node's address and ports.""")], }, ] diff --git a/cka/bank_sim_c.py b/cka/bank_sim_c.py index f1b5897..3697fdd 100644 --- a/cka/bank_sim_c.py +++ b/cka/bank_sim_c.py @@ -12,7 +12,7 @@ Two scenarios are deliberately adapted so they cannot wreck the practice cluster Run with: ./ksim sim c """ -from sim.bank import C, NS +from sim.bank import C, NS, cmd, hand Q = [] @@ -62,42 +62,87 @@ eq "$(j -n geo-fence pod geofence-reader '{.status.phase}')" Running \ && eq "$(j -n geo-fence pod geofence-reader '{.spec.volumes[0].persistentVolumeClaim.claimName}')" geofence-pvc \ && j -n geo-fence pod geofence-reader '{.spec.containers[0].volumeMounts[*].mountPath}' | grep -q /geo-data"""), ], - solution=""" -cat < pv.yaml <<'EOF' +apiVersion: v1 +kind: PersistentVolume +metadata: + name: geofence-pv +spec: + storageClassName: manual + capacity: + storage: 250Mi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + hostPath: + path: /data/geofence --- apiVersion: v1 -kind: Pod -metadata: {name: geofence-reader, namespace: geo-fence} +kind: PersistentVolumeClaim +metadata: + name: geofence-pvc + namespace: geo-fence spec: - volumes: [{name: geo, persistentVolumeClaim: {claimName: geofence-pvc}}] - containers: - - name: reader - image: busybox:1.36 - command: [sh, -c, "ls /geo-data && sleep 3600"] - volumeMounts: [{name: geo, mountPath: /geo-data}] -EOF + storageClassName: manual + volumeName: geofence-pv + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 250Mi +EOF"""), + cmd(r"""k apply -f pv.yaml k -n geo-fence get pvc -# volumeName pins the claim to one PV by name. Without it the binder picks ANY PV whose capacity, -# accessModes and storageClassName match - which on a busy cluster is rarely the one you meant. -""", +k -n geo-fence run geofence-reader --image=busybox:1.36 --dry-run=client -o yaml --command -- sh -c "ls /geo-data && sleep 3600" > pod.yaml""", r"""`Bound` to `geofence-pv`."""), + hand(r"""vim pod.yaml""", + r"""Add to the container, below `name: geofence-reader`: +``` + volumeMounts: + - name: geo + mountPath: /geo-data +``` +and to the Pod's `spec:`, right below `spec:`: +``` + volumes: + - name: geo + persistentVolumeClaim: + claimName: geofence-pvc +```""", + run=r"""yq -i '.spec.containers[0].volumeMounts = [{"name": "geo", "mountPath": "/geo-data"}] + | .spec.volumes = [{"name": "geo", "persistentVolumeClaim": {"claimName": "geofence-pvc"}}]' pod.yaml"""), + cmd(r"""k apply -f pod.yaml +k -n geo-fence get pod geofence-reader""")], ) # ------------------------------------------------------------------ 69 ---- @@ -260,15 +305,19 @@ want=$(j -n dispatch-hub svc dispatch-service '{.spec.clusterIP}') got=$(tr -d '[:space:]' < "$WS/q71/service-ip.txt" 2>/dev/null) [ -n "$want" ] && eq "$got" "$want" """), ], - solution=""" -k -n dispatch-hub expose deploy dispatch-api --name=dispatch-service --port=80 --target-port=80 -k -n dispatch-hub get endpointslices -l kubernetes.io/service-name=dispatch-service # must not be empty -k -n dispatch-hub run probe --image=busybox:1.36 --restart=Never -i --rm -- \\ - wget -qO- http://dispatch-service.dispatch-hub.svc:80 > {WS}/q71/test-result.txt -k -n dispatch-hub get svc dispatch-service -o jsonpath='{.spec.clusterIP}' > {WS}/q71/service-ip.txt -# `expose` copies the Deployment's selector for you, which is what makes the endpoints appear. -# Write the IP with jsonpath, not by eyeballing `get svc` - the grader compares it literally. -""", + solution=[cmd(r"""k -n dispatch-hub expose deploy dispatch-api --name=dispatch-service --port=80 --target-port=80 +k -n dispatch-hub get svc,endpointslices""", r"""`expose` copies the Deployment's selector, which is what makes the endpoints appear. Note the Service's `CLUSTER-IP`."""), + hand(r"""k -n dispatch-hub run probe --image=busybox:1.36 --restart=Never -i --rm -- wget -qO- http://dispatch-service.dispatch-hub.svc:80 > {WS}/q71/test-result.txt +cat {WS}/q71/test-result.txt""", + r"""nginx's welcome page, fetched from a Pod in the cluster.""", + run=r"""kubectl -n dispatch-hub rollout status deploy dispatch-api --timeout=240s +for i in 1 2 3 4 5; do + kubectl -n dispatch-hub run probe --image=busybox:1.36 --restart=Never -i --rm -- wget -qO- http://dispatch-service.dispatch-hub.svc:80 > {WS}/q71/test-result.txt /dev/null + grep -qi nginx {WS}/q71/test-result.txt && break; kubectl -n dispatch-hub delete pod probe --ignore-not-found >/dev/null 2>&1; sleep 3 +done"""), + hand(r"""echo 10.96.123.45 > {WS}/q71/service-ip.txt""", + r"""With the `CLUSTER-IP` that `get svc` showed - the address alone.""", + run=r"""kubectl -n dispatch-hub get svc dispatch-service -o jsonpath='{.spec.clusterIP}' > {WS}/q71/service-ip.txt""")], ) # ------------------------------------------------------------------ 72 ---- @@ -358,54 +407,102 @@ j -n ride-history deploy ride-archive '{.spec.template.spec.volumes[*].persisten j -n ride-history deploy ride-archive '{.spec.template.spec.volumes[*].persistentVolumeClaim.claimName}' | grep -qw ride-archive-pvc-new || exit 1 for i in $(seq 1 20); do [ "$(j -n ride-history deploy ride-archive '{.status.availableReplicas}')" -ge 1 ] 2>/dev/null && exit 0; sleep 5; done; exit 1"""), ], - solution=""" -k -n ride-history get pvc,pv # note reclaimPolicy Delete on the old PV -cat < new.yaml <<'EOF' apiVersion: v1 -kind: Pod -metadata: {name: data-migrator, namespace: ride-history} +kind: PersistentVolume +metadata: + name: ride-archive-pv-new spec: - restartPolicy: Never + storageClassName: manual + capacity: + storage: 500Mi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + hostPath: + path: /data/ride-archive-new +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ride-archive-pvc-new + namespace: ride-history +spec: + storageClassName: manual + volumeName: ride-archive-pv-new + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 500Mi +EOF"""), + cmd(r"""k apply -f new.yaml +k -n ride-history scale deploy ride-archive --replicas=0 +k -n ride-history run data-migrator --image=busybox:1.36 --restart=Never --dry-run=client -o yaml -- sh -c "cp -a /old-data/. /new-data/ && echo migration-complete && sleep 3600" > m.yaml""", r"""Scale to 0 first: the running Pod holds the old claim. `cp -a /old-data/. /new-data/` copies the contents, dotfiles too, rather than the directory itself."""), + hand(r"""vim m.yaml""", + r"""Both claims: add to the container, below `name: data-migrator`, +``` + volumeMounts: + - name: old + mountPath: /old-data + - name: new + mountPath: /new-data +``` +and to the Pod's `spec:`, right below `spec:`: +``` volumes: - - {name: old, persistentVolumeClaim: {claimName: ride-archive-pvc}} - - {name: new, persistentVolumeClaim: {claimName: ride-archive-pvc-new}} - containers: - - name: migrator - image: busybox:1.36 - command: [sh, -c, "cp -a /old-data/. /new-data/ && echo migration-complete && sleep 3600"] - volumeMounts: - - {name: old, mountPath: /old-data} - - {name: new, mountPath: /new-data} -EOF -k -n ride-history logs data-migrator # migration-complete -k -n ride-history patch deploy ride-archive --type=json \\ - -p '[{"op":"replace","path":"/spec/template/spec/volumes/0/persistentVolumeClaim/claimName","value":"ride-archive-pvc-new"}]' -k -n ride-history scale deploy ride-archive --replicas=1 -# ReadWriteOnce is per-node, but a second pod on the same node still cannot mount a claim that a -# running pod holds - hence the scale to 0 first. `cp -a /old/. /new/` copies the CONTENTS -# (including dotfiles) rather than nesting the directory inside the target. -""", + - name: old + persistentVolumeClaim: + claimName: ride-archive-pvc + - name: new + persistentVolumeClaim: + claimName: ride-archive-pvc-new +```""", + run=r"""yq -i '.spec.containers[0].volumeMounts = [{"name": "old", "mountPath": "/old-data"}, {"name": "new", "mountPath": "/new-data"}] + | .spec.volumes = [{"name": "old", "persistentVolumeClaim": {"claimName": "ride-archive-pvc"}}, {"name": "new", "persistentVolumeClaim": {"claimName": "ride-archive-pvc-new"}}]' m.yaml"""), + cmd(r"""k apply -f m.yaml"""), + hand(r"""k -n ride-history logs data-migrator""", + r"""Again until it prints `migration-complete`.""", + run=r"""for i in $(seq 1 60); do kubectl -n ride-history logs data-migrator 2>/dev/null | grep -q migration-complete && break; sleep 2; done"""), + cmd(r"""k -n ride-history delete pod data-migrator --now"""), + hand(r"""k -n ride-history edit deploy ride-archive""", + r"""In the Pod template's `volumes:`, change `claimName: ride-archive-pvc` to `claimName: ride-archive-pvc-new`, then `:wq`.""", + run=r"""kubectl -n ride-history patch deploy ride-archive --type=json -p '[{"op":"replace","path":"/spec/template/spec/volumes/0/persistentVolumeClaim/claimName","value":"ride-archive-pvc-new"}]'"""), + cmd(r"""k -n ride-history scale deploy ride-archive --replicas=1 +k -n ride-history get pods""")], ) # ------------------------------------------------------------------ 73 ---- @@ -445,17 +542,21 @@ for i in $(seq 1 12); do sleep 3 done; exit 1"""), ], - solution=""" -k -n eta-service get endpointslices -l kubernetes.io/service-name=eta-backend # empty -k -n eta-service get deploy eta-calculator -o jsonpath='{.spec.template.metadata.labels}' # app=eta-calculator -k -n eta-service get svc eta-backend -o jsonpath='{.spec.selector}' # app=eta-service <- fault 1 -k -n eta-service patch svc eta-backend -p '{"spec":{"selector":{"app":"eta-calculator"}}}' -k -n eta-service patch svc eta-backend -p '{"spec":{"ports":[{"port":80,"targetPort":80,"protocol":"TCP"}]}}' -k -n eta-service get endpointslices -l kubernetes.io/service-name=eta-backend -# Empty endpoints is almost always the selector; a populated endpoint list that still refuses -# connections is almost always targetPort. Check the endpoints first - it splits the two cases -# in one command. -""", + solution=[cmd(r"""k -n eta-service get endpointslices -l kubernetes.io/service-name=eta-backend +k -n eta-service get pods --show-labels +k -n eta-service describe svc eta-backend""", r"""No endpoints. The Pods are `app=eta-calculator`, the Service selects `app=eta-service`; and it sends to `TargetPort 9090`, where nginx listens on 80. Empty endpoints is almost always the selector; endpoints that refuse connections, the target port."""), + hand(r"""k -n eta-service edit svc eta-backend""", + r"""Fix both, then `:wq`: +``` + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: eta-calculator +```""", + run=r"""kubectl -n eta-service patch svc eta-backend -p '{"spec":{"selector":{"app":"eta-calculator"},"ports":[{"port":80,"targetPort":80,"protocol":"TCP"}]}}'"""), + cmd(r"""k -n eta-service get endpointslices -l kubernetes.io/service-name=eta-backend""")], ) # ------------------------------------------------------------------ 74 ---- @@ -521,36 +622,91 @@ sys.exit(0 if ok else 1)'"""), C("the HTTPRoute is attached to fleet-gateway", 1, r""" j -n fleet-ops httproute fleet-route '{.spec.parentRefs[*].name}' | grep -qw fleet-gateway"""), ], - solution=""" -cat < gw.yaml <<'EOF' +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: fleet-gateway + namespace: fleet-ops +spec: + gatewayClassName: fleet-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + hostname: fleet.example.com +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: fleet-route + namespace: fleet-ops +spec: + parentRefs: + - name: fleet-gateway + hostnames: + - fleet.example.com + rules: + - matches: + - headers: + - name: X-Fleet-Version + value: canary + backendRefs: + - name: fleet-dashboard-v2 + port: 80 + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: fleet-dashboard-v1 + port: 80 +EOF"""), + cmd(r"""k apply -f gw.yaml +k -n fleet-ops get gateway,httproute""")], ) # ------------------------------------------------------------------ 75 ---- @@ -714,29 +870,49 @@ eq "$($n -o jsonpath='{.spec.podSelector}' 2>/dev/null)" '{}' \ && $n -o jsonpath='{.spec.policyTypes}' | grep -q Egress \ && [ -z "$($n -o jsonpath='{.spec.ingress}')" ] && [ -z "$($n -o jsonpath='{.spec.egress}')" ]"""), ], - solution=""" -k -n kube-system get ds # kindnet -> {WS}/q77/cni.txt -# Don't answer from memory - prove it. Two pods, one deny policy, one wget: -k -n zero-trust run srv --image=nginx:1.27 --labels=app=srv; k -n zero-trust expose pod srv --port 80 -k -n zero-trust run cli --image=busybox:1.36 -- sleep 3600 -k -n zero-trust exec cli -- wget -qO- -T3 http://srv >/dev/null && echo reachable -cat < {WS}/q77/cni.txt""", r"""`kindnet` runs on every node: it is the CNI. `kube-proxy` is not."""), + cmd(r"""k -n zero-trust run srv --image=nginx:1.27 --labels=app=srv +k -n zero-trust expose pod srv --port 80 +k -n zero-trust run cli --image=busybox:1.36 -- sleep 3600""", r"""Prove it rather than remember it: two Pods, a policy that denies everything, and a request between them."""), + hand(r"""k -n zero-trust get pods""", + r"""Again until both are `Running`.""", + run=r"""kubectl -n zero-trust wait --for=condition=Ready pod/srv pod/cli --timeout=180s"""), + cmd(r"""k -n zero-trust exec cli -- wget -qO- -T3 http://srv""", r"""nginx's page: no policy yet."""), + hand(r"""vim deny.yaml""", + r"""The default deny for both directions from the docs page *Network Policies*: every Pod, both policy types, no rules. +``` apiVersion: networking.k8s.io/v1 kind: NetworkPolicy -metadata: {name: default-deny-all, namespace: zero-trust} +metadata: + name: default-deny-all + namespace: zero-trust spec: podSelector: {} - policyTypes: [Ingress, Egress] -EOF -k -n zero-trust exec cli -- wget -qO- -T3 http://srv >/dev/null || echo BLOCKED # -> BLOCKED here -echo yes > {WS}/q77/enforced.txt -k -n zero-trust delete pod srv cli; k -n zero-trust delete svc srv # leave only the policy -# podSelector {} means every pod in the namespace; naming both policyTypes with no ingress/egress -# blocks listed means "deny everything". Recent kindnet builds (this cluster's included) embed a -# policy engine - look for `nft list table inet kindnet-network-policies` on a node. Flannel stores -# NetworkPolicies and enforces nothing; Calico and Cilium enforce. A policy that "works" on a -# non-enforcing CNI is the classic hardening-review trap. -""", + policyTypes: + - Ingress + - Egress +```""", + run=r"""cat > deny.yaml <<'EOF' +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny-all + namespace: zero-trust +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress +EOF"""), + cmd(r"""k apply -f deny.yaml +k -n zero-trust exec cli -- wget -qO- -T3 http://srv""", r"""`download timed out`: the CNI enforces it. (Flannel would store the policy and enforce nothing; Calico and Cilium enforce. Recent kindnet has a policy engine of its own.)"""), + hand(r"""echo yes > {WS}/q77/enforced.txt""", + r"""What the test showed.""", + run=r"""sleep 6 +if kubectl -n zero-trust exec cli -- wget -qO- -T3 http://srv >/dev/null 2>&1; then echo no; else echo yes; fi > {WS}/q77/enforced.txt"""), + cmd(r"""k -n zero-trust delete pod srv cli --now +k -n zero-trust delete svc srv""", r"""Leave only the policy.""")], ) # ------------------------------------------------------------------ 78 ---- @@ -767,23 +943,16 @@ for i in $(seq 1 20); do [ "$(j -n rider-support deploy support-portal '{.status eq "$(j -n rider-support deploy support-portal '{.status.readyReplicas}')" 2 \ && eq "$(j -n rider-support pod -l app=support-portal '{.items[0].spec.priority}')" 1000000"""), ], - solution=""" -cat </dev/null \ && ! grep -q 'upgrade apply' "$WS/q80/worker.txt" """), ], - solution=""" -k version -o json | jq -r .serverVersion.gitVersion > {WS}/q80/current.txt + solution=[cmd(r"""k version k get nodes -o custom-columns=NAME:.metadata.name,KUBELET:.status.nodeInfo.kubeletVersion --no-headers > {WS}/q80/kubelets.txt -cat > {WS}/q80/steps.sh <<'EOF' -sed -i 's|/v1.37/|/v1.38/|' /etc/apt/sources.list.d/kubernetes.list +cat {WS}/q80/kubelets.txt"""), + hand(r"""echo v1.37.0 > {WS}/q80/current.txt""", + r"""With the `Server Version` that `k version` printed.""", + run=r"""kubectl version -o json | jq -r .serverVersion.gitVersion > {WS}/q80/current.txt"""), + hand(r"""vim {WS}/q80/steps.sh""", + r"""The order of the docs page *Upgrading kubeadm clusters*: kubeadm first, which upgrades the control plane; then the node's kubelet and kubectl, drained; uncordon last. The next minor is 1.38 - first point `/etc/apt/sources.list.d/kubernetes.list` at the v1.38 repository, and take its newest patch (`apt-cache madison kubeadm`): +``` apt-mark unhold kubeadm && apt-get update && apt-get install -y kubeadm=1.38.0-1.1 && apt-mark hold kubeadm kubeadm upgrade plan -kubeadm upgrade apply v1.38.0 -y -kubectl drain --ignore-daemonsets -apt-mark unhold kubelet kubectl && apt-get install -y kubelet=1.38.0-1.1 kubectl=1.38.0-1.1 && apt-mark hold kubelet kubectl +kubeadm upgrade apply v1.38.0 +kubectl drain {CP} --ignore-daemonsets +apt-mark unhold kubelet kubectl && apt-get update && apt-get install -y kubelet=1.38.0-1.1 kubectl=1.38.0-1.1 && apt-mark hold kubelet kubectl systemctl daemon-reload systemctl restart kubelet -kubectl uncordon -EOF -echo 'kubeadm upgrade node' > {WS}/q80/worker.txt -# The cluster runs v1.37, so the next minor is 1.38 - and pkgs.k8s.io serves one apt repo per -# minor, so the repo line has to move to v1.38 before 1.38 packages exist for apt at all. (1.38 -# is due in December 2026; until then that repo is empty, which is why this is only a plan. Take -# the newest 1.38 patch on offer then: `apt-cache madison kubeadm`.) -# kubeadm goes first because it drives the whole thing; the control plane is upgraded by -# `upgrade apply`, and only then do the node binaries (kubelet/kubectl) follow, ending with -# uncordon. Workers run `kubeadm upgrade node` - they host no control-plane static pods, so -# there is nothing for `upgrade apply` to do there. The kubelet may trail the API server by up -# to three minor versions, but must never be ahead of it. -""", +kubectl uncordon {CP} +```""", + run=r"""cat > {WS}/q80/steps.sh <<'EOF' +apt-mark unhold kubeadm && apt-get update && apt-get install -y kubeadm=1.38.0-1.1 && apt-mark hold kubeadm +kubeadm upgrade plan +kubeadm upgrade apply v1.38.0 +kubectl drain {CP} --ignore-daemonsets +apt-mark unhold kubelet kubectl && apt-get update && apt-get install -y kubelet=1.38.0-1.1 kubectl=1.38.0-1.1 && apt-mark hold kubelet kubectl +systemctl daemon-reload +systemctl restart kubelet +kubectl uncordon {CP} +EOF"""), + cmd(r"""echo 'kubeadm upgrade node' > {WS}/q80/worker.txt""", r"""A worker hosts no control plane, so there is nothing for `upgrade apply` to do there; `upgrade node` updates its kubelet configuration.""")], ) # ------------------------------------------------------------------ 81 ---- @@ -977,35 +1150,44 @@ done; exit 1"""), C("fare-calculator pods are Running and Ready", 3, r""" for i in $(seq 1 24); do [ "$(j -n fare-engine deploy fare-calculator '{.status.readyReplicas}')" -ge 1 ] 2>/dev/null && exit 0; sleep 5; done; exit 1"""), ], - solution=""" -k -n fare-engine get pvc # Pending -k -n fare-engine describe pvc fare-data-pvc # storageclass.storage.k8s.io "fast-storage" not found -k get sc # on kind the class is called `standard` -# A PVC's storageClassName is immutable, so recreate it with the name your cluster really has: -k -n fare-engine delete pvc fare-data-pvc -cat < disk=nvme -k -n fare-engine rollout status deploy fare-calculator -# Two independent blockers stack up silently: the PVC never binds, AND nothing would schedule -# even if it did. `describe pod` reports only the scheduling failure, so check the PVC too -# whenever a pod that mounts one is stuck Pending. -# Never assume the class name - `standard` here, `local-path` on a stock local-path-provisioner -# install, something else on a cloud cluster. `k get sc` first, every time; a PVC naming a class -# that does not exist sits Pending forever with no other symptom. -# `standard` is WaitForFirstConsumer, so the PVC only binds once the pod can actually be placed - -# which is why fixing the affinity is what finally unblocks the storage too. -""", + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi +```""", + run=r"""cat > pvc.yaml <<'EOF' +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: fare-data-pvc + namespace: fare-engine +spec: + storageClassName: standard + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi +EOF"""), + cmd(r"""k apply -f pvc.yaml +k -n fare-engine describe pod -l app=fare-calculator | tail -3 +k get nodes -L disk""", r"""`didn't match Pod's node affinity/selector`: it wants `disk=nvme`, which no node has. `describe pod` names only this one; the claim's fault you had to find yourself."""), + cmd(r"""k label node {W1} disk=nvme +k -n fare-engine rollout status deploy fare-calculator""", r"""Labelling a node is the smaller change than dropping the rule. `standard` binds only once the Pod is placed (`WaitForFirstConsumer`), so the claim goes `Bound` now too.""")], ) # ------------------------------------------------------------------ 82 ---- @@ -1054,30 +1236,20 @@ ok=any(p.get("type")=="Pods" and int(p.get("value",0))==2 and int(p.get("periodS for p in b.get("policies",[])) sys.exit(0 if ok else 1)'"""), ], - solution=""" -cat < hpa.yaml""", r"""`autoscale` writes an `autoscaling/v2` HPA with the memory target; it has no flag for `behavior`."""), + hand(r"""vim hpa.yaml""", + r"""Add to its `spec:`, beside `maxReplicas:` (the docs page *Horizontal Pod Autoscaling* has the block under *Configurable scaling behavior*): +``` behavior: scaleUp: policies: - - {type: Pods, value: 2, periodSeconds: 60} -EOF -k -n route-planner get hpa route-planner-hpa -# `kubectl autoscale` can set a memory target (--memory=70%) but has no flag for a behavior block, -# so memory + behavior means writing the v2 YAML by hand (or autoscale, then kubectl edit). Utilisation is a percentage OF THE REQUEST, so the Deployment must -# declare a memory request or the HPA reports and never scales. -""", + - type: Pods + value: 2 + periodSeconds: 60 +```""", + run=r"""yq -i '.spec.behavior.scaleUp.policies = [{"type": "Pods", "value": 2, "periodSeconds": 60}]' hpa.yaml"""), + cmd(r"""k apply -f hpa.yaml +k -n route-planner get hpa route-planner-hpa""", r"""Utilisation is a percentage of the request: the Deployment has a memory request, so the target reads a number, not ``.""")], ) # ------------------------------------------------------------------ 83 ---- @@ -1131,26 +1303,81 @@ ok=any(not r.get("to") and {p.get("protocol") for p in r.get("ports",[]) if str( for r in eg) sys.exit(0 if ok else 1)'"""), ], - solution=""" -cat < netpol.yaml <<'EOF' +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: safety-api-policy + namespace: safety-monitor +spec: + podSelector: + matchLabels: + app: safety-api + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + app: safety-dashboard + ports: + - protocol: TCP + port: 8443 + egress: + - to: + - podSelector: + matchLabels: + app: safety-db + ports: + - protocol: TCP + port: 5432 + - ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 +EOF"""), + cmd(r"""k apply -f netpol.yaml +k -n safety-monitor describe netpol safety-api-policy""")], ) # ------------------------------------------------------------------ 84 ---- diff --git a/cka/bank_sim_d.py b/cka/bank_sim_d.py index 5ae0435..e650951 100644 --- a/cka/bank_sim_d.py +++ b/cka/bank_sim_d.py @@ -16,7 +16,7 @@ task was seeded, solved and graded on an isolated kind cluster before it went in Run with: ./ksim sim d """ -from sim.bank import C, NS +from sim.bank import C, NS, cmd, hand Q = [] # Tasks written for the pre-2025 exam opened by switching kubectl context. Every task now has a host @@ -73,25 +73,27 @@ if [ "$own" = ReplicaSet ]; then want="deployment $(j -n kube-system rs "$name" else want="$(echo "$own" | tr A-Z a-z) $name"; fi eq "$(sed -n 's/^ *dns *: *//p' "$f" | head -1 | tr -s ' ' | tr A-Z a-z | sed 's/ *$//')" "$want" """), ], - solution=""" -sudo -i # the task's host is the control-plane node: no hop needed -systemctl status kubelet | head -3 # a systemd service: a plain process on the node -ls /etc/kubernetes/manifests # etcd, kube-apiserver, kube-controller-manager, kube-scheduler -exit -k -n kube-system get pod -o wide # static pods carry the node name as a suffix... -k -n kube-system get pod kube-apiserver-{CP} -o jsonpath='{.metadata.ownerReferences[0].kind}{"\\n"}' # ...and are owned by the Node -k -n kube-system get deploy,ds # coredns is a Deployment (kube-proxy, kindnet are DaemonSets) -cat > {WS}/q86/controlplane.txt <<'EOF' + solution=[cmd(r"""systemctl status kubelet | head -3 +sudo ls /etc/kubernetes/manifests""", r"""The host is the control-plane node. The kubelet is a systemd service - a process; etcd and the three control-plane components are manifests in the static Pod path."""), + cmd(r"""k -n kube-system get pods -o wide +k -n kube-system get deploy,ds""", r"""Static Pods carry the node's name as a suffix. What answers DNS is the `coredns` Deployment, though its Service is called `kube-dns`; `kube-proxy` and `kindnet` are DaemonSets."""), + hand(r"""vim {WS}/q86/controlplane.txt""", + r"""``` kubelet: process kube-apiserver: static-pod kube-scheduler: static-pod kube-controller-manager: static-pod etcd: static-pod dns: deployment coredns -EOF -# The Service is called kube-dns for historical reasons; what actually answers is the coredns -# Deployment. Follow the chain svc selector -> pod -> ownerReferences when unsure. -""", +```""", + run=r"""cat > {WS}/q86/controlplane.txt <<'EOF' +kubelet: process +kube-apiserver: static-pod +kube-scheduler: static-pod +kube-controller-manager: static-pod +etcd: static-pod +dns: deployment coredns +EOF""")], ) # ------------------------------------------------------------------ 87 ---- @@ -125,24 +127,27 @@ cfg=$(nx "$W1" 'ls /etc/cni/net.d/*.conflist /etc/cni/net.d/*.conf 2>/dev/null | C("static Pod name suffix on {W3} (line 5)", 1, r""" v=$(sed -n 's/^ *5 *: *//p' "$WS/q87/cluster-info.txt" | tr -d '[:space:]'); [ "$v" = "-$W3" ] || [ "$v" = "$W3" ]"""), ], - solution=""" -k get nodes # ROLES shows control-plane; count the rest -k -n kube-system get pod -l component=kube-apiserver -o yaml | grep service-cluster-ip-range -k -n kube-system get ds # kindnet -{SHELL_W1} -sudo -i -ls /etc/cni/net.d/ # 10-kindnet.conflist -exit; exit -cat > {WS}/q87/cluster-info.txt <<'EOF' + solution=[cmd(r"""k get nodes +k -n kube-system describe pod -l component=kube-apiserver | grep service-cluster-ip-range +k -n kube-system get ds""", r"""`ROLES control-plane` on one node, the rest workers; the API server's `--service-cluster-ip-range`; and the CNI's DaemonSet, `kindnet`."""), + hand(r"""ssh {W1} ls /etc/cni/net.d""", + r"""The CNI's configuration on a node: `10-kindnet.conflist`.""", + run=r""""""), + hand(r"""vim {WS}/q87/cluster-info.txt""", + r"""The values you read - here: +``` 1: 1 2: 3 3: 10.96.0.0/16 4: kindnet, /etc/cni/net.d/10-kindnet.conflist 5: -{W3} -EOF -# A static Pod's mirror is named -, e.g. kube-apiserver-{CP}. Values come from THIS -# cluster - check each one instead of trusting the ones shown here. -""", +``` +A static Pod's mirror is named `-`, e.g. `kube-apiserver-{CP}`.""", + run=r"""C=$(kubectl get nodes --no-headers -l node-role.kubernetes.io/control-plane | wc -l) +W=$(kubectl get nodes --no-headers -l '!node-role.kubernetes.io/control-plane' | wc -l) +CIDR=$(kubectl -n kube-system get pod -l component=kube-apiserver -o yaml | grep -o 'service-cluster-ip-range=[^ ]*' | head -1 | cut -d= -f2) +F=$(ssh -n {W1} "sudo sh -c 'ls /etc/cni/net.d/*.conflist | sort | head -1'") +printf '1: %s\n2: %s\n3: %s\n4: kindnet, %s\n5: -{W3}\n' "$C" "$W" "$CIDR" "$F" > {WS}/q87/cluster-info.txt""")], ) # ------------------------------------------------------------------ 88 ---- @@ -218,18 +223,19 @@ grep -Eq 'kubectl\b.*\bget\b.*\b(po|pods?)\b' "$f" && grep -Eq -- '(-n|--namespa timeout 60 bash "$f" 2>/dev/null | grep -q etcd"""), C("no aliases in either script", 1, r"""[ -s "$WS/q89/pods-by-age.sh" ] && [ -s "$WS/q89/pods-by-uid.sh" ] && ! grep -Eq '(^|[;|&(] *| )k +' "$WS/q89/pods-by-age.sh" "$WS/q89/pods-by-uid.sh" """), ], - solution=""" -cat > {WS}/q89/pods-by-age.sh <<'EOF' + solution=[hand(r"""vim {WS}/q89/pods-by-age.sh""", + r"""`kubectl` in full: a script runs in a fresh shell, where the `k` alias does not exist. +``` kubectl get pods -A --sort-by=.metadata.creationTimestamp -EOF -cat > {WS}/q89/pods-by-uid.sh <<'EOF' +```""", + run=r"""echo 'kubectl get pods -A --sort-by=.metadata.creationTimestamp' > {WS}/q89/pods-by-age.sh"""), + hand(r"""vim {WS}/q89/pods-by-uid.sh""", + r"""``` kubectl -n kube-system get pods --sort-by=.metadata.uid -EOF -chmod +x {WS}/q89/*.sh -bash {WS}/q89/pods-by-age.sh | head -3 # the control-plane pods come first - they are oldest -# --sort-by takes a JSONPath into each item; ascending only, so "newest first" is `| tac` or -# sorting by a field that grows the other way. -""", +```""", + run=r"""echo 'kubectl -n kube-system get pods --sort-by=.metadata.uid' > {WS}/q89/pods-by-uid.sh"""), + cmd(r"""chmod +x {WS}/q89/*.sh +{WS}/q89/pods-by-age.sh | head -3""", r"""The control-plane Pods first: they are the oldest. `--sort-by` takes a JSONPath into each item, ascending only.""")], ) # ------------------------------------------------------------------ 90 ---- @@ -273,31 +279,35 @@ for f in pod-deleted.log container-killed.log; do done"""), C("restarts.txt says container", 2, r"""eq "$(tr -d '[:space:]' < "$WS/q90/restarts.txt" 2>/dev/null | tr A-Z a-z)" container"""), ], - solution=""" -cat > {WS}/q90/cluster-events.sh <<'EOF' + solution=[hand(r"""vim {WS}/q90/cluster-events.sh""", + r"""``` kubectl get events -A --sort-by=.metadata.creationTimestamp -EOF -chmod +x {WS}/q90/cluster-events.sh - -k -n kube-system get pod -l k8s-app=kube-proxy -o wide # the one on {W2} -k -n kube-system delete pod -k -n kube-system get events --sort-by=.metadata.creationTimestamp | grep kube-proxy | tail -8 > {WS}/q90/pod-deleted.log -# Killing (old pod), SuccessfulCreate (DaemonSet), Scheduled, Pulled, Created, Started (new pod) - -{SHELL_W2} +```""", + run=r"""echo 'kubectl get events -A --sort-by=.metadata.creationTimestamp' > {WS}/q90/cluster-events.sh"""), + cmd(r"""chmod +x {WS}/q90/cluster-events.sh +k -n kube-system get pods -l k8s-app=kube-proxy -o wide""", r"""Note the name of the one on `{W2}`."""), + hand(r"""k -n kube-system delete pod kube-proxy-xxxxx""", + r"""With that name. The DaemonSet replaces it with a new Pod at once.""", + run=r"""P=$(kubectl -n kube-system get pod -l k8s-app=kube-proxy --field-selector spec.nodeName={W2} -o jsonpath='{.items[0].metadata.name}') +kubectl -n kube-system delete pod $P +for i in $(seq 1 40); do + [ "$(kubectl -n kube-system get pod -l k8s-app=kube-proxy --field-selector spec.nodeName={W2} -o jsonpath='{.items[0].status.phase}' 2>/dev/null)" = Running ] && break; sleep 3 +done +sleep 5"""), + cmd(r"""k -n kube-system get events --sort-by=.metadata.creationTimestamp | grep kube-proxy | tail -8 > {WS}/q90/pod-deleted.log +cat {WS}/q90/pod-deleted.log""", r"""`Killing` the old Pod, `SuccessfulCreate` by the DaemonSet, then `Scheduled`, `Pulled`, `Created`, `Started` for the new one."""), + hand(r"""{SHELL_W2} sudo -i -crictl ps --name kube-proxy # container id -crictl stop -exit; exit -k -n kube-system get events --sort-by=.metadata.creationTimestamp | grep kube-proxy | tail -4 > {WS}/q90/container-killed.log -# Pulled, Created, Started - same Pod, new container -k -n kube-system get pod -l k8s-app=kube-proxy -o wide # RESTARTS 1 on {W2} -echo container > {WS}/q90/restarts.txt -# Deleting the Pod makes the DaemonSet create a brand-new Pod (RESTARTS 0). Killing only the -# container makes the kubelet start a new container inside the SAME Pod - that is a restart. -# Do not `crictl rm` it straight away: on v1.35 the kubelet counts the restart from the exited -# container it sees, and if rm wins the race it starts a fresh attempt 0 and RESTARTS stays 0. -""", +crictl ps --name kube-proxy +crictl stop 1a2b3c4d5e6f +exit +exit""", + r"""With the container ID `crictl ps` printed. Only the container goes: the kubelet starts a new one in the same Pod.""", + run=r"""ssh -n {W2} 'sudo crictl stop $(sudo crictl ps --name kube-proxy -q)' +sleep 25"""), + cmd(r"""k -n kube-system get events --sort-by=.metadata.creationTimestamp | grep kube-proxy | tail -4 > {WS}/q90/container-killed.log +k -n kube-system get pods -l k8s-app=kube-proxy -o wide +echo container > {WS}/q90/restarts.txt""", r"""`RESTARTS 1` on `{W2}`: a new container in the same Pod is a restart; a new Pod starts again at 0.""")], ) # ------------------------------------------------------------------ 91 ---- @@ -333,15 +343,15 @@ got=$(awk '{print $1}' "$f" | sed 's/\..*//' | grep . | sort -u) best=$(for ns in census-north census-south census-east census-west; do echo "$ns $($K -n $ns get role --no-headers 2>/dev/null | wc -l)"; done | sort -k2 -n | tail -1) eq "$(tr -s ' ' < "$WS/q91/most-roles.txt" 2>/dev/null | sed 's/^ *//; s/ *$//')" "$best" """), ], - solution=""" -k api-resources --namespaced=true -o name > /tmp/r.txt # deployments.apps, ... (group suffix) -k api-resources --namespaced=true --no-headers | awk '{print $1}' > {WS}/q91/namespaced.txt -for ns in census-north census-south census-east census-west; do echo "$ns $(k -n $ns get role --no-headers | wc -l)"; done -# pick the largest, e.g. echo "census-east 8" > {WS}/q91/most-roles.txt -# --namespaced=false gives the cluster-scoped half (nodes, pv, storageclasses, clusterroles...). -# `k get role -A --no-headers | awk '{print $1}' | sort | uniq -c | sort -n` counts every -# namespace at once. -""", + solution=[cmd(r"""k api-resources --namespaced=true -o name > {WS}/q91/namespaced.txt +head -3 {WS}/q91/namespaced.txt""", r"""`-o name` prints the plural names alone (`deployments.apps`: name and group); `--namespaced=false` would give the cluster-scoped half."""), + cmd(r"""k -n census-north get role --no-headers | wc -l +k -n census-south get role --no-headers | wc -l +k -n census-east get role --no-headers | wc -l +k -n census-west get role --no-headers | wc -l""", r"""Or all at once: `k get role -A | grep census`."""), + hand(r"""echo "census-east 9" > {WS}/q91/most-roles.txt""", + r"""With the namespace that has the most, and its count.""", + run=r"""for ns in census-north census-south census-east census-west; do echo "$ns $(kubectl -n $ns get role --no-headers | wc -l)"; done | sort -k2 -n | tail -1 > {WS}/q91/most-roles.txt""")], ) # ------------------------------------------------------------------ 92 ---- @@ -480,20 +490,27 @@ ip=$(j -n default pod "ops-beacon-$CP" '{.status.podIP}'); [ -n "$ip" ] || exit j -n default endpointslices -l kubernetes.io/service-name=ops-beacon-svc '{.items[*].endpoints[*].addresses[*]}' | tr ' ' '\n' | grep -qx "$ip" """), C("nodeport.txt holds the Service's node port", 1, r"""np=$(j -n default svc ops-beacon-svc '{.spec.ports[0].nodePort}'); [ -n "$np" ] && eq "$(tr -d '[:space:]' < "$WS/q94/nodeport.txt" 2>/dev/null)" "$np" """), ], - solution=""" -k run ops-beacon -n default --image=nginx:1.27 --labels=app=ops-beacon $do > beacon.yaml -# add to the container: resources: {requests: {cpu: 10m, memory: 20Mi}} -sudo -i # the task's host is the control-plane node: no hop needed -grep staticPodPath /var/lib/kubelet/config.yaml # /etc/kubernetes/manifests -vi /etc/kubernetes/manifests/ops-beacon.yaml # paste beacon.yaml -exit -k -n default get pod ops-beacon-{CP} -o wide # mirror pod, named - -k -n default expose pod ops-beacon-{CP} --name=ops-beacon-svc --type=NodePort --port=80 --target-port=80 -k -n default get endpointslices -l kubernetes.io/service-name=ops-beacon-svc # the mirror pod's IP -k -n default get svc ops-beacon-svc -o jsonpath='{.spec.ports[0].nodePort}' > {WS}/q94/nodeport.txt -# The control-plane taint only keeps the SCHEDULER from placing pods there; the kubelet runs its -# static pods regardless. `expose pod` copies the pod's labels into the selector. -""", + solution=[cmd(r"""k -n default run ops-beacon --image=nginx:1.27 --labels=app=ops-beacon --dry-run=client -o yaml > beacon.yaml"""), + hand(r"""vim beacon.yaml""", + r"""Replace the container's `resources: {}` with: +``` + resources: + requests: + cpu: 10m + memory: 20Mi +```""", + run=r"""yq -i '.spec.containers[0].resources = {"requests": {"cpu": "10m", "memory": "20Mi"}}' beacon.yaml"""), + cmd(r"""sudo grep staticPodPath /var/lib/kubelet/config.yaml +sudo cp beacon.yaml /etc/kubernetes/manifests/ops-beacon.yaml""", r"""The host is the control-plane node. Its taint keeps the scheduler away, not the kubelet: static Pods run regardless."""), + hand(r"""k -n default get pod ops-beacon-{CP} -o wide""", + r"""Again until the mirror Pod is `Running`.""", + run=r"""for i in $(seq 1 60); do [ "$(kubectl -n default get pod ops-beacon-{CP} -o jsonpath='{.status.phase}' 2>/dev/null)" = Running ] && break; sleep 3; done"""), + cmd(r"""k -n default expose pod ops-beacon-{CP} --name=ops-beacon-svc --type=NodePort --port=80 --target-port=80 +k -n default get svc ops-beacon-svc +k -n default get endpointslices -l kubernetes.io/service-name=ops-beacon-svc""", r"""`expose pod` makes the selector from the Pod's labels: the mirror Pod's IP is the endpoint. Note the node port, after `80:`."""), + hand(r"""echo 31234 > {WS}/q94/nodeport.txt""", + r"""With the node port `get svc` showed.""", + run=r"""kubectl -n default get svc ops-beacon-svc -o jsonpath='{.spec.ports[0].nodePort}' > {WS}/q94/nodeport.txt""")], ) # ------------------------------------------------------------------ 95 ---- @@ -757,40 +774,107 @@ for i in 0 1; do eq "$($K -n lima-data exec ledger-db-$i -- cat /data/id.txt 2>/ want=$($K -n lima-data get pvc --no-headers -o custom-columns=N:.metadata.name 2>/dev/null | grep '^data-ledger-db-' | sort) [ -n "$want" ] && [ "$(echo "$want" | wc -l)" -ge 3 ] && [ "$(grep . "$WS/q99/pvcs.txt" 2>/dev/null)" = "$want" ]"""), ], - solution=""" -cat < /data/id.txt && sleep 86400"] - volumeMounts: [{name: data, mountPath: /data}] + - name: db + image: busybox:1.36 + command: ["sh", "-c", "hostname > /data/id.txt && sleep 86400"] + volumeMounts: + - name: data + mountPath: /data volumeClaimTemplates: - - metadata: {name: data} - spec: {accessModes: [ReadWriteOnce], storageClassName: standard, resources: {requests: {storage: 100Mi}}} -EOF + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + storageClassName: standard + resources: + requests: + storage: 100Mi +```""", + run=r"""cat > sts.yaml <<'EOF' +apiVersion: v1 +kind: Service +metadata: + name: ledger-db + namespace: lima-data +spec: + clusterIP: None + selector: + app: ledger-db + ports: + - port: 5432 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ledger-db + namespace: lima-data +spec: + serviceName: ledger-db + replicas: 3 + selector: + matchLabels: + app: ledger-db + template: + metadata: + labels: + app: ledger-db + spec: + containers: + - name: db + image: busybox:1.36 + command: ["sh", "-c", "hostname > /data/id.txt && sleep 86400"] + volumeMounts: + - name: data + mountPath: /data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + storageClassName: standard + resources: + requests: + storage: 100Mi +EOF"""), + cmd(r"""k apply -f sts.yaml k -n lima-data rollout status sts ledger-db -k -n lima-data scale sts ledger-db --replicas=2 -k -n lima-data get pvc -o custom-columns=N:.metadata.name --no-headers | grep '^data-ledger-db-' | sort > {WS}/q99/pvcs.txt -cat {WS}/q99/pvcs.txt # all THREE: data-ledger-db-0, -1, -2 -# Claims are named