diff --git a/README.md b/README.md index 2ee30ad..5f7d208 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,9 @@ Entry point is `./exam` (renamed from `cka-sim` so it can't collide with the dir Hands-on exam tasks graded **per check** against a real cluster, an exam clock, a browser front end modelled on the real PSI exam, attempt history, and drills that rotate through the domains before you sit a full exam. Tracks: **CKA** (102 tasks: a random pool of 43 normal + -24 hard, plus two fixed simulator sets, C and D, of 18 and 17 tasks), CKAD and CKS (scaffolded — -same format, add questions when you get there). +24 hard, plus two fixed simulator sets, C and D, of 18 and 17 tasks), **CKAD** (37 tasks, at +least one for every competency in the curriculum) and CKS (scaffolded — same format, add +questions when you get there). ## How a session is built @@ -73,10 +74,25 @@ The same state is available from the terminal if you prefer: ./exam list | show N | timer | status # during ./exam end # submit ./exam weak | history # afterwards -./exam --track ckad web # other tracks +./exam --track ckad web # other tracks (or CKA_TRACK=ckad) ./teardown.sh # delete cluster + state (history kept) ``` +## CKAD track + +`./exam --track ckad ...` (every command takes it). A mock draws **16** tasks in 120 minutes, +weighted like the exam: Design and Build 3, Deployment 3, Observability 3, Environment and +Security 4, Networking 3, from a pool of 37 (8/7/7/10/5), so no two mocks are the same set. Every +line of the curriculum has at least one task: image builds, Jobs, DaemonSets, native sidecars, +volumes; blue/green, canary, rolling-update limits, Helm, Kustomize; API deprecations, probes, +`kubectl top`, logs, debugging (CrashLoopBackOff, OOMKilled, config errors); CRDs, RBAC, Pod +Security admission, quotas and LimitRanges, QoS, ConfigMaps, Secrets, ServiceAccounts; +NetworkPolicy (tested with real traffic), Services, Ingress with TLS. + +A few tasks need more than kubectl on this machine: `podman` (Q13 builds an image), `helm` +(Q6, Q21), and network access the first time Q23 runs on a fresh cluster (it installs +metrics-server). Drill a domain with `./exam --track ckad drill ENV`, as for CKA. + ## Scoring, history, drills Each task is a list of named, weighted checks (`tracks/cka/questions.py`). You get the points @@ -158,5 +174,5 @@ Banks live in `tracks/cka/{questions,bank_extra,bank_hard,bank_sim_*}.py`; a tas Scripts get `$K` (pinned to the grading context), `$WS`, `$CP/$W1/$W2/$W3`, `nx NODE 'cmd'`, and helpers `j '{jsonpath}'` and `eq a b`. Task text may use `{WS}`, `{CP}`, `{W2}`, `{W3}`, `{SHELL_CP}` etc. `TRACK["quota"]` / `["quota_hard"]` set how many tasks per domain a -session draws. For CKAD/CKS, fill in `QUESTIONS` in the +session draws. For CKS, fill in `QUESTIONS` in the existing placeholder track; `TRACK` already carries the domain table and pass mark. diff --git a/exam b/exam index efdaa6a..3bc97e2 100755 --- a/exam +++ b/exam @@ -136,6 +136,20 @@ ensure_gateway_crds() { && $K wait --for=condition=Established --timeout=60s crd/gatewayclasses.gateway.networking.k8s.io \ crd/gateways.gateway.networking.k8s.io crd/httproutes.gateway.networking.k8s.io >/dev/null 2>&1 } +# ensure_metrics_server -> a working metrics API for `kubectl top` (CKAD Q23). kind ships none, and +# kind's kubelet serving certs have no IP SANs, so it only becomes Ready with --kubelet-insecure-tls. +# CKA Q23 strips that flag on purpose; the baseline puts it back before every session. +ensure_metrics_server() { + if ! $K -n kube-system get deploy metrics-server >/dev/null 2>&1; then + curl -sfL --max-time 60 https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml -o /tmp/ms.yaml \ + && $K apply -f /tmp/ms.yaml >/dev/null 2>&1 || return 1 + fi + if ! $K -n kube-system get deploy metrics-server -o jsonpath='{.spec.template.spec.containers[0].args}' | grep -q kubelet-insecure-tls; then + $K -n kube-system patch deploy metrics-server --type=json \ + -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-insecure-tls"}]' >/dev/null 2>&1 + fi + $K -n kube-system rollout status deploy metrics-server --timeout=150s >/dev/null 2>&1 +} """ def _pick_track_arg(argv): diff --git a/tracks/ckad/bank_extra.py b/tracks/ckad/bank_extra.py new file mode 100644 index 0000000..7f4f5f3 --- /dev/null +++ b/tracks/ckad/bank_extra.py @@ -0,0 +1,1595 @@ +"""CKAD tasks 13-37: one or more per curriculum competency the starter bank (1-12) left uncovered. + +Same format as questions.py. Parameters (%{...}) are drawn per session; every setup is safe to +re-run, and every check reads the live cluster (or the files the task asks for). +""" +from tracks.cka.common import C, NS, ns_pick, name_pick + +QUESTIONS = [ +# ---------------------------------------------------------------- DESIGN ---- +{ + "id": 13, "domain": "DESIGN", "title": "Build, change and export a container image", + "params": lambda r: {"img": name_pick(r, r.choice(["ticker", "beacon", "courier"])), + "greet": r.choice(["hola", "ciao", "salut", "hej"])}, + "task": """ +The build context for a small image is in `{WS}/q13/image` (a `Dockerfile` and `app.sh`). +Use `podman` on this machine. + +1. Build the image, unchanged, as `localhost/%{img}:v1`. +2. Change the **Dockerfile** so the image sets `GREETING=%{greet}` and a new environment + variable `APP_VERSION=v2`, then build it as `localhost/%{img}:v2`. `v1` must stay as it was. +3. Save `localhost/%{img}:v2` as an **OCI archive** to `{WS}/q13/%{img}-v2.tar`. +4. Run `v2` once and write its output to `{WS}/q13/run.txt`. +""", + "setup": r""" +rm -rf "$WS/q13"; mkdir -p "$WS/q13/image" +command -v podman >/dev/null || { echo "podman is not installed on this machine - the task needs it"; exit 1; } +podman rmi -f localhost/%{img}:v1 localhost/%{img}:v2 >/dev/null 2>&1 || true +cat > "$WS/q13/image/Dockerfile" <<'EOF' +FROM docker.io/library/busybox:1.36 +ENV GREETING=hello +COPY app.sh /app/app.sh +CMD ["sh", "/app/app.sh"] +EOF +cat > "$WS/q13/image/app.sh" <<'EOF' +echo "${GREETING} from %{img} ${APP_VERSION:-unversioned}" +EOF +""", + "checks": [ + C("v1 exists and is the unchanged build (GREETING=hello, no APP_VERSION)", 2, r""" +env=$(podman image inspect localhost/%{img}:v1 --format '{{json .Config.Env}}' 2>/dev/null) || exit 1 +echo "$env" | grep -q '"GREETING=hello"' && ! echo "$env" | grep -q APP_VERSION"""), + C("v2 sets GREETING=%{greet} and APP_VERSION=v2", 2, r""" +env=$(podman image inspect localhost/%{img}:v2 --format '{{json .Config.Env}}' 2>/dev/null) || exit 1 +echo "$env" | grep -q '"GREETING=%{greet}"' && echo "$env" | grep -q '"APP_VERSION=v2"'"""), + C("the change is in the Dockerfile itself", 1, r""" +f="$WS/q13/image/Dockerfile" +grep -Eq '^ENV.*APP_VERSION[= ]+"?v2' "$f" && grep -Eq '^ENV.*GREETING[= ]+"?%{greet}' "$f" """), + C("OCI archive of v2 at {WS}/q13/%{img}-v2.tar", 2, r""" +f="$WS/q13/%{img}-v2.tar"; [ -s "$f" ] || exit 1 +tar -xOf "$f" oci-layout >/dev/null 2>&1 || exit 1 +tar -xOf "$f" index.json 2>/dev/null | grep -q '"org.opencontainers.image.ref.name":"localhost/%{img}:v2"'"""), + C("run.txt holds v2's output", 1, r"""tr -d '\r' < "$WS/q13/run.txt" 2>/dev/null | grep -qx '%{greet} from %{img} v2'"""), + ], + "solution": """ +cd {WS}/q13/image +podman build -t localhost/%{img}:v1 . # build BEFORE editing: v1 must be the original +vi Dockerfile # ENV GREETING=%{greet} + # ENV APP_VERSION=v2 +podman build -t localhost/%{img}:v2 . +podman images | grep %{img} +podman save --format oci-archive -o {WS}/q13/%{img}-v2.tar localhost/%{img}:v2 +podman run --rm localhost/%{img}:v2 > {WS}/q13/run.txt # %{greet} from %{img} v2 +# `podman save` writes docker-archive unless told otherwise; "OCI format" means +# --format oci-archive (an index.json + oci-layout + blobs/ tarball). With docker: +# docker build -t ... . && docker save ... (docker-archive only) - on the exam, read which tool +# and which format the task names. `--env`/`--build-arg` at build time would not change the +# Dockerfile, which is what "change the Dockerfile" asks for. +""", +}, +{ + "id": 14, "domain": "DESIGN", "title": "Parallel batch Job", + "params": lambda r: {"ns": ns_pick(r, "batch"), "job": name_pick(r, "render"), "n": r.choice([6, 8]), "p": r.choice([2, 3])}, + "task": """ +In Namespace `%{ns}` create a Job `%{job}` that renders %{n} frames: + +- container name `render`, image `busybox:1.36`, command `sh -c 'echo frame done; sleep 3'` +- %{n} successful completions in total, at most %{p} Pods running at the same time +- give up after 2 failed retries +- kill the whole Job if it is still running after 5 minutes +- its Pods carry the label `batch=%{job}` + +The Job must finish successfully. +""", + "setup": NS + r"""mkns %{ns}; $K -n %{ns} delete job %{job} --ignore-not-found --wait=true >/dev/null""", + "checks": [ + C("completions %{n}, parallelism %{p}, backoffLimit 2", 3, r""" +s="$K -n %{ns} get job %{job}" +eq "$($s -o jsonpath='{.spec.completions}' 2>/dev/null)" %{n} && eq "$($s -o jsonpath='{.spec.parallelism}')" %{p} && eq "$($s -o jsonpath='{.spec.backoffLimit}')" 2"""), + C("activeDeadlineSeconds 300", 1, r"""eq "$(j -n %{ns} job %{job} '{.spec.activeDeadlineSeconds}')" 300"""), + C("container render (busybox:1.36), Pod label batch=%{job}", 1, r""" +eq "$(j -n %{ns} job %{job} '{.spec.template.spec.containers[0].name}')" render \ + && eq "$(j -n %{ns} job %{job} '{.spec.template.spec.containers[0].image}')" busybox:1.36 \ + && eq "$(j -n %{ns} job %{job} '{.spec.template.metadata.labels.batch}')" %{job}"""), + C("Job complete: %{n} succeeded", 3, r""" +for i in $(seq 1 40); do [ "$(j -n %{ns} job %{job} '{.status.succeeded}')" = %{n} ] && exit 0; sleep 3; done; exit 1"""), + ], + "solution": """ +k -n %{ns} create job %{job} --image=busybox:1.36 $do -- sh -c 'echo frame done; sleep 3' > job.yaml +vi job.yaml +# spec.completions: %{n} +# spec.parallelism: %{p} +# spec.backoffLimit: 2 +# spec.activeDeadlineSeconds: 300 +# spec.template.metadata.labels: {batch: %{job}} +# container name: render (create job names the container after the Job) +k apply -f job.yaml +k -n %{ns} get job,pod -l batch=%{job} -w # %{p} at a time until %{n}/%{n} +# A Job's pod template is immutable: if you got a field wrong, delete the Job and apply again. +""", +}, +{ + "id": 15, "domain": "DESIGN", "title": "DaemonSet on every node", + "params": lambda r: {"ns": ns_pick(r, "fleet"), "ds": name_pick(r, "node-agent")}, + "task": """ +Create a DaemonSet `%{ds}` in Namespace `%{ns}` that runs one Pod on **every** node of the +cluster, the control plane included: + +- image `busybox:1.36`, command + `sh -c 'while true; do echo "$(date) $NODE_NAME ok"; sleep 30; done'` +- environment variable `NODE_NAME` filled from the Pod's own `spec.nodeName` +- requests cpu `10m`, memory `16Mi` +- Pod label `app=%{ds}` +""", + "setup": NS + r"""mkns %{ns}; $K -n %{ns} delete ds %{ds} --ignore-not-found --wait=true >/dev/null""", + "checks": [ + C("one ready Pod per node", 3, r""" +n=$($K get nodes --no-headers 2>/dev/null | wc -l) +for i in $(seq 1 20); do + [ "$(j -n %{ns} ds %{ds} '{.status.numberReady}')" = "$n" ] && [ "$(j -n %{ns} ds %{ds} '{.status.desiredNumberScheduled}')" = "$n" ] && exit 0; sleep 3 +done; exit 1"""), + C("a Pod runs on the control-plane node", 2, r"""j -n %{ns} pod -l app=%{ds} '{.items[*].spec.nodeName}' | tr ' ' '\n' | grep -qx "$CP" """), + C("NODE_NAME comes from spec.nodeName (and shows in the logs)", 2, r""" +eq "$(j -n %{ns} ds %{ds} '{.spec.template.spec.containers[0].env[?(@.name=="NODE_NAME")].valueFrom.fieldRef.fieldPath}')" spec.nodeName || exit 1 +p=$(j -n %{ns} pod -l app=%{ds} '{.items[0].metadata.name}'); n=$(j -n %{ns} pod "$p" '{.spec.nodeName}') +[ -n "$n" ] && $K -n %{ns} logs "$p" --tail=5 2>/dev/null | grep -q " $n ok" """), + C("requests cpu 10m / memory 16Mi", 1, r""" +eq "$(j -n %{ns} ds %{ds} '{.spec.template.spec.containers[0].resources.requests.cpu}')" 10m \ + && eq "$(j -n %{ns} ds %{ds} '{.spec.template.spec.containers[0].resources.requests.memory}')" 16Mi"""), + ], + "solution": """ +k -n %{ns} create deploy %{ds} --image=busybox:1.36 $do -- sh -c 'while true; do echo "$(date) $NODE_NAME ok"; sleep 30; done' > ds.yaml +vi ds.yaml +# kind: Deployment -> DaemonSet; delete replicas, strategy and status +# container: +# env: [{name: NODE_NAME, valueFrom: {fieldRef: {fieldPath: spec.nodeName}}}] +# resources: {requests: {cpu: 10m, memory: 16Mi}} +# pod spec: +# tolerations: [{key: node-role.kubernetes.io/control-plane, operator: Exists, effect: NoSchedule}] +k apply -f ds.yaml +k -n %{ns} get ds,pod -o wide # DESIRED = number of nodes +# There is no `kubectl create daemonset`; starting from a Deployment and changing the kind is the +# fast way. Without the toleration the control-plane node's NoSchedule taint keeps it at 3 of 4. +""", +}, +{ + "id": 16, "domain": "DESIGN", "title": "Native sidecar lets a Job finish", + "params": lambda r: {"ns": ns_pick(r, "nightly"), "job": name_pick(r, "export")}, + "task": """ +Job `%{job}` in Namespace `%{ns}` never completes: its log-shipping container `shipper` keeps +running after the `worker` container has finished. Its manifest is `{WS}/q16/job.yaml`. + +Keep `shipper` - it must still start **before** `worker` and receive all of its output - but make +it a **native sidecar** container, so the Job completes by itself once `worker` is done. +Keep the Job's name. The Job must end up Complete. +""", + "setup": NS + r""" +mkns %{ns}; rm -rf "$WS/q16"; mkdir -p "$WS/q16" +cat > "$WS/q16/job.yaml" <<'EOF' +apiVersion: batch/v1 +kind: Job +metadata: {name: %{job}, namespace: %{ns}} +spec: + backoffLimit: 1 + template: + spec: + restartPolicy: Never + volumes: [{name: data, emptyDir: {}}] + containers: + - name: worker + image: busybox:1.36 + command: [sh, -c, "for i in 1 2 3 4 5; do echo \"batch line $i\" >> /data/out.log; sleep 2; done"] + volumeMounts: [{name: data, mountPath: /data}] + - name: shipper + image: busybox:1.36 + command: [sh, -c, "touch /data/out.log; tail -F /data/out.log"] + volumeMounts: [{name: data, mountPath: /data}] +EOF +$K -n %{ns} delete job %{job} --ignore-not-found --wait=true >/dev/null +$K apply -f "$WS/q16/job.yaml" >/dev/null +""", + "checks": [ + C("shipper is a native sidecar (init container with restartPolicy Always)", 3, r""" +eq "$(j -n %{ns} job %{job} '{.spec.template.spec.initContainers[?(@.name=="shipper")].restartPolicy}')" Always"""), + C("worker is the only regular container", 1, r"""eq "$(j -n %{ns} job %{job} '{.spec.template.spec.containers[*].name}')" worker"""), + C("Job Complete", 3, r""" +for i in $(seq 1 30); do [ "$(j -n %{ns} job %{job} '{.status.succeeded}')" = 1 ] && exit 0; sleep 3; done; exit 1"""), + C("shipper, now a sidecar, received all of worker's output", 1, r""" +eq "$(j -n %{ns} job %{job} '{.spec.template.spec.initContainers[?(@.name=="shipper")].restartPolicy}')" Always \ + && $K -n %{ns} logs job/%{job} -c shipper 2>/dev/null | grep -q 'batch line 5'"""), + ], + "solution": """ +k -n %{ns} get job,pod # 0/1 complete; the Pod sits at 1/2 Running forever +# A Job counts a Pod as done only when ALL its containers have exited, and `tail -F` never does. +# A native sidecar is an init container with restartPolicy: Always - it starts before the main +# containers, keeps running beside them, and is stopped automatically once they have finished. +vi {WS}/q16/job.yaml +# spec.template.spec: +# initContainers: +# - name: shipper +# restartPolicy: Always # <- this line is what makes it a sidecar +# image: busybox:1.36 +# command: [sh, -c, "touch /data/out.log; tail -F /data/out.log"] +# volumeMounts: [{name: data, mountPath: /data}] +# containers: [ worker only ] +k -n %{ns} delete job %{job} # a Job's pod template is immutable +k apply -f {WS}/q16/job.yaml +k -n %{ns} get job %{job} -w # 1/1 +k -n %{ns} logs job/%{job} -c shipper # batch line 1..5 +""", +}, +{ + "id": 17, "domain": "DESIGN", "title": "Persistent and ephemeral volumes", + "params": lambda r: {"ns": ns_pick(r, "store"), "dep": name_pick(r, "journal"), "pvc": name_pick(r, "journal-data")}, + "task": """ +In Namespace `%{ns}`: + +1. Create a PersistentVolumeClaim `%{pvc}`: `100Mi`, `ReadWriteOnce`, StorageClass `standard`. +2. Create Deployment `%{dep}` (1 replica, image `busybox:1.36`) whose container runs + `sh -c 'date >> /data/boot.log; sleep 86400'` and mounts + - the claim `%{pvc}` at `/data`, + - an `emptyDir` volume named `scratch` at `/scratch`, held in memory and capped at `32Mi`. +3. Delete the Deployment's Pod once, so that `/data/boot.log` in the replacement Pod has two lines. +""", + "setup": NS + r""" +mkns %{ns} +$K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null +$K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null +""", + "checks": [ + C("claim %{pvc} Bound: 100Mi, ReadWriteOnce, class standard", 2, r""" +p="$K -n %{ns} get pvc %{pvc}" +eq "$($p -o jsonpath='{.status.phase}' 2>/dev/null)" Bound && eq "$($p -o jsonpath='{.spec.resources.requests.storage}')" 100Mi \ + && eq "$($p -o jsonpath='{.spec.accessModes[0]}')" ReadWriteOnce && eq "$($p -o jsonpath='{.spec.storageClassName}')" standard"""), + C("Deployment mounts the claim at /data", 2, r""" +v=$(j -n %{ns} deploy %{dep} '{.spec.template.spec.volumes[?(@.persistentVolumeClaim.claimName=="%{pvc}")].name}'); [ -n "$v" ] || exit 1 +eq "$(j -n %{ns} deploy %{dep} "{.spec.template.spec.containers[0].volumeMounts[?(@.name==\"$v\")].mountPath}")" /data"""), + C("emptyDir scratch: medium Memory, sizeLimit 32Mi, at /scratch", 2, r""" +d='{.spec.template.spec' +eq "$(j -n %{ns} deploy %{dep} "$d.volumes[?(@.name==\"scratch\")].emptyDir.medium}")" Memory \ + && eq "$(j -n %{ns} deploy %{dep} "$d.volumes[?(@.name==\"scratch\")].emptyDir.sizeLimit}")" 32Mi \ + && eq "$(j -n %{ns} deploy %{dep} "$d.containers[0].volumeMounts[?(@.name==\"scratch\")].mountPath}")" /scratch"""), + C("boot.log survived a Pod replacement (2+ lines)", 2, r""" +n=$($K -n %{ns} exec deploy/%{dep} -- sh -c 'wc -l < /data/boot.log' 2>/dev/null | tr -dc '0-9'); [ "${n:-0}" -ge 2 ]"""), + ], + "solution": """ +cat <> /data/boot.log; sleep 86400' > d.yaml +vi d.yaml +# pod spec: +# volumes: +# - {name: data, persistentVolumeClaim: {claimName: %{pvc}}} +# - {name: scratch, emptyDir: {medium: Memory, sizeLimit: 32Mi}} +# container: +# volumeMounts: [{name: data, mountPath: /data}, {name: scratch, mountPath: /scratch}] +k apply -f d.yaml +k -n %{ns} get pvc # Pending until the Pod exists: WaitForFirstConsumer +k -n %{ns} delete pod -l app=%{dep} +k -n %{ns} exec deploy/%{dep} -- cat /data/boot.log # two lines: the PVC outlived the Pod +# The emptyDir lives exactly as long as the Pod; medium: Memory makes it tmpfs, and what is +# written there counts against the container's memory limit. +""", +}, +# ---------------------------------------------------------------- DEPLOY ---- +{ + "id": 18, "domain": "DEPLOY", "title": "Canary by replica ratio", + "params": lambda r: {"ns": ns_pick(r, "canary"), "app": r.choice(["checkout", "search", "catalog"])}, + "task": """ +Namespace `%{ns}` runs Deployment `%{app}-stable` (3 replicas, `nginx:1.25`) behind Service `%{app}`. + +Start a canary of `nginx:1.27` that gets **20%** of the Service's traffic by replica count: + +- a new Deployment `%{app}-canary` whose Pods the existing Service already selects, with the + label `track=canary`, +- the Service must end up with exactly **5** ready endpoints, **1** of them a canary Pod, +- do not change the Service. +""", + "setup": NS + r""" +mkns %{ns} +$K -n %{ns} delete deploy %{app}-stable %{app}-canary --ignore-not-found --wait=true >/dev/null +$K -n %{ns} delete svc %{app} --ignore-not-found >/dev/null +$K apply -f - >/dev/null <<'EOF' +apiVersion: apps/v1 +kind: Deployment +metadata: {name: %{app}-stable, namespace: %{ns}} +spec: + replicas: 3 + selector: {matchLabels: {app: %{app}, track: stable}} + template: + metadata: {labels: {app: %{app}, track: stable}} + spec: {containers: [{name: web, image: nginx:1.25, ports: [{containerPort: 80}]}]} +--- +apiVersion: v1 +kind: Service +metadata: {name: %{app}, namespace: %{ns}} +spec: {selector: {app: %{app}}, ports: [{port: 80, targetPort: 80}]} +EOF +""", + "checks": [ + C("canary Deployment: nginx:1.27, 1 ready, label track=canary", 2, r""" +d="$K -n %{ns} get deploy %{app}-canary" +eq "$($d -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null)" nginx:1.27 \ + && eq "$($d -o jsonpath='{.spec.template.metadata.labels.track}')" canary && eq "$($d -o jsonpath='{.status.readyReplicas}')" 1"""), + C("stable scaled to 4 and still on nginx:1.25", 2, r""" +eq "$(j -n %{ns} deploy %{app}-stable '{.status.readyReplicas}')" 4 && eq "$(j -n %{ns} deploy %{app}-stable '{.spec.template.spec.containers[0].image}')" nginx:1.25"""), + C("Service unchanged: still selects only app=%{app}", 1, r""" +[ -n "$(j -n %{ns} deploy %{app}-canary '{.metadata.name}')" ] && eq "$(j -n %{ns} svc %{app} '{.spec.selector}')" '{"app":"%{app}"}'"""), + C("Service has 5 ready endpoints, exactly 1 of them canary", 3, r""" +ips=$(j -n %{ns} endpointslices -l kubernetes.io/service-name=%{app} '{range .items[*].endpoints[?(@.conditions.ready==true)]}{.addresses[0]}{" "}{end}') +[ "$(echo $ips | wc -w)" = 5 ] || exit 1 +can=" $(j -n %{ns} pod -l app=%{app},track=canary '{.items[*].status.podIP}') "; c=0 +for ip in $ips; do case "$can" in *" $ip "*) c=$((c+1));; esac; done; [ "$c" = 1 ]"""), + ], + "solution": """ +k -n %{ns} get deploy %{app}-stable -o yaml > canary.yaml +vi canary.yaml +# metadata.name: %{app}-canary, replicas: 1, image: nginx:1.27 +# track: stable -> track: canary in BOTH spec.selector.matchLabels and the template labels +# delete status, uid, resourceVersion, creationTimestamp, generation +k apply -f canary.yaml +k -n %{ns} scale deploy %{app}-stable --replicas=4 +k -n %{ns} get endpointslices -l kubernetes.io/service-name=%{app} # 5 addresses +# The Service selects only app=%{app}, so it load-balances over stable AND canary Pods in +# proportion to their ready replicas: 4 + 1 = 20% canary. Promote by shifting the ratio; +# roll back by scaling the canary to 0 - the Service never changes. +""", +}, +{ + "id": 19, "domain": "DEPLOY", "title": "Rolling update without losing capacity", + "params": lambda r: {"ns": ns_pick(r, "roll"), "dep": name_pick(r, "payments")}, + "task": """ +Deployment `%{dep}` in Namespace `%{ns}` (4 replicas) must never lose capacity during an update. + +1. Change its update strategy so that during a rollout **no** Pod is ever unavailable and **at + most one** extra Pod is created, and a new Pod only counts as available once it has been Ready + for **10 seconds**. Keep at most **3** old ReplicaSets around. +2. Then update the image to `nginx:1.27` and let the rollout finish. +""", + "setup": NS + r""" +mkns %{ns} +$K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null +$K -n %{ns} create deploy %{dep} --image=nginx:1.25 --replicas=4 >/dev/null +$K -n %{ns} rollout status deploy %{dep} --timeout=150s >/dev/null 2>&1 +""", + "checks": [ + C("RollingUpdate with maxUnavailable 0 and maxSurge 1", 3, r""" +s='{.spec.strategy' +eq "$(j -n %{ns} deploy %{dep} "$s.type}")" RollingUpdate || exit 1 +case "$(j -n %{ns} deploy %{dep} "$s.rollingUpdate.maxUnavailable}")" in 0|0%) ;; *) exit 1;; esac +case "$(j -n %{ns} deploy %{dep} "$s.rollingUpdate.maxSurge}")" in 1|25%) ;; *) exit 1;; esac"""), + C("minReadySeconds 10 and revisionHistoryLimit 3", 2, r""" +eq "$(j -n %{ns} deploy %{dep} '{.spec.minReadySeconds}')" 10 && eq "$(j -n %{ns} deploy %{dep} '{.spec.revisionHistoryLimit}')" 3"""), + C("rolled out: 4/4 updated and available on nginx:1.27", 3, r""" +d="$K -n %{ns} get deploy %{dep}" +eq "$($d -o jsonpath='{.spec.template.spec.containers[0].image}')" nginx:1.27 || exit 1 +for i in $(seq 1 30); do + [ "$($d -o jsonpath='{.status.updatedReplicas}/{.status.availableReplicas}/{.status.replicas}')" = 4/4/4 ] \ + && [ "$($d -o jsonpath='{.status.observedGeneration}')" = "$($d -o jsonpath='{.metadata.generation}')" ] && exit 0; sleep 3 +done; exit 1"""), + ], + "solution": """ +k -n %{ns} patch deploy %{dep} -p '{"spec":{"minReadySeconds":10,"revisionHistoryLimit":3, + "strategy":{"type":"RollingUpdate","rollingUpdate":{"maxSurge":1,"maxUnavailable":0}}}}' +k -n %{ns} set image deploy %{dep} nginx=nginx:1.27 +k -n %{ns} rollout status deploy %{dep} # one new Pod at a time, 10 s apart +# maxUnavailable: 0 + maxSurge: 1 means "add one, wait until it is available, remove one old". +# minReadySeconds is what "available" waits for; it also slows every rollout down by design. +""", +}, +{ + "id": 20, "domain": "DEPLOY", "title": "Kustomize overlay", + "params": lambda r: {"ns": ns_pick(r, "prod"), "n": r.choice([2, 3])}, + "task": """ +A Kustomize base for an app called `web` is in `{WS}/q20/base`. Do **not** edit the base. + +Create an overlay in `{WS}/q20/overlays/prod` that +- puts everything into Namespace `%{ns}`, +- prefixes every resource name with `prod-`, +- runs **%{n}** replicas, +- uses image `nginx:1.27`, +- adds the label `env=prod` to every resource. + +Apply the overlay with kubectl's built-in Kustomize. +""", + "setup": NS + r""" +mkns %{ns} +$K -n %{ns} delete deploy prod-web --ignore-not-found >/dev/null; $K -n %{ns} delete svc prod-web --ignore-not-found >/dev/null +rm -rf "$WS/q20"; mkdir -p "$WS/q20/base" +cat > "$WS/q20/base/deployment.yaml" <<'EOF' +apiVersion: apps/v1 +kind: Deployment +metadata: {name: web, labels: {app: web}} +spec: + replicas: 1 + selector: {matchLabels: {app: web}} + template: + metadata: {labels: {app: web}} + spec: {containers: [{name: web, image: nginx:1.25, ports: [{containerPort: 80}]}]} +EOF +cat > "$WS/q20/base/service.yaml" <<'EOF' +apiVersion: v1 +kind: Service +metadata: {name: web, labels: {app: web}} +spec: {selector: {app: web}, ports: [{port: 80, targetPort: 80}]} +EOF +cat > "$WS/q20/base/kustomization.yaml" <<'EOF' +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: [deployment.yaml, service.yaml] +EOF +(cd "$WS/q20/base" && md5sum *) > "$WS/.q20-base.md5" +""", + "checks": [ + C("overlay renders prod-web in %{ns}: %{n} replicas, nginx:1.27, env=prod on everything", 3, r""" +$K kustomize "$WS/q20/overlays/prod" 2>/dev/null | $K create --dry-run=client -o json -f - 2>/dev/null | python3 -c ' +import sys, json +s = sys.stdin.read(); dec = json.JSONDecoder(); i = 0; items = [] +while i < len(s): + while i < len(s) and s[i].isspace(): i += 1 + if i >= len(s): break + o, i = dec.raw_decode(s, i); items.extend(o.get("items", [o])) +dep = [x for x in items if x["kind"] == "Deployment"] +ok = len(items) == 2 and len(dep) == 1 and dep[0]["metadata"]["name"] == "prod-web" \ + and all(x["metadata"].get("namespace") == "%{ns}" for x in items) \ + and dep[0]["spec"]["replicas"] == %{n} and dep[0]["spec"]["template"]["spec"]["containers"][0]["image"] == "nginx:1.27" \ + and all(x["metadata"].get("labels", {}).get("env") == "prod" for x in items) +sys.exit(0 if ok else 1)'"""), + C("applied: Deployment prod-web (%{n} available, nginx:1.27, env=prod) and Service prod-web", 3, r""" +d="$K -n %{ns} get deploy prod-web" +eq "$($d -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null)" nginx:1.27 && eq "$($d -o jsonpath='{.metadata.labels.env}')" prod || exit 1 +eq "$(j -n %{ns} svc prod-web '{.metadata.labels.env}')" prod || exit 1 +for i in $(seq 1 30); do [ "$($d -o jsonpath='{.status.availableReplicas}')" = %{n} ] && exit 0; sleep 3; done; exit 1"""), + C("base left untouched", 1, r""" +[ -f "$WS/q20/overlays/prod/kustomization.yaml" ] && (cd "$WS/q20/base" && md5sum -c --quiet "$WS/.q20-base.md5" >/dev/null 2>&1)"""), + ], + "solution": """ +mkdir -p {WS}/q20/overlays/prod +cat > {WS}/q20/overlays/prod/kustomization.yaml </dev/null || { echo "helm is not installed on this machine - the task needs it"; exit 1; } +mkns %{ns} +for r in $(helm -n %{ns} list -aq 2>/dev/null || helm -n %{ns} list -q 2>/dev/null); do helm -n %{ns} uninstall "$r" >/dev/null 2>&1; done +rm -rf "$WS/q21"; mkdir -p "$WS/q21/charts" +src=$(mktemp -d) +for v in 1.2.0 1.3.0; do + d="$src/$v/webapp"; mkdir -p "$d/templates" + tag=1.25; [ "$v" = 1.3.0 ] && tag=1.27 + printf 'apiVersion: v2\nname: webapp\nversion: %s\nappVersion: "%s"\n' "$v" "$tag" > "$d/Chart.yaml" + printf 'replicaCount: 1\nimage:\n repository: nginx\n tag: "%s"\nservice:\n type: ClusterIP\n port: 80\n' "$tag" > "$d/values.yaml" + cat > "$d/templates/deployment.yaml" <<'EOF' +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-webapp + labels: {app.kubernetes.io/instance: {{ .Release.Name }}, chart: {{ .Chart.Name }}-{{ .Chart.Version }}} +spec: + replicas: {{ .Values.replicaCount }} + selector: {matchLabels: {app.kubernetes.io/instance: {{ .Release.Name }}}} + template: + metadata: {labels: {app.kubernetes.io/instance: {{ .Release.Name }}}} + spec: + containers: + - name: web + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + ports: [{containerPort: 80}] +EOF + cat > "$d/templates/service.yaml" <<'EOF' +apiVersion: v1 +kind: Service +metadata: {name: {{ .Release.Name }}-webapp} +spec: + type: {{ .Values.service.type }} + selector: {app.kubernetes.io/instance: {{ .Release.Name }}} + ports: [{port: {{ .Values.service.port }}, targetPort: 80}] +EOF + helm package "$d" -d "$WS/q21/charts" >/dev/null +done +rm -rf "$src" +helm -n %{ns} install %{old} "$WS/q21/charts/webapp-1.2.0.tgz" --set replicaCount=3 >/dev/null +""", + "checks": [ + C("defaults-1.3.0.yaml is the chart's default values", 1, r""" +f="$WS/q21/defaults-1.3.0.yaml"; [ -s "$f" ] && diff -Bw <(helm show values "$WS/q21/charts/webapp-1.3.0.tgz" 2>/dev/null) "$f" >/dev/null"""), + C("%{rel} deployed from webapp-1.3.0 with replicaCount 2 and a NodePort Service", 3, r""" +helm -n %{ns} list -o json 2>/dev/null | python3 -c ' +import sys, json +r = {x["name"]: x for x in json.load(sys.stdin)}.get("%{rel}") or {} +sys.exit(0 if r.get("chart") == "webapp-1.3.0" and r.get("status") == "deployed" else 1)' || exit 1 +helm -n %{ns} get values %{rel} -o json 2>/dev/null | python3 -c ' +import sys, json +d = json.load(sys.stdin) or {} +sys.exit(0 if d.get("replicaCount") == 2 and (d.get("service") or {}).get("type") == "NodePort" else 1)' || exit 1 +eq "$(j -n %{ns} svc %{rel}-webapp '{.spec.type}')" NodePort || exit 1 +for i in $(seq 1 20); do [ "$(j -n %{ns} deploy %{rel}-webapp '{.status.availableReplicas}')" = 2 ] && exit 0; sleep 3; done; exit 1"""), + C("the values file sets both", 1, r""" +f="$WS/q21/%{rel}-values.yaml"; grep -Eq '^replicaCount: *2 *$' "$f" 2>/dev/null && grep -Eq '^ +type: *"?NodePort"? *$' "$f" """), + C("%{old} upgraded to webapp-1.3.0 and still runs 3 replicas", 3, r""" +helm -n %{ns} list -o json 2>/dev/null | python3 -c ' +import sys, json +r = {x["name"]: x for x in json.load(sys.stdin)}.get("%{old}") or {} +sys.exit(0 if r.get("chart") == "webapp-1.3.0" and r.get("status") == "deployed" and int(r.get("revision", 0)) >= 2 else 1)' || exit 1 +for i in $(seq 1 20); do [ "$(j -n %{ns} deploy %{old}-webapp '{.spec.replicas}/{.status.availableReplicas}')" = 3/3 ] && exit 0; sleep 3; done; exit 1"""), + ], + "solution": """ +helm show values {WS}/q21/charts/webapp-1.3.0.tgz > {WS}/q21/defaults-1.3.0.yaml +cat > {WS}/q21/%{rel}-values.yaml </dev/null 2>&1 +$K -n %{ns} create deploy %{app} --image=nginx:1.27 --replicas=2 --dry-run=client -o yaml | $K apply -f - >/dev/null +$K -n %{ns} expose deploy %{app} --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null +cat > "$WS/q22/legacy.yaml" <<'EOF' +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: {name: %{app}, namespace: %{ns}} +spec: + rules: + - host: %{app}.example.com + http: + paths: + - path: / + backend: {serviceName: %{app}, servicePort: 80} +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: {name: %{app}-report, namespace: %{ns}} +spec: + schedule: "15 2 * * *" + jobTemplate: + spec: + template: + spec: + restartPolicy: OnFailure + containers: [{name: report, image: busybox:1.36, command: [sh, -c, "echo report"]}] +--- +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: {name: %{app}, namespace: %{ns}} +spec: + minAvailable: 1 + selector: {matchLabels: {app: %{app}}} +--- +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: {name: %{app}, namespace: %{ns}} +spec: + scaleTargetRef: {apiVersion: apps/v1, kind: Deployment, name: %{app}} + minReplicas: 2 + maxReplicas: 5 + metrics: + - type: Resource + resource: {name: cpu, target: {type: Utilization, averageUtilization: 70}} +EOF +""", + "checks": [ + C("removed.txt lists exactly the four removed versions", 2, r""" +python3 - "$WS/q22/removed.txt" <<'PY' +import sys +want = {"extensions/v1beta1", "batch/v1beta1", "policy/v1beta1", "autoscaling/v2beta2"} +try: + got = {l.strip() for l in open(sys.argv[1]) if l.strip()} +except OSError: + sys.exit(1) +sys.exit(0 if got == want else 1) +PY"""), + C("Ingress %{app}: host %{app}.example.com, / -> Service %{app}:80", 2, r""" +i="$K -n %{ns} get ingress %{app}" +eq "$($i -o jsonpath='{.spec.rules[0].host}' 2>/dev/null)" %{app}.example.com \ + && eq "$($i -o jsonpath='{.spec.rules[0].http.paths[0].path}')" / \ + && eq "$($i -o jsonpath='{.spec.rules[0].http.paths[0].backend.service.name}')" %{app} \ + && eq "$($i -o jsonpath='{.spec.rules[0].http.paths[0].backend.service.port.number}')" 80"""), + C("CronJob %{app}-report on '15 2 * * *' and PDB %{app} minAvailable 1", 2, r""" +eq "$(j -n %{ns} cronjob %{app}-report '{.spec.schedule}')" "15 2 * * *" \ + && eq "$(j -n %{ns} pdb %{app} '{.spec.minAvailable}/{.spec.selector.matchLabels.app}')" 1/%{app}"""), + C("HPA %{app}: 2-5 replicas at 70% CPU", 1, r""" +eq "$(j -n %{ns} hpa %{app} '{.spec.minReplicas}/{.spec.maxReplicas}/{.spec.metrics[0].resource.target.averageUtilization}/{.spec.scaleTargetRef.name}')" 2/5/70/%{app}"""), + C("fixed.yaml holds all four objects and applies cleanly", 1, r""" +f="$WS/q22/fixed.yaml"; [ "$(grep -c '^kind:' "$f" 2>/dev/null)" -ge 4 ] && $K apply --dry-run=server -f "$f" >/dev/null 2>&1"""), + ], + "solution": """ +k apply -f {WS}/q22/legacy.yaml --dry-run=server # no matches for kind "Ingress" in version "extensions/v1beta1" ... +k api-resources | grep -Ei 'ingresses|cronjobs|poddisruptionbudgets|horizontalpodautoscalers' # APIVERSION column +k explain ingress.spec.rules.http.paths.backend # the v1 backend shape +cp {WS}/q22/legacy.yaml {WS}/q22/fixed.yaml && vi {WS}/q22/fixed.yaml +# extensions/v1beta1 Ingress -> networking.k8s.io/v1, and the backend changes shape: +# - path: / +# pathType: Prefix (now required) +# backend: {service: {name: %{app}, port: {number: 80}}} +# batch/v1beta1 CronJob -> batch/v1 spec unchanged +# policy/v1beta1 PDB -> policy/v1 spec unchanged +# autoscaling/v2beta2 HPA -> autoscaling/v2 spec unchanged +printf '%s\\n' extensions/v1beta1 batch/v1beta1 policy/v1beta1 autoscaling/v2beta2 > {WS}/q22/removed.txt +k apply -f {WS}/q22/fixed.yaml +# The `kubectl convert` plugin can do the rewrite, but it is not installed by default - the +# Ingress backend change is the one to know by heart. +""", +}, +{ + "id": 23, "domain": "OBS", "title": "Find the heaviest Pods with kubectl top", + "params": lambda r: dict(zip(("cpu", "mem", "idle"), + [f"{x}-{r.randint(10, 99)}" for x in r.sample(["orders", "billing", "search", "profile", "ledger"], 3)]), + ns=ns_pick(r, "watch")), + "task": """ +Namespace `%{ns}` runs three Pods. Using the metrics API (metrics-server is installed): + +1. Write the name of the Pod using the most **CPU** to `{WS}/q23/cpu.txt`. +2. Write the name of the Pod using the most **memory** to `{WS}/q23/memory.txt`. +3. Write a command that shows the current CPU and memory usage of every **node** to + `{WS}/q23/nodes.sh`. +4. Write a command that lists the Pods of **all** Namespaces sorted by memory usage to + `{WS}/q23/pods.sh`. +""", + "setup": NS + r""" +ensure_metrics_server || echo "warn: metrics-server could not be installed (no network?) - kubectl top will not work" +mkns %{ns}; rm -rf "$WS/q23"; mkdir -p "$WS/q23" +$K -n %{ns} delete pod %{cpu} %{mem} %{idle} --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1 +$K apply -f - >/dev/null <<'EOF' +apiVersion: v1 +kind: Pod +metadata: {name: %{cpu}, namespace: %{ns}, labels: {app: watch}} +spec: + containers: + - name: main + image: busybox:1.36 + command: [sh, -c, "timeout 7200 sh -c 'while :; do :; done'; sleep 86400"] + resources: {requests: {cpu: 50m, memory: 16Mi}, limits: {cpu: 200m, memory: 64Mi}} +--- +apiVersion: v1 +kind: Pod +metadata: {name: %{mem}, namespace: %{ns}, labels: {app: watch}} +spec: + volumes: [{name: cache, emptyDir: {medium: Memory, sizeLimit: 160Mi}}] + containers: + - name: main + image: busybox:1.36 + command: [sh, -c, "dd if=/dev/zero of=/cache/fill bs=1M count=120 2>/dev/null; sleep 86400"] + volumeMounts: [{name: cache, mountPath: /cache}] + resources: {requests: {cpu: 10m, memory: 160Mi}, limits: {cpu: 100m, memory: 256Mi}} +--- +apiVersion: v1 +kind: Pod +metadata: {name: %{idle}, namespace: %{ns}, labels: {app: watch}} +spec: + containers: + - name: main + image: busybox:1.36 + command: [sleep, "86400"] + resources: {requests: {cpu: 10m, memory: 16Mi}, limits: {cpu: 100m, memory: 64Mi}} +EOF +$K -n %{ns} get pod %{cpu} %{mem} %{idle} >/dev/null +""", + "checks": [ + C("cpu.txt names %{cpu}", 2, r"""eq "$(tr -d '[:space:]' < "$WS/q23/cpu.txt" 2>/dev/null)" %{cpu}"""), + C("memory.txt names %{mem}", 2, r"""eq "$(tr -d '[:space:]' < "$WS/q23/memory.txt" 2>/dev/null)" %{mem}"""), + C("nodes.sh uses kubectl top for nodes", 1, r"""grep -Eq '(kubectl|k) +top +(node|nodes|no)( |$)' "$WS/q23/nodes.sh" 2>/dev/null"""), + C("pods.sh: top pods of all namespaces sorted by memory", 1, r""" +f="$WS/q23/pods.sh"; grep -Eq '(kubectl|k) +top +(pod|pods|po)( |$)' "$f" 2>/dev/null \ + && grep -Eq '(-A( |$)|--all-namespaces)' "$f" && grep -Eq -- '--sort-by[= ]memory' "$f" """), + ], + "solution": """ +k top pod -n %{ns} # give metrics-server ~30 s after the Pods start +k top pod -n %{ns} --sort-by=cpu --no-headers | head -1 | awk '{print $1}' > {WS}/q23/cpu.txt +k top pod -n %{ns} --sort-by=memory --no-headers | head -1 | awk '{print $1}' > {WS}/q23/memory.txt +echo 'kubectl top node' > {WS}/q23/nodes.sh +echo 'kubectl top pod -A --sort-by=memory' > {WS}/q23/pods.sh +# `kubectl top` reads the metrics API (metrics.k8s.io), served by metrics-server; without it you get +# "Metrics API not available". `--containers` breaks a Pod's usage down per container. +""", +}, +{ + "id": 24, "domain": "OBS", "title": "Logs of a container that keeps crashing", + "params": lambda r: {"ns": ns_pick(r, "ingest"), "pod": name_pick(r, "ingest"), "cfg": r.choice(["pipeline", "routes", "sinks", "tenants"])}, + "task": """ +Pod `%{pod}` in Namespace `%{ns}` has two containers, and one of them keeps restarting. + +1. Save the complete log of the **previous** (crashed) run of the failing container to + `{WS}/q24/crash.log`. +2. Write the full path of the file that the crashed run could not open to `{WS}/q24/missing.txt`. +3. Save the last **3** log lines of the other container, **with timestamps**, to + `{WS}/q24/helper.log`. +""", + "setup": NS + r""" +mkns %{ns}; rm -rf "$WS/q24"; mkdir -p "$WS/q24" +$K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null +$K apply -f - >/dev/null <<'EOF' +apiVersion: v1 +kind: Pod +metadata: {name: %{pod}, namespace: %{ns}} +spec: + containers: + - name: ingest + image: busybox:1.36 + command: [sh, -c, "echo \"ingest starting (pid $$)\"; echo \"loading configuration\"; sleep 12; echo \"FATAL: cannot open /etc/ingest/%{cfg}.yaml: no such file or directory\"; exit 1"] + - name: helper + image: busybox:1.36 + command: [sh, -c, "while true; do echo \"helper tick $(date +%s)\"; sleep 5; done"] +EOF +$K -n %{ns} get pod %{pod} >/dev/null +""", + "checks": [ + C("crash.log is one full crashed run of container ingest", 3, r""" +f="$WS/q24/crash.log"; grep -q 'ingest starting' "$f" 2>/dev/null && grep -q 'FATAL: cannot open /etc/ingest/%{cfg}.yaml' "$f" """), + C("missing.txt holds /etc/ingest/%{cfg}.yaml", 2, r"""eq "$(tr -d '[:space:]' < "$WS/q24/missing.txt" 2>/dev/null)" /etc/ingest/%{cfg}.yaml"""), + C("helper.log: its last 3 lines, timestamped", 2, r""" +f="$WS/q24/helper.log"; [ "$(grep -c . "$f" 2>/dev/null)" = 3 ] \ + && [ "$(grep -Ec '^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z helper tick [0-9]+$' "$f")" = 3 ]"""), + ], + "solution": """ +k -n %{ns} get pod %{pod} # 1/2, RESTARTS climbing: container ingest +k -n %{ns} logs %{pod} -c ingest --previous > {WS}/q24/crash.log +cat {WS}/q24/crash.log # FATAL: cannot open /etc/ingest/%{cfg}.yaml ... +echo /etc/ingest/%{cfg}.yaml > {WS}/q24/missing.txt +k -n %{ns} logs %{pod} -c helper --tail=3 --timestamps > {WS}/q24/helper.log +# Without -c, logs of a multi-container Pod asks you to choose a container. --previous reads the +# last terminated instance - while the new one is still running it is the only place the crash +# message exists. `describe pod` shows Last State: Terminated, Exit Code: 1 as well. +# For a few seconds right after a crash (STATUS Error, before CrashLoopBackOff) --previous can fail +# with "unable to retrieve container logs": it points at the run BEFORE the one that just died, and +# that one is already garbage-collected. Plain `logs` shows the crashed run at that moment - read +# STATUS first, or wait a few seconds and try --previous again. +""", +}, +{ + "id": 25, "domain": "OBS", "title": "Two faults behind a stuck Deployment", + "params": lambda r: {"ns": ns_pick(r, "fix"), "dep": name_pick(r, "catalog"), "cm": name_pick(r, "catalog-config"), + "lvl": r.choice(["info", "warn", "debug"])}, + "task": """ +Deployment `%{dep}` in Namespace `%{ns}` has 0 of 2 Pods ready. + +Fix it so both replicas are Running and Ready. Do not modify ConfigMap `%{cm}`: the container must +keep taking its `LOG_LEVEL` environment variable from that ConfigMap. + +Then write the value the containers see for `LOG_LEVEL` to `{WS}/q25/log-level.txt`. +""", + "setup": NS + r""" +mkns %{ns}; rm -rf "$WS/q25"; mkdir -p "$WS/q25" +$K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null +$K -n %{ns} delete cm %{cm} --ignore-not-found >/dev/null +$K apply -f - >/dev/null <<'EOF' +apiVersion: v1 +kind: ConfigMap +metadata: {name: %{cm}, namespace: %{ns}} +data: {log_level: %{lvl}, cache_size: "256"} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: %{dep}, namespace: %{ns}} +spec: + replicas: 2 + selector: {matchLabels: {app: %{dep}}} + template: + metadata: {labels: {app: %{dep}}} + spec: + containers: + - name: web + image: nginx:1.27 + command: [nginxx, -g, "daemon off;"] + env: + - name: LOG_LEVEL + valueFrom: {configMapKeyRef: {name: %{cm}, key: log-level}} +EOF +$K -n %{ns} get cm %{cm} -o jsonpath='{.metadata.resourceVersion}' > "$WS/.q25-cm-rv" +""", + "checks": [ + C("2/2 Pods Ready", 3, r""" +for i in $(seq 1 30); do [ "$(j -n %{ns} deploy %{dep} '{.status.readyReplicas}/{.status.updatedReplicas}/{.status.replicas}')" = 2/2/2 ] && exit 0; sleep 3; done; exit 1"""), + C("LOG_LEVEL still comes from %{cm} (key log_level) and reads %{lvl}", 2, r""" +e='{.spec.template.spec.containers[0].env[?(@.name=="LOG_LEVEL")].valueFrom.configMapKeyRef' +eq "$(j -n %{ns} deploy %{dep} "$e.name}")" %{cm} && eq "$(j -n %{ns} deploy %{dep} "$e.key}")" log_level \ + && eq "$($K -n %{ns} exec deploy/%{dep} -- printenv LOG_LEVEL 2>/dev/null)" %{lvl}"""), + C("ConfigMap %{cm} untouched", 1, r""" +[ "$(j -n %{ns} deploy %{dep} '{.metadata.generation}')" -gt 1 ] 2>/dev/null && eq "$(j -n %{ns} cm %{cm} '{.metadata.resourceVersion}')" "$(cat "$WS/.q25-cm-rv" 2>/dev/null)" """), + C("log-level.txt holds %{lvl}", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q25/log-level.txt" 2>/dev/null)" %{lvl}"""), + ], + "solution": """ +k -n %{ns} get pod # CreateContainerConfigError +k -n %{ns} describe pod -l app=%{dep} | grep -A3 Events # couldn't find key log-level in ConfigMap %{ns}/%{cm} +k -n %{ns} get cm %{cm} -o yaml # the key is log_level (underscore) +k -n %{ns} edit deploy %{dep} # key: log-level -> key: log_level +k -n %{ns} get pod # now RunContainerError / CrashLoopBackOff +k -n %{ns} describe pod -l app=%{dep} | grep -iE 'reason|message' | head # exec: "nginxx": executable file not found +k -n %{ns} edit deploy %{dep} # delete the command: line (the image's own entrypoint runs nginx) +k -n %{ns} rollout status deploy %{dep} +k -n %{ns} exec deploy/%{dep} -- printenv LOG_LEVEL > {WS}/q25/log-level.txt +# The second fault only shows once the first is fixed: a container whose config cannot be built is +# never started, so its command is never tried. Fix, re-read the status, repeat. +""", +}, +{ + "id": 26, "domain": "OBS", "title": "OOMKilled", + "params": lambda r: {"ns": ns_pick(r, "media"), "dep": name_pick(r, "thumbnailer")}, + "task": """ +Pods of Deployment `%{dep}` in Namespace `%{ns}` keep restarting. + +1. Write the reason the container was last terminated, exactly as Kubernetes reports it, to + `{WS}/q26/reason.txt`. +2. Fix it by raising only the container's **memory limit**, to `256Mi` - leave its requests + alone. The new Pod must stay up without restarting. +""", + "setup": NS + r""" +mkns %{ns}; rm -rf "$WS/q26"; mkdir -p "$WS/q26" +$K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null +$K apply -f - >/dev/null <<'EOF' +apiVersion: apps/v1 +kind: Deployment +metadata: {name: %{dep}, namespace: %{ns}} +spec: + replicas: 1 + selector: {matchLabels: {app: %{dep}}} + template: + metadata: {labels: {app: %{dep}}} + spec: + containers: + - name: resize + image: busybox:1.36 + command: [sh, -c, "dd if=/dev/zero of=/dev/null bs=150M count=1 2>/dev/null && echo warmed up; sleep 86400"] + resources: {requests: {cpu: 50m, memory: 32Mi}, limits: {memory: 64Mi}} +EOF +""", + "checks": [ + C("reason.txt holds OOMKilled", 2, r"""eq "$(tr -d '[:space:]' < "$WS/q26/reason.txt" 2>/dev/null)" OOMKilled"""), + C("memory limit 256Mi, requests unchanged (cpu 50m, memory 32Mi)", 2, r""" +r='{.spec.template.spec.containers[0].resources' +eq "$(j -n %{ns} deploy %{dep} "$r.limits.memory}")" 256Mi && eq "$(j -n %{ns} deploy %{dep} "$r.requests.memory}")" 32Mi \ + && eq "$(j -n %{ns} deploy %{dep} "$r.requests.cpu}")" 50m"""), + C("the new Pod is Ready and has not restarted", 3, r""" +for i in $(seq 1 40); do + if [ "$(j -n %{ns} deploy %{dep} '{.status.updatedReplicas}/{.status.availableReplicas}/{.status.replicas}')" = 1/1/1 ] \ + && [ "$(j -n %{ns} pod -l app=%{dep} '{.items[*].status.containerStatuses[0].restartCount}')" = 0 ]; then + [ "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].resources.limits.memory}')" = 256Mi ] && exit 0 + fi; sleep 3 +done; exit 1"""), + ], + "solution": """ +k -n %{ns} get pod # RESTARTS climbing, CrashLoopBackOff +k -n %{ns} get pod -l app=%{dep} -o jsonpath='{.items[0].status.containerStatuses[0].lastState.terminated.reason}' +# OOMKilled -> {WS}/q26/reason.txt (describe pod: Last State: Terminated, Reason: OOMKilled, Exit Code: 137) +k -n %{ns} set resources deploy %{dep} -c resize --limits=memory=256Mi +k -n %{ns} rollout status deploy %{dep} +k -n %{ns} get pod # 0 restarts +# `set resources --limits` only touches limits; requests stay as they were. The kernel kills a +# container that crosses its memory limit - that is OOMKilled, not a crash in the app itself. +""", +}, +# ------------------------------------------------------------------- ENV ---- +{ + "id": 27, "domain": "ENV", "title": "Discover and use a custom resource", + "params": lambda r: {"ns": ns_pick(r, "dataops"), "name": name_pick(r, "nightly"), "n": r.choice([3, 4, 5]), + "target": r.choice(["orders-db", "billing-db", "users-db"])}, + "task": """ +An operator's CustomResourceDefinition for **backups** is installed in this cluster. + +1. Write the CRD's full name to `{WS}/q27/crd.txt`. +2. Before you create anything, write how many objects of that kind exist in the **whole + cluster** to `{WS}/q27/count.txt`. +3. In Namespace `%{ns}` create one named `%{name}` that backs up `%{target}` every day at 03:30 + (cron `30 3 * * *`) and keeps each backup for 14 days. Use the field names the CRD defines. +""", + "setup": NS + r""" +rm -rf "$WS/q27"; mkdir -p "$WS/q27" +$K apply -f - >/dev/null <<'EOF' +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: {name: backups.data.example.com} +spec: + group: data.example.com + scope: Namespaced + names: {plural: backups, singular: backup, kind: Backup, shortNames: [bkp]} + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - {name: Target, type: string, jsonPath: .spec.target} + - {name: Schedule, type: string, jsonPath: .spec.schedule} + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + required: [target, schedule, retentionDays] + properties: + target: {type: string, description: "Name of the database to back up"} + schedule: {type: string, description: "Cron expression"} + retentionDays: {type: integer, minimum: 1, maximum: 90, description: "Days each backup is kept"} +EOF +$K wait --for=condition=Established crd/backups.data.example.com --timeout=60s >/dev/null +mkns %{ns}; mkns %{ns}-archive +$K delete backups.data.example.com --all -A --wait=true >/dev/null 2>&1 +i=0 +for spec in "orders-db|0 1 * * *|7" "users-db|30 1 * * *|30" "billing-db|0 2 * * 0|90" "search-db|15 4 * * *|14" "audit-db|0 5 1 * *|60"; do + i=$((i+1)); [ "$i" -gt %{n} ] && break + t=${spec%%|*}; rest=${spec#*|}; s=${rest%%|*}; d=${rest#*|} + bns=%{ns}-archive; [ "$i" = 1 ] && bns=%{ns} + printf 'apiVersion: data.example.com/v1\nkind: Backup\nmetadata: {name: %s-weekly, namespace: %s}\nspec: {target: %s, schedule: "%s", retentionDays: %s}\n' \ + "$t" "$bns" "$t" "$s" "$d" | $K apply -f - >/dev/null +done +""", + "checks": [ + C("crd.txt holds backups.data.example.com", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q27/crd.txt" 2>/dev/null)" backups.data.example.com"""), + C("count.txt holds %{n}", 2, r"""eq "$(tr -d '[:space:]' < "$WS/q27/count.txt" 2>/dev/null)" %{n}"""), + C("Backup %{name}: target %{target}, schedule 30 3 * * *, retentionDays 14", 3, r""" +b="$K -n %{ns} get backups.data.example.com %{name}" +eq "$($b -o jsonpath='{.spec.target}' 2>/dev/null)" %{target} && eq "$($b -o jsonpath='{.spec.schedule}')" "30 3 * * *" \ + && eq "$($b -o jsonpath='{.spec.retentionDays}')" 14"""), + ], + "solution": """ +k get crd | grep -i backup # backups.data.example.com +echo backups.data.example.com > {WS}/q27/crd.txt +k api-resources --api-group=data.example.com # kind Backup, short name bkp, namespaced +k get backups -A --no-headers | wc -l > {WS}/q27/count.txt +k explain backup.spec # target, schedule, retentionDays (all required) +cat </dev/null; $K -n %{ns} delete role %{sa}-role --ignore-not-found >/dev/null +$K -n %{ns} delete sa %{sa} --ignore-not-found >/dev/null +""", + "checks": [ + C("Role %{sa}-role bound to ServiceAccount %{sa} by %{sa}-binding", 1, r""" +b="$K -n %{ns} get rolebinding %{sa}-binding" +eq "$($b -o jsonpath='{.roleRef.kind}/{.roleRef.name}' 2>/dev/null)" Role/%{sa}-role \ + && eq "$($b -o jsonpath='{.subjects[0].kind}/{.subjects[0].name}/{.subjects[0].namespace}')" ServiceAccount/%{sa}/%{ns}"""), + C("can get, list and watch Pods and read pods/log", 2, r""" +a="--as=system:serviceaccount:%{ns}:%{sa} -n %{ns}" +for v in get list watch; do eq "$($K auth can-i $v pods $a 2>/dev/null)" yes || exit 1; done +eq "$($K auth can-i get pods --subresource=log $a 2>/dev/null)" yes"""), + C("can get, list, update, patch and scale Deployments", 2, r""" +a="--as=system:serviceaccount:%{ns}:%{sa} -n %{ns}" +for v in get list update patch; do eq "$($K auth can-i $v deployments.apps $a 2>/dev/null)" yes || exit 1; done +eq "$($K auth can-i patch deployments.apps --subresource=scale $a 2>/dev/null)" yes || eq "$($K auth can-i update deployments.apps --subresource=scale $a 2>/dev/null)" yes"""), + C("nothing else: no deleting Pods, creating Deployments, reading Secrets, or other Namespaces", 2, r""" +a="--as=system:serviceaccount:%{ns}:%{sa}" +eq "$($K auth can-i list pods $a -n %{ns} 2>/dev/null)" yes || exit 1 +for t in "delete pods -n %{ns}" "create deployments.apps -n %{ns}" "list secrets -n %{ns}" "list pods -n default" "delete deployments.apps -n %{ns}"; do + eq "$($K auth can-i $t $a 2>/dev/null)" no || exit 1 +done"""), + ], + "solution": """ +k -n %{ns} create sa %{sa} +k -n %{ns} create role %{sa}-role --verb=get,list,watch --resource=pods,pods/log $do > role.yaml +vi role.yaml # add a second rule: +# - apiGroups: [apps] +# resources: [deployments, deployments/scale] +# verbs: [get, list, update, patch] +k apply -f role.yaml +k -n %{ns} create rolebinding %{sa}-binding --role=%{sa}-role --serviceaccount=%{ns}:%{sa} +k auth can-i watch pods -n %{ns} --as=system:serviceaccount:%{ns}:%{sa} # yes +k auth can-i get pods --subresource=log -n %{ns} --as=system:serviceaccount:%{ns}:%{sa} # yes +k auth can-i patch deployments --subresource=scale -n %{ns} --as=system:serviceaccount:%{ns}:%{sa} # yes +k auth can-i delete pods -n %{ns} --as=system:serviceaccount:%{ns}:%{sa} # no +# Subresources are separate resources to RBAC: `pods` does not include `pods/log`, and +# `deployments` does not include `deployments/scale` (what `kubectl scale` calls). +# One `create role` can only hold one rule's verbs, hence the edit. +""", +}, +{ + "id": 29, "domain": "ENV", "title": "Pod Security admission blocks a Deployment", + "params": lambda r: {"ns": ns_pick(r, "secure"), "dep": name_pick(r, "vault-agent"), "uid": r.choice([1000, 2000, 3000])}, + "task": """ +Deployment `%{dep}` in Namespace `%{ns}` shows `0/2` and none of its Pods exist. + +1. Find out why, and write the Pod Security level the Namespace enforces to + `{WS}/q29/level.txt`. +2. Fix the **Deployment** (leave the Namespace as it is) so both replicas run. Its container must + run as user `%{uid}`. +""", + "setup": NS + r""" +mkns %{ns}; rm -rf "$WS/q29"; mkdir -p "$WS/q29" +$K label ns %{ns} pod-security.kubernetes.io/enforce=restricted pod-security.kubernetes.io/enforce-version=latest --overwrite >/dev/null +$K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null +$K apply -f - >/dev/null 2>&1 <<'EOF' +apiVersion: apps/v1 +kind: Deployment +metadata: {name: %{dep}, namespace: %{ns}} +spec: + replicas: 2 + selector: {matchLabels: {app: %{dep}}} + template: + metadata: {labels: {app: %{dep}}} + spec: + containers: + - name: agent + image: busybox:1.36 + command: [sh, -c, "echo agent up as $(id -u); sleep 86400"] +EOF +""", + "checks": [ + C("level.txt holds restricted", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q29/level.txt" 2>/dev/null)" restricted"""), + C("2/2 available and the Namespace still enforces restricted", 4, r""" +eq "$(j ns %{ns} '{.metadata.labels.pod-security\.kubernetes\.io/enforce}')" restricted || exit 1 +for i in $(seq 1 30); do [ "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" = 2 ] && exit 0; sleep 3; done; exit 1"""), + C("the container runs as UID %{uid}", 2, r"""eq "$($K -n %{ns} exec deploy/%{dep} -- id -u 2>/dev/null)" %{uid}"""), + ], + "solution": """ +k -n %{ns} get deploy,rs,pod # desired 2, current 0 +k -n %{ns} describe rs -l app=%{dep} | tail -5 +# Error creating: pods ... is forbidden: violates PodSecurity "restricted:latest": +# allowPrivilegeEscalation != false, unrestricted capabilities, runAsNonRoot != true, seccompProfile ... +k get ns %{ns} --show-labels # pod-security.kubernetes.io/enforce=restricted +echo restricted > {WS}/q29/level.txt +k -n %{ns} edit deploy %{dep} +# template.spec: +# securityContext: {runAsNonRoot: true, runAsUser: %{uid}, seccompProfile: {type: RuntimeDefault}} +# containers[0].securityContext: {allowPrivilegeEscalation: false, capabilities: {drop: [ALL]}} +k -n %{ns} get deploy %{dep} -w +# Admission rejects the Pods, not the Deployment, so the only symptom on the Deployment is 0/2 - +# the reason is in the ReplicaSet's events. busybox runs as root by default, so runAsNonRoot +# alone is not enough: it needs a non-zero runAsUser too. +""", +}, +{ + "id": 30, "domain": "ENV", "title": "Quota with LimitRange defaults", + "params": lambda r: {"ns": ns_pick(r, "capped"), "dep": name_pick(r, "api")}, + "task": """ +Namespace `%{ns}` must be capped: + +1. ResourceQuota `compute-quota`: `requests.cpu` 1, `requests.memory` 1Gi, `limits.cpu` 2, + `limits.memory` 2Gi, and at most 6 Pods. +2. LimitRange `container-defaults`: a container that declares no resources gets requests + cpu `100m` / memory `128Mi` and limits cpu `250m` / memory `256Mi`; no container may set a + limit above cpu `500m` / memory `512Mi`. +3. Create Deployment `%{dep}` (`nginx:1.27`, 3 replicas) **without** a resources section. It must run. +""", + "setup": NS + r""" +mkns %{ns} +$K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null +$K -n %{ns} delete resourcequota --all >/dev/null 2>&1; $K -n %{ns} delete limitrange --all >/dev/null 2>&1 +""", + "checks": [ + C("ResourceQuota compute-quota: 1 / 1Gi requests, 2 / 2Gi limits, 6 pods", 2, r""" +$K -n %{ns} get resourcequota compute-quota -o json 2>/dev/null | python3 -c ' +import sys, json +h = json.load(sys.stdin)["spec"]["hard"] +want = {"requests.cpu": "1", "requests.memory": "1Gi", "limits.cpu": "2", "limits.memory": "2Gi", "pods": "6"} +sys.exit(0 if all(h.get(k) == v for k, v in want.items()) else 1)'"""), + C("LimitRange container-defaults: defaults and maximums", 2, r""" +$K -n %{ns} get limitrange container-defaults -o json 2>/dev/null | python3 -c ' +import sys, json +c = [l for l in json.load(sys.stdin)["spec"]["limits"] if l.get("type") == "Container"] +ok = len(c) == 1 and c[0].get("defaultRequest") == {"cpu": "100m", "memory": "128Mi"} \ + and c[0].get("default") == {"cpu": "250m", "memory": "256Mi"} and c[0].get("max") == {"cpu": "500m", "memory": "512Mi"} +sys.exit(0 if ok else 1)'"""), + C("Deployment %{dep}: 3 available, no resources of its own, Pods got the defaults", 2, r""" +[ -n "$(j -n %{ns} deploy %{dep} '{.metadata.name}')" ] || exit 1 +[ -z "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].resources.requests}{.spec.template.spec.containers[0].resources.limits}')" ] || exit 1 +for i in $(seq 1 30); do [ "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" = 3 ] && break; sleep 3; done +eq "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" 3 \ + && eq "$(j -n %{ns} pod -l app=%{dep} '{.items[0].spec.containers[0].resources.requests.cpu}/{.items[0].spec.containers[0].resources.limits.memory}')" 100m/256Mi"""), + C("the quota counts the three Pods (300m CPU requested)", 1, r""" +eq "$(j -n %{ns} resourcequota compute-quota '{.status.used.requests\.cpu}/{.status.used.pods}')" 300m/3"""), + ], + "solution": """ +k -n %{ns} create quota compute-quota --hard=requests.cpu=1,requests.memory=1Gi,limits.cpu=2,limits.memory=2Gi,pods=6 +cat </dev/null +$K apply -f - >/dev/null <<'EOF' +apiVersion: apps/v1 +kind: Deployment +metadata: {name: %{dep}, namespace: %{ns}} +spec: + replicas: 2 + selector: {matchLabels: {app: %{dep}}} + template: + metadata: {labels: {app: %{dep}}} + spec: + containers: + - {name: app, image: nginx:1.27} + - {name: metrics, image: busybox:1.36, command: [sleep, "86400"]} +EOF +""", + "checks": [ + C("app: cpu 250m / memory 128Mi, requests equal to limits", 2, r""" +r='{.spec.template.spec.containers[?(@.name=="app")].resources' +eq "$(j -n %{ns} deploy %{dep} "$r.limits.cpu}/$r.limits.memory}")" 250m/128Mi || exit 1 +q=$(j -n %{ns} deploy %{dep} "$r.requests.cpu}/$r.requests.memory}"); [ "$q" = / ] || [ "$q" = 250m/128Mi ]"""), + C("metrics: cpu 50m / memory 32Mi, requests equal to limits", 2, r""" +r='{.spec.template.spec.containers[?(@.name=="metrics")].resources' +eq "$(j -n %{ns} deploy %{dep} "$r.limits.cpu}/$r.limits.memory}")" 50m/32Mi || exit 1 +q=$(j -n %{ns} deploy %{dep} "$r.requests.cpu}/$r.requests.memory}"); [ "$q" = / ] || [ "$q" = 50m/32Mi ]"""), + C("rolled out; its running Pods are Guaranteed", 2, r""" +for i in $(seq 1 30); do [ "$(j -n %{ns} deploy %{dep} '{.status.updatedReplicas}/{.status.availableReplicas}/{.status.replicas}')" = 2/2/2 ] && break; sleep 3; done +# old Pods linger in Terminating for the grace period (sleep ignores SIGTERM) - judge only live ones +$K -n %{ns} get pod -l app=%{dep} -o json 2>/dev/null | python3 -c ' +import sys, json +live = [p for p in json.load(sys.stdin)["items"] if not p["metadata"].get("deletionTimestamp") and p["status"].get("phase") == "Running"] +sys.exit(0 if len(live) == 2 and all(p["status"].get("qosClass") == "Guaranteed" for p in live) else 1)'"""), + C("qos.txt holds Guaranteed", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q31/qos.txt" 2>/dev/null)" Guaranteed"""), + ], + "solution": """ +k -n %{ns} set resources deploy %{dep} -c app --requests=cpu=250m,memory=128Mi --limits=cpu=250m,memory=128Mi +k -n %{ns} set resources deploy %{dep} -c metrics --requests=cpu=50m,memory=32Mi --limits=cpu=50m,memory=32Mi +k -n %{ns} rollout status deploy %{dep} +k -n %{ns} get pod -l app=%{dep} # wait until the old Pods are gone, not just Terminating: + # `sleep` ignores SIGTERM, so they linger ~30 s, still BestEffort +k -n %{ns} get pod -l app=%{dep} -o jsonpath='{.items[0].status.qosClass}' > {WS}/q31/qos.txt +# Guaranteed = EVERY container has cpu and memory limits, and requests equal to them (limits +# alone also works: requests default to the limits). One container short of that makes the whole +# Pod Burstable; nothing set anywhere is BestEffort. +""", +}, +{ + "id": 32, "domain": "ENV", "title": "ConfigMap as an env var and as a file", + "params": lambda r: {"ns": ns_pick(r, "conf"), "cm": name_pick(r, "app-config"), "pod": name_pick(r, "reader"), + "color": r.choice(["teal", "amber", "violet", "coral"])}, + "task": """ +In Namespace `%{ns}`: + +1. Create ConfigMap `%{cm}` holding the file `{WS}/q32/app.properties` under the key + `app.properties`, plus a key `TIER` with the value `backend`. +2. Create Pod `%{pod}` (`busybox:1.36`, command `sleep 86400`) that + - gets the environment variable `APP_TIER` from the ConfigMap's `TIER` key, + - mounts **only** the `app.properties` key, as the file `/etc/app/app.properties`. +""", + "setup": NS + r""" +mkns %{ns}; rm -rf "$WS/q32"; mkdir -p "$WS/q32" +$K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null; $K -n %{ns} delete cm %{cm} --ignore-not-found >/dev/null +printf 'color=%{color}\nretries=3\nmode=strict\n' > "$WS/q32/app.properties" +""", + "checks": [ + C("ConfigMap: app.properties matches the file, TIER=backend", 2, r""" +[ "$(j -n %{ns} cm %{cm} '{.data.app\.properties}')" = "$(cat "$WS/q32/app.properties")" ] && eq "$(j -n %{ns} cm %{cm} '{.data.TIER}')" backend"""), + C("APP_TIER comes from key TIER and reads backend", 2, r""" +e='{.spec.containers[0].env[?(@.name=="APP_TIER")].valueFrom.configMapKeyRef' +eq "$(j -n %{ns} pod %{pod} "$e.name}/$e.key}")" %{cm}/TIER && eq "$($K -n %{ns} exec %{pod} -- printenv APP_TIER 2>/dev/null)" backend"""), + C("/etc/app holds only app.properties, with the file's content", 2, r""" +eq "$($K -n %{ns} exec %{pod} -- ls /etc/app 2>/dev/null)" app.properties \ + && [ "$($K -n %{ns} exec %{pod} -- cat /etc/app/app.properties 2>/dev/null)" = "$(cat "$WS/q32/app.properties")" ]"""), + ], + "solution": """ +k -n %{ns} create cm %{cm} --from-file=app.properties={WS}/q32/app.properties --from-literal=TIER=backend +cat </dev/null +$K -n %{ns} delete secret %{old} %{sec} --ignore-not-found >/dev/null +$K -n %{ns} create secret generic %{old} --from-literal=username=legacy-app --from-literal=password=%{oldpw} >/dev/null +""", + "checks": [ + C("old-password.txt holds the decoded password", 2, r"""eq "$(tr -d '[:space:]' < "$WS/q33/old-password.txt" 2>/dev/null)" %{oldpw}"""), + C("Secret %{sec}: username and password", 1, r""" +eq "$(j -n %{ns} secret %{sec} '{.data.username}' | base64 -d 2>/dev/null)" %{user} && eq "$(j -n %{ns} secret %{sec} '{.data.password}' | base64 -d 2>/dev/null)" %{pw}"""), + C("API_USER comes from the Secret and reads %{user}", 2, r""" +e='{.spec.containers[0].env[?(@.name=="API_USER")].valueFrom.secretKeyRef' +eq "$(j -n %{ns} pod %{pod} "$e.name}/$e.key}")" %{sec}/username && eq "$($K -n %{ns} exec %{pod} -- printenv API_USER 2>/dev/null)" %{user}"""), + C("mounted read-only at /etc/api-creds, files mode 0400", 2, r""" +v=$(j -n %{ns} pod %{pod} '{.spec.volumes[?(@.secret.secretName=="%{sec}")].name}'); [ -n "$v" ] || exit 1 +eq "$(j -n %{ns} pod %{pod} "{.spec.containers[0].volumeMounts[?(@.name==\"$v\")].mountPath}")" /etc/api-creds || exit 1 +eq "$(j -n %{ns} pod %{pod} "{.spec.containers[0].volumeMounts[?(@.name==\"$v\")].readOnly}")" true || exit 1 +eq "$($K -n %{ns} exec %{pod} -- cat /etc/api-creds/password 2>/dev/null)" %{pw} \ + && eq "$($K -n %{ns} exec %{pod} -- stat -L -c %a /etc/api-creds/password 2>/dev/null)" 400"""), + ], + "solution": """ +k -n %{ns} get secret %{old} -o jsonpath='{.data.password}' | base64 -d > {WS}/q33/old-password.txt +k -n %{ns} create secret generic %{sec} --from-literal=username=%{user} --from-literal=password=%{pw} +cat </dev/null +$K -n %{ns} delete secret %{sa}-token --ignore-not-found >/dev/null; $K -n %{ns} delete sa %{sa} --ignore-not-found >/dev/null +""", + "checks": [ + C("ServiceAccount %{sa} with automountServiceAccountToken: false", 1, r"""eq "$(j -n %{ns} sa %{sa} '{.automountServiceAccountToken}')" false"""), + C("Secret %{sa}-token is a service-account token for %{sa}, issued", 2, r""" +s="$K -n %{ns} get secret %{sa}-token" +eq "$($s -o jsonpath='{.type}' 2>/dev/null)" kubernetes.io/service-account-token \ + && eq "$($s -o jsonpath='{.metadata.annotations.kubernetes\.io/service-account\.name}')" %{sa} && [ -n "$($s -o jsonpath='{.data.token}')" ]"""), + C("token.txt holds the decoded token", 2, r""" +t=$(j -n %{ns} secret %{sa}-token '{.data.token}' | base64 -d 2>/dev/null); [ -n "$t" ] && eq "$(tr -d '[:space:]' < "$WS/q34/token.txt" 2>/dev/null)" "$t" """), + C("Deployment runs as %{sa} with no API token volume", 2, r""" +eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.serviceAccountName}')" %{sa} || exit 1 +for i in $(seq 1 30); do [ "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" = 1 ] && break; sleep 3; done +p=$(j -n %{ns} pod -l app=%{dep} --field-selector=status.phase=Running '{.items[0].metadata.name}'); [ -n "$p" ] || exit 1 +eq "$(j -n %{ns} pod "$p" '{.spec.serviceAccountName}')" %{sa} && [ -z "$(j -n %{ns} pod "$p" '{.spec.volumes[*].projected.sources[*].serviceAccountToken.path}')" ]"""), + ], + "solution": """ +k -n %{ns} create sa %{sa} +k -n %{ns} patch sa %{sa} -p '{"automountServiceAccountToken": false}' +cat < {WS}/q34/token.txt +k -n %{ns} create deploy %{dep} --image=nginx:1.27 $do > d.yaml +vi d.yaml # template.spec.serviceAccountName: %{sa} +k apply -f d.yaml +k -n %{ns} get pod -l app=%{dep} -o jsonpath='{.items[0].spec.volumes}' # no kube-api-access-* +# Since 1.24 a ServiceAccount gets no token Secret on its own; the token controller fills one in +# when you create it with that type and annotation. `kubectl create token %{sa}` makes a +# short-lived token instead, with no Secret at all. +""", +}, +# ------------------------------------------------------------------- NET ---- +{ + "id": 35, "domain": "NET", "title": "Egress lockdown that keeps DNS working", + "params": lambda r: {"ns": ns_pick(r, "mesh")}, + "task": """ +In Namespace `%{ns}`, Pods labelled `app=api` must be locked down for **outgoing** traffic: + +- they may connect to Pods labelled `app=db` on TCP port 80, +- they may resolve names through the cluster DNS (UDP and TCP port 53), +- every other outgoing connection from them must be blocked. + +Create NetworkPolicy `api-egress` for this, without restricting incoming traffic. kindnet +enforces NetworkPolicy in this cluster, so it is tested for real from an `api` Pod. +""", + "setup": NS + r""" +mkns %{ns}; mkns %{ns}-ext +$K -n %{ns} delete netpol --all >/dev/null 2>&1 +for a in db cache; do + $K -n %{ns} create deploy $a --image=nginx:1.27 --dry-run=client -o yaml | $K apply -f - >/dev/null + $K -n %{ns} expose deploy $a --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null +done +$K -n %{ns} create deploy api --image=busybox:1.36 --dry-run=client -o yaml -- sleep 86400 | $K apply -f - >/dev/null +$K -n %{ns}-ext create deploy web --image=nginx:1.27 --dry-run=client -o yaml | $K apply -f - >/dev/null +$K -n %{ns}-ext expose deploy web --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null +for d in api db cache; do $K -n %{ns} rollout status deploy $d --timeout=150s >/dev/null 2>&1; done +""", + "checks": [ + C("api-egress selects app=api and restricts Egress only", 1, r""" +eq "$(j -n %{ns} netpol api-egress '{.spec.podSelector.matchLabels.app}')" api && eq "$(j -n %{ns} netpol api-egress '{.spec.policyTypes}')" '["Egress"]'"""), + C("api reaches db by its Service name (DNS + TCP 80)", 3, r""" +[ -n "$(j -n %{ns} netpol api-egress '{.metadata.name}')" ] || exit 1 +for i in 1 2 3; do $K -n %{ns} exec deploy/api -- wget -T 3 -qO- http://db 2>/dev/null | grep -qi nginx && exit 0; sleep 2; done; exit 1"""), + # a failed connection proves nothing on its own (a Pod not ready yet fails too): only count it as + # blocked while the policy exists and the allowed path to db demonstrably works + C("api can no longer reach cache", 2, r""" +ok() { $K -n %{ns} exec deploy/api -- wget -T 3 -qO- "$1" >/dev/null 2>&1; } +[ -n "$(j -n %{ns} netpol api-egress '{.metadata.name}')" ] && ok http://db && ! ok http://cache"""), + C("api can no longer reach another Namespace", 1, r""" +ok() { $K -n %{ns} exec deploy/api -- wget -T 3 -qO- "$1" >/dev/null 2>&1; } +[ -n "$(j -n %{ns} netpol api-egress '{.metadata.name}')" ] && ok http://db && ! ok http://web.%{ns}-ext"""), + ], + "solution": """ +cat </dev/null +$K -n %{ns} delete svc %{app} --ignore-not-found >/dev/null +$K -n %{ns} delete pod probe --ignore-not-found --wait=true --grace-period=1 >/dev/null +$K apply -f - >/dev/null <<'EOF' +apiVersion: apps/v1 +kind: Deployment +metadata: {name: %{app}, namespace: %{ns}} +spec: + replicas: 2 + selector: {matchLabels: {app: %{app}, tier: web}} + template: + metadata: {labels: {app: %{app}, tier: web}} + spec: + containers: + - name: web + image: nginx:1.27 + ports: [{name: http, containerPort: 80}] +--- +apiVersion: v1 +kind: Service +metadata: {name: %{app}, namespace: %{ns}} +spec: + selector: {app: %{app}, tier: frontend} + ports: [{name: http, port: 80, targetPort: web}] +--- +apiVersion: v1 +kind: Pod +metadata: {name: probe, namespace: %{ns}} +spec: + containers: [{name: probe, image: busybox:1.36, command: [sleep, "86400"]}] +EOF +$K -n %{ns} rollout status deploy %{app} --timeout=150s >/dev/null 2>&1 +$K -n %{ns} wait --for=condition=Ready pod/probe --timeout=120s >/dev/null # fail the seed loudly if it is missing +""", + "checks": [ + C("probe gets the nginx page from http://%{app}", 3, r""" +for i in 1 2 3; do $K -n %{ns} exec probe -- wget -T 3 -qO- http://%{app} 2>/dev/null | grep -qi nginx && exit 0; sleep 2; done; exit 1"""), + C("Deployment untouched, Service still on port 80", 2, r""" +$K -n %{ns} exec probe -- wget -T 3 -qO- http://%{app} >/dev/null 2>&1 || exit 1 +eq "$(j -n %{ns} deploy %{app} '{.metadata.generation}')" 1 && eq "$(j -n %{ns} svc %{app} '{.spec.ports[0].port}')" 80"""), + C("endpoints.txt lists exactly the Pods' IPs", 2, r""" +want=$(j -n %{ns} pod -l app=%{app},tier=web --field-selector=status.phase=Running '{range .items[*]}{.status.podIP}{"\n"}{end}' | sort) +[ -n "$want" ] && [ "$(grep -v '^[[:space:]]*$' "$WS/q36/endpoints.txt" 2>/dev/null | tr -d ' \r' | sort)" = "$want" ]"""), + ], + "solution": """ +k -n %{ns} get svc %{app} -o wide # SELECTOR app=%{app},tier=frontend +k -n %{ns} get endpointslices -l kubernetes.io/service-name=%{app} # no endpoints +k -n %{ns} get pod --show-labels # the Pods say tier=web +k -n %{ns} get deploy %{app} -o jsonpath='{.spec.template.spec.containers[0].ports}' # port name: http +k -n %{ns} edit svc %{app} +# selector: tier: frontend -> tier: web +# targetPort: web -> targetPort: http (or 80) +k -n %{ns} exec probe -- wget -T 3 -qO- http://%{app} | head -3 +k -n %{ns} get endpointslices -l kubernetes.io/service-name=%{app} -o jsonpath='{range .items[*].endpoints[*]}{.addresses[0]}{"\\n"}{end}' > {WS}/q36/endpoints.txt +# Two faults, one symptom. Fixing the selector gives the Service endpoints, but a named targetPort +# that no container port carries still leaves them unusable - check both, every time. +""", +}, +{ + "id": 37, "domain": "NET", "title": "Ingress with two paths and TLS", + "params": lambda r: {"ns": ns_pick(r, "edge"), "dom": r.choice(["shop", "store", "market", "bazaar"])}, + "task": """ +In Namespace `%{ns}`, Services `web` (port 80) and `api` (port 8080) exist. + +1. Create a TLS Secret `%{dom}-tls` from `{WS}/q37/tls.crt` and `{WS}/q37/tls.key`. +2. Create Ingress `%{dom}` with IngressClass `nginx` for host `%{dom}.example.com`: + - path `/api` (Prefix) goes to Service `api` on port 8080, + - path `/` (Prefix) goes to Service `web` on port 80, + - TLS for that host uses the Secret from step 1. + +(No Ingress controller runs in this cluster; the objects are graded.) +""", + "setup": NS + r""" +mkns %{ns}; rm -rf "$WS/q37"; mkdir -p "$WS/q37" +$K -n %{ns} delete ingress %{dom} --ignore-not-found >/dev/null; $K -n %{ns} delete secret %{dom}-tls --ignore-not-found >/dev/null +for s in web api; do $K -n %{ns} create deploy $s --image=nginx:1.27 --dry-run=client -o yaml | $K apply -f - >/dev/null; done +$K -n %{ns} expose deploy web --port=80 --target-port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null +$K -n %{ns} expose deploy api --port=8080 --target-port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null +subj="/CN=%{dom}.example.com" +if command -v openssl >/dev/null; then + openssl req -x509 -nodes -newkey rsa:2048 -days 30 -subj "$subj" -keyout "$WS/q37/tls.key" -out "$WS/q37/tls.crt" >/dev/null 2>&1 +else # no openssl on this machine: make the pair on the control-plane node instead + nx "$CP" "openssl req -x509 -nodes -newkey rsa:2048 -days 30 -subj $subj -keyout /tmp/q37.key -out /tmp/q37.crt >/dev/null 2>&1; cat /tmp/q37.crt" > "$WS/q37/tls.crt" "$WS/q37/tls.key" /dev/null)" = "$(cat "$WS/q37/tls.crt")" ] \ + && [ "$(j -n %{ns} secret %{dom}-tls '{.data.tls\.key}' | base64 -d 2>/dev/null)" = "$(cat "$WS/q37/tls.key")" ]"""), + C("class nginx; %{dom}.example.com: /api -> api:8080 and / -> web:80, both Prefix", 3, r""" +$K -n %{ns} get ingress %{dom} -o json 2>/dev/null | python3 -c ' +import sys, json +s = json.load(sys.stdin)["spec"] +rules = [r for r in s.get("rules", []) if r.get("host") == "%{dom}.example.com"] +paths = {p.get("path"): (p.get("pathType"), p["backend"]["service"]["name"], p["backend"]["service"]["port"].get("number")) + for r in rules for p in r.get("http", {}).get("paths", [])} +ok = s.get("ingressClassName") == "nginx" and paths == {"/api": ("Prefix", "api", 8080), "/": ("Prefix", "web", 80)} +sys.exit(0 if ok else 1)'"""), + C("TLS for %{dom}.example.com uses %{dom}-tls", 2, r""" +$K -n %{ns} get ingress %{dom} -o json 2>/dev/null | python3 -c ' +import sys, json +t = json.load(sys.stdin)["spec"].get("tls", []) +sys.exit(0 if any(x.get("secretName") == "%{dom}-tls" and "%{dom}.example.com" in x.get("hosts", []) for x in t) else 1)'"""), + ], + "solution": """ +k -n %{ns} create secret tls %{dom}-tls --cert={WS}/q37/tls.crt --key={WS}/q37/tls.key +k -n %{ns} create ingress %{dom} --class=nginx \\ + --rule="%{dom}.example.com/api*=api:8080,tls=%{dom}-tls" \\ + --rule="%{dom}.example.com/*=web:80,tls=%{dom}-tls" +k -n %{ns} get ingress %{dom} -o yaml +# In --rule, a trailing * means pathType Prefix (no * = Exact), and tls=SECRET adds the host to +# spec.tls with that Secret. The YAML equivalent: +# spec: +# ingressClassName: nginx +# tls: [{hosts: [%{dom}.example.com], secretName: %{dom}-tls}] +# rules: +# - host: %{dom}.example.com +# http: +# paths: +# - {path: /api, pathType: Prefix, backend: {service: {name: api, port: {number: 8080}}}} +# - {path: /, pathType: Prefix, backend: {service: {name: web, port: {number: 80}}}} +""", +}, +] diff --git a/tracks/ckad/questions.py b/tracks/ckad/questions.py index e3ab285..25c56c2 100644 --- a/tracks/ckad/questions.py +++ b/tracks/ckad/questions.py @@ -1,4 +1,5 @@ -"""CKAD track. Starter bank - same format as the CKA banks, extend it the same way. +"""CKAD track: 37 tasks, at least one per competency in the CKAD curriculum (1-12 here, 13-37 in +bank_extra.py). Same format as the CKA banks. CKAD is a build-things exam: less cluster surgery, more "produce the right object, fast". The clock is the adversary, so these are mostly single-object tasks graded exactingly. @@ -17,8 +18,10 @@ TRACK = { "ENV": "Application Environment, Configuration and Security (25%)", "NET": "Services and Networking (20%)", }, - "quota": {"DESIGN": 3, "DEPLOY": 3, "OBS": 2, "ENV": 2, "NET": 2}, - "quota_hard": {"DESIGN": 3, "DEPLOY": 3, "OBS": 2, "ENV": 2, "NET": 2}, + # 16 tasks in 120 minutes, weighted like the exam (20/20/15/25/20 %). The pool per domain is + # 8/7/7/10/5, so no two mocks are the same set. + "quota": {"DESIGN": 3, "DEPLOY": 3, "OBS": 3, "ENV": 4, "NET": 3}, + "quota_hard": {"DESIGN": 3, "DEPLOY": 3, "OBS": 3, "ENV": 4, "NET": 3}, } QUESTIONS = [ @@ -35,7 +38,7 @@ mounted at `/spool` in both: `kubectl logs %{pod} -c adapter` must show the writer's lines. """, - "setup": NS + r"""mkns %{ns}; $K -n %{ns} delete pod %{pod} --ignore-not-found --wait=false >/dev/null""", + "setup": NS + r"""mkns %{ns}; $K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null""", "checks": [ C("pod Running with containers writer and adapter", 2, r""" p="$K -n %{ns} get pod %{pod}" @@ -86,7 +89,7 @@ connection instead. """, "setup": NS + r""" mkns %{ns} -$K -n %{ns} delete pod %{pod} --ignore-not-found --wait=false >/dev/null +$K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null $K -n %{ns} delete deploy backend --ignore-not-found >/dev/null $K apply -f - >/dev/null </dev/null """, "checks": [ C("reason.txt records the image pull failure", 3, r"""grep -Eqi 'ImagePull|ErrImagePull|ImagePullBackOff' "$WS/q8/reason.txt" 2>/dev/null"""), @@ -394,7 +398,7 @@ Create Pod `%{pod}` in `%{ns}` (image `busybox:1.36`, command `sleep 86400`) tha - drops **all** capabilities and adds back only `NET_BIND_SERVICE`, - cannot gain new privileges. """, - "setup": NS + r"""mkns %{ns}; $K -n %{ns} delete pod %{pod} --ignore-not-found --wait=false >/dev/null""", + "setup": NS + r"""mkns %{ns}; $K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null""", "checks": [ C("runs as UID %{uid} with runAsNonRoot", 3, r""" p="$K -n %{ns} get pod %{pod}" @@ -444,7 +448,7 @@ In `%{ns}`: """, "setup": NS + r""" mkns %{ns} -$K -n %{ns} delete pod %{pod} --ignore-not-found --wait=false >/dev/null +$K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null $K -n %{ns} delete cm app-cfg --ignore-not-found >/dev/null; $K -n %{ns} delete secret app-sec --ignore-not-found >/dev/null """, "checks": [ @@ -573,3 +577,8 @@ EOF """, }, ] + +# 13-37: the rest of the curriculum, one or more tasks per competency the starter bank left out. +from tracks.ckad import bank_extra # noqa: E402 + +QUESTIONS = QUESTIONS + bank_extra.QUESTIONS