"""CKAD tasks 13-37: one or more per curriculum competency the starter bank (1-12) left uncovered. Same format as questions.py. Parameters (%{...}) are drawn per session; every setup is safe to re-run, and every check reads the live cluster (or the files the task asks for). """ from sim.bank import C, NS, ns_pick, name_pick QUESTIONS = [ # ---------------------------------------------------------------- DESIGN ---- { "id": 13, "domain": "DESIGN", "title": "Build, change and export a container image", "params": lambda r: {"img": name_pick(r, r.choice(["ticker", "beacon", "courier"])), "greet": r.choice(["hola", "ciao", "salut", "hej"])}, "task": """ The build context for a small image is in `{WS}/q13/image` (a `Dockerfile` and `app.sh`). Use `podman` on this machine. 1. Build the image, unchanged, as `localhost/%{img}:v1`. 2. Change the **Dockerfile** so the image sets `GREETING=%{greet}` and a new environment variable `APP_VERSION=v2`, then build it as `localhost/%{img}:v2`. `v1` must stay as it was. 3. Save `localhost/%{img}:v2` as an **OCI archive** to `{WS}/q13/%{img}-v2.tar`. 4. Run `v2` once and write its output to `{WS}/q13/run.txt`. """, "setup": r""" rm -rf "$WS/q13"; mkdir -p "$WS/q13/image" command -v podman >/dev/null || { echo "podman is not installed on this machine - the task needs it"; exit 1; } podman rmi -f localhost/%{img}:v1 localhost/%{img}:v2 >/dev/null 2>&1 || true cat > "$WS/q13/image/Dockerfile" <<'EOF' FROM docker.io/library/busybox:1.36 ENV GREETING=hello COPY app.sh /app/app.sh CMD ["sh", "/app/app.sh"] EOF cat > "$WS/q13/image/app.sh" <<'EOF' echo "${GREETING} from %{img} ${APP_VERSION:-unversioned}" EOF """, "checks": [ C("v1 exists and is the unchanged build (GREETING=hello, no APP_VERSION)", 2, r""" env=$(podman image inspect localhost/%{img}:v1 --format '{{json .Config.Env}}' 2>/dev/null) || exit 1 echo "$env" | grep -q '"GREETING=hello"' && ! echo "$env" | grep -q APP_VERSION"""), C("v2 sets GREETING=%{greet} and APP_VERSION=v2", 2, r""" env=$(podman image inspect localhost/%{img}:v2 --format '{{json .Config.Env}}' 2>/dev/null) || exit 1 echo "$env" | grep -q '"GREETING=%{greet}"' && echo "$env" | grep -q '"APP_VERSION=v2"'"""), C("the change is in the Dockerfile itself", 1, r""" f="$WS/q13/image/Dockerfile" grep -Eq '^ENV.*APP_VERSION[= ]+"?v2' "$f" && grep -Eq '^ENV.*GREETING[= ]+"?%{greet}' "$f" """), C("OCI archive of v2 at {WS}/q13/%{img}-v2.tar", 2, r""" f="$WS/q13/%{img}-v2.tar"; [ -s "$f" ] || exit 1 tar -xOf "$f" oci-layout >/dev/null 2>&1 || exit 1 tar -xOf "$f" index.json 2>/dev/null | grep -q '"org.opencontainers.image.ref.name":"localhost/%{img}:v2"'"""), C("run.txt holds v2's output", 1, r"""tr -d '\r' < "$WS/q13/run.txt" 2>/dev/null | grep -qx '%{greet} from %{img} v2'"""), ], "solution": """ cd {WS}/q13/image podman build -t localhost/%{img}:v1 . # build BEFORE editing: v1 must be the original vi Dockerfile # ENV GREETING=%{greet} # ENV APP_VERSION=v2 podman build -t localhost/%{img}:v2 . podman images | grep %{img} podman save --format oci-archive -o {WS}/q13/%{img}-v2.tar localhost/%{img}:v2 podman run --rm localhost/%{img}:v2 > {WS}/q13/run.txt # %{greet} from %{img} v2 # `podman save` writes docker-archive unless told otherwise; "OCI format" means # --format oci-archive (an index.json + oci-layout + blobs/ tarball). With docker: # docker build -t ... . && docker save -o x.tar ... (Docker Engine 25+ writes an OCI layout # alongside the legacy manifest.json) - on the exam, read which tool and which format the task names. `--env`/`--build-arg` at build time would not change the # Dockerfile, which is what "change the Dockerfile" asks for. """, }, { "id": 14, "domain": "DESIGN", "title": "Parallel batch Job", "params": lambda r: {"ns": ns_pick(r, "batch"), "job": name_pick(r, "render"), "n": r.choice([6, 8]), "p": r.choice([2, 3])}, "task": """ In Namespace `%{ns}` create a Job `%{job}` that renders %{n} frames: - container name `render`, image `busybox:1.36`, command `sh -c 'echo frame done; sleep 3'` - %{n} successful completions in total, at most %{p} Pods running at the same time - give up after 2 failed retries - kill the whole Job if it is still running after 5 minutes - its Pods carry the label `batch=%{job}` The Job must finish successfully. """, "setup": NS + r"""mkns %{ns}; $K -n %{ns} delete job %{job} --ignore-not-found --wait=true >/dev/null""", "checks": [ C("completions %{n}, parallelism %{p}, backoffLimit 2", 3, r""" s="$K -n %{ns} get job %{job}" eq "$($s -o jsonpath='{.spec.completions}' 2>/dev/null)" %{n} && eq "$($s -o jsonpath='{.spec.parallelism}')" %{p} && eq "$($s -o jsonpath='{.spec.backoffLimit}')" 2"""), C("activeDeadlineSeconds 300", 1, r"""eq "$(j -n %{ns} job %{job} '{.spec.activeDeadlineSeconds}')" 300"""), C("container render (busybox:1.36), Pod label batch=%{job}", 1, r""" eq "$(j -n %{ns} job %{job} '{.spec.template.spec.containers[0].name}')" render \ && eq "$(j -n %{ns} job %{job} '{.spec.template.spec.containers[0].image}')" busybox:1.36 \ && eq "$(j -n %{ns} job %{job} '{.spec.template.metadata.labels.batch}')" %{job}"""), C("Job complete: %{n} succeeded", 3, r""" for i in $(seq 1 40); do [ "$(j -n %{ns} job %{job} '{.status.succeeded}')" = %{n} ] && exit 0; sleep 3; done; exit 1"""), ], "solution": """ k -n %{ns} create job %{job} --image=busybox:1.36 $do -- sh -c 'echo frame done; sleep 3' > job.yaml vi job.yaml # spec.completions: %{n} # spec.parallelism: %{p} # spec.backoffLimit: 2 # spec.activeDeadlineSeconds: 300 # spec.template.metadata.labels: {batch: %{job}} # container name: render (create job names the container after the Job) k apply -f job.yaml k -n %{ns} get job,pod -l batch=%{job} -w # %{p} at a time until %{n}/%{n} # A Job's pod template is immutable: if you got a field wrong, delete the Job and apply again. """, }, { "id": 15, "domain": "DESIGN", "title": "DaemonSet on every node", "params": lambda r: {"ns": ns_pick(r, "fleet"), "ds": name_pick(r, "node-agent")}, "task": """ Create a DaemonSet `%{ds}` in Namespace `%{ns}` that runs one Pod on **every** node of the cluster, the control plane included: - image `busybox:1.36`, command `sh -c 'while true; do echo "$(date) $NODE_NAME ok"; sleep 30; done'` - environment variable `NODE_NAME` filled from the Pod's own `spec.nodeName` - requests cpu `10m`, memory `16Mi` - Pod label `app=%{ds}` """, "setup": NS + r"""mkns %{ns}; $K -n %{ns} delete ds %{ds} --ignore-not-found --wait=true >/dev/null""", "checks": [ C("one ready Pod per node", 3, r""" n=$($K get nodes --no-headers 2>/dev/null | wc -l) for i in $(seq 1 20); do [ "$(j -n %{ns} ds %{ds} '{.status.numberReady}')" = "$n" ] && [ "$(j -n %{ns} ds %{ds} '{.status.desiredNumberScheduled}')" = "$n" ] && exit 0; sleep 3 done; exit 1"""), C("a Pod runs on the control-plane node", 2, r"""j -n %{ns} pod -l app=%{ds} '{.items[*].spec.nodeName}' | tr ' ' '\n' | grep -qx "$CP" """), C("NODE_NAME comes from spec.nodeName (and shows in the logs)", 2, r""" eq "$(j -n %{ns} ds %{ds} '{.spec.template.spec.containers[0].env[?(@.name=="NODE_NAME")].valueFrom.fieldRef.fieldPath}')" spec.nodeName || exit 1 p=$(j -n %{ns} pod -l app=%{ds} '{.items[0].metadata.name}'); n=$(j -n %{ns} pod "$p" '{.spec.nodeName}') [ -n "$n" ] && $K -n %{ns} logs "$p" --tail=5 2>/dev/null | grep -q " $n ok" """), C("requests cpu 10m / memory 16Mi", 1, r""" eq "$(j -n %{ns} ds %{ds} '{.spec.template.spec.containers[0].resources.requests.cpu}')" 10m \ && eq "$(j -n %{ns} ds %{ds} '{.spec.template.spec.containers[0].resources.requests.memory}')" 16Mi"""), ], "solution": """ k -n %{ns} create deploy %{ds} --image=busybox:1.36 $do -- sh -c 'while true; do echo "$(date) $NODE_NAME ok"; sleep 30; done' > ds.yaml vi ds.yaml # kind: Deployment -> DaemonSet; delete replicas, strategy and status # container: # env: [{name: NODE_NAME, valueFrom: {fieldRef: {fieldPath: spec.nodeName}}}] # resources: {requests: {cpu: 10m, memory: 16Mi}} # pod spec: # tolerations: [{key: node-role.kubernetes.io/control-plane, operator: Exists, effect: NoSchedule}] k apply -f ds.yaml k -n %{ns} get ds,pod -o wide # DESIRED = number of nodes # There is no `kubectl create daemonset`; starting from a Deployment and changing the kind is the # fast way. Without the toleration the control-plane node's NoSchedule taint keeps it at 3 of 4. """, }, { "id": 16, "domain": "DESIGN", "title": "Native sidecar lets a Job finish", "params": lambda r: {"ns": ns_pick(r, "nightly"), "job": name_pick(r, "export")}, "task": """ Job `%{job}` in Namespace `%{ns}` never completes: its log-shipping container `shipper` keeps running after the `worker` container has finished. Its manifest is `{WS}/q16/job.yaml`. Keep `shipper` - it must still start **before** `worker` and receive all of its output - but make it a **native sidecar** container, so the Job completes by itself once `worker` is done. Keep the Job's name. The Job must end up Complete. """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q16"; mkdir -p "$WS/q16" cat > "$WS/q16/job.yaml" <<'EOF' apiVersion: batch/v1 kind: Job metadata: {name: %{job}, namespace: %{ns}} spec: backoffLimit: 1 template: spec: restartPolicy: Never volumes: [{name: data, emptyDir: {}}] containers: - name: worker image: busybox:1.36 command: [sh, -c, "for i in 1 2 3 4 5; do echo \"batch line $i\" >> /data/out.log; sleep 2; done"] volumeMounts: [{name: data, mountPath: /data}] - name: shipper image: busybox:1.36 command: [sh, -c, "touch /data/out.log; tail -F /data/out.log"] volumeMounts: [{name: data, mountPath: /data}] EOF $K -n %{ns} delete job %{job} --ignore-not-found --wait=true >/dev/null $K apply -f "$WS/q16/job.yaml" >/dev/null """, "checks": [ C("shipper is a native sidecar (init container with restartPolicy Always)", 3, r""" eq "$(j -n %{ns} job %{job} '{.spec.template.spec.initContainers[?(@.name=="shipper")].restartPolicy}')" Always"""), C("worker is the only regular container", 1, r"""eq "$(j -n %{ns} job %{job} '{.spec.template.spec.containers[*].name}')" worker"""), C("Job Complete", 3, r""" for i in $(seq 1 30); do [ "$(j -n %{ns} job %{job} '{.status.succeeded}')" = 1 ] && exit 0; sleep 3; done; exit 1"""), C("shipper, now a sidecar, received all of worker's output", 1, r""" eq "$(j -n %{ns} job %{job} '{.spec.template.spec.initContainers[?(@.name=="shipper")].restartPolicy}')" Always \ && $K -n %{ns} logs job/%{job} -c shipper 2>/dev/null | grep -q 'batch line 5'"""), ], "solution": """ k -n %{ns} get job,pod # 0/1 complete; the Pod sits at 1/2 Running forever # A Job counts a Pod as done only when ALL its containers have exited, and `tail -F` never does. # A native sidecar is an init container with restartPolicy: Always - it starts before the main # containers, keeps running beside them, and is stopped automatically once they have finished. vi {WS}/q16/job.yaml # spec.template.spec: # initContainers: # - name: shipper # restartPolicy: Always # <- this line is what makes it a sidecar # image: busybox:1.36 # command: [sh, -c, "touch /data/out.log; tail -F /data/out.log"] # volumeMounts: [{name: data, mountPath: /data}] # containers: [ worker only ] k -n %{ns} delete job %{job} # a Job's pod template is immutable k apply -f {WS}/q16/job.yaml k -n %{ns} get job %{job} -w # 1/1 k -n %{ns} logs job/%{job} -c shipper # batch line 1..5 """, }, { "id": 17, "domain": "DESIGN", "title": "Persistent and ephemeral volumes", "params": lambda r: {"ns": ns_pick(r, "store"), "dep": name_pick(r, "journal"), "pvc": name_pick(r, "journal-data")}, "task": """ In Namespace `%{ns}`: 1. Create a PersistentVolumeClaim `%{pvc}`: `100Mi`, `ReadWriteOnce`, StorageClass `standard`. 2. Create Deployment `%{dep}` (1 replica, image `busybox:1.36`) whose container runs `sh -c 'date >> /data/boot.log; sleep 86400'` and mounts - the claim `%{pvc}` at `/data`, - an `emptyDir` volume named `scratch` at `/scratch`, held in memory and capped at `32Mi`. 3. Delete the Deployment's Pod once, so that `/data/boot.log` in the replacement Pod has two lines. """, "setup": NS + r""" mkns %{ns} $K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null $K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null """, "checks": [ C("claim %{pvc} Bound: 100Mi, ReadWriteOnce, class standard", 2, r""" p="$K -n %{ns} get pvc %{pvc}" eq "$($p -o jsonpath='{.status.phase}' 2>/dev/null)" Bound && eq "$($p -o jsonpath='{.spec.resources.requests.storage}')" 100Mi \ && eq "$($p -o jsonpath='{.spec.accessModes[0]}')" ReadWriteOnce && eq "$($p -o jsonpath='{.spec.storageClassName}')" standard"""), C("Deployment mounts the claim at /data", 2, r""" v=$(j -n %{ns} deploy %{dep} '{.spec.template.spec.volumes[?(@.persistentVolumeClaim.claimName=="%{pvc}")].name}'); [ -n "$v" ] || exit 1 eq "$(j -n %{ns} deploy %{dep} "{.spec.template.spec.containers[0].volumeMounts[?(@.name==\"$v\")].mountPath}")" /data"""), C("emptyDir scratch: medium Memory, sizeLimit 32Mi, at /scratch", 2, r""" d='{.spec.template.spec' eq "$(j -n %{ns} deploy %{dep} "$d.volumes[?(@.name==\"scratch\")].emptyDir.medium}")" Memory \ && eq "$(j -n %{ns} deploy %{dep} "$d.volumes[?(@.name==\"scratch\")].emptyDir.sizeLimit}")" 32Mi \ && eq "$(j -n %{ns} deploy %{dep} "$d.containers[0].volumeMounts[?(@.name==\"scratch\")].mountPath}")" /scratch"""), C("boot.log survived a Pod replacement (2+ lines)", 2, r""" n=$($K -n %{ns} exec deploy/%{dep} -- sh -c 'wc -l < /data/boot.log' 2>/dev/null | tr -dc '0-9'); [ "${n:-0}" -ge 2 ]"""), ], "solution": """ cat <> /data/boot.log; sleep 86400' > d.yaml vi d.yaml # pod spec: # volumes: # - {name: data, persistentVolumeClaim: {claimName: %{pvc}}} # - {name: scratch, emptyDir: {medium: Memory, sizeLimit: 32Mi}} # container: # volumeMounts: [{name: data, mountPath: /data}, {name: scratch, mountPath: /scratch}] k apply -f d.yaml k -n %{ns} get pvc # Pending until the Pod exists: WaitForFirstConsumer k -n %{ns} delete pod -l app=%{dep} k -n %{ns} exec deploy/%{dep} -- cat /data/boot.log # two lines: the PVC outlived the Pod # The emptyDir lives exactly as long as the Pod; medium: Memory makes it tmpfs, and what is # written there counts against the container's memory limit. """, }, # ---------------------------------------------------------------- DEPLOY ---- { "id": 18, "domain": "DEPLOY", "title": "Canary by replica ratio", "params": lambda r: {"ns": ns_pick(r, "canary"), "app": r.choice(["checkout", "search", "catalog"])}, "task": """ Namespace `%{ns}` runs Deployment `%{app}-stable` (3 replicas, `nginx:1.25`) behind Service `%{app}`. Start a canary of `nginx:1.27` that gets **20%** of the Service's traffic by replica count: - a new Deployment `%{app}-canary` whose Pods the existing Service already selects, with the label `track=canary`, - the Service must end up with exactly **5** ready endpoints, **1** of them a canary Pod, - do not change the Service. """, "setup": NS + r""" mkns %{ns} $K -n %{ns} delete deploy %{app}-stable %{app}-canary --ignore-not-found --wait=true >/dev/null $K -n %{ns} delete svc %{app} --ignore-not-found >/dev/null $K apply -f - >/dev/null <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: {name: %{app}-stable, namespace: %{ns}} spec: replicas: 3 selector: {matchLabels: {app: %{app}, track: stable}} template: metadata: {labels: {app: %{app}, track: stable}} spec: {containers: [{name: web, image: nginx:1.25, ports: [{containerPort: 80}]}]} --- apiVersion: v1 kind: Service metadata: {name: %{app}, namespace: %{ns}} spec: {selector: {app: %{app}}, ports: [{port: 80, targetPort: 80}]} EOF """, "checks": [ C("canary Deployment: nginx:1.27, 1 ready, label track=canary", 2, r""" d="$K -n %{ns} get deploy %{app}-canary" eq "$($d -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null)" nginx:1.27 \ && eq "$($d -o jsonpath='{.spec.template.metadata.labels.track}')" canary && eq "$($d -o jsonpath='{.status.readyReplicas}')" 1"""), C("stable scaled to 4 and still on nginx:1.25", 2, r""" eq "$(j -n %{ns} deploy %{app}-stable '{.status.readyReplicas}')" 4 && eq "$(j -n %{ns} deploy %{app}-stable '{.spec.template.spec.containers[0].image}')" nginx:1.25"""), C("Service unchanged: still selects only app=%{app}", 1, r""" [ -n "$(j -n %{ns} deploy %{app}-canary '{.metadata.name}')" ] && eq "$(j -n %{ns} svc %{app} '{.spec.selector}')" '{"app":"%{app}"}'"""), C("Service has 5 ready endpoints, exactly 1 of them canary", 3, r""" ips=$(j -n %{ns} endpointslices -l kubernetes.io/service-name=%{app} '{range .items[*].endpoints[?(@.conditions.ready==true)]}{.addresses[0]}{" "}{end}') [ "$(echo $ips | wc -w)" = 5 ] || exit 1 can=" $(j -n %{ns} pod -l app=%{app},track=canary '{.items[*].status.podIP}') "; c=0 for ip in $ips; do case "$can" in *" $ip "*) c=$((c+1));; esac; done; [ "$c" = 1 ]"""), ], "solution": """ k -n %{ns} get deploy %{app}-stable -o yaml > canary.yaml vi canary.yaml # metadata.name: %{app}-canary, replicas: 1, image: nginx:1.27 # track: stable -> track: canary in BOTH spec.selector.matchLabels and the template labels # delete status, uid, resourceVersion, creationTimestamp, generation k apply -f canary.yaml k -n %{ns} scale deploy %{app}-stable --replicas=4 k -n %{ns} get endpointslices -l kubernetes.io/service-name=%{app} # 5 addresses # The Service selects only app=%{app}, so it load-balances over stable AND canary Pods in # proportion to their ready replicas: 4 + 1 = 20% canary. Promote by shifting the ratio; # roll back by scaling the canary to 0 - the Service never changes. """, }, { "id": 19, "domain": "DEPLOY", "title": "Rolling update without losing capacity", "params": lambda r: {"ns": ns_pick(r, "roll"), "dep": name_pick(r, "payments")}, "task": """ Deployment `%{dep}` in Namespace `%{ns}` (4 replicas) must never lose capacity during an update. 1. Change its update strategy so that during a rollout **no** Pod is ever unavailable and **at most one** extra Pod is created, and a new Pod only counts as available once it has been Ready for **10 seconds**. Keep at most **3** old ReplicaSets around. 2. Then update the image to `nginx:1.27` and let the rollout finish. """, "setup": NS + r""" mkns %{ns} $K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null $K -n %{ns} create deploy %{dep} --image=nginx:1.25 --replicas=4 >/dev/null $K -n %{ns} rollout status deploy %{dep} --timeout=150s >/dev/null 2>&1 """, "checks": [ C("RollingUpdate with maxUnavailable 0 and maxSurge 1", 3, r""" s='{.spec.strategy' eq "$(j -n %{ns} deploy %{dep} "$s.type}")" RollingUpdate || exit 1 case "$(j -n %{ns} deploy %{dep} "$s.rollingUpdate.maxUnavailable}")" in 0|0%) ;; *) exit 1;; esac case "$(j -n %{ns} deploy %{dep} "$s.rollingUpdate.maxSurge}")" in 1|25%) ;; *) exit 1;; esac"""), C("minReadySeconds 10 and revisionHistoryLimit 3", 2, r""" eq "$(j -n %{ns} deploy %{dep} '{.spec.minReadySeconds}')" 10 && eq "$(j -n %{ns} deploy %{dep} '{.spec.revisionHistoryLimit}')" 3"""), C("rolled out: 4/4 updated and available on nginx:1.27", 3, r""" d="$K -n %{ns} get deploy %{dep}" eq "$($d -o jsonpath='{.spec.template.spec.containers[0].image}')" nginx:1.27 || exit 1 for i in $(seq 1 30); do [ "$($d -o jsonpath='{.status.updatedReplicas}/{.status.availableReplicas}/{.status.replicas}')" = 4/4/4 ] \ && [ "$($d -o jsonpath='{.status.observedGeneration}')" = "$($d -o jsonpath='{.metadata.generation}')" ] && exit 0; sleep 3 done; exit 1"""), ], "solution": """ k -n %{ns} patch deploy %{dep} -p '{"spec":{"minReadySeconds":10,"revisionHistoryLimit":3, "strategy":{"type":"RollingUpdate","rollingUpdate":{"maxSurge":1,"maxUnavailable":0}}}}' k -n %{ns} set image deploy %{dep} nginx=nginx:1.27 k -n %{ns} rollout status deploy %{dep} # one new Pod at a time, 10 s apart # maxUnavailable: 0 + maxSurge: 1 means "add one, wait until it is available, remove one old". # minReadySeconds is what "available" waits for; it also slows every rollout down by design. """, }, { "id": 20, "domain": "DEPLOY", "title": "Kustomize overlay", "params": lambda r: {"ns": ns_pick(r, "prod"), "n": r.choice([2, 3])}, "task": """ A Kustomize base for an app called `web` is in `{WS}/q20/base`. Do **not** edit the base. Create an overlay in `{WS}/q20/overlays/prod` that - puts everything into Namespace `%{ns}`, - prefixes every resource name with `prod-`, - runs **%{n}** replicas, - uses image `nginx:1.27`, - adds the label `env=prod` to every resource. Apply the overlay with kubectl's built-in Kustomize. """, "setup": NS + r""" mkns %{ns} $K -n %{ns} delete deploy prod-web --ignore-not-found >/dev/null; $K -n %{ns} delete svc prod-web --ignore-not-found >/dev/null rm -rf "$WS/q20"; mkdir -p "$WS/q20/base" cat > "$WS/q20/base/deployment.yaml" <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: {name: web, labels: {app: web}} spec: replicas: 1 selector: {matchLabels: {app: web}} template: metadata: {labels: {app: web}} spec: {containers: [{name: web, image: nginx:1.25, ports: [{containerPort: 80}]}]} EOF cat > "$WS/q20/base/service.yaml" <<'EOF' apiVersion: v1 kind: Service metadata: {name: web, labels: {app: web}} spec: {selector: {app: web}, ports: [{port: 80, targetPort: 80}]} EOF cat > "$WS/q20/base/kustomization.yaml" <<'EOF' apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: [deployment.yaml, service.yaml] EOF (cd "$WS/q20/base" && md5sum *) > "$WS/.q20-base.md5" """, "checks": [ C("overlay renders prod-web in %{ns}: %{n} replicas, nginx:1.27, env=prod on everything", 3, r""" $K kustomize "$WS/q20/overlays/prod" 2>/dev/null | $K create --dry-run=client -o json -f - 2>/dev/null | python3 -c ' import sys, json s = sys.stdin.read(); dec = json.JSONDecoder(); i = 0; items = [] while i < len(s): while i < len(s) and s[i].isspace(): i += 1 if i >= len(s): break o, i = dec.raw_decode(s, i); items.extend(o.get("items", [o])) dep = [x for x in items if x["kind"] == "Deployment"] ok = len(items) == 2 and len(dep) == 1 and dep[0]["metadata"]["name"] == "prod-web" \ and all(x["metadata"].get("namespace") == "%{ns}" for x in items) \ and dep[0]["spec"]["replicas"] == %{n} and dep[0]["spec"]["template"]["spec"]["containers"][0]["image"] == "nginx:1.27" \ and all(x["metadata"].get("labels", {}).get("env") == "prod" for x in items) sys.exit(0 if ok else 1)'"""), C("applied: Deployment prod-web (%{n} available, nginx:1.27, env=prod) and Service prod-web", 3, r""" d="$K -n %{ns} get deploy prod-web" eq "$($d -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null)" nginx:1.27 && eq "$($d -o jsonpath='{.metadata.labels.env}')" prod || exit 1 eq "$(j -n %{ns} svc prod-web '{.metadata.labels.env}')" prod || exit 1 for i in $(seq 1 30); do [ "$($d -o jsonpath='{.status.availableReplicas}')" = %{n} ] && exit 0; sleep 3; done; exit 1"""), C("base left untouched", 1, r""" [ -f "$WS/q20/overlays/prod/kustomization.yaml" ] && (cd "$WS/q20/base" && md5sum -c --quiet "$WS/.q20-base.md5" >/dev/null 2>&1)"""), ], "solution": """ mkdir -p {WS}/q20/overlays/prod cat > {WS}/q20/overlays/prod/kustomization.yaml </dev/null || { echo "helm is not installed on this machine - the task needs it"; exit 1; } mkns %{ns} for r in $(helm -n %{ns} list -aq 2>/dev/null || helm -n %{ns} list -q 2>/dev/null); do helm -n %{ns} uninstall "$r" >/dev/null 2>&1; done rm -rf "$WS/q21"; mkdir -p "$WS/q21/charts" src=$(mktemp -d) for v in 1.2.0 1.3.0; do d="$src/$v/webapp"; mkdir -p "$d/templates" tag=1.25; [ "$v" = 1.3.0 ] && tag=1.27 printf 'apiVersion: v2\nname: webapp\nversion: %s\nappVersion: "%s"\n' "$v" "$tag" > "$d/Chart.yaml" printf 'replicaCount: 1\nimage:\n repository: nginx\n tag: "%s"\nservice:\n type: ClusterIP\n port: 80\n' "$tag" > "$d/values.yaml" cat > "$d/templates/deployment.yaml" <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: name: {{ .Release.Name }}-webapp labels: {app.kubernetes.io/instance: {{ .Release.Name }}, chart: {{ .Chart.Name }}-{{ .Chart.Version }}} spec: replicas: {{ .Values.replicaCount }} selector: {matchLabels: {app.kubernetes.io/instance: {{ .Release.Name }}}} template: metadata: {labels: {app.kubernetes.io/instance: {{ .Release.Name }}}} spec: containers: - name: web image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" ports: [{containerPort: 80}] EOF cat > "$d/templates/service.yaml" <<'EOF' apiVersion: v1 kind: Service metadata: {name: {{ .Release.Name }}-webapp} spec: type: {{ .Values.service.type }} selector: {app.kubernetes.io/instance: {{ .Release.Name }}} ports: [{port: {{ .Values.service.port }}, targetPort: 80}] EOF helm package "$d" -d "$WS/q21/charts" >/dev/null done rm -rf "$src" helm -n %{ns} install %{old} "$WS/q21/charts/webapp-1.2.0.tgz" --set replicaCount=3 >/dev/null """, "checks": [ C("defaults-1.3.0.yaml is the chart's default values", 1, r""" f="$WS/q21/defaults-1.3.0.yaml"; [ -s "$f" ] && diff -Bw <(helm show values "$WS/q21/charts/webapp-1.3.0.tgz" 2>/dev/null) "$f" >/dev/null"""), C("%{rel} deployed from webapp-1.3.0 with replicaCount 2 and a NodePort Service", 3, r""" helm -n %{ns} list -o json 2>/dev/null | python3 -c ' import sys, json r = {x["name"]: x for x in json.load(sys.stdin)}.get("%{rel}") or {} sys.exit(0 if r.get("chart") == "webapp-1.3.0" and r.get("status") == "deployed" else 1)' || exit 1 helm -n %{ns} get values %{rel} -o json 2>/dev/null | python3 -c ' import sys, json d = json.load(sys.stdin) or {} sys.exit(0 if d.get("replicaCount") == 2 and (d.get("service") or {}).get("type") == "NodePort" else 1)' || exit 1 eq "$(j -n %{ns} svc %{rel}-webapp '{.spec.type}')" NodePort || exit 1 for i in $(seq 1 20); do [ "$(j -n %{ns} deploy %{rel}-webapp '{.status.availableReplicas}')" = 2 ] && exit 0; sleep 3; done; exit 1"""), C("the values file sets both", 1, r""" f="$WS/q21/%{rel}-values.yaml"; grep -Eq '^replicaCount: *2 *$' "$f" 2>/dev/null && grep -Eq '^ +type: *"?NodePort"? *$' "$f" """), C("%{old} upgraded to webapp-1.3.0 and still runs 3 replicas", 3, r""" helm -n %{ns} list -o json 2>/dev/null | python3 -c ' import sys, json r = {x["name"]: x for x in json.load(sys.stdin)}.get("%{old}") or {} sys.exit(0 if r.get("chart") == "webapp-1.3.0" and r.get("status") == "deployed" and int(r.get("revision", 0)) >= 2 else 1)' || exit 1 for i in $(seq 1 20); do [ "$(j -n %{ns} deploy %{old}-webapp '{.spec.replicas}/{.status.availableReplicas}')" = 3/3 ] && exit 0; sleep 3; done; exit 1"""), ], "solution": """ helm show values {WS}/q21/charts/webapp-1.3.0.tgz > {WS}/q21/defaults-1.3.0.yaml cat > {WS}/q21/%{rel}-values.yaml </dev/null 2>&1 $K -n %{ns} create deploy %{app} --image=nginx:1.27 --replicas=2 --dry-run=client -o yaml | $K apply -f - >/dev/null $K -n %{ns} expose deploy %{app} --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null cat > "$WS/q22/legacy.yaml" <<'EOF' apiVersion: extensions/v1beta1 kind: Ingress metadata: {name: %{app}, namespace: %{ns}} spec: rules: - host: %{app}.example.com http: paths: - path: / backend: {serviceName: %{app}, servicePort: 80} --- apiVersion: batch/v1beta1 kind: CronJob metadata: {name: %{app}-report, namespace: %{ns}} spec: schedule: "15 2 * * *" jobTemplate: spec: template: spec: restartPolicy: OnFailure containers: [{name: report, image: busybox:1.36, command: [sh, -c, "echo report"]}] --- apiVersion: policy/v1beta1 kind: PodDisruptionBudget metadata: {name: %{app}, namespace: %{ns}} spec: minAvailable: 1 selector: {matchLabels: {app: %{app}}} --- apiVersion: autoscaling/v2beta2 kind: HorizontalPodAutoscaler metadata: {name: %{app}, namespace: %{ns}} spec: scaleTargetRef: {apiVersion: apps/v1, kind: Deployment, name: %{app}} minReplicas: 2 maxReplicas: 5 metrics: - type: Resource resource: {name: cpu, target: {type: Utilization, averageUtilization: 70}} EOF """, "checks": [ C("removed.txt lists exactly the four removed versions", 2, r""" python3 - "$WS/q22/removed.txt" <<'PY' import sys want = {"extensions/v1beta1", "batch/v1beta1", "policy/v1beta1", "autoscaling/v2beta2"} try: got = {l.strip() for l in open(sys.argv[1]) if l.strip()} except OSError: sys.exit(1) sys.exit(0 if got == want else 1) PY"""), C("Ingress %{app}: host %{app}.example.com, / -> Service %{app}:80", 2, r""" i="$K -n %{ns} get ingress %{app}" eq "$($i -o jsonpath='{.spec.rules[0].host}' 2>/dev/null)" %{app}.example.com \ && eq "$($i -o jsonpath='{.spec.rules[0].http.paths[0].path}')" / \ && eq "$($i -o jsonpath='{.spec.rules[0].http.paths[0].backend.service.name}')" %{app} \ && eq "$($i -o jsonpath='{.spec.rules[0].http.paths[0].backend.service.port.number}')" 80"""), C("CronJob %{app}-report on '15 2 * * *' and PDB %{app} minAvailable 1", 2, r""" eq "$(j -n %{ns} cronjob %{app}-report '{.spec.schedule}')" "15 2 * * *" \ && eq "$(j -n %{ns} pdb %{app} '{.spec.minAvailable}/{.spec.selector.matchLabels.app}')" 1/%{app}"""), C("HPA %{app}: 2-5 replicas at 70% CPU", 1, r""" eq "$(j -n %{ns} hpa %{app} '{.spec.minReplicas}/{.spec.maxReplicas}/{.spec.metrics[0].resource.target.averageUtilization}/{.spec.scaleTargetRef.name}')" 2/5/70/%{app}"""), C("fixed.yaml holds all four objects and applies cleanly", 1, r""" f="$WS/q22/fixed.yaml"; [ "$(grep -c '^kind:' "$f" 2>/dev/null)" -ge 4 ] && $K apply --dry-run=server -f "$f" >/dev/null 2>&1"""), ], "solution": """ k apply -f {WS}/q22/legacy.yaml --dry-run=server # no matches for kind "Ingress" in version "extensions/v1beta1" ... k api-resources | grep -Ei 'ingresses|cronjobs|poddisruptionbudgets|horizontalpodautoscalers' # APIVERSION column k explain ingress.spec.rules.http.paths.backend # the v1 backend shape cp {WS}/q22/legacy.yaml {WS}/q22/fixed.yaml && vi {WS}/q22/fixed.yaml # extensions/v1beta1 Ingress -> networking.k8s.io/v1, and the backend changes shape: # - path: / # pathType: Prefix (now required) # backend: {service: {name: %{app}, port: {number: 80}}} # batch/v1beta1 CronJob -> batch/v1 spec unchanged # policy/v1beta1 PDB -> policy/v1 spec unchanged # autoscaling/v2beta2 HPA -> autoscaling/v2 spec unchanged printf '%s\\n' extensions/v1beta1 batch/v1beta1 policy/v1beta1 autoscaling/v2beta2 > {WS}/q22/removed.txt k apply -f {WS}/q22/fixed.yaml # The `kubectl convert` plugin can do the rewrite, but it is not installed by default - the # Ingress backend change is the one to know by heart. """, }, { "id": 23, "domain": "OBS", "title": "Find the heaviest Pods with kubectl top", "params": lambda r: dict(zip(("cpu", "mem", "idle"), [f"{x}-{r.randint(10, 99)}" for x in r.sample(["orders", "billing", "search", "profile", "ledger"], 3)]), ns=ns_pick(r, "watch")), "task": """ Namespace `%{ns}` runs three Pods. Using the metrics API (metrics-server is installed): 1. Write the name of the Pod using the most **CPU** to `{WS}/q23/cpu.txt`. 2. Write the name of the Pod using the most **memory** to `{WS}/q23/memory.txt`. 3. Write a command that shows the current CPU and memory usage of every **node** to `{WS}/q23/nodes.sh`. 4. Write a command that lists the Pods of **all** Namespaces sorted by memory usage to `{WS}/q23/pods.sh`. """, "setup": NS + r""" ensure_metrics_server || echo "warn: metrics-server could not be installed (no network?) - kubectl top will not work" mkns %{ns}; rm -rf "$WS/q23"; mkdir -p "$WS/q23" $K -n %{ns} delete pod %{cpu} %{mem} %{idle} --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1 $K apply -f - >/dev/null <<'EOF' apiVersion: v1 kind: Pod metadata: {name: %{cpu}, namespace: %{ns}, labels: {app: watch}} spec: containers: - name: main image: busybox:1.36 command: [sh, -c, "timeout 7200 sh -c 'while :; do :; done'; sleep 86400"] resources: {requests: {cpu: 50m, memory: 16Mi}, limits: {cpu: 200m, memory: 64Mi}} --- apiVersion: v1 kind: Pod metadata: {name: %{mem}, namespace: %{ns}, labels: {app: watch}} spec: volumes: [{name: cache, emptyDir: {medium: Memory, sizeLimit: 160Mi}}] containers: - name: main image: busybox:1.36 command: [sh, -c, "dd if=/dev/zero of=/cache/fill bs=1M count=120 2>/dev/null; sleep 86400"] volumeMounts: [{name: cache, mountPath: /cache}] resources: {requests: {cpu: 10m, memory: 160Mi}, limits: {cpu: 100m, memory: 256Mi}} --- apiVersion: v1 kind: Pod metadata: {name: %{idle}, namespace: %{ns}, labels: {app: watch}} spec: containers: - name: main image: busybox:1.36 command: [sleep, "86400"] resources: {requests: {cpu: 10m, memory: 16Mi}, limits: {cpu: 100m, memory: 64Mi}} EOF $K -n %{ns} get pod %{cpu} %{mem} %{idle} >/dev/null """, "checks": [ C("cpu.txt names %{cpu}", 2, r"""eq "$(tr -d '[:space:]' < "$WS/q23/cpu.txt" 2>/dev/null)" %{cpu}"""), C("memory.txt names %{mem}", 2, r"""eq "$(tr -d '[:space:]' < "$WS/q23/memory.txt" 2>/dev/null)" %{mem}"""), C("nodes.sh uses kubectl top for nodes", 1, r"""grep -Eq '(kubectl|k) +top +(node|nodes|no)( |$)' "$WS/q23/nodes.sh" 2>/dev/null"""), C("pods.sh: top pods of all namespaces sorted by memory", 1, r""" f="$WS/q23/pods.sh"; grep -Eq '(kubectl|k) +top +(pod|pods|po)( |$)' "$f" 2>/dev/null \ && grep -Eq '(-A( |$)|--all-namespaces)' "$f" && grep -Eq -- '--sort-by[= ]memory' "$f" """), ], "solution": """ k top pod -n %{ns} # give metrics-server ~30 s after the Pods start k top pod -n %{ns} --sort-by=cpu --no-headers | head -1 | awk '{print $1}' > {WS}/q23/cpu.txt k top pod -n %{ns} --sort-by=memory --no-headers | head -1 | awk '{print $1}' > {WS}/q23/memory.txt echo 'kubectl top node' > {WS}/q23/nodes.sh echo 'kubectl top pod -A --sort-by=memory' > {WS}/q23/pods.sh # `kubectl top` reads the metrics API (metrics.k8s.io), served by metrics-server; without it you get # "Metrics API not available". `--containers` breaks a Pod's usage down per container. """, }, { "id": 24, "domain": "OBS", "title": "Logs of a container that keeps crashing", "params": lambda r: {"ns": ns_pick(r, "ingest"), "pod": name_pick(r, "ingest"), "cfg": r.choice(["pipeline", "routes", "sinks", "tenants"])}, "task": """ Pod `%{pod}` in Namespace `%{ns}` has two containers, and one of them keeps restarting. 1. Save the complete log of the **previous** (crashed) run of the failing container to `{WS}/q24/crash.log`. 2. Write the full path of the file that the crashed run could not open to `{WS}/q24/missing.txt`. 3. Save the last **3** log lines of the other container, **with timestamps**, to `{WS}/q24/helper.log`. """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q24"; mkdir -p "$WS/q24" $K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null $K apply -f - >/dev/null <<'EOF' apiVersion: v1 kind: Pod metadata: {name: %{pod}, namespace: %{ns}} spec: containers: - name: ingest image: busybox:1.36 command: [sh, -c, "echo \"ingest starting (pid $$)\"; echo \"loading configuration\"; sleep 12; echo \"FATAL: cannot open /etc/ingest/%{cfg}.yaml: no such file or directory\"; exit 1"] - name: helper image: busybox:1.36 command: [sh, -c, "while true; do echo \"helper tick $(date +%s)\"; sleep 5; done"] EOF $K -n %{ns} get pod %{pod} >/dev/null """, "checks": [ C("crash.log is one full crashed run of container ingest", 3, r""" f="$WS/q24/crash.log"; grep -q 'ingest starting' "$f" 2>/dev/null && grep -q 'FATAL: cannot open /etc/ingest/%{cfg}.yaml' "$f" """), C("missing.txt holds /etc/ingest/%{cfg}.yaml", 2, r"""eq "$(tr -d '[:space:]' < "$WS/q24/missing.txt" 2>/dev/null)" /etc/ingest/%{cfg}.yaml"""), C("helper.log: its last 3 lines, timestamped", 2, r""" f="$WS/q24/helper.log"; [ "$(grep -c . "$f" 2>/dev/null)" = 3 ] \ && [ "$(grep -Ec '^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z helper tick [0-9]+$' "$f")" = 3 ]"""), ], "solution": """ k -n %{ns} get pod %{pod} # 1/2, RESTARTS climbing: container ingest k -n %{ns} logs %{pod} -c ingest --previous > {WS}/q24/crash.log cat {WS}/q24/crash.log # FATAL: cannot open /etc/ingest/%{cfg}.yaml ... echo /etc/ingest/%{cfg}.yaml > {WS}/q24/missing.txt k -n %{ns} logs %{pod} -c helper --tail=3 --timestamps > {WS}/q24/helper.log # Without -c, logs of a multi-container Pod asks you to choose a container. --previous reads the # last terminated instance - while the new one is still running it is the only place the crash # message exists. `describe pod` shows Last State: Terminated, Exit Code: 1 as well. # For a few seconds right after a crash (STATUS Error, before CrashLoopBackOff) --previous can fail # with "unable to retrieve container logs": it points at the run BEFORE the one that just died, and # that one is already garbage-collected. Plain `logs` shows the crashed run at that moment - read # STATUS first, or wait a few seconds and try --previous again. """, }, { "id": 25, "domain": "OBS", "title": "Two faults behind a stuck Deployment", "params": lambda r: {"ns": ns_pick(r, "fix"), "dep": name_pick(r, "catalog"), "cm": name_pick(r, "catalog-config"), "lvl": r.choice(["info", "warn", "debug"])}, "task": """ Deployment `%{dep}` in Namespace `%{ns}` has 0 of 2 Pods ready. Fix it so both replicas are Running and Ready. Do not modify ConfigMap `%{cm}`: the container must keep taking its `LOG_LEVEL` environment variable from that ConfigMap. Then write the value the containers see for `LOG_LEVEL` to `{WS}/q25/log-level.txt`. """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q25"; mkdir -p "$WS/q25" $K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null $K -n %{ns} delete cm %{cm} --ignore-not-found >/dev/null $K apply -f - >/dev/null <<'EOF' apiVersion: v1 kind: ConfigMap metadata: {name: %{cm}, namespace: %{ns}} data: {log_level: %{lvl}, cache_size: "256"} --- apiVersion: apps/v1 kind: Deployment metadata: {name: %{dep}, namespace: %{ns}} spec: replicas: 2 selector: {matchLabels: {app: %{dep}}} template: metadata: {labels: {app: %{dep}}} spec: containers: - name: web image: nginx:1.27 command: [nginxx, -g, "daemon off;"] env: - name: LOG_LEVEL valueFrom: {configMapKeyRef: {name: %{cm}, key: log-level}} EOF $K -n %{ns} get cm %{cm} -o jsonpath='{.metadata.resourceVersion}' > "$WS/.q25-cm-rv" """, "checks": [ C("2/2 Pods Ready", 3, r""" for i in $(seq 1 30); do [ "$(j -n %{ns} deploy %{dep} '{.status.readyReplicas}/{.status.updatedReplicas}/{.status.replicas}')" = 2/2/2 ] && exit 0; sleep 3; done; exit 1"""), C("LOG_LEVEL still comes from %{cm} (key log_level) and reads %{lvl}", 2, r""" e='{.spec.template.spec.containers[0].env[?(@.name=="LOG_LEVEL")].valueFrom.configMapKeyRef' eq "$(j -n %{ns} deploy %{dep} "$e.name}")" %{cm} && eq "$(j -n %{ns} deploy %{dep} "$e.key}")" log_level \ && eq "$($K -n %{ns} exec deploy/%{dep} -- printenv LOG_LEVEL 2>/dev/null)" %{lvl}"""), C("ConfigMap %{cm} untouched", 1, r""" [ "$(j -n %{ns} deploy %{dep} '{.metadata.generation}')" -gt 1 ] 2>/dev/null && eq "$(j -n %{ns} cm %{cm} '{.metadata.resourceVersion}')" "$(cat "$WS/.q25-cm-rv" 2>/dev/null)" """), C("log-level.txt holds %{lvl}", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q25/log-level.txt" 2>/dev/null)" %{lvl}"""), ], "solution": """ k -n %{ns} get pod # CreateContainerConfigError k -n %{ns} events --types=Warning # couldn't find key log-level in ConfigMap %{ns}/%{cm} # (kubectl 1.36+ describe drops Events when a selector matches several Pods - add --show-events) k -n %{ns} get cm %{cm} -o yaml # the key is log_level (underscore) k -n %{ns} edit deploy %{dep} # key: log-level -> key: log_level k -n %{ns} get pod # now RunContainerError / CrashLoopBackOff k -n %{ns} describe pod -l app=%{dep} | grep -iE 'reason|message' | head # exec: "nginxx": executable file not found k -n %{ns} edit deploy %{dep} # delete the command: line (the image's own entrypoint runs nginx) k -n %{ns} rollout status deploy %{dep} k -n %{ns} exec deploy/%{dep} -- printenv LOG_LEVEL > {WS}/q25/log-level.txt # The second fault only shows once the first is fixed: a container whose config cannot be built is # never started, so its command is never tried. Fix, re-read the status, repeat. """, }, { "id": 26, "domain": "OBS", "title": "OOMKilled", "params": lambda r: {"ns": ns_pick(r, "media"), "dep": name_pick(r, "thumbnailer")}, "task": """ Pods of Deployment `%{dep}` in Namespace `%{ns}` keep restarting. 1. Write the reason the container was last terminated, exactly as Kubernetes reports it, to `{WS}/q26/reason.txt`. 2. Fix it by raising only the container's **memory limit**, to `256Mi` - leave its requests alone. The new Pod must stay up without restarting. """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q26"; mkdir -p "$WS/q26" $K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null $K apply -f - >/dev/null <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: {name: %{dep}, namespace: %{ns}} spec: replicas: 1 selector: {matchLabels: {app: %{dep}}} template: metadata: {labels: {app: %{dep}}} spec: containers: - name: resize image: busybox:1.36 command: [sh, -c, "dd if=/dev/zero of=/dev/null bs=150M count=1 2>/dev/null && echo warmed up; sleep 86400"] resources: {requests: {cpu: 50m, memory: 32Mi}, limits: {memory: 64Mi}} EOF """, "checks": [ C("reason.txt holds OOMKilled", 2, r"""eq "$(tr -d '[:space:]' < "$WS/q26/reason.txt" 2>/dev/null)" OOMKilled"""), C("memory limit 256Mi, requests unchanged (cpu 50m, memory 32Mi)", 2, r""" r='{.spec.template.spec.containers[0].resources' eq "$(j -n %{ns} deploy %{dep} "$r.limits.memory}")" 256Mi && eq "$(j -n %{ns} deploy %{dep} "$r.requests.memory}")" 32Mi \ && eq "$(j -n %{ns} deploy %{dep} "$r.requests.cpu}")" 50m"""), C("the new Pod is Ready and has not restarted", 3, r""" for i in $(seq 1 40); do if [ "$(j -n %{ns} deploy %{dep} '{.status.updatedReplicas}/{.status.availableReplicas}/{.status.replicas}')" = 1/1/1 ] \ && [ "$(j -n %{ns} pod -l app=%{dep} '{.items[*].status.containerStatuses[0].restartCount}')" = 0 ]; then [ "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].resources.limits.memory}')" = 256Mi ] && exit 0 fi; sleep 3 done; exit 1"""), ], "solution": """ k -n %{ns} get pod # RESTARTS climbing, CrashLoopBackOff k -n %{ns} get pod -l app=%{dep} -o jsonpath='{.items[0].status.containerStatuses[0].lastState.terminated.reason}' # OOMKilled -> {WS}/q26/reason.txt (describe pod: Last State: Terminated, Reason: OOMKilled, Exit Code: 137) k -n %{ns} set resources deploy %{dep} -c resize --limits=memory=256Mi k -n %{ns} rollout status deploy %{dep} k -n %{ns} get pod # 0 restarts # `set resources --limits` only touches limits; requests stay as they were. The kernel kills a # container that crosses its memory limit - that is OOMKilled, not a crash in the app itself. """, }, # ------------------------------------------------------------------- ENV ---- { "id": 27, "domain": "ENV", "title": "Discover and use a custom resource", "params": lambda r: {"ns": ns_pick(r, "dataops"), "name": name_pick(r, "nightly"), "n": r.choice([3, 4, 5]), "target": r.choice(["orders-db", "billing-db", "users-db"])}, "task": """ An operator's CustomResourceDefinition for **backups** is installed in this cluster. 1. Write the CRD's full name to `{WS}/q27/crd.txt`. 2. Before you create anything, write how many objects of that kind exist in the **whole cluster** to `{WS}/q27/count.txt`. 3. In Namespace `%{ns}` create one named `%{name}` that backs up `%{target}` every day at 03:30 (cron `30 3 * * *`) and keeps each backup for 14 days. Use the field names the CRD defines. """, "setup": NS + r""" rm -rf "$WS/q27"; mkdir -p "$WS/q27" $K apply -f - >/dev/null <<'EOF' apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: {name: backups.data.example.com} spec: group: data.example.com scope: Namespaced names: {plural: backups, singular: backup, kind: Backup, shortNames: [bkp]} versions: - name: v1 served: true storage: true additionalPrinterColumns: - {name: Target, type: string, jsonPath: .spec.target} - {name: Schedule, type: string, jsonPath: .spec.schedule} schema: openAPIV3Schema: type: object properties: spec: type: object required: [target, schedule, retentionDays] properties: target: {type: string, description: "Name of the database to back up"} schedule: {type: string, description: "Cron expression"} retentionDays: {type: integer, minimum: 1, maximum: 90, description: "Days each backup is kept"} EOF $K wait --for=condition=Established crd/backups.data.example.com --timeout=60s >/dev/null mkns %{ns}; mkns %{ns}-archive $K delete backups.data.example.com --all -A --wait=true >/dev/null 2>&1 i=0 for spec in "orders-db|0 1 * * *|7" "users-db|30 1 * * *|30" "billing-db|0 2 * * 0|90" "search-db|15 4 * * *|14" "audit-db|0 5 1 * *|60"; do i=$((i+1)); [ "$i" -gt %{n} ] && break t=${spec%%|*}; rest=${spec#*|}; s=${rest%%|*}; d=${rest#*|} bns=%{ns}-archive; [ "$i" = 1 ] && bns=%{ns} printf 'apiVersion: data.example.com/v1\nkind: Backup\nmetadata: {name: %s-weekly, namespace: %s}\nspec: {target: %s, schedule: "%s", retentionDays: %s}\n' \ "$t" "$bns" "$t" "$s" "$d" | $K apply -f - >/dev/null done """, "checks": [ C("crd.txt holds backups.data.example.com", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q27/crd.txt" 2>/dev/null)" backups.data.example.com"""), C("count.txt holds %{n}", 2, r"""eq "$(tr -d '[:space:]' < "$WS/q27/count.txt" 2>/dev/null)" %{n}"""), C("Backup %{name}: target %{target}, schedule 30 3 * * *, retentionDays 14", 3, r""" b="$K -n %{ns} get backups.data.example.com %{name}" eq "$($b -o jsonpath='{.spec.target}' 2>/dev/null)" %{target} && eq "$($b -o jsonpath='{.spec.schedule}')" "30 3 * * *" \ && eq "$($b -o jsonpath='{.spec.retentionDays}')" 14"""), ], "solution": """ k get crd | grep -i backup # backups.data.example.com echo backups.data.example.com > {WS}/q27/crd.txt k api-resources --api-group=data.example.com # kind Backup, short name bkp, namespaced k get backups -A --no-headers | wc -l > {WS}/q27/count.txt k explain backup.spec # target, schedule, retentionDays (all required) cat </dev/null; $K -n %{ns} delete role %{sa}-role --ignore-not-found >/dev/null $K -n %{ns} delete sa %{sa} --ignore-not-found >/dev/null """, "checks": [ C("Role %{sa}-role bound to ServiceAccount %{sa} by %{sa}-binding", 1, r""" b="$K -n %{ns} get rolebinding %{sa}-binding" eq "$($b -o jsonpath='{.roleRef.kind}/{.roleRef.name}' 2>/dev/null)" Role/%{sa}-role \ && eq "$($b -o jsonpath='{.subjects[0].kind}/{.subjects[0].name}/{.subjects[0].namespace}')" ServiceAccount/%{sa}/%{ns}"""), C("can get, list and watch Pods and read pods/log", 2, r""" a="--as=system:serviceaccount:%{ns}:%{sa} -n %{ns}" for v in get list watch; do eq "$($K auth can-i $v pods $a 2>/dev/null)" yes || exit 1; done eq "$($K auth can-i get pods --subresource=log $a 2>/dev/null)" yes"""), C("can get, list, update, patch and scale Deployments", 2, r""" a="--as=system:serviceaccount:%{ns}:%{sa} -n %{ns}" for v in get list update patch; do eq "$($K auth can-i $v deployments.apps $a 2>/dev/null)" yes || exit 1; done eq "$($K auth can-i patch deployments.apps --subresource=scale $a 2>/dev/null)" yes || eq "$($K auth can-i update deployments.apps --subresource=scale $a 2>/dev/null)" yes"""), C("nothing else: no deleting Pods, creating Deployments, reading Secrets, or other Namespaces", 2, r""" a="--as=system:serviceaccount:%{ns}:%{sa}" eq "$($K auth can-i list pods $a -n %{ns} 2>/dev/null)" yes || exit 1 for t in "delete pods -n %{ns}" "create deployments.apps -n %{ns}" "list secrets -n %{ns}" "list pods -n default" "delete deployments.apps -n %{ns}"; do eq "$($K auth can-i $t $a 2>/dev/null)" no || exit 1 done"""), ], "solution": """ k -n %{ns} create sa %{sa} k -n %{ns} create role %{sa}-role --verb=get,list,watch --resource=pods,pods/log $do > role.yaml vi role.yaml # add a second rule: # - apiGroups: [apps] # resources: [deployments, deployments/scale] # verbs: [get, list, update, patch] k apply -f role.yaml k -n %{ns} create rolebinding %{sa}-binding --role=%{sa}-role --serviceaccount=%{ns}:%{sa} k auth can-i watch pods -n %{ns} --as=system:serviceaccount:%{ns}:%{sa} # yes k auth can-i get pods --subresource=log -n %{ns} --as=system:serviceaccount:%{ns}:%{sa} # yes k auth can-i patch deployments --subresource=scale -n %{ns} --as=system:serviceaccount:%{ns}:%{sa} # yes k auth can-i delete pods -n %{ns} --as=system:serviceaccount:%{ns}:%{sa} # no # Subresources are separate resources to RBAC: `pods` does not include `pods/log`, and # `deployments` does not include `deployments/scale` (what `kubectl scale` calls). # One `create role` can only hold one rule's verbs, hence the edit. """, }, { "id": 29, "domain": "ENV", "title": "Pod Security admission blocks a Deployment", "params": lambda r: {"ns": ns_pick(r, "secure"), "dep": name_pick(r, "vault-agent"), "uid": r.choice([1000, 2000, 3000])}, "task": """ Deployment `%{dep}` in Namespace `%{ns}` shows `0/2` and none of its Pods exist. 1. Find out why, and write the Pod Security level the Namespace enforces to `{WS}/q29/level.txt`. 2. Fix the **Deployment** (leave the Namespace as it is) so both replicas run. Its container must run as user `%{uid}`. """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q29"; mkdir -p "$WS/q29" $K label ns %{ns} pod-security.kubernetes.io/enforce=restricted pod-security.kubernetes.io/enforce-version=latest --overwrite >/dev/null $K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null $K apply -f - >/dev/null 2>&1 <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: {name: %{dep}, namespace: %{ns}} spec: replicas: 2 selector: {matchLabels: {app: %{dep}}} template: metadata: {labels: {app: %{dep}}} spec: containers: - name: agent image: busybox:1.36 command: [sh, -c, "echo agent up as $(id -u); sleep 86400"] EOF """, "checks": [ C("level.txt holds restricted", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q29/level.txt" 2>/dev/null)" restricted"""), C("2/2 available and the Namespace still enforces restricted", 4, r""" eq "$(j ns %{ns} '{.metadata.labels.pod-security\.kubernetes\.io/enforce}')" restricted || exit 1 for i in $(seq 1 30); do [ "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" = 2 ] && exit 0; sleep 3; done; exit 1"""), C("the container runs as UID %{uid}", 2, r"""eq "$($K -n %{ns} exec deploy/%{dep} -- id -u 2>/dev/null)" %{uid}"""), ], "solution": """ k -n %{ns} get deploy,rs,pod # desired 2, current 0 k -n %{ns} describe rs -l app=%{dep} | tail -5 # Error creating: pods ... is forbidden: violates PodSecurity "restricted:latest": # allowPrivilegeEscalation != false, unrestricted capabilities, runAsNonRoot != true, seccompProfile ... k get ns %{ns} --show-labels # pod-security.kubernetes.io/enforce=restricted echo restricted > {WS}/q29/level.txt k -n %{ns} edit deploy %{dep} # template.spec: # securityContext: {runAsNonRoot: true, runAsUser: %{uid}, seccompProfile: {type: RuntimeDefault}} # containers[0].securityContext: {allowPrivilegeEscalation: false, capabilities: {drop: [ALL]}} k -n %{ns} get deploy %{dep} -w # Admission rejects the Pods, not the Deployment, so the only symptom on the Deployment is 0/2 - # the reason is in the ReplicaSet's events. busybox runs as root by default, so runAsNonRoot # alone is not enough: it needs a non-zero runAsUser too. """, }, { "id": 30, "domain": "ENV", "title": "Quota with LimitRange defaults", "params": lambda r: {"ns": ns_pick(r, "capped"), "dep": name_pick(r, "api")}, "task": """ Namespace `%{ns}` must be capped: 1. ResourceQuota `compute-quota`: `requests.cpu` 1, `requests.memory` 1Gi, `limits.cpu` 2, `limits.memory` 2Gi, and at most 6 Pods. 2. LimitRange `container-defaults`: a container that declares no resources gets requests cpu `100m` / memory `128Mi` and limits cpu `250m` / memory `256Mi`; no container may set a limit above cpu `500m` / memory `512Mi`. 3. Create Deployment `%{dep}` (`nginx:1.27`, 3 replicas) **without** a resources section. It must run. """, "setup": NS + r""" mkns %{ns} $K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null $K -n %{ns} delete resourcequota --all >/dev/null 2>&1; $K -n %{ns} delete limitrange --all >/dev/null 2>&1 """, "checks": [ C("ResourceQuota compute-quota: 1 / 1Gi requests, 2 / 2Gi limits, 6 pods", 2, r""" $K -n %{ns} get resourcequota compute-quota -o json 2>/dev/null | python3 -c ' import sys, json h = json.load(sys.stdin)["spec"]["hard"] want = {"requests.cpu": "1", "requests.memory": "1Gi", "limits.cpu": "2", "limits.memory": "2Gi", "pods": "6"} sys.exit(0 if all(h.get(k) == v for k, v in want.items()) else 1)'"""), C("LimitRange container-defaults: defaults and maximums", 2, r""" $K -n %{ns} get limitrange container-defaults -o json 2>/dev/null | python3 -c ' import sys, json c = [l for l in json.load(sys.stdin)["spec"]["limits"] if l.get("type") == "Container"] ok = len(c) == 1 and c[0].get("defaultRequest") == {"cpu": "100m", "memory": "128Mi"} \ and c[0].get("default") == {"cpu": "250m", "memory": "256Mi"} and c[0].get("max") == {"cpu": "500m", "memory": "512Mi"} sys.exit(0 if ok else 1)'"""), C("Deployment %{dep}: 3 available, no resources of its own, Pods got the defaults", 2, r""" [ -n "$(j -n %{ns} deploy %{dep} '{.metadata.name}')" ] || exit 1 [ -z "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].resources.requests}{.spec.template.spec.containers[0].resources.limits}')" ] || exit 1 for i in $(seq 1 30); do [ "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" = 3 ] && break; sleep 3; done eq "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" 3 \ && eq "$(j -n %{ns} pod -l app=%{dep} '{.items[0].spec.containers[0].resources.requests.cpu}/{.items[0].spec.containers[0].resources.limits.memory}')" 100m/256Mi"""), C("the quota counts the three Pods (300m CPU requested)", 1, r""" eq "$(j -n %{ns} resourcequota compute-quota '{.status.used.requests\.cpu}/{.status.used.pods}')" 300m/3"""), ], "solution": """ k -n %{ns} create quota compute-quota --hard=requests.cpu=1,requests.memory=1Gi,limits.cpu=2,limits.memory=2Gi,pods=6 cat </dev/null $K apply -f - >/dev/null <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: {name: %{dep}, namespace: %{ns}} spec: replicas: 2 selector: {matchLabels: {app: %{dep}}} template: metadata: {labels: {app: %{dep}}} spec: containers: - {name: app, image: nginx:1.27} - {name: metrics, image: busybox:1.36, command: [sleep, "86400"]} EOF """, "checks": [ C("app: cpu 250m / memory 128Mi, requests equal to limits", 2, r""" r='{.spec.template.spec.containers[?(@.name=="app")].resources' eq "$(j -n %{ns} deploy %{dep} "$r.limits.cpu}/$r.limits.memory}")" 250m/128Mi || exit 1 q=$(j -n %{ns} deploy %{dep} "$r.requests.cpu}/$r.requests.memory}"); [ "$q" = / ] || [ "$q" = 250m/128Mi ]"""), C("metrics: cpu 50m / memory 32Mi, requests equal to limits", 2, r""" r='{.spec.template.spec.containers[?(@.name=="metrics")].resources' eq "$(j -n %{ns} deploy %{dep} "$r.limits.cpu}/$r.limits.memory}")" 50m/32Mi || exit 1 q=$(j -n %{ns} deploy %{dep} "$r.requests.cpu}/$r.requests.memory}"); [ "$q" = / ] || [ "$q" = 50m/32Mi ]"""), C("rolled out; its running Pods are Guaranteed", 2, r""" for i in $(seq 1 30); do [ "$(j -n %{ns} deploy %{dep} '{.status.updatedReplicas}/{.status.availableReplicas}/{.status.replicas}')" = 2/2/2 ] && break; sleep 3; done # old Pods linger in Terminating for the grace period (sleep ignores SIGTERM) - judge only live ones $K -n %{ns} get pod -l app=%{dep} -o json 2>/dev/null | python3 -c ' import sys, json live = [p for p in json.load(sys.stdin)["items"] if not p["metadata"].get("deletionTimestamp") and p["status"].get("phase") == "Running"] sys.exit(0 if len(live) == 2 and all(p["status"].get("qosClass") == "Guaranteed" for p in live) else 1)'"""), C("qos.txt holds Guaranteed", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q31/qos.txt" 2>/dev/null)" Guaranteed"""), ], "solution": """ k -n %{ns} set resources deploy %{dep} -c app --requests=cpu=250m,memory=128Mi --limits=cpu=250m,memory=128Mi k -n %{ns} set resources deploy %{dep} -c metrics --requests=cpu=50m,memory=32Mi --limits=cpu=50m,memory=32Mi k -n %{ns} rollout status deploy %{dep} k -n %{ns} get pod -l app=%{dep} # wait until the old Pods are gone, not just Terminating: # `sleep` ignores SIGTERM, so they linger ~30 s, still BestEffort k -n %{ns} get pod -l app=%{dep} -o jsonpath='{.items[0].status.qosClass}' > {WS}/q31/qos.txt # Guaranteed = EVERY container has cpu and memory limits, and requests equal to them (limits # alone also works: requests default to the limits). One container short of that makes the whole # Pod Burstable; nothing set anywhere is BestEffort. """, }, { "id": 32, "domain": "ENV", "title": "ConfigMap as an env var and as a file", "params": lambda r: {"ns": ns_pick(r, "conf"), "cm": name_pick(r, "app-config"), "pod": name_pick(r, "reader"), "color": r.choice(["teal", "amber", "violet", "coral"])}, "task": """ In Namespace `%{ns}`: 1. Create ConfigMap `%{cm}` holding the file `{WS}/q32/app.properties` under the key `app.properties`, plus a key `TIER` with the value `backend`. 2. Create Pod `%{pod}` (`busybox:1.36`, command `sleep 86400`) that - gets the environment variable `APP_TIER` from the ConfigMap's `TIER` key, - mounts **only** the `app.properties` key, as the file `/etc/app/app.properties`. """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q32"; mkdir -p "$WS/q32" $K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null; $K -n %{ns} delete cm %{cm} --ignore-not-found >/dev/null printf 'color=%{color}\nretries=3\nmode=strict\n' > "$WS/q32/app.properties" """, "checks": [ C("ConfigMap: app.properties matches the file, TIER=backend", 2, r""" [ "$(j -n %{ns} cm %{cm} '{.data.app\.properties}')" = "$(cat "$WS/q32/app.properties")" ] && eq "$(j -n %{ns} cm %{cm} '{.data.TIER}')" backend"""), C("APP_TIER comes from key TIER and reads backend", 2, r""" e='{.spec.containers[0].env[?(@.name=="APP_TIER")].valueFrom.configMapKeyRef' eq "$(j -n %{ns} pod %{pod} "$e.name}/$e.key}")" %{cm}/TIER && eq "$($K -n %{ns} exec %{pod} -- printenv APP_TIER 2>/dev/null)" backend"""), C("/etc/app holds only app.properties, with the file's content", 2, r""" eq "$($K -n %{ns} exec %{pod} -- ls /etc/app 2>/dev/null)" app.properties \ && [ "$($K -n %{ns} exec %{pod} -- cat /etc/app/app.properties 2>/dev/null)" = "$(cat "$WS/q32/app.properties")" ]"""), ], "solution": """ k -n %{ns} create cm %{cm} --from-file=app.properties={WS}/q32/app.properties --from-literal=TIER=backend cat </dev/null $K -n %{ns} delete secret %{old} %{sec} --ignore-not-found >/dev/null $K -n %{ns} create secret generic %{old} --from-literal=username=legacy-app --from-literal=password=%{oldpw} >/dev/null """, "checks": [ C("old-password.txt holds the decoded password", 2, r"""eq "$(tr -d '[:space:]' < "$WS/q33/old-password.txt" 2>/dev/null)" %{oldpw}"""), C("Secret %{sec}: username and password", 1, r""" eq "$(j -n %{ns} secret %{sec} '{.data.username}' | base64 -d 2>/dev/null)" %{user} && eq "$(j -n %{ns} secret %{sec} '{.data.password}' | base64 -d 2>/dev/null)" %{pw}"""), C("API_USER comes from the Secret and reads %{user}", 2, r""" e='{.spec.containers[0].env[?(@.name=="API_USER")].valueFrom.secretKeyRef' eq "$(j -n %{ns} pod %{pod} "$e.name}/$e.key}")" %{sec}/username && eq "$($K -n %{ns} exec %{pod} -- printenv API_USER 2>/dev/null)" %{user}"""), C("mounted read-only at /etc/api-creds, files mode 0400", 2, r""" v=$(j -n %{ns} pod %{pod} '{.spec.volumes[?(@.secret.secretName=="%{sec}")].name}'); [ -n "$v" ] || exit 1 eq "$(j -n %{ns} pod %{pod} "{.spec.containers[0].volumeMounts[?(@.name==\"$v\")].mountPath}")" /etc/api-creds || exit 1 eq "$(j -n %{ns} pod %{pod} "{.spec.containers[0].volumeMounts[?(@.name==\"$v\")].readOnly}")" true || exit 1 eq "$($K -n %{ns} exec %{pod} -- cat /etc/api-creds/password 2>/dev/null)" %{pw} \ && eq "$($K -n %{ns} exec %{pod} -- stat -L -c %a /etc/api-creds/password 2>/dev/null)" 400"""), ], "solution": """ k -n %{ns} get secret %{old} -o jsonpath='{.data.password}' | base64 -d > {WS}/q33/old-password.txt k -n %{ns} create secret generic %{sec} --from-literal=username=%{user} --from-literal=password=%{pw} cat </dev/null $K -n %{ns} delete secret %{sa}-token --ignore-not-found >/dev/null; $K -n %{ns} delete sa %{sa} --ignore-not-found >/dev/null """, "checks": [ C("ServiceAccount %{sa} with automountServiceAccountToken: false", 1, r"""eq "$(j -n %{ns} sa %{sa} '{.automountServiceAccountToken}')" false"""), C("Secret %{sa}-token is a service-account token for %{sa}, issued", 2, r""" s="$K -n %{ns} get secret %{sa}-token" eq "$($s -o jsonpath='{.type}' 2>/dev/null)" kubernetes.io/service-account-token \ && eq "$($s -o jsonpath='{.metadata.annotations.kubernetes\.io/service-account\.name}')" %{sa} && [ -n "$($s -o jsonpath='{.data.token}')" ]"""), C("token.txt holds the decoded token", 2, r""" t=$(j -n %{ns} secret %{sa}-token '{.data.token}' | base64 -d 2>/dev/null); [ -n "$t" ] && eq "$(tr -d '[:space:]' < "$WS/q34/token.txt" 2>/dev/null)" "$t" """), C("Deployment runs as %{sa} with no API token volume", 2, r""" eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.serviceAccountName}')" %{sa} || exit 1 for i in $(seq 1 30); do [ "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" = 1 ] && break; sleep 3; done p=$(j -n %{ns} pod -l app=%{dep} --field-selector=status.phase=Running '{.items[0].metadata.name}'); [ -n "$p" ] || exit 1 eq "$(j -n %{ns} pod "$p" '{.spec.serviceAccountName}')" %{sa} && [ -z "$(j -n %{ns} pod "$p" '{.spec.volumes[*].projected.sources[*].serviceAccountToken.path}')" ]"""), ], "solution": """ k -n %{ns} create sa %{sa} k -n %{ns} patch sa %{sa} -p '{"automountServiceAccountToken": false}' cat < {WS}/q34/token.txt k -n %{ns} create deploy %{dep} --image=nginx:1.27 $do > d.yaml vi d.yaml # template.spec.serviceAccountName: %{sa} k apply -f d.yaml k -n %{ns} get pod -l app=%{dep} -o jsonpath='{.items[0].spec.volumes}' # no kube-api-access-* # Since 1.24 a ServiceAccount gets no token Secret on its own; the token controller fills one in # when you create it with that type and annotation. `kubectl create token %{sa}` makes a # short-lived token instead, with no Secret at all. """, }, # ------------------------------------------------------------------- NET ---- { "id": 35, "domain": "NET", "title": "Egress lockdown that keeps DNS working", "params": lambda r: {"ns": ns_pick(r, "mesh")}, "task": """ In Namespace `%{ns}`, Pods labelled `app=api` must be locked down for **outgoing** traffic: - they may connect to Pods labelled `app=db` on TCP port 80, - they may resolve names through the cluster DNS (UDP and TCP port 53), - every other outgoing connection from them must be blocked. Create NetworkPolicy `api-egress` for this, without restricting incoming traffic. kindnet enforces NetworkPolicy in this cluster, so it is tested for real from an `api` Pod. """, "setup": NS + r""" mkns %{ns}; mkns %{ns}-ext $K -n %{ns} delete netpol --all >/dev/null 2>&1 for a in db cache; do $K -n %{ns} create deploy $a --image=nginx:1.27 --dry-run=client -o yaml | $K apply -f - >/dev/null $K -n %{ns} expose deploy $a --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null done $K -n %{ns} create deploy api --image=busybox:1.36 --dry-run=client -o yaml -- sleep 86400 | $K apply -f - >/dev/null $K -n %{ns}-ext create deploy web --image=nginx:1.27 --dry-run=client -o yaml | $K apply -f - >/dev/null $K -n %{ns}-ext expose deploy web --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null for d in api db cache; do $K -n %{ns} rollout status deploy $d --timeout=150s >/dev/null 2>&1; done """, "checks": [ C("api-egress selects app=api and restricts Egress only", 1, r""" eq "$(j -n %{ns} netpol api-egress '{.spec.podSelector.matchLabels.app}')" api && eq "$(j -n %{ns} netpol api-egress '{.spec.policyTypes}')" '["Egress"]'"""), C("api reaches db by its Service name (DNS + TCP 80)", 3, r""" [ -n "$(j -n %{ns} netpol api-egress '{.metadata.name}')" ] || exit 1 for i in 1 2 3; do $K -n %{ns} exec deploy/api -- wget -T 3 -qO- http://db 2>/dev/null | grep -qi nginx && exit 0; sleep 2; done; exit 1"""), # a failed connection proves nothing on its own (a Pod not ready yet fails too): only count it as # blocked while the policy exists and the allowed path to db demonstrably works C("api can no longer reach cache", 2, r""" ok() { $K -n %{ns} exec deploy/api -- wget -T 3 -qO- "$1" >/dev/null 2>&1; } [ -n "$(j -n %{ns} netpol api-egress '{.metadata.name}')" ] && ok http://db && ! ok http://cache"""), C("api can no longer reach another Namespace", 1, r""" ok() { $K -n %{ns} exec deploy/api -- wget -T 3 -qO- "$1" >/dev/null 2>&1; } [ -n "$(j -n %{ns} netpol api-egress '{.metadata.name}')" ] && ok http://db && ! ok http://web.%{ns}-ext"""), ], "solution": """ cat </dev/null $K -n %{ns} delete svc %{app} --ignore-not-found >/dev/null $K -n %{ns} delete pod probe --ignore-not-found --wait=true --grace-period=1 >/dev/null $K apply -f - >/dev/null <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: {name: %{app}, namespace: %{ns}} spec: replicas: 2 selector: {matchLabels: {app: %{app}, tier: web}} template: metadata: {labels: {app: %{app}, tier: web}} spec: containers: - name: web image: nginx:1.27 ports: [{name: http, containerPort: 80}] --- apiVersion: v1 kind: Service metadata: {name: %{app}, namespace: %{ns}} spec: selector: {app: %{app}, tier: frontend} ports: [{name: http, port: 80, targetPort: web}] --- apiVersion: v1 kind: Pod metadata: {name: probe, namespace: %{ns}} spec: containers: [{name: probe, image: busybox:1.36, command: [sleep, "86400"]}] EOF $K -n %{ns} rollout status deploy %{app} --timeout=150s >/dev/null 2>&1 $K -n %{ns} wait --for=condition=Ready pod/probe --timeout=120s >/dev/null # fail the seed loudly if it is missing """, "checks": [ C("probe gets the nginx page from http://%{app}", 3, r""" for i in 1 2 3; do $K -n %{ns} exec probe -- wget -T 3 -qO- http://%{app} 2>/dev/null | grep -qi nginx && exit 0; sleep 2; done; exit 1"""), C("Deployment untouched, Service still on port 80", 2, r""" $K -n %{ns} exec probe -- wget -T 3 -qO- http://%{app} >/dev/null 2>&1 || exit 1 eq "$(j -n %{ns} deploy %{app} '{.metadata.generation}')" 1 && eq "$(j -n %{ns} svc %{app} '{.spec.ports[0].port}')" 80"""), C("endpoints.txt lists exactly the Pods' IPs", 2, r""" want=$(j -n %{ns} pod -l app=%{app},tier=web --field-selector=status.phase=Running '{range .items[*]}{.status.podIP}{"\n"}{end}' | sort) [ -n "$want" ] && [ "$(grep -v '^[[:space:]]*$' "$WS/q36/endpoints.txt" 2>/dev/null | tr -d ' \r' | sort)" = "$want" ]"""), ], "solution": """ k -n %{ns} get svc %{app} -o wide # SELECTOR app=%{app},tier=frontend k -n %{ns} get endpointslices -l kubernetes.io/service-name=%{app} # no endpoints k -n %{ns} get pod --show-labels # the Pods say tier=web k -n %{ns} get deploy %{app} -o jsonpath='{.spec.template.spec.containers[0].ports}' # port name: http k -n %{ns} edit svc %{app} # selector: tier: frontend -> tier: web # targetPort: web -> targetPort: http (or 80) k -n %{ns} exec probe -- wget -T 3 -qO- http://%{app} | head -3 k -n %{ns} get endpointslices -l kubernetes.io/service-name=%{app} -o jsonpath='{range .items[*].endpoints[*]}{.addresses[0]}{"\\n"}{end}' > {WS}/q36/endpoints.txt # Two faults, one symptom. Fixing the selector gives the Service endpoints, but a named targetPort # that no container port carries still leaves them unusable - check both, every time. """, }, { "id": 37, "domain": "NET", "title": "Ingress with two paths and TLS", "params": lambda r: {"ns": ns_pick(r, "edge"), "dom": r.choice(["shop", "store", "market", "bazaar"])}, "task": """ In Namespace `%{ns}`, Services `web` (port 80) and `api` (port 8080) exist. 1. Create a TLS Secret `%{dom}-tls` from `{WS}/q37/tls.crt` and `{WS}/q37/tls.key`. 2. Create Ingress `%{dom}` with IngressClass `nginx` for host `%{dom}.example.com`: - path `/api` (Prefix) goes to Service `api` on port 8080, - path `/` (Prefix) goes to Service `web` on port 80, - TLS for that host uses the Secret from step 1. (No Ingress controller runs in this cluster; the objects are graded.) """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q37"; mkdir -p "$WS/q37" $K -n %{ns} delete ingress %{dom} --ignore-not-found >/dev/null; $K -n %{ns} delete secret %{dom}-tls --ignore-not-found >/dev/null for s in web api; do $K -n %{ns} create deploy $s --image=nginx:1.27 --dry-run=client -o yaml | $K apply -f - >/dev/null; done $K -n %{ns} expose deploy web --port=80 --target-port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null $K -n %{ns} expose deploy api --port=8080 --target-port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null subj="/CN=%{dom}.example.com" if command -v openssl >/dev/null; then openssl req -x509 -nodes -newkey rsa:2048 -days 30 -subj "$subj" -keyout "$WS/q37/tls.key" -out "$WS/q37/tls.crt" >/dev/null 2>&1 else # no openssl on this machine: make the pair on the control-plane node instead nx "$CP" "openssl req -x509 -nodes -newkey rsa:2048 -days 30 -subj $subj -keyout /tmp/q37.key -out /tmp/q37.crt >/dev/null 2>&1; cat /tmp/q37.crt" > "$WS/q37/tls.crt" "$WS/q37/tls.key" /dev/null)" = "$crt" ] \ && [ "$(j -n %{ns} secret %{dom}-tls '{.data.tls\.key}' | base64 -d 2>/dev/null)" = "$key" ]"""), C("class nginx; %{dom}.example.com: /api -> api:8080 and / -> web:80, both Prefix", 3, r""" $K -n %{ns} get ingress %{dom} -o json 2>/dev/null | python3 -c ' import sys, json s = json.load(sys.stdin)["spec"] rules = [r for r in s.get("rules", []) if r.get("host") == "%{dom}.example.com"] paths = {p.get("path"): (p.get("pathType"), p["backend"]["service"]["name"], p["backend"]["service"]["port"].get("number")) for r in rules for p in r.get("http", {}).get("paths", [])} ok = s.get("ingressClassName") == "nginx" and paths == {"/api": ("Prefix", "api", 8080), "/": ("Prefix", "web", 80)} sys.exit(0 if ok else 1)'"""), C("TLS for %{dom}.example.com uses %{dom}-tls", 2, r""" $K -n %{ns} get ingress %{dom} -o json 2>/dev/null | python3 -c ' import sys, json t = json.load(sys.stdin)["spec"].get("tls", []) sys.exit(0 if any(x.get("secretName") == "%{dom}-tls" and "%{dom}.example.com" in x.get("hosts", []) for x in t) else 1)'"""), ], "solution": """ k -n %{ns} create secret tls %{dom}-tls --cert={WS}/q37/tls.crt --key={WS}/q37/tls.key k -n %{ns} create ingress %{dom} --class=nginx \\ --rule="%{dom}.example.com/api*=api:8080,tls=%{dom}-tls" \\ --rule="%{dom}.example.com/*=web:80,tls=%{dom}-tls" k -n %{ns} get ingress %{dom} -o yaml # In --rule, a trailing * means pathType Prefix (no * = Exact), and tls=SECRET adds the host to # spec.tls with that Secret. The YAML equivalent: # spec: # ingressClassName: nginx # tls: [{hosts: [%{dom}.example.com], secretName: %{dom}-tls}] # rules: # - host: %{dom}.example.com # http: # paths: # - {path: /api, pathType: Prefix, backend: {service: {name: api, port: {number: 8080}}}} # - {path: /, pathType: Prefix, backend: {service: {name: web, port: {number: 80}}}} """, }, ]