"""CKAD Simulator Set A - 20 tasks, fixed set, killer.sh-style. Original scenarios, not copies of any commercial simulator: a fictional river-freight company, Riverline, one Namespace per river. They exercise the techniques the killer.sh CKAD environment is known for - exact names and file outputs, commands written to scripts, several steps per task - across the whole curriculum: a Pod with a status one-liner, a suspended CronJob run by hand, an init container building the page from the downward API, a podman build with a build argument run detached, rolling back to the last good revision, Helm housekeeping (a failed release, an upgrade that keeps values, a rollback), a single Pod turned into a hardened Deployment, a Kustomize patch with a hashed configMapGenerator, probes that match an app's warm-up, debugging a container with no shell through an ephemeral container, restart and event forensics, RBAC proven by the app's own log, an immutable Secret mounted with subPath, a ConfigMap-driven nginx that needs a restart, fsGroup on a shared volume, Pods rejected by a LimitRange, a Service tested from a temporary Pod, the two-source NetworkPolicy trap (enforced here - kindnet enforces), StatefulSet DNS names, and an Ingress pointing at the wrong backends. Every value a grader compares against is derived from the live cluster, and every task was seeded, solved and graded on an isolated kind cluster before it went in. Run with: ./ksim sim a (./ksim --track ckad sim a when CKAD is not your default track) """ from sim.bank import C, NS Q = [] def task(**kw): kw.setdefault("difficulty", "hard") kw["set"] = "a" Q.append(kw) # ------------------------------------------------------------------ 38 ---- task( id=38, domain="DESIGN", title="A Pod and a status one-liner", task=""" Team Danube needs a quick check Pod. 1. In Namespace `danube` create a Pod `ferry` from image `httpd:2-alpine`. Its container must be named `ferry-web` and the Pod must carry the label `tier=edge`. 2. Write a command into `{WS}/q38/ferry-status.sh` that prints only the status phase of Pod `ferry` (for example `Running`), using `kubectl`. It must work whatever Namespace your current context points at. """, setup=NS + r""" mkns danube; rm -rf "$WS/q38"; mkdir -p "$WS/q38" $K -n danube delete pod ferry --ignore-not-found --wait=true --grace-period=1 >/dev/null """, checks=[ C("Pod ferry Running: container ferry-web from httpd:2-alpine", 3, r""" eq "$(j -n danube pod ferry '{.status.phase}')" Running \ && eq "$(j -n danube pod ferry '{.spec.containers[0].name}/{.spec.containers[0].image}')" ferry-web/httpd:2-alpine"""), C("label tier=edge", 1, r"""eq "$(j -n danube pod ferry '{.metadata.labels.tier}')" edge"""), C("ferry-status.sh prints Running", 2, r""" f="$WS/q38/ferry-status.sh"; [ -s "$f" ] || exit 1 eq "$(cd / && bash "$f" 2>/dev/null | tr -d '[:space:]')" Running"""), C("the command names the Namespace itself", 1, r"""grep -Eq -- '(-n|--namespace)[= ]+danube( |$)' "$WS/q38/ferry-status.sh" 2>/dev/null"""), ], solution=""" k -n danube run ferry --image=httpd:2-alpine --labels=tier=edge $do > ferry.yaml vi ferry.yaml # containers[0].name: ferry-web (`run` names it after the Pod) k apply -f ferry.yaml echo "kubectl -n danube get pod ferry -o jsonpath='{.status.phase}'" > {WS}/q38/ferry-status.sh bash {WS}/q38/ferry-status.sh # Running # The script runs in a plain shell: no `k` alias there, so spell out kubectl. An explicit -n makes # it independent of whatever Namespace your context happens to point at. """, ) # ------------------------------------------------------------------ 39 ---- task( id=39, domain="DESIGN", title="A suspended CronJob and a manual run", task=""" In Namespace `rhine`: 1. Create a CronJob `tide-report` that runs every 30 minutes, image `busybox:1.36`, command `sh -c 'echo "tide report $(date +%H:%M)"'`. Create it **suspended**, so it never starts a Job on its own. 2. Start one run right now: a Job named `tide-report-manual`, created **from the CronJob**. 3. Write that Job's log output to `{WS}/q39/manual-run.log`. """, setup=NS + r""" mkns rhine; rm -rf "$WS/q39"; mkdir -p "$WS/q39" $K -n rhine delete cronjob tide-report --ignore-not-found >/dev/null $K -n rhine delete job tide-report-manual --ignore-not-found --wait=true >/dev/null """, checks=[ C("CronJob tide-report: every 30 minutes, suspended, busybox:1.36", 2, r""" c="$K -n rhine get cronjob tide-report" case "$($c -o jsonpath='{.spec.schedule}' 2>/dev/null)" in "*/30 * * * *"|"0,30 * * * *"|"0/30 * * * *") ;; *) exit 1;; esac eq "$($c -o jsonpath='{.spec.suspend}')" true && eq "$($c -o jsonpath='{.spec.jobTemplate.spec.template.spec.containers[0].image}')" busybox:1.36"""), C("Job tide-report-manual was created from the CronJob", 2, r""" eq "$(j -n rhine job tide-report-manual '{.metadata.ownerReferences[0].kind}/{.metadata.ownerReferences[0].name}')" CronJob/tide-report \ || eq "$(j -n rhine job tide-report-manual '{.metadata.annotations.cronjob\.kubernetes\.io/instantiate}')" manual"""), C("the manual run completed", 2, r""" for i in $(seq 1 20); do [ "$(j -n rhine job tide-report-manual '{.status.succeeded}')" = 1 ] && exit 0; sleep 3; done; exit 1"""), C("manual-run.log holds its output", 1, r"""grep -Eq '^tide report [0-9]{2}:[0-9]{2}$' "$WS/q39/manual-run.log" 2>/dev/null"""), ], solution=""" k -n rhine create cronjob tide-report --image=busybox:1.36 --schedule='*/30 * * * *' $do \\ -- sh -c 'echo "tide report $(date +%H:%M)"' > cj.yaml vi cj.yaml # spec.suspend: true k apply -f cj.yaml k -n rhine create job tide-report-manual --from=cronjob/tide-report k -n rhine wait --for=condition=complete job/tide-report-manual k -n rhine logs job/tide-report-manual > {WS}/q39/manual-run.log # `suspend` only stops the schedule; `create job --from=cronjob/...` still copies its jobTemplate. # The Job it makes is owned by the CronJob and annotated cronjob.kubernetes.io/instantiate=manual. """, ) # ------------------------------------------------------------------ 40 ---- task( id=40, domain="DESIGN", title="An init container writes the page", task=""" In Namespace `volga` create Deployment `harbor-web` (2 replicas, image `nginx:1.27`) whose Pods serve a page generated as each Pod starts: - an init container `page-gen` (`busybox:1.36`) writes the line `harbor online: ` to `index.html` in an `emptyDir` volume named `html`, taking the Pod's own name from the downward API, - the nginx container serves that volume at `/usr/share/nginx/html`. Expose the Deployment as ClusterIP Service `harbor-web` on port 80. """, setup=NS + r""" mkns volga $K -n volga delete deploy harbor-web --ignore-not-found --wait=true >/dev/null; $K -n volga delete svc harbor-web --ignore-not-found >/dev/null """, checks=[ C("2 available, with init container page-gen (busybox:1.36)", 2, r""" eq "$(j -n volga deploy harbor-web '{.spec.template.spec.initContainers[?(@.name=="page-gen")].image}')" busybox:1.36 || exit 1 for i in $(seq 1 30); do [ "$(j -n volga deploy harbor-web '{.status.availableReplicas}')" = 2 ] && exit 0; sleep 3; done; exit 1"""), C("volume html: an emptyDir, written by page-gen, served at /usr/share/nginx/html", 1, r""" t='{.spec.template.spec' [ -n "$(j -n volga deploy harbor-web "$t.volumes[?(@.name==\"html\")].emptyDir}")" ] || exit 1 [ -n "$(j -n volga deploy harbor-web "$t.initContainers[?(@.name==\"page-gen\")].volumeMounts[?(@.name==\"html\")].mountPath}")" ] || exit 1 j -n volga deploy harbor-web "$t.containers[*].volumeMounts[?(@.name==\"html\")].mountPath}" | tr ' ' '\n' | grep -qx /usr/share/nginx/html"""), C("page-gen reads the Pod name from the downward API", 1, r""" t='{.spec.template.spec' { j -n volga deploy harbor-web "$t.initContainers[?(@.name==\"page-gen\")].env[*].valueFrom.fieldRef.fieldPath}" j -n volga deploy harbor-web "$t.volumes[*].downwardAPI.items[*].fieldRef.fieldPath}"; } | tr ' ' '\n' | grep -qx metadata.name"""), C("Service harbor-web:80 returns 'harbor online: '", 3, r""" [ "$(j -n volga svc harbor-web '{.spec.ports[0].port}')" = 80 ] || exit 1 page=$($K -n volga exec deploy/harbor-web -- curl -s -m 3 http://harbor-web 2>/dev/null | tr -d '\r' | head -1) name=${page#harbor online: }; [ "$name" != "$page" ] && [ -n "$name" ] || exit 1 echo " $(j -n volga pod '{.items[*].metadata.name}') " | grep -q " $name " """), ], solution=""" k -n volga create deploy harbor-web --image=nginx:1.27 --replicas=2 $do > web.yaml vi web.yaml # template.spec: # volumes: [{name: html, emptyDir: {}}] # initContainers: # - name: page-gen # image: busybox:1.36 # env: [{name: POD_NAME, valueFrom: {fieldRef: {fieldPath: metadata.name}}}] # command: [sh, -c, 'echo "harbor online: $POD_NAME" > /work/index.html'] # volumeMounts: [{name: html, mountPath: /work}] # containers[0].volumeMounts: [{name: html, mountPath: /usr/share/nginx/html}] k apply -f web.yaml k -n volga expose deploy harbor-web --port=80 k -n volga exec deploy/harbor-web -- curl -s http://harbor-web # harbor online: harbor-web-... """, ) # ------------------------------------------------------------------ 41 ---- task( id=41, domain="DESIGN", title="Build with a build argument, run it detached", task=""" `{WS}/q41/image` holds a Dockerfile that takes the build argument `APP_VERSION`. Use `podman` on this machine. 1. Build it with `APP_VERSION=2.1` and tag the result as both `localhost/beacon-relay:2.1` and `localhost/beacon-relay:latest`. 2. Start a **detached** container named `beacon-relay` from `localhost/beacon-relay:2.1`. 3. Write that container's logs to `{WS}/q41/relay.log`. """, setup=r""" command -v podman >/dev/null || { echo "podman is not installed on this machine - the task needs it"; exit 1; } podman rm -f beacon-relay >/dev/null 2>&1 || true podman rmi -f localhost/beacon-relay:2.1 localhost/beacon-relay:latest >/dev/null 2>&1 || true rm -rf "$WS/q41"; mkdir -p "$WS/q41/image" cat > "$WS/q41/image/Dockerfile" <<'EOF' FROM docker.io/library/busybox:1.36 ARG APP_VERSION=dev ENV APP_VERSION=${APP_VERSION} # stops by itself after an hour, so a forgotten container does not run forever CMD ["sh", "-c", "echo \"beacon relay $APP_VERSION starting\"; i=0; while [ $i -lt 360 ]; do echo \"beacon relay $APP_VERSION heartbeat $i\"; i=$((i+1)); sleep 10; done"] EOF """, checks=[ C("tags 2.1 and latest name the same image", 2, r""" a=$(podman image inspect localhost/beacon-relay:2.1 --format '{{.Id}}' 2>/dev/null) [ -n "$a" ] && eq "$a" "$(podman image inspect localhost/beacon-relay:latest --format '{{.Id}}' 2>/dev/null)" """), C("built with APP_VERSION=2.1", 1, r"""podman image inspect localhost/beacon-relay:2.1 --format '{{json .Config.Env}}' 2>/dev/null | grep -q '"APP_VERSION=2.1"'"""), C("container beacon-relay running from localhost/beacon-relay:2.1", 2, r""" eq "$(podman inspect beacon-relay --format '{{.State.Running}} {{.ImageName}}' 2>/dev/null)" "true localhost/beacon-relay:2.1" """), C("relay.log holds its logs", 2, r"""grep -q '^beacon relay 2.1 starting$' "$WS/q41/relay.log" 2>/dev/null"""), ], solution=""" cd {WS}/q41/image podman build --build-arg APP_VERSION=2.1 -t localhost/beacon-relay:2.1 -t localhost/beacon-relay:latest . podman run -d --name beacon-relay localhost/beacon-relay:2.1 podman logs beacon-relay > {WS}/q41/relay.log podman ps; podman images | grep beacon-relay # ARG exists only while building; copying it into ENV is what makes it visible to the running # container. `podman tag localhost/beacon-relay:2.1 localhost/beacon-relay:latest` adds a tag later. """, ) # ------------------------------------------------------------------ 42 ---- task( id=42, domain="DEPLOY", title="Back to the last good revision", task=""" Deployment `dispatch-api` in Namespace `yukon` has been updated a few times, and the latest update broke it. 1. Roll it back to the most recent revision whose Pods actually worked, with all 3 replicas available. 2. Write the image of the broken revision to `{WS}/q42/broken-image.txt`. 3. Write the revision number that is active after your rollback to `{WS}/q42/revision.txt`. """, setup=NS + r""" mkns yukon; rm -rf "$WS/q42"; mkdir -p "$WS/q42" $K -n yukon delete deploy dispatch-api --ignore-not-found --wait=true >/dev/null $K -n yukon create deploy dispatch-api --image=nginx:1.25 --replicas=3 >/dev/null $K -n yukon annotate deploy dispatch-api kubernetes.io/change-cause="initial release" >/dev/null $K -n yukon rollout status deploy dispatch-api --timeout=150s >/dev/null 2>&1 $K -n yukon set image deploy dispatch-api nginx=nginx:1.27 >/dev/null $K -n yukon annotate deploy dispatch-api kubernetes.io/change-cause="move to nginx 1.27" --overwrite >/dev/null $K -n yukon rollout status deploy dispatch-api --timeout=150s >/dev/null 2>&1 $K -n yukon set image deploy dispatch-api nginx=nginx:1.27-slim >/dev/null $K -n yukon annotate deploy dispatch-api kubernetes.io/change-cause="slimmer image" --overwrite >/dev/null $K -n yukon get deploy dispatch-api -o jsonpath='{.metadata.uid}' > "$WS/.q42-uid" """, checks=[ C("back on nginx:1.27 with 3/3 updated and available", 3, r""" eq "$(j -n yukon deploy dispatch-api '{.spec.template.spec.containers[0].image}')" nginx:1.27 || exit 1 for i in $(seq 1 30); do [ "$(j -n yukon deploy dispatch-api '{.status.updatedReplicas}/{.status.availableReplicas}/{.status.replicas}')" = 3/3/3 ] && exit 0; sleep 3; done; exit 1"""), C("rolled back in place, not recreated", 1, r""" eq "$(j -n yukon deploy dispatch-api '{.metadata.uid}')" "$(cat "$WS/.q42-uid" 2>/dev/null)" \ && [ "$(j -n yukon deploy dispatch-api '{.metadata.annotations.deployment\.kubernetes\.io/revision}')" -ge 4 ] 2>/dev/null"""), C("broken-image.txt holds nginx:1.27-slim", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q42/broken-image.txt" 2>/dev/null)" nginx:1.27-slim"""), C("revision.txt holds the revision now active", 2, r""" r=$(j -n yukon deploy dispatch-api '{.metadata.annotations.deployment\.kubernetes\.io/revision}') [ -n "$r" ] && eq "$(tr -d '[:space:]' < "$WS/q42/revision.txt" 2>/dev/null)" "$r" """), ], solution=""" k -n yukon rollout history deploy dispatch-api # 1 initial release, 2 nginx 1.27, 3 slimmer image k -n yukon get rs,pod # the newest ReplicaSet's Pod: ImagePullBackOff k -n yukon rollout history deploy dispatch-api --revision=3 | grep Image # nginx:1.27-slim echo nginx:1.27-slim > {WS}/q42/broken-image.txt k -n yukon rollout undo deploy dispatch-api # to the previous revision, 2 k -n yukon rollout status deploy dispatch-api k -n yukon get deploy dispatch-api -o jsonpath='{.metadata.annotations.deployment\\.kubernetes\\.io/revision}' > {WS}/q42/revision.txt # undo re-uses revision 2's ReplicaSet and renumbers it as the newest revision - 4, not 2. # The old Pods kept serving meanwhile: maxUnavailable 25% of 3 rounds down to 0. """, ) # ------------------------------------------------------------------ 43 ---- task( id=43, domain="DEPLOY", title="Helm housekeeping", task=""" Namespace `mekong` runs three Helm releases of the chart `buoy`, whose packaged versions are in `{WS}/q43/charts/`. 1. One release is in status `failed`. Write its name to `{WS}/q43/failed.txt`, then uninstall it. 2. Upgrade `buoy-west` to chart version 0.2.0 with `replicaCount=3`, keeping every other value it was installed with. 3. `buoy-east` was upgraded recently. Roll it back to its first revision. """, setup=NS + r""" command -v helm >/dev/null || { echo "helm is not installed on this machine - the task needs it"; exit 1; } mkns mekong for r in $(helm -n mekong list -aq 2>/dev/null || helm -n mekong list -q 2>/dev/null); do helm -n mekong uninstall "$r" --wait >/dev/null 2>&1; done rm -rf "$WS/q43"; mkdir -p "$WS/q43/charts" src=$(mktemp -d) for v in 0.1.0 0.2.0; do d="$src/$v/buoy"; mkdir -p "$d/templates" tag=1.25; [ "$v" = 0.2.0 ] && tag=1.27 printf 'apiVersion: v2\nname: buoy\nversion: %s\nappVersion: "%s"\n' "$v" "$tag" > "$d/Chart.yaml" printf 'replicaCount: 1\nregion: default\nimage:\n repository: nginx\n tag: "%s"\n' "$tag" > "$d/values.yaml" cat > "$d/templates/deployment.yaml" <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: name: {{ .Release.Name }} labels: {app.kubernetes.io/instance: {{ .Release.Name }}} spec: replicas: {{ .Values.replicaCount }} selector: {matchLabels: {app.kubernetes.io/instance: {{ .Release.Name }}}} template: metadata: {labels: {app.kubernetes.io/instance: {{ .Release.Name }}}} spec: containers: - name: buoy image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" env: [{name: REGION, value: {{ .Values.region | quote }}}] EOF helm package "$d" -d "$WS/q43/charts" >/dev/null done rm -rf "$src" c="$WS/q43/charts" helm -n mekong install buoy-east "$c/buoy-0.1.0.tgz" --set replicaCount=2 >/dev/null helm -n mekong upgrade buoy-east "$c/buoy-0.2.0.tgz" --reuse-values >/dev/null helm -n mekong install buoy-west "$c/buoy-0.1.0.tgz" --set region=west >/dev/null # its Pods can never start, and --wait makes Helm give up and mark the release failed helm -n mekong install buoy-north "$c/buoy-0.1.0.tgz" --set image.tag=0.0-missing --wait --timeout 10s >/dev/null 2>&1 helm -n mekong status buoy-north -o json 2>/dev/null | grep -q '"status":"failed"' || { echo "buoy-north did not end up failed"; exit 1; } """, checks=[ C("failed.txt names buoy-north, and it is uninstalled", 2, r""" eq "$(tr -d '[:space:]' < "$WS/q43/failed.txt" 2>/dev/null)" buoy-north && ! helm -n mekong status buoy-north >/dev/null 2>&1"""), C("buoy-west on buoy-0.2.0 with 3 replicas, region still west", 3, r""" helm -n mekong list -o json 2>/dev/null | python3 -c ' import sys, json r = {x["name"]: x for x in json.load(sys.stdin)}.get("buoy-west") or {} sys.exit(0 if r.get("chart") == "buoy-0.2.0" and r.get("status") == "deployed" else 1)' || exit 1 eq "$(j -n mekong deploy buoy-west '{.spec.template.spec.containers[0].env[?(@.name=="REGION")].value}')" west || exit 1 for i in $(seq 1 30); do [ "$(j -n mekong deploy buoy-west '{.spec.replicas}/{.status.availableReplicas}')" = 3/3 ] && exit 0; sleep 3; done; exit 1"""), C("buoy-east rolled back to revision 1", 2, r""" helm -n mekong history buoy-east -o json 2>/dev/null | python3 -c ' import sys, json last = json.load(sys.stdin)[-1] d = str(last.get("description", "")).lower() sys.exit(0 if last.get("status") == "deployed" and last.get("chart") == "buoy-0.1.0" and "rollback to 1" in d else 1)'"""), ], solution=""" helm -n mekong list --failed # buoy-north (plain `list` shows it too; Helm 4 has no -a) echo buoy-north > {WS}/q43/failed.txt helm -n mekong uninstall buoy-north helm -n mekong get values buoy-west # region: west - the value to keep helm -n mekong upgrade buoy-west {WS}/q43/charts/buoy-0.2.0.tgz --reuse-values --set replicaCount=3 helm -n mekong history buoy-east helm -n mekong rollback buoy-east 1 # A plain upgrade would start from the new chart's defaults and silently drop region=west. # A rollback is itself a new revision, described as "Rollback to 1". """, ) # ------------------------------------------------------------------ 44 ---- task( id=44, domain="DEPLOY", title="From a single Pod to a hardened Deployment", task=""" Pod `gauge-sensor` in Namespace `ganges` was started by hand from `{WS}/q44/gauge-pod.yaml`. Turn it into a Deployment: 1. Deployment `gauge-sensor` in `ganges`, 3 replicas, with the same container (image, command, environment) and the Pod label `app=gauge-sensor`. 2. Its container must neither allow privilege escalation nor run privileged. 3. Save the Deployment manifest you applied to `{WS}/q44/gauge-deployment.yaml`. 4. Delete the original single Pod. """, setup=NS + r""" mkns ganges; rm -rf "$WS/q44"; mkdir -p "$WS/q44" $K -n ganges delete deploy gauge-sensor --ignore-not-found --wait=true >/dev/null $K -n ganges delete pod gauge-sensor --ignore-not-found --wait=true --grace-period=1 >/dev/null cat > "$WS/q44/gauge-pod.yaml" <<'EOF' apiVersion: v1 kind: Pod metadata: name: gauge-sensor namespace: ganges labels: {app: gauge-sensor} spec: containers: - name: sensor image: busybox:1.36 command: [sh, -c, "while true; do echo \"gauge $SENSOR_ID reading $(date +%s)\"; sleep 15; done"] env: [{name: SENSOR_ID, value: G-7}] EOF $K apply -f "$WS/q44/gauge-pod.yaml" >/dev/null """, checks=[ C("Deployment gauge-sensor: 3 available, same container, label app=gauge-sensor", 3, r""" d='{.spec.template.spec.containers[0]' eq "$(j -n ganges deploy gauge-sensor "$d.image}")" busybox:1.36 && eq "$(j -n ganges deploy gauge-sensor "$d.env[?(@.name==\"SENSOR_ID\")].value}")" G-7 \ && j -n ganges deploy gauge-sensor "$d.command}" | grep -q 'gauge \$SENSOR_ID reading' \ && eq "$(j -n ganges deploy gauge-sensor '{.spec.template.metadata.labels.app}')" gauge-sensor || exit 1 for i in $(seq 1 30); do [ "$(j -n ganges deploy gauge-sensor '{.status.availableReplicas}')" = 3 ] && exit 0; sleep 3; done; exit 1"""), C("no privilege escalation, not privileged", 2, r""" s='{.spec.template.spec.containers[0].securityContext' eq "$(j -n ganges deploy gauge-sensor "$s.allowPrivilegeEscalation}/$s.privileged}")" false/false"""), C("gauge-deployment.yaml is that Deployment", 1, r""" f="$WS/q44/gauge-deployment.yaml"; [ -s "$f" ] && eq "$($K create --dry-run=client -f "$f" -o jsonpath='{.kind}/{.metadata.name}' 2>/dev/null)" Deployment/gauge-sensor"""), C("the original single Pod is gone", 1, r""" [ -n "$(j -n ganges deploy gauge-sensor '{.metadata.name}')" ] && ! $K -n ganges get pod gauge-sensor >/dev/null 2>&1"""), ], solution=""" k -n ganges create deploy gauge-sensor --image=busybox:1.36 --replicas=3 $do > {WS}/q44/gauge-deployment.yaml vi {WS}/q44/gauge-deployment.yaml # copy the container from {WS}/q44/gauge-pod.yaml: name sensor, command, env SENSOR_ID=G-7 # (create deploy already labels the template app=gauge-sensor) # container securityContext: {allowPrivilegeEscalation: false, privileged: false} k apply -f {WS}/q44/gauge-deployment.yaml k -n ganges delete pod gauge-sensor k -n ganges get deploy,pod # The bare Pod is not adopted by the new ReplicaSet - its selector includes pod-template-hash, # which a hand-made Pod does not carry - so it has to be deleted separately. """, ) # ------------------------------------------------------------------ 45 ---- task( id=45, domain="DEPLOY", title="Kustomize: a patch and a generated ConfigMap", task=""" A Kustomize base for `river-api` is in `{WS}/q45/base`. Do not edit the base. Finish the overlay in `{WS}/q45/overlays/staging` (it only holds `settings.env` so far) so that it - deploys everything into Namespace `orinoco`, - generates a ConfigMap `river-api-settings` from `settings.env`, keeping Kustomize's default hash suffix on its name, - patches the Deployment so its container gets all of that ConfigMap's keys as environment variables and a memory limit of `128Mi`, - runs 2 replicas. Apply the overlay. """, setup=NS + r""" mkns orinoco $K -n orinoco delete deploy river-api --ignore-not-found --wait=true >/dev/null; $K -n orinoco delete svc river-api --ignore-not-found >/dev/null for cm in $($K -n orinoco get cm -o name 2>/dev/null | grep river-api-settings); do $K -n orinoco delete "$cm" >/dev/null; done rm -rf "$WS/q45"; mkdir -p "$WS/q45/base" "$WS/q45/overlays/staging" cat > "$WS/q45/base/deployment.yaml" <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: {name: river-api, labels: {app: river-api}} spec: replicas: 1 selector: {matchLabels: {app: river-api}} template: metadata: {labels: {app: river-api}} spec: {containers: [{name: river-api, image: nginx:1.27, ports: [{containerPort: 80}]}]} EOF cat > "$WS/q45/base/service.yaml" <<'EOF' apiVersion: v1 kind: Service metadata: {name: river-api} spec: {selector: {app: river-api}, ports: [{port: 80}]} EOF cat > "$WS/q45/base/kustomization.yaml" <<'EOF' apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: [deployment.yaml, service.yaml] EOF printf 'LOG_LEVEL=info\nFEATURE_FLAGS=search,export\n' > "$WS/q45/overlays/staging/settings.env" (cd "$WS/q45/base" && md5sum *) > "$WS/.q45-base.md5" """, checks=[ C("renders: ConfigMap river-api-settings-, used by the container via envFrom, 128Mi limit, 2 replicas, all in orinoco", 3, r""" $K kustomize "$WS/q45/overlays/staging" 2>/dev/null | $K create --dry-run=client -o json -f - 2>/dev/null | python3 -c ' import sys, json, re s = sys.stdin.read(); dec = json.JSONDecoder(); i = 0; items = [] while i < len(s): while i < len(s) and s[i].isspace(): i += 1 if i >= len(s): break o, i = dec.raw_decode(s, i); items.extend(o.get("items", [o])) cms = [x for x in items if x["kind"] == "ConfigMap" and re.fullmatch(r"river-api-settings-[a-z0-9]{6,}", x["metadata"]["name"])] dep = [x for x in items if x["kind"] == "Deployment"] if len(cms) != 1 or len(dep) != 1: sys.exit(1) cm, d = cms[0], dep[0]; c = d["spec"]["template"]["spec"]["containers"][0] ok = cm.get("data", {}).get("LOG_LEVEL") == "info" and d["spec"].get("replicas") == 2 \ and any((e.get("configMapRef") or {}).get("name") == cm["metadata"]["name"] for e in c.get("envFrom", [])) \ and ((c.get("resources") or {}).get("limits") or {}).get("memory") == "128Mi" \ and all(x["metadata"].get("namespace") == "orinoco" for x in items) sys.exit(0 if ok else 1)'"""), C("applied: river-api in orinoco, 2 available, Pods see LOG_LEVEL=info", 3, r""" for i in $(seq 1 30); do [ "$(j -n orinoco deploy river-api '{.status.updatedReplicas}/{.status.availableReplicas}')" = 2/2 ] && break; sleep 3; done eq "$(j -n orinoco deploy river-api '{.status.availableReplicas}')" 2 && eq "$($K -n orinoco exec deploy/river-api -- printenv LOG_LEVEL 2>/dev/null)" info"""), C("base left untouched", 1, r""" [ -f "$WS/q45/overlays/staging/kustomization.yaml" ] && (cd "$WS/q45/base" && md5sum -c --quiet "$WS/.q45-base.md5" >/dev/null 2>&1)"""), ], solution=""" cd {WS}/q45/overlays/staging cat > kustomization.yaml < patch.yaml <, and so is the envFrom reference k apply -k . k -n orinoco exec deploy/river-api -- printenv LOG_LEVEL # The patch names the ConfigMap without its suffix; Kustomize rewrites every reference to the # hashed name. Change settings.env and the name changes too - which is what rolls the Deployment. """, ) # ------------------------------------------------------------------ 46 ---- task( id=46, domain="OBS", title="Probes that match the app's warm-up", task=""" Pods of Deployment `tide-api` in Namespace `tagus` report Ready long before they can serve: the app needs about 20 seconds to warm up, and only then writes the file `/tmp/warm` and starts serving HTTP on port 8080. Give its container: - a readiness probe that passes only once `/tmp/warm` exists, checked with a command every 5 seconds, - a liveness probe: HTTP GET `/healthz` on port 8080, first run 30 seconds after the container starts, then every 10 seconds, failing after 3 misses. Both replicas must end up Ready, without restarts. """, setup=NS + r""" mkns tagus $K -n tagus delete deploy tide-api --ignore-not-found --wait=true >/dev/null $K apply -f - >/dev/null <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: {name: tide-api, namespace: tagus} spec: replicas: 2 selector: {matchLabels: {app: tide-api}} template: metadata: {labels: {app: tide-api}} spec: containers: - name: api image: busybox:1.36 command: [sh, -c, "sleep 20; mkdir -p /www; echo ok > /www/healthz; touch /tmp/warm; exec httpd -f -p 8080 -h /www"] ports: [{containerPort: 8080}] EOF """, checks=[ C("readiness: a command checking /tmp/warm, every 5 seconds", 2, r""" r='{.spec.template.spec.containers[0].readinessProbe' j -n tagus deploy tide-api "$r.exec.command}" | grep -q '/tmp/warm' && eq "$(j -n tagus deploy tide-api "$r.periodSeconds}")" 5"""), C("liveness: GET /healthz on 8080, delay 30, period 10, 3 failures", 3, r""" l='{.spec.template.spec.containers[0].livenessProbe' eq "$(j -n tagus deploy tide-api "$l.httpGet.path}/$l.httpGet.port}/$l.initialDelaySeconds}/$l.periodSeconds}/$l.failureThreshold}")" /healthz/8080/30/10/3"""), C("rolled out: 2/2 Ready with no restarts", 2, r""" [ -n "$(j -n tagus deploy tide-api '{.spec.template.spec.containers[0].readinessProbe}')" ] || exit 1 for i in $(seq 1 40); do [ "$(j -n tagus deploy tide-api '{.status.updatedReplicas}/{.status.readyReplicas}/{.status.replicas}')" = 2/2/2 ] && break; sleep 3; done [ "$(j -n tagus deploy tide-api '{.status.updatedReplicas}/{.status.readyReplicas}/{.status.replicas}')" = 2/2/2 ] || exit 1 $K -n tagus get pod -l app=tide-api -o json 2>/dev/null | python3 -c ' import sys, json live = [p for p in json.load(sys.stdin)["items"] if not p["metadata"].get("deletionTimestamp")] sys.exit(0 if len(live) == 2 and all(c.get("restartCount", 0) == 0 for p in live for c in p["status"].get("containerStatuses", [])) else 1)'"""), ], solution=""" k -n tagus edit deploy tide-api # under containers[0]: # readinessProbe: # exec: {command: [test, -f, /tmp/warm]} # periodSeconds: 5 # livenessProbe: # httpGet: {path: /healthz, port: 8080} # initialDelaySeconds: 30 # periodSeconds: 10 # failureThreshold: 3 k -n tagus rollout status deploy tide-api # ~25 s: Ready only once /tmp/warm exists # A liveness probe that fires before the app listens restarts it forever - that is what the # 30 s delay protects against. Readiness has no such risk: failing it only removes the Pod from # the Service's endpoints. """, ) # ------------------------------------------------------------------ 47 ---- task( id=47, domain="OBS", title="Debugging a container with no shell", params=lambda r: {"token": "".join(r.choice("abcdefghjkmnpqrstuvwxyz23456789") for _ in range(16))}, task=""" Pod `vault-shim` in Namespace `elbe` runs an image with no shell and no tools at all. Its main process was started with an environment variable `SHIM_TOKEN` taken from a Secret that has since been deleted. Without restarting or recreating the Pod, find the value that the running process has and write it to `{WS}/q47/token.txt`. """, setup=NS + r""" mkns elbe; rm -rf "$WS/q47"; mkdir -p "$WS/q47" $K -n elbe delete pod vault-shim --ignore-not-found --wait=true --grace-period=1 >/dev/null $K -n elbe delete secret vault-shim-boot --ignore-not-found >/dev/null $K -n elbe create secret generic vault-shim-boot --from-literal=token=%{token} >/dev/null $K apply -f - >/dev/null <<'EOF' apiVersion: v1 kind: Pod metadata: {name: vault-shim, namespace: elbe} spec: containers: - name: main image: registry.k8s.io/pause:3.10 env: [{name: SHIM_TOKEN, valueFrom: {secretKeyRef: {name: vault-shim-boot, key: token}}}] EOF $K -n elbe wait --for=condition=Ready pod/vault-shim --timeout=150s >/dev/null || exit 1 # once the process runs, the value lives only in its environment $K -n elbe delete secret vault-shim-boot >/dev/null $K -n elbe get pod vault-shim -o jsonpath='{.metadata.uid}' > "$WS/.q47-uid" """, checks=[ C("token.txt holds the running process's SHIM_TOKEN", 4, r"""eq "$(tr -d '[:space:]' < "$WS/q47/token.txt" 2>/dev/null)" %{token}"""), C("found through an ephemeral debug container", 2, r"""[ -n "$(j -n elbe pod vault-shim '{.spec.ephemeralContainers[*].name}')" ]"""), C("the Pod was neither restarted nor recreated", 1, r""" eq "$(tr -d '[:space:]' < "$WS/q47/token.txt" 2>/dev/null)" %{token} || exit 1 eq "$(j -n elbe pod vault-shim '{.metadata.uid}')" "$(cat "$WS/.q47-uid" 2>/dev/null)" && eq "$(j -n elbe pod vault-shim '{.status.containerStatuses[0].restartCount}')" 0"""), ], solution=""" k -n elbe get pod vault-shim -o yaml | grep -A4 SHIM_TOKEN # only a secretKeyRef - and that Secret is gone k -n elbe exec vault-shim -- sh # fails: there is no shell in the image k -n elbe debug -it vault-shim --image=busybox:1.36 --target=main -- sh # inside the ephemeral container: --target shares main's process namespace, so its process is PID 1 tr '\\0' '\\n' < /proc/1/environ | grep SHIM_TOKEN exit echo '' > {WS}/q47/token.txt # Without --target you only see your own shell's processes. Ephemeral containers are never # restarted and cannot be removed; they stay listed in the Pod spec, which is harmless. """, ) # ------------------------------------------------------------------ 48 ---- task( id=48, domain="OBS", title="Restart and event forensics", task=""" Namespaces `seine` and `amur` run Riverline's port services. 1. Write the Pod with the most container restarts across both Namespaces to `{WS}/q48/most-restarts.txt`, as `/`. 2. Write the exit code of that Pod's last terminated container to `{WS}/q48/exit-code.txt`. 3. Write a command into `{WS}/q48/warnings.sh` that lists only the **Warning** events of Namespace `seine`, newest last. """, setup=NS + r""" mkns seine; mkns amur; rm -rf "$WS/q48"; mkdir -p "$WS/q48" for n in seine amur; do $K label ns $n team=riverline --overwrite >/dev/null; done $K -n seine delete deploy crane-a crane-b --ignore-not-found --wait=true >/dev/null $K -n amur delete deploy pump --ignore-not-found --wait=true >/dev/null $K -n seine create deploy crane-a --image=busybox:1.36 -- sleep 86400 >/dev/null $K -n seine create deploy crane-b --image=busybox:1.36 -- sh -c 'echo "crane b: hydraulic fault"; sleep 40; exit 1' >/dev/null $K -n amur create deploy pump --image=busybox:1.36 -- sh -c 'echo "pump: pressure lost"; sleep 8; exit 3' >/dev/null """, checks=[ C("most-restarts.txt names the Pod with the most restarts", 3, r""" best=$(for n in seine amur; do $K -n $n get pod -o json; done 2>/dev/null | python3 -c ' import sys, json s = sys.stdin.read(); dec = json.JSONDecoder(); i = 0; pods = [] while i < len(s): while i < len(s) and s[i].isspace(): i += 1 if i >= len(s): break o, i = dec.raw_decode(s, i); pods.extend(o["items"]) p = max(pods, key=lambda p: sum(c.get("restartCount", 0) for c in p["status"].get("containerStatuses", []))) print(p["metadata"]["namespace"] + "/" + p["metadata"]["name"])') [ -n "$best" ] && eq "$(tr -d '[:space:]' < "$WS/q48/most-restarts.txt" 2>/dev/null)" "$best" """), C("exit-code.txt holds its last exit code", 2, r""" p=$(tr -d '[:space:]' < "$WS/q48/most-restarts.txt" 2>/dev/null); n=${p%%/*}; pod=${p#*/} case "$n" in seine|amur) ;; *) exit 1;; esac [ "$(j -n "$n" pod "$pod" '{.status.containerStatuses[0].restartCount}')" -ge 1 ] 2>/dev/null || exit 1 eq "$(tr -d '[:space:]' < "$WS/q48/exit-code.txt" 2>/dev/null)" "$(j -n "$n" pod "$pod" '{.status.containerStatuses[0].lastState.terminated.exitCode}')" """), C("warnings.sh lists only seine's Warning events, sorted by time", 2, r""" f="$WS/q48/warnings.sh"; [ -s "$f" ] && grep -q seine "$f" || exit 1 grep -Eq -- '--sort-by|kubectl +events|k +events' "$f" || exit 1 out=$(cd / && bash "$f" 2>/dev/null | grep -Ev '^(LAST SEEN|NAMESPACE)'); [ -n "$out" ] || exit 1 [ -z "$(echo "$out" | grep -v Warning)" ]"""), ], solution=""" k get pod -n seine; k get pod -n amur # RESTARTS column k get pod -A --sort-by='.status.containerStatuses[0].restartCount' | grep -E '^(seine|amur) ' echo amur/ > {WS}/q48/most-restarts.txt k -n amur get pod -o jsonpath='{.status.containerStatuses[0].lastState.terminated.exitCode}' > {WS}/q48/exit-code.txt echo 'kubectl -n seine get events --field-selector type=Warning --sort-by=.lastTimestamp' > {WS}/q48/warnings.sh # (`kubectl events -n seine --types=Warning` works too: it sorts oldest to newest by itself.) # `describe pod` shows the same exit code under Last State: Terminated. """, ) # ------------------------------------------------------------------ 49 ---- task( id=49, domain="ENV", title="RBAC, proven by the app's own log", task=""" Deployment `ledger-sync` in Namespace `nile` runs as ServiceAccount `ledger-sync` and polls the Kubernetes API for the ConfigMaps of its own Namespace. Its log only says `forbidden`. 1. Allow that ServiceAccount to `get`, `list` and `watch` ConfigMaps in `nile` - nothing more - using a Role `configmap-reader` and a RoleBinding `ledger-sync-configmaps`. 2. Once the app picks it up, write the latest line of its log to `{WS}/q49/log.txt`. """, setup=NS + r""" mkns nile; rm -rf "$WS/q49"; mkdir -p "$WS/q49" $K -n nile delete deploy ledger-sync --ignore-not-found --wait=true >/dev/null $K -n nile delete rolebinding ledger-sync-configmaps --ignore-not-found >/dev/null; $K -n nile delete role configmap-reader --ignore-not-found >/dev/null $K -n nile create sa ledger-sync --dry-run=client -o yaml | $K apply -f - >/dev/null $K -n nile create cm river-rates --from-literal=eur=1.00 --dry-run=client -o yaml | $K apply -f - >/dev/null $K apply -f - >/dev/null <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: {name: ledger-sync, namespace: nile} spec: replicas: 1 selector: {matchLabels: {app: ledger-sync}} template: metadata: {labels: {app: ledger-sync}} spec: serviceAccountName: ledger-sync containers: - name: sync image: nginx:1.27 command: - sh - -c - | trap 'exit 0' TERM t=/var/run/secrets/kubernetes.io/serviceaccount while true; do # curl's http_code write-out variable, split by '' so the exam's placeholder substitution never matches it code=$(curl -s -o /tmp/out -w '%''{http_code}' --cacert $t/ca.crt -H "Authorization: Bearer $(cat $t/token)" https://kubernetes.default.svc/api/v1/namespaces/nile/configmaps) if [ "$code" = 200 ]; then echo "ledger-sync: can list configmaps ($(grep -o '"name":' /tmp/out | wc -l) names seen)"; else echo "ledger-sync: forbidden (HTTP $code)"; fi sleep 5 done EOF """, checks=[ C("the ServiceAccount can get, list and watch ConfigMaps in nile", 2, r""" a="--as=system:serviceaccount:nile:ledger-sync -n nile" for v in get list watch; do eq "$($K auth can-i $v configmaps $a 2>/dev/null)" yes || exit 1; done"""), C("... and nothing more", 2, r""" a="--as=system:serviceaccount:nile:ledger-sync" eq "$($K auth can-i list configmaps $a -n nile 2>/dev/null)" yes || exit 1 for t in "create configmaps -n nile" "delete configmaps -n nile" "list secrets -n nile" "list configmaps -n default"; do eq "$($K auth can-i $t $a 2>/dev/null)" no || exit 1 done"""), C("Role configmap-reader, bound to ledger-sync by RoleBinding ledger-sync-configmaps", 1, r""" b="$K -n nile get rolebinding ledger-sync-configmaps" eq "$($b -o jsonpath='{.roleRef.kind}/{.roleRef.name}' 2>/dev/null)" Role/configmap-reader \ && eq "$($b -o jsonpath='{.subjects[0].kind}/{.subjects[0].name}')" ServiceAccount/ledger-sync"""), C("the app now logs success, and log.txt holds that line", 2, r""" for i in $(seq 1 10); do $K -n nile logs deploy/ledger-sync --tail=1 2>/dev/null | grep -q 'can list configmaps' && break; sleep 3; done $K -n nile logs deploy/ledger-sync --tail=1 2>/dev/null | grep -q 'can list configmaps' && grep -q 'can list configmaps' "$WS/q49/log.txt" 2>/dev/null"""), ], solution=""" k -n nile logs deploy/ledger-sync --tail=3 # ledger-sync: forbidden (HTTP 403) k -n nile create role configmap-reader --verb=get,list,watch --resource=configmaps k -n nile create rolebinding ledger-sync-configmaps --role=configmap-reader --serviceaccount=nile:ledger-sync k auth can-i list configmaps -n nile --as=system:serviceaccount:nile:ledger-sync # yes k auth can-i list secrets -n nile --as=system:serviceaccount:nile:ledger-sync # no k -n nile logs deploy/ledger-sync --tail=1 > {WS}/q49/log.txt # within ~5 s: can list configmaps # RBAC takes effect on the next request - no restart. The --serviceaccount form is NAMESPACE:NAME. """, ) # ------------------------------------------------------------------ 50 ---- task( id=50, domain="ENV", title="An immutable Secret, one file into a busy directory", task=""" In Namespace `indus`: 1. Create an **immutable** Secret `gatekeeper-tls-settings` that holds the file `{WS}/q50/tls-settings.conf` under the key `tls-settings.conf`. 2. Mount just that key into Deployment `gatekeeper`'s container as the file `/etc/nginx/conf.d/tls-settings.conf`, without hiding the files already in that directory. """, setup=NS + r""" mkns indus; rm -rf "$WS/q50"; mkdir -p "$WS/q50" $K -n indus delete deploy gatekeeper --ignore-not-found --wait=true >/dev/null $K -n indus delete secret gatekeeper-tls-settings --ignore-not-found >/dev/null printf '# managed by the Riverline security team\nssl_session_timeout 10m;\n' > "$WS/q50/tls-settings.conf" $K -n indus create deploy gatekeeper --image=nginx:1.27 >/dev/null $K -n indus rollout status deploy gatekeeper --timeout=150s >/dev/null 2>&1 """, checks=[ C("Secret gatekeeper-tls-settings: immutable, key tls-settings.conf = the file", 2, r""" eq "$(j -n indus secret gatekeeper-tls-settings '{.immutable}')" true \ && [ "$(j -n indus secret gatekeeper-tls-settings '{.data.tls-settings\.conf}' | base64 -d 2>/dev/null)" = "$(cat "$WS/q50/tls-settings.conf")" ]"""), C("mounted with subPath at /etc/nginx/conf.d/tls-settings.conf", 2, r""" v=$(j -n indus deploy gatekeeper '{.spec.template.spec.volumes[?(@.secret.secretName=="gatekeeper-tls-settings")].name}'); [ -n "$v" ] || exit 1 m="{.spec.template.spec.containers[0].volumeMounts[?(@.name==\"$v\")]" eq "$(j -n indus deploy gatekeeper "$m.mountPath}/$m.subPath}")" /etc/nginx/conf.d/tls-settings.conf/tls-settings.conf"""), C("the running Pod has default.conf and tls-settings.conf, nginx serving", 3, r""" for i in $(seq 1 30); do [ "$(j -n indus deploy gatekeeper '{.status.updatedReplicas}/{.status.availableReplicas}/{.status.replicas}')" = 1/1/1 ] && break; sleep 3; done l=" $($K -n indus exec deploy/gatekeeper -- ls /etc/nginx/conf.d 2>/dev/null | tr '\n' ' ')" case "$l" in *" default.conf "*) ;; *) exit 1;; esac; case "$l" in *" tls-settings.conf "*) ;; *) exit 1;; esac [ "$($K -n indus exec deploy/gatekeeper -- cat /etc/nginx/conf.d/tls-settings.conf 2>/dev/null)" = "$(cat "$WS/q50/tls-settings.conf")" ] \ && $K -n indus exec deploy/gatekeeper -- curl -s -m 3 -o /dev/null http://localhost/ 2>/dev/null"""), ], solution=""" k -n indus create secret generic gatekeeper-tls-settings --from-file=tls-settings.conf={WS}/q50/tls-settings.conf $do > s.yaml echo 'immutable: true' >> s.yaml k apply -f s.yaml k -n indus edit deploy gatekeeper # template.spec.volumes: [{name: tls, secret: {secretName: gatekeeper-tls-settings}}] # containers[0].volumeMounts: # - {name: tls, mountPath: /etc/nginx/conf.d/tls-settings.conf, subPath: tls-settings.conf} k -n indus exec deploy/gatekeeper -- ls /etc/nginx/conf.d # default.conf tls-settings.conf # Mounting the Secret on the directory itself would hide default.conf; subPath mounts one file. # The catch - a subPath mount never sees updates - cannot bite an immutable Secret. """, ) # ------------------------------------------------------------------ 51 ---- task( id=51, domain="ENV", title="A ConfigMap-driven nginx that needs a restart", task=""" Deployment `harbor-proxy` in Namespace `niger` takes its whole nginx configuration from ConfigMap `harbor-nginx`. 1. Change that configuration so that `/status` answers `harbor v2 ok`, while `/` keeps working. 2. Make the running Pods serve the new configuration - nginx does not reload by itself. 3. From inside the cluster, fetch `http://harbor-proxy.niger.svc.cluster.local/status` and save the response to `{WS}/q51/status.txt`. """, setup=NS + r""" mkns niger; rm -rf "$WS/q51"; mkdir -p "$WS/q51" $K -n niger delete deploy harbor-proxy --ignore-not-found --wait=true >/dev/null # a test Pod left by an interrupted earlier attempt would make `kubectl run tmp` fail "AlreadyExists" $K -n niger delete pod tmp --ignore-not-found --wait=false >/dev/null $K apply -f - >/dev/null <<'EOF' apiVersion: v1 kind: ConfigMap metadata: {name: harbor-nginx, namespace: niger} data: default.conf: | server { listen 80; location / { return 200 "harbor v1"; } } --- apiVersion: apps/v1 kind: Deployment metadata: {name: harbor-proxy, namespace: niger} spec: replicas: 2 selector: {matchLabels: {app: harbor-proxy}} template: metadata: {labels: {app: harbor-proxy}} spec: volumes: [{name: conf, configMap: {name: harbor-nginx}}] containers: - name: nginx image: nginx:1.27 volumeMounts: [{name: conf, mountPath: /etc/nginx/conf.d}] --- apiVersion: v1 kind: Service metadata: {name: harbor-proxy, namespace: niger} spec: {selector: {app: harbor-proxy}, ports: [{port: 80}]} EOF $K -n niger rollout status deploy harbor-proxy --timeout=150s >/dev/null 2>&1 """, checks=[ C("the ConfigMap has a /status location", 1, r"""j -n niger cm harbor-nginx '{.data.default\.conf}' | grep -Eq 'location +(= +|\^~ +)?/status'"""), C("every Pod answers /status with 'harbor v2 ok' and / with 'harbor v1'", 4, r""" for i in $(seq 1 30); do [ "$(j -n niger deploy harbor-proxy '{.status.updatedReplicas}/{.status.availableReplicas}/{.status.replicas}')" = 2/2/2 ] && break; sleep 3; done live=$($K -n niger get pod -l app=harbor-proxy -o json 2>/dev/null | python3 -c ' import sys, json for p in json.load(sys.stdin)["items"]: if not p["metadata"].get("deletionTimestamp") and p["status"].get("podIP"): print(p["metadata"]["name"], p["status"]["podIP"])') [ "$(echo "$live" | grep -c .)" = 2 ] || exit 1 src=$(echo "$live" | head -1 | cut -d' ' -f1) for ip in $(echo "$live" | cut -d' ' -f2); do case "$($K -n niger exec "$src" -- curl -s -m 3 "http://$ip/status" 2>/dev/null)" in *"harbor v2 ok"*) ;; *) exit 1;; esac case "$($K -n niger exec "$src" -- curl -s -m 3 "http://$ip/" 2>/dev/null)" in *"harbor v1"*) ;; *) exit 1;; esac done"""), C("status.txt holds the /status response", 2, r"""grep -q 'harbor v2 ok' "$WS/q51/status.txt" 2>/dev/null"""), ], solution=""" k -n niger edit cm harbor-nginx # add inside the server block: # location /status { return 200 "harbor v2 ok"; } k -n niger rollout restart deploy harbor-proxy k -n niger rollout status deploy harbor-proxy k -n niger run tmp --rm -i --restart=Never --image=nginx:1.27 -- \\ curl -s http://harbor-proxy.niger.svc.cluster.local/status > {WS}/q51/status.txt # The kubelet does update a mounted ConfigMap in place (after up to a minute), but nginx reads its # config only at start - so restart the Pods (or `nginx -s reload` in each one). """, ) # ------------------------------------------------------------------ 52 ---- task( id=52, domain="ENV", title="Two users, one volume: fsGroup", task=""" In Namespace `murray` create Pod `ledger-pair`: - container `writer-a` (`busybox:1.36`) runs as user **1000**, writes `/data/a.txt`, then sleeps, - container `writer-b` (`busybox:1.36`) runs as user **2000**, writes `/data/b.txt`, then sleeps, - both share an `emptyDir` volume `data` mounted at `/data`, - every file they create in that volume must belong to group **3000**, - neither container may run as root. """, setup=NS + r""" mkns murray $K -n murray delete pod ledger-pair --ignore-not-found --wait=true --grace-period=1 >/dev/null """, checks=[ C("Running: writer-a as uid 1000 and writer-b as uid 2000, runAsNonRoot", 2, r""" eq "$(j -n murray pod ledger-pair '{.status.phase}')" Running || exit 1 eq "$($K -n murray exec ledger-pair -c writer-a -- id -u 2>/dev/null)/$($K -n murray exec ledger-pair -c writer-b -- id -u 2>/dev/null)" 1000/2000 || exit 1 j -n murray pod ledger-pair '{.spec.securityContext.runAsNonRoot} {.spec.containers[*].securityContext.runAsNonRoot}' | grep -q true"""), C("fsGroup 3000 on the Pod", 2, r"""eq "$(j -n murray pod ledger-pair '{.spec.securityContext.fsGroup}')" 3000"""), C("a.txt is 1000:3000 and b.txt is 2000:3000", 3, r""" eq "$($K -n murray exec ledger-pair -c writer-a -- stat -c '%u:%g' /data/a.txt 2>/dev/null)" 1000:3000 \ && eq "$($K -n murray exec ledger-pair -c writer-a -- stat -c '%u:%g' /data/b.txt 2>/dev/null)" 2000:3000"""), ], solution=""" cat < /data/a.txt; sleep 86400"] volumeMounts: [{name: data, mountPath: /data}] - name: writer-b image: busybox:1.36 securityContext: {runAsUser: 2000} command: [sh, -c, "echo b > /data/b.txt; sleep 86400"] volumeMounts: [{name: data, mountPath: /data}] EOF k -n murray exec ledger-pair -c writer-a -- ls -ln /data # 1000 3000 a.txt / 2000 3000 b.txt # fsGroup makes the volume group-owned by 3000 with the setgid bit, so new files inherit group 3000, # and adds 3000 to every container's supplementary groups. runAsUser is per container here. """, ) # ------------------------------------------------------------------ 53 ---- task( id=53, domain="ENV", title="Pods rejected by the Namespace's limits", task=""" Deployment `batch-runner` in Namespace `douro` has no Pods at all. 1. Find out why, and write the object that rejects them to `{WS}/q53/rejected-by.txt` as `/`. 2. Fix the Deployment's container resources so both replicas run: set the memory **limit** as high as the Namespace allows and **request** half of that. Do not change anything but the Deployment. """, setup=NS + r""" mkns douro; rm -rf "$WS/q53"; mkdir -p "$WS/q53" $K -n douro delete deploy batch-runner --ignore-not-found --wait=true >/dev/null $K -n douro delete limitrange --all >/dev/null 2>&1; $K -n douro delete resourcequota --all >/dev/null 2>&1 $K apply -f - >/dev/null <<'EOF' apiVersion: v1 kind: LimitRange metadata: {name: douro-limits, namespace: douro} spec: limits: - type: Container max: {cpu: 500m, memory: 256Mi} defaultRequest: {cpu: 100m, memory: 64Mi} default: {cpu: 200m, memory: 128Mi} --- apiVersion: v1 kind: ResourceQuota metadata: {name: douro-quota, namespace: douro} spec: hard: {requests.memory: 512Mi, limits.memory: 1Gi, pods: "4"} --- apiVersion: apps/v1 kind: Deployment metadata: {name: batch-runner, namespace: douro} spec: replicas: 2 selector: {matchLabels: {app: batch-runner}} template: metadata: {labels: {app: batch-runner}} spec: containers: - name: runner image: busybox:1.36 command: [sleep, "86400"] resources: {requests: {cpu: 100m, memory: 400Mi}, limits: {cpu: 200m, memory: 600Mi}} EOF # specs, not resourceVersions: the quota controller rewrites the ResourceQuota's status (and with it # its resourceVersion) as soon as Pods start, so only the spec shows whether the candidate changed it for o in limitrange/douro-limits resourcequota/douro-quota; do $K -n douro get $o -o jsonpath='{.spec}'; echo; done > "$WS/.q53-spec" """, checks=[ C("rejected-by.txt names the LimitRange", 1, r""" case "$(tr -d '[:space:]' < "$WS/q53/rejected-by.txt" 2>/dev/null | tr A-Z a-z)" in limitrange/douro-limits|limitranges/douro-limits) ;; *) exit 1;; esac"""), C("container memory: limit 256Mi, request 128Mi", 2, r""" r='{.spec.template.spec.containers[0].resources' eq "$(j -n douro deploy batch-runner "$r.limits.memory}/$r.requests.memory}")" 256Mi/128Mi"""), C("both replicas run", 3, r""" for i in $(seq 1 30); do [ "$(j -n douro deploy batch-runner '{.status.availableReplicas}')" = 2 ] && exit 0; sleep 3; done; exit 1"""), C("LimitRange and ResourceQuota untouched", 1, r""" [ "$(j -n douro deploy batch-runner '{.metadata.generation}')" -gt 1 ] 2>/dev/null || exit 1 [ "$(for o in limitrange/douro-limits resourcequota/douro-quota; do $K -n douro get $o -o jsonpath='{.spec}'; echo; done)" = "$(cat "$WS/.q53-spec" 2>/dev/null)" ]"""), ], solution=""" k -n douro get deploy,rs,pod # desired 2, current 0 k -n douro describe rs -l app=batch-runner | tail -3 # Error creating: pods ... forbidden: maximum memory usage per Container is 256Mi, but limit is 600Mi k -n douro get limitrange,resourcequota # the message names no object - the LimitRange is douro-limits echo LimitRange/douro-limits > {WS}/q53/rejected-by.txt k -n douro set resources deploy batch-runner -c runner --limits=memory=256Mi --requests=memory=128Mi k -n douro get pod -w # 2 x 128Mi requested fits the quota's 512Mi too - 2 x 400Mi would have been refused by the quota # next, even with the LimitRange satisfied. """, ) # ------------------------------------------------------------------ 54 ---- task( id=54, domain="NET", title="Expose a Pod and prove it from inside", task=""" In Namespace `oder`, Pod `buoy-api` serves HTTP on port 80. 1. Create a ClusterIP Service `buoy-svc` that exposes it on port **8080**. 2. From a **temporary** Pod in the same Namespace, request `http://buoy-svc:8080` and save the response body to `{WS}/q54/response.html`. The temporary Pod must not be left behind. 3. Afterwards save the log of Pod `buoy-api` to `{WS}/q54/buoy.log` - it must show your request. """, setup=NS + r""" mkns oder; rm -rf "$WS/q54"; mkdir -p "$WS/q54" $K -n oder delete svc buoy-svc --ignore-not-found >/dev/null $K -n oder delete pod --all --wait=true --grace-period=1 >/dev/null 2>&1 $K -n oder run buoy-api --image=nginx:1.27 --labels=app=buoy-api >/dev/null $K -n oder wait --for=condition=Ready pod/buoy-api --timeout=150s >/dev/null """, checks=[ C("Service buoy-svc: ClusterIP, 8080 -> 80, its endpoint is buoy-api", 3, r""" eq "$(j -n oder svc buoy-svc '{.spec.type}/{.spec.ports[0].port}/{.spec.ports[0].targetPort}')" ClusterIP/8080/80 || exit 1 ip=$(j -n oder pod buoy-api '{.status.podIP}') [ -n "$ip" ] && j -n oder endpointslices -l kubernetes.io/service-name=buoy-svc '{.items[*].endpoints[*].addresses[*]}' | tr ' ' '\n' | grep -qx "$ip" """), C("response.html holds the page", 2, r"""grep -qi 'Welcome to nginx' "$WS/q54/response.html" 2>/dev/null"""), C("buoy.log shows the request, and no temporary Pod is left", 2, r""" grep -q '"GET / HTTP/1.1" 200' "$WS/q54/buoy.log" 2>/dev/null && eq "$(j -n oder pod '{.items[*].metadata.name}')" buoy-api"""), ], solution=""" k -n oder expose pod buoy-api --name=buoy-svc --port=8080 --target-port=80 k -n oder run tmp --rm -i --restart=Never --image=nginx:1.27 -- curl -s http://buoy-svc:8080 > {WS}/q54/response.html # (busybox: --image=busybox:1.36 -- wget -qO- http://buoy-svc:8080) k -n oder logs buoy-api > {WS}/q54/buoy.log # ... "GET / HTTP/1.1" 200 ... # --rm needs -i (attach) to know when to delete the Pod; --restart=Never makes it a plain Pod. # Check the saved file: kubectl may append 'pod "tmp" deleted' - to stderr, but look anyway. """, ) # ------------------------------------------------------------------ 55 ---- task( id=55, domain="NET", title="Two sources, one port", task=""" In Namespace `zambezi`, Pods labelled `app=vault` may accept connections on TCP port 80 only from - Pods labelled `app=teller` in the same Namespace, and - any Pod in a Namespace labelled `purpose=monitoring`. Create NetworkPolicy `vault-ingress` for this. Nothing else may reach `app=vault`, and outgoing traffic must stay unrestricted. kindnet enforces NetworkPolicy here, so it is tested for real. """, setup=NS + r""" mkns zambezi; mkns zambezi-mon; mkns zambezi-ext $K label ns zambezi-mon purpose=monitoring --overwrite >/dev/null; $K label ns zambezi-ext purpose- >/dev/null 2>&1 $K -n zambezi delete netpol --all >/dev/null 2>&1 $K -n zambezi run vault --image=nginx:1.27 --labels=app=vault --dry-run=client -o yaml | $K apply -f - >/dev/null for p in teller auditor; do $K -n zambezi run $p --image=busybox:1.36 --labels=app=$p --dry-run=client -o yaml -- sleep 86400 | $K apply -f - >/dev/null; done $K -n zambezi-mon run prober --image=busybox:1.36 --dry-run=client -o yaml -- sleep 86400 | $K apply -f - >/dev/null $K -n zambezi-ext run outsider --image=busybox:1.36 --dry-run=client -o yaml -- sleep 86400 | $K apply -f - >/dev/null for np in zambezi/vault zambezi/teller zambezi/auditor zambezi-mon/prober zambezi-ext/outsider; do $K -n "${np%/*}" wait --for=condition=Ready "pod/${np#*/}" --timeout=150s >/dev/null || exit 1 done """, checks=[ C("vault-ingress selects app=vault, restricts Ingress only, port 80", 1, r""" $K -n zambezi get netpol vault-ingress -o json 2>/dev/null | python3 -c ' import sys, json s = json.load(sys.stdin)["spec"] ports = [p.get("port") for r in s.get("ingress", []) for p in r.get("ports", [])] ok = s.get("podSelector", {}).get("matchLabels", {}).get("app") == "vault" and s.get("policyTypes") == ["Ingress"] \ and ports and all(str(p) == "80" for p in ports) and all(r.get("ports") for r in s.get("ingress", [])) sys.exit(0 if ok else 1)'"""), C("teller reaches vault", 2, r""" [ -n "$(j -n zambezi netpol vault-ingress '{.metadata.name}')" ] || exit 1 ip=$(j -n zambezi pod vault '{.status.podIP}'); $K -n zambezi exec teller -- wget -T 3 -qO- "http://$ip" >/dev/null 2>&1"""), C("a Pod in a purpose=monitoring Namespace reaches vault", 2, r""" [ -n "$(j -n zambezi netpol vault-ingress '{.metadata.name}')" ] || exit 1 ip=$(j -n zambezi pod vault '{.status.podIP}'); $K -n zambezi-mon exec prober -- wget -T 3 -qO- "http://$ip" >/dev/null 2>&1"""), C("auditor and Pods in other Namespaces are blocked", 2, r""" # only meaningful while the policy exists and an allowed path demonstrably works ip=$(j -n zambezi pod vault '{.status.podIP}') [ -n "$(j -n zambezi netpol vault-ingress '{.metadata.name}')" ] && $K -n zambezi exec teller -- wget -T 3 -qO- "http://$ip" >/dev/null 2>&1 || exit 1 ! $K -n zambezi exec auditor -- wget -T 3 -qO- "http://$ip" >/dev/null 2>&1 && ! $K -n zambezi-ext exec outsider -- wget -T 3 -qO- "http://$ip" >/dev/null 2>&1"""), ], solution=""" cat < # works k -n zambezi exec auditor -- wget -T 3 -qO- http:// # times out # Two list items under `from` are OR. Writing `- podSelector: ... namespaceSelector: ...` as ONE item # would mean "teller Pods inside a monitoring Namespace" - and block both real sources. """, ) # ------------------------------------------------------------------ 56 ---- task( id=56, domain="NET", title="Stable DNS names for a StatefulSet", task=""" In Namespace `ebro`: 1. Create a headless Service `lighthouse` (port 80) and a StatefulSet `lighthouse` (3 replicas, image `nginx:1.27`) that uses it, so that each Pod gets its own DNS name. 2. Write the fully qualified DNS name of the StatefulSet's second Pod to `{WS}/q56/fqdn.txt`. 3. From a temporary Pod, resolve that name and write the IP address it returns to `{WS}/q56/ip.txt`. """, setup=NS + r""" mkns ebro; rm -rf "$WS/q56"; mkdir -p "$WS/q56" $K -n ebro delete sts lighthouse --ignore-not-found --wait=true >/dev/null; $K -n ebro delete svc lighthouse --ignore-not-found >/dev/null """, checks=[ C("headless Service lighthouse with the 3 Pods as endpoints", 2, r""" eq "$(j -n ebro svc lighthouse '{.spec.clusterIP}')" None || exit 1 for i in $(seq 1 20); do [ "$(j -n ebro endpointslices -l kubernetes.io/service-name=lighthouse '{range .items[*].endpoints[?(@.conditions.ready==true)]}{.addresses[0]}{" "}{end}' | wc -w)" = 3 ] && exit 0; sleep 3 done; exit 1"""), C("StatefulSet lighthouse: serviceName lighthouse, 3 ready", 2, r""" eq "$(j -n ebro sts lighthouse '{.spec.serviceName}')" lighthouse || exit 1 for i in $(seq 1 30); do [ "$(j -n ebro sts lighthouse '{.status.readyReplicas}')" = 3 ] && exit 0; sleep 3; done; exit 1"""), C("fqdn.txt holds lighthouse-1's FQDN", 2, r"""eq "$(tr -d '[:space:]' < "$WS/q56/fqdn.txt" 2>/dev/null | sed 's/\.$//')" lighthouse-1.lighthouse.ebro.svc.cluster.local"""), C("ip.txt holds lighthouse-1's IP", 1, r"""ip=$(j -n ebro pod lighthouse-1 '{.status.podIP}'); [ -n "$ip" ] && eq "$(tr -d '[:space:]' < "$WS/q56/ip.txt" 2>/dev/null)" "$ip" """), ], solution=""" k -n ebro create svc clusterip lighthouse --clusterip=None --tcp=80:80 $do > sts.yaml # (the selector it writes is app=lighthouse - keep the StatefulSet's Pod labels matching) echo --- >> sts.yaml k -n ebro create deploy lighthouse --image=nginx:1.27 --replicas=3 $do >> sts.yaml vi sts.yaml # Deployment -> StatefulSet, add spec.serviceName: lighthouse, drop strategy/status k apply -f sts.yaml echo lighthouse-1.lighthouse.ebro.svc.cluster.local > {WS}/q56/fqdn.txt k -n ebro run tmp --rm -i --restart=Never --image=busybox:1.36 -- nslookup lighthouse-1.lighthouse.ebro.svc.cluster.local echo > {WS}/q56/ip.txt # ...svc.: the Pod name is the StatefulSet name plus its # ordinal, and the second Pod is ordinal 1. busybox nslookup is only reliable with the FQDN. """, ) # ------------------------------------------------------------------ 57 ---- task( id=57, domain="NET", title="An Ingress pointing at the wrong backends", task=""" Ingress `portal` in Namespace `dnieper` is meant to send `portal.riverline.test/api...` to Service `portal-api` and everything else on that host to Service `portal-ui`, but both of its backends are wrong. Fix the Ingress - not the Services - so that each path points at an existing Service on a port that Service really exposes, both paths with pathType `Prefix`. (No Ingress controller runs here; the object is graded.) """, setup=NS + r""" mkns dnieper for a in portal-ui portal-api; do $K -n dnieper create deploy $a --image=nginx:1.27 --dry-run=client -o yaml | $K apply -f - >/dev/null; done $K -n dnieper expose deploy portal-ui --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null $K -n dnieper expose deploy portal-api --port=8080 --target-port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null $K -n dnieper delete ingress portal --ignore-not-found >/dev/null $K apply -f - >/dev/null <<'EOF' apiVersion: networking.k8s.io/v1 kind: Ingress metadata: {name: portal, namespace: dnieper} spec: ingressClassName: nginx rules: - host: portal.riverline.test http: paths: - {path: /api, pathType: ImplementationSpecific, backend: {service: {name: portal-api, port: {number: 80}}}} - {path: /, pathType: Prefix, backend: {service: {name: portal-web, port: {number: 80}}}} EOF for o in svc/portal-ui svc/portal-api; do $K -n dnieper get $o -o jsonpath='{.metadata.resourceVersion}'; echo; done > "$WS/.q57-rv" """, checks=[ C("/api -> portal-api:8080, Prefix", 3, r""" $K -n dnieper get ingress portal -o json 2>/dev/null | python3 -c ' import sys, json r = [r for r in json.load(sys.stdin)["spec"].get("rules", []) if r.get("host") == "portal.riverline.test"] p = {x.get("path"): x for x in (r[0].get("http", {}).get("paths", []) if r else [])}.get("/api") or {} b = (p.get("backend") or {}).get("service") or {} sys.exit(0 if p.get("pathType") == "Prefix" and b.get("name") == "portal-api" and (b.get("port") or {}).get("number") == 8080 else 1)'"""), C("/ -> portal-ui:80, Prefix", 3, r""" $K -n dnieper get ingress portal -o json 2>/dev/null | python3 -c ' import sys, json r = [r for r in json.load(sys.stdin)["spec"].get("rules", []) if r.get("host") == "portal.riverline.test"] p = {x.get("path"): x for x in (r[0].get("http", {}).get("paths", []) if r else [])}.get("/") or {} b = (p.get("backend") or {}).get("service") or {} sys.exit(0 if p.get("pathType") == "Prefix" and b.get("name") == "portal-ui" and (b.get("port") or {}).get("number") == 80 else 1)'"""), C("Services untouched, host and class unchanged", 1, r""" eq "$(j -n dnieper ingress portal '{.spec.rules[0].http.paths[*].backend.service.name}' | tr ' ' '\n' | sort | tr '\n' ' ')" "portal-api portal-ui " || exit 1 eq "$(j -n dnieper ingress portal '{.spec.ingressClassName}/{.spec.rules[0].host}')" nginx/portal.riverline.test || exit 1 [ "$(for o in svc/portal-ui svc/portal-api; do $K -n dnieper get $o -o jsonpath='{.metadata.resourceVersion}'; echo; done)" = "$(cat "$WS/.q57-rv" 2>/dev/null)" ]"""), ], solution=""" k -n dnieper get svc # portal-ui 80, portal-api 8080 - and no portal-web k -n dnieper describe ingress portal # backends: portal-api:80 (no such port), portal-web:80 (no such Service) k -n dnieper edit ingress portal # /api: pathType Prefix, backend service portal-api port number 8080 # / : backend service portal-ui (port 80 is right) k -n dnieper describe ingress portal # both backends now resolve to endpoints # ImplementationSpecific leaves matching to the controller; Prefix matches /api, /api/ and /api/x # (whole path segments - not /apix). """, ) QUESTIONS = Q