"""CKAD tasks 94-99: Application Observability and Maintenance - the pool tasks that take the domain from 11 to 17. 3.2 Probes 94 readiness takes a Pod out of the Service 95 a slow starter that liveness keeps killing 3.3 CLI monitoring 96 why is it Pending? ask the events 3.4 Logs 97 the logs of Jobs 98 the init container said why 3.5 Debugging 99 a container with no shell, on the wrong port Probes are judged by what they do - which Pods are Ready, what the Service's endpoints are, whether the container was restarted; logs by the lines themselves; the debugging task by the Service answering. Every task has a scripted solution in solve.py. """ from ckad.common import C, NET, ONE, PY, ROLL, NS, ns_pick, name_pick QUESTIONS = [ # ========================================================================= 3.2 Probes ==== { "id": 94, "domain": "OBS", "title": "Readiness takes a Pod out of the Service", "params": lambda r: {"ns": ns_pick(r, "store"), "app": name_pick(r, "catalog")}, "task": """ Deployment `%{app}` in Namespace `%{ns}` (3 replicas, Service `%{app}`) serves the directory `/www` on port 80. A replica is able to take requests exactly as long as the file `/www/ready` exists in it — but nothing tells Kubernetes so. 1. Add a **readiness** probe to the Deployment: HTTP GET `/ready` on port 80, every 3 seconds; one failed check makes the Pod not ready. Do not add a liveness probe. 2. When the rollout is finished, make **one** replica unable to serve: delete `/www/ready` in it. Write that Pod's name to `{WS}/q94/unready.txt`. The Pod must stay Running and must not be restarted; the Service must be left with 2 endpoints. """, "setup": NS + ROLL + r""" mkns %{ns}; rm -rf "$WS/q94"; mkdir -p "$WS/q94" $K apply -f - >/dev/null <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: {name: %{app}, namespace: %{ns}} spec: replicas: 3 selector: {matchLabels: {app: %{app}}} template: metadata: {labels: {app: %{app}}} spec: containers: - name: web image: busybox:1.36 command: ["sh", "-c", "mkdir -p /www && hostname > /www/index.html && touch /www/ready && exec httpd -f -p 80 -h /www"] ports: [{containerPort: 80}] EOF $K -n %{ns} expose deploy %{app} --port=80 >/dev/null rolled %{ns} %{app} 3 240 """, "checks": [ C("the Deployment has the readiness probe, and no liveness probe", 3, PY + r""" py -n %{ns} deploy %{app} ' c = o["spec"]["template"]["spec"]["containers"][0]; p = c.get("readinessProbe", {}); g = p.get("httpGet", {}) g.get("path") == "/ready" and str(g.get("port")) == "80" and p.get("periodSeconds") == 3 and p.get("failureThreshold") == 1 and "livenessProbe" not in c'"""), C("one of its 3 Pods is not ready, none was restarted, and the Service has the other 2 as endpoints", 4, r""" [ -n "$(j -n %{ns} deploy %{app} '{.spec.template.spec.containers[0].readinessProbe.httpGet.path}')" ] || exit 1 bad=$($K -n %{ns} get pod -l app=%{app} -o json | python3 -c ' import json, sys p = [x for x in json.load(sys.stdin)["items"] if "deletionTimestamp" not in x["metadata"]] nr = [x for x in p if not all(c["ready"] for c in x["status"].get("containerStatuses", [{"ready": False}]))] if len(p) == 3 and len(nr) == 1 and all(x["status"]["phase"] == "Running" and x["status"]["containerStatuses"][0]["restartCount"] == 0 for x in p): print(nr[0]["metadata"]["name"], nr[0]["status"].get("podIP", ""))') || exit 1 [ -n "$bad" ] || exit 1; echo "${bad% *}" > "$PRIV/q94.bad" $K -n %{ns} get endpointslice -l kubernetes.io/service-name=%{app} -o json | BAD="${bad#* }" python3 -c ' import json, os, sys e = [a for s in json.load(sys.stdin)["items"] for x in s.get("endpoints", []) if x.get("conditions", {}).get("ready") for a in x["addresses"]] sys.exit(0 if len(e) == 2 and os.environ["BAD"] not in e else 1)'"""), C("unready.txt names that Pod", 3, ONE + r""" bad=$($K -n %{ns} get pod -l app=%{app} -o json | python3 -c ' import json, sys p = [x for x in json.load(sys.stdin)["items"] if "deletionTimestamp" not in x["metadata"]] nr = [x["metadata"]["name"] for x in p if not all(c["ready"] for c in x["status"].get("containerStatuses", [{"ready": False}]))] print(nr[0] if len(p) == 3 and len(nr) == 1 else "")') [ -n "$bad" ] && eq "$(one "$WS/q94/unready.txt" | sed 's#.*/##')" "$bad" """), ], "solution": """ k -n %{ns} edit deploy %{app} # containers: # - name: web # readinessProbe: # httpGet: {path: /ready, port: 80} # periodSeconds: 3 # failureThreshold: 1 k -n %{ns} rollout status deploy/%{app} POD=$(k -n %{ns} get pod -l app=%{app} -o jsonpath='{.items[0].metadata.name}') k -n %{ns} exec $POD -- rm /www/ready; echo $POD > {WS}/q94/unready.txt k -n %{ns} get pod; k -n %{ns} get endpointslices -l kubernetes.io/service-name=%{app} # the Pod: 0/1 Running, RESTARTS 0 - and two addresses left behind the Service # Readiness answers "may this Pod receive traffic now?": failing it removes the Pod from the # Service's endpoints and nothing else. Liveness answers "is this container beyond repair?": # failing it restarts the container. A dependency that is down belongs in readiness - with a # liveness probe on it, every replica would be restarted for a problem a restart cannot fix. """, }, { "id": 95, "domain": "OBS", "title": "A slow starter that liveness keeps killing", "params": lambda r: {"ns": ns_pick(r, "reports"), "dep": name_pick(r, "renderer")}, "task": """ The container of Deployment `%{dep}` in Namespace `%{ns}` needs about 40 seconds before it answers on port 80. Its liveness probe does not wait that long: the container is killed and restarted again and again, and never becomes Ready. Fix it **without changing the liveness probe**, the image or the command: add a **startup** probe with the same HTTP check (GET `/` on port 80) that checks every 5 seconds and gives the container 90 seconds in all to come up. The Pod must become Ready without a single restart. """, "setup": NS + r""" mkns %{ns} $K apply -f - >/dev/null <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: {name: %{dep}, namespace: %{ns}} spec: replicas: 1 selector: {matchLabels: {app: %{dep}}} template: metadata: {labels: {app: %{dep}}} spec: terminationGracePeriodSeconds: 2 containers: - name: app image: busybox:1.36 command: ["sh", "-c", "sleep 40; mkdir -p /www; echo ok > /www/index.html; exec httpd -f -p 80 -h /www"] livenessProbe: httpGet: {path: /, port: 80} initialDelaySeconds: 5 periodSeconds: 5 failureThreshold: 2 EOF """, "checks": [ C("the liveness probe, the image and the command are as they were", 3, PY + r""" py -n %{ns} deploy %{dep} ' c = o["spec"]["template"]["spec"]["containers"][0]; p = c.get("livenessProbe", {}) p.get("httpGet", {}).get("path") == "/" and p.get("initialDelaySeconds") == 5 and p.get("periodSeconds") == 5 and p.get("failureThreshold") == 2 and c["image"] == "busybox:1.36" and "sleep 40" in c["command"][-1] and "startupProbe" in c'"""), C("a startup probe: GET / on port 80, every 5 seconds, 90 seconds in all", 3, PY + r""" py -n %{ns} deploy %{dep} ' p = o["spec"]["template"]["spec"]["containers"][0].get("startupProbe", {}); g = p.get("httpGet", {}) g.get("path") == "/" and str(g.get("port")) == "80" and p.get("periodSeconds") == 5 and p.get("failureThreshold") == 18'"""), C("the Pod became Ready without a restart", 4, ROLL + r""" [ -n "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].startupProbe}')" ] && rolled %{ns} %{dep} 1 110 || exit 1 p=$(newest %{ns} app=%{dep}); [ -n "$p" ] && eq "$(j -n %{ns} pod "$p" '{.status.containerStatuses[0].restartCount}')" 0 && eq "$(j -n %{ns} pod "$p" '{.status.containerStatuses[0].ready}')" true"""), ], "solution": """ k -n %{ns} get pod -w # Running, restarted every 20 seconds or so, never READY k -n %{ns} describe pod -l app=%{dep} | grep -A3 -i liveness k -n %{ns} edit deploy %{dep} # startupProbe: # httpGet: {path: /, port: 80} # periodSeconds: 5 # failureThreshold: 18 # 18 x 5 s = 90 s k -n %{ns} rollout status deploy/%{dep} # about 45 seconds k -n %{ns} get pod # 1/1, RESTARTS 0 # While a startup probe has not succeeded, liveness and readiness probes are not run at all - it is # the place for "this takes long once". Raising the liveness probe's initialDelaySeconds would work # too, but then a hung container is also discovered that much later for the rest of its life. # The time a startup probe allows is failureThreshold x periodSeconds. """, }, # ================================================================= 3.3 CLI monitoring ==== { "id": 96, "domain": "OBS", "title": "Why is it Pending? Ask the events", "params": lambda r: {"ns": ns_pick(r, "ingest"), "dep": name_pick(r, "parser")}, "task": """ The Pod of Deployment `%{dep}` in Namespace `%{ns}` has been `Pending` since it was created. 1. Find the event that says why. Write its `reason` to `{WS}/q96/reason.txt` and its message to `{WS}/q96/message.txt`. 2. The cause is a typo in the Deployment: a quantity that should be in `Mi` was written in `Gi`. Correct it — nothing else — so that the Pod runs. """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q96"; mkdir -p "$WS/q96" $K apply -f - >/dev/null <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: {name: %{dep}, namespace: %{ns}} spec: replicas: 1 selector: {matchLabels: {app: %{dep}}} template: metadata: {labels: {app: %{dep}}} spec: containers: - name: parser image: busybox:1.36 command: ["sleep", "86400"] resources: requests: {cpu: 50m, memory: 640Gi} limits: {memory: 640Gi} EOF for i in $(seq 1 40); do $K -n %{ns} get events --field-selector reason=FailedScheduling -o name 2>/dev/null | grep -q . && exit 0; sleep 2; done echo "no FailedScheduling event appeared" >&2; exit 1 """, "checks": [ C("reason.txt holds the event's reason", 3, ONE + r"""eq "$(one "$WS/q96/reason.txt")" FailedScheduling"""), C("message.txt holds its message", 3, r"""grep -qi 'insufficient memory' "$WS/q96/message.txt" 2>/dev/null"""), C("the request and limit are 640Mi, and the Pod runs", 4, ROLL + r""" eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].resources.requests.memory}')" 640Mi \ && eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].resources.limits.memory}')" 640Mi \ && eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].resources.requests.cpu}')" 50m && rolled %{ns} %{dep} 1 90"""), ], "solution": """ k -n %{ns} get events --sort-by=.lastTimestamp | tail -5 # Warning FailedScheduling pod/%{dep}-... 0/1 nodes are available: 1 Insufficient memory. ... k -n %{ns} get events --field-selector reason=FailedScheduling -o jsonpath='{.items[-1:].message}' | tee {WS}/q96/message.txt echo FailedScheduling > {WS}/q96/reason.txt k -n %{ns} set resources deploy/%{dep} --requests=cpu=50m,memory=640Mi --limits=memory=640Mi k -n %{ns} rollout status deploy/%{dep} # A Pending Pod has no node, so it has no logs and no container state: the events are all there # is - `k describe pod` prints them at the bottom, `k get events` lists them for the Namespace # (unsorted unless you ask). The scheduler compares the REQUEST with what is free on each node. """, }, # =========================================================================== 3.4 Logs ==== { "id": 97, "domain": "OBS", "title": "The logs of Jobs", "params": lambda r: {"ns": ns_pick(r, "nightly"), "job": name_pick(r, "export"), "salt": "".join(r.choice("abcdefghjkmnpqrstuvwxyz") for _ in range(6)), "boot": "".join(r.choice("0123456789abcdef") for _ in range(8)), "file": r.choice(["ledger.csv", "customers.db", "orders.json", "prices.xml"])}, "task": """ Two Jobs ran in Namespace `%{ns}` last night. 1. Job `%{job}` ran three Pods. Each printed one line that starts with `result`. Save those three lines to `{WS}/q97/results.txt` (in any order). 2. Job `%{job}-load` failed. Save the line with which its Pod gave up — it starts with `FATAL` — to `{WS}/q97/fatal.txt`, and the **first** line that Pod logged to `{WS}/q97/first.txt`. """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q97"; mkdir -p "$WS/q97" $K apply -f - >/dev/null <<'EOF' apiVersion: batch/v1 kind: Job metadata: {name: %{job}, namespace: %{ns}} spec: completions: 3 parallelism: 3 completionMode: Indexed template: spec: restartPolicy: Never containers: - name: export image: busybox:1.36 command: ["sh", "-c", "i=1; while [ $i -le 15 ]; do echo \"step $i of 15\"; i=$((i+1)); done; echo \"result index=$JOB_COMPLETION_INDEX checksum=$(echo %{salt}-$JOB_COMPLETION_INDEX | md5sum | cut -c1-8)\""] --- apiVersion: batch/v1 kind: Job metadata: {name: %{job}-load, namespace: %{ns}} spec: backoffLimit: 0 template: spec: restartPolicy: Never containers: - name: load image: busybox:1.36 command: ["sh", "-c", "echo \"boot id=%{boot}\"; i=1; while [ $i -le 14 ]; do echo \"loading chunk $i\"; i=$((i+1)); done; echo \"FATAL cannot open /import/%{file}\"; exit 1"] EOF $K -n %{ns} wait --for=condition=Complete job/%{job} --timeout=240s >/dev/null || exit 1 $K -n %{ns} wait --for=condition=Failed job/%{job}-load --timeout=240s >/dev/null || exit 1 for i in 0 1 2; do echo "result index=$i checksum=$(echo %{salt}-$i | md5sum | cut -c1-8)"; done | sort > "$PRIV/q97.results" """, "checks": [ C("results.txt holds the three result lines", 4, r""" [ -s "$WS/q97/results.txt" ] && diff <(tr -d '\r' < "$WS/q97/results.txt" | sed 's/^.*\(result index=\)/\1/; s/ *$//' | grep . | sort) "$PRIV/q97.results" >/dev/null"""), C("fatal.txt holds the line the failed Pod gave up with", 3, r""" grep -qx 'FATAL cannot open /import/%{file}' <(tr -d '\r' < "$WS/q97/fatal.txt" 2>/dev/null | sed 's/^.*\(FATAL \)/\1/; s/ *$//')"""), C("first.txt holds the first line it logged", 3, r""" grep -qx 'boot id=%{boot}' <(tr -d '\r' < "$WS/q97/first.txt" 2>/dev/null | sed 's/^.*\(boot id=\)/\1/; s/ *$//')"""), ], "solution": """ k -n %{ns} get job,pod # the Pods of a Job carry the label job-name= k -n %{ns} logs -l job-name=%{job} --tail=-1 | grep '^result' | tee {WS}/q97/results.txt k -n %{ns} logs job/%{job}-load | grep '^FATAL' | tee {WS}/q97/fatal.txt k -n %{ns} logs job/%{job}-load | head -1 | tee {WS}/q97/first.txt # Two traps: `logs job/NAME` shows ONE Pod of the Job, so three Pods need the label selector; and # with a selector kubectl prints only the last 10 lines of each Pod unless told --tail=-1 - the # result lines happen to be last, the first line of a 16-line log is not among them. # --prefix puts the Pod's name in front of every line when it matters which one said what. """, }, { "id": 98, "domain": "OBS", "title": "The init container said why", "params": lambda r: {"ns": ns_pick(r, "crm"), "dep": name_pick(r, "sync")}, "task": """ The Pod of Deployment `%{dep}` in Namespace `%{ns}` never starts. 1. Save the log of the container that fails to `{WS}/q98/why.log`. 2. The log names the cause. It lies in the Deployment's **ConfigMap**, not in the Deployment: correct the value there. 3. Get the Pod running. """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q98"; mkdir -p "$WS/q98" $K -n %{ns} create configmap %{dep}-config --from-literal=DB_HOST=db.%{ns}.svc.local --from-literal=DB_NAME=crm >/dev/null $K apply -f - >/dev/null <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: {name: %{dep}, namespace: %{ns}} spec: replicas: 1 selector: {matchLabels: {app: %{dep}}} template: metadata: {labels: {app: %{dep}}} spec: initContainers: - name: migrate image: busybox:1.36 command: ["sh", "-c", "echo \"migrate: connecting to $DB_HOST\"; [ \"$DB_HOST\" = db.%{ns}.svc ] || { echo \"FATAL: unknown database host $DB_HOST (expected db.%{ns}.svc)\"; exit 1; }; echo \"migrate: schema is current\""] envFrom: [{configMapRef: {name: %{dep}-config}}] containers: - name: sync image: busybox:1.36 command: ["sleep", "86400"] envFrom: [{configMapRef: {name: %{dep}-config}}] EOF for i in $(seq 1 60); do $K -n %{ns} get pod -l app=%{dep} -o jsonpath='{.items[0].status.initContainerStatuses[0].restartCount}' 2>/dev/null | grep -q '[1-9]' && exit 0; sleep 2; done echo "the init container did not fail" >&2; exit 1 """, "checks": [ C("why.log holds the init container's log", 3, r"""grep -q 'FATAL: unknown database host db.%{ns}.svc.local' "$WS/q98/why.log" 2>/dev/null"""), C("the ConfigMap has the right host, and the Deployment's containers are as they were", 3, PY + r""" eq "$(j -n %{ns} cm %{dep}-config '{.data.DB_HOST}')" db.%{ns}.svc && eq "$(j -n %{ns} cm %{dep}-config '{.data.DB_NAME}')" crm || exit 1 py -n %{ns} deploy %{dep} ' t = o["spec"]["template"]["spec"]; i = t["initContainers"][0] i["name"] == "migrate" and "FATAL: unknown database host" in i["command"][-1] and i["envFrom"][0]["configMapRef"]["name"] == "%{dep}-config" and t["containers"][0]["command"] == ["sleep", "86400"]'"""), C("the Pod runs", 4, ROLL + r""" eq "$(j -n %{ns} cm %{dep}-config '{.data.DB_HOST}')" db.%{ns}.svc && rolled %{ns} %{dep} 1 90"""), ], "solution": """ k -n %{ns} get pod # Init:Error / Init:CrashLoopBackOff k -n %{ns} logs deploy/%{dep} -c migrate | tee {WS}/q98/why.log # FATAL: unknown database host db.%{ns}.svc.local (expected db.%{ns}.svc) k -n %{ns} edit cm %{dep}-config # DB_HOST: db.%{ns}.svc k -n %{ns} rollout restart deploy/%{dep} # or delete the Pod: do not wait for the back-off k -n %{ns} rollout status deploy/%{dep} # "Init:" in the STATUS column means an init container has not finished: `logs` needs -c with its # name, since the main container has not started and has nothing to say. A crash-looping container # is restarted after a growing delay (up to five minutes) - after fixing the cause, a new Pod is # quicker than waiting for the next attempt. """, }, # ====================================================================== 3.5 Debugging ==== { "id": 99, "domain": "OBS", "title": "A container with no shell, on the wrong port", "difficulty": "hard", "params": lambda r: {"ns": ns_pick(r, "edge"), "pod": name_pick(r, "whoami"), "port": r.choice([8088, 9191, 7070, 8765])}, "task": """ Pod `%{pod}` in Namespace `%{ns}` runs an image with no shell and no tools. It answers HTTP — but not on port 80, where Service `%{pod}` sends its requests: the port was set through a Secret that has since been deleted, and nobody remembers the number. Without restarting or re-creating the Pod: 1. Find the port its process listens on, and write the number to `{WS}/q99/port.txt`. 2. Repair the **Service** so that `http://%{pod}` (the Service's port 80) reaches the Pod. Pod `probe` in the same Namespace has `wget` for your tests. """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q99"; mkdir -p "$WS/q99" $K -n %{ns} create secret generic %{pod}-boot --from-literal=port=%{port} >/dev/null $K apply -f - >/dev/null <<'EOF' apiVersion: v1 kind: Pod metadata: {name: %{pod}, namespace: %{ns}, labels: {app: %{pod}}} spec: containers: - name: whoami image: traefik/whoami:v1.10 env: [{name: WHOAMI_PORT_NUMBER, valueFrom: {secretKeyRef: {name: %{pod}-boot, key: port}}}] --- apiVersion: v1 kind: Service metadata: {name: %{pod}, namespace: %{ns}} spec: selector: {app: %{pod}} ports: [{port: 80, targetPort: 80}] EOF $K -n %{ns} run probe --image=busybox:1.36 --command -- sleep 86400 >/dev/null $K -n %{ns} wait --for=condition=Ready pod/%{pod} pod/probe --timeout=240s >/dev/null || exit 1 ip=$(j -n %{ns} pod %{pod} '{.status.podIP}') for i in $(seq 1 15); do $K -n %{ns} exec probe -- wget -T 2 -qO- "http://$ip:%{port}" 2>/dev/null | grep -q 'Hostname: %{pod}' && break; sleep 2; done $K -n %{ns} exec probe -- wget -T 2 -qO- "http://$ip:%{port}" 2>/dev/null | grep -q 'Hostname: %{pod}' || { echo "the Pod does not answer on its port" >&2; exit 1; } # once the process runs, the number lives only in its environment $K -n %{ns} delete secret %{pod}-boot >/dev/null j -n %{ns} pod %{pod} '{.metadata.uid}' > "$PRIV/q99.uid" """, "checks": [ C("port.txt holds the port the process listens on", 3, ONE + r"""eq "$(one "$WS/q99/port.txt")" %{port}"""), C("found with an ephemeral debug container, in the same Pod, which was not restarted", 2, r""" eq "$(j -n %{ns} pod %{pod} '{.metadata.uid}')" "$(cat "$PRIV/q99.uid")" && eq "$(j -n %{ns} pod %{pod} '{.status.containerStatuses[0].restartCount}')" 0 \ && [ -n "$(j -n %{ns} pod %{pod} '{.spec.ephemeralContainers[*].name}')" ]"""), C("the Service's port 80 now reaches the Pod", 5, NET + r""" eq "$(j -n %{ns} pod %{pod} '{.metadata.uid}')" "$(cat "$PRIV/q99.uid")" && eq "$(j -n %{ns} svc %{pod} '{.spec.ports[0].port}')" 80 || exit 1 for i in 1 2 3 4; do $K -n %{ns} exec probe -- wget -T 3 -qO- http://%{pod} 2>/dev/null | grep -q 'Hostname: %{pod}' && exit 0; sleep 2; done; exit 1"""), ], "solution": """ k -n %{ns} exec %{pod} -- sh # ... executable file not found: no shell in there k -n %{ns} debug -it %{pod} --image=busybox:1.36 --target=whoami -- sh # netstat -tln # the debug container shares the Pod's network: :::%{port} # tr '\\0' '\\n' < /proc/1/environ | grep PORT # --target also shares its processes: WHOAMI_PORT_NUMBER=%{port} # exit echo %{port} > {WS}/q99/port.txt k -n %{ns} patch svc %{pod} -p '{"spec":{"ports":[{"port":80,"targetPort":%{port}}]}}' k -n %{ns} exec probe -- wget -qO- http://%{pod} | head -2 # An ephemeral container is added to the running Pod - nothing restarts - and brings the tools the # image lacks. It always shares the Pod's network; with --target it shares the named container's # process namespace too, so that process is PID 1 there and /proc/1 is its environment, its files # (/proc/1/root) and its command line. The Pod's spec shows nothing: the value came from a Secret. """, }, ]