f3d0e536b6
kubestronaut-sim runs each exam task on a host of its own - a kind (kubeadm) cluster in rootless podman, or a QEMU/KVM Ubuntu VM for the CKS tasks that need their own kernel (Falco, AppArmor) - and puts a browser desktop in front of them, the way killer.sh and sailor.sh do. - ksim, sim/: the engine (sessions, drills, study mode, grading, coverage, audit), host and VM lifecycle, the desktop - images/: exam host images (common, CKS variant with its tools, VM guest) - cka/ (131 tasks), ckad/ (105), cks/ (110): task banks, curricula with study notes, reference solutions and scripted solutions Every domain of every track has at least 17 pool tasks. `./ksim audit` proves a task three ways: it seeds cleanly on its own host, scores nothing before anyone has worked on it, and reaches full marks when its scripted solution is run on the host as `candidate`. All 346 tasks pass (CKA audited 2026-10-13, CKAD and CKS 2026-10-12). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01JZPd3GGWyxTgAkL6a4gAJz
417 lines
22 KiB
Python
417 lines
22 KiB
Python
"""CKAD tasks 94-99: Application Observability and Maintenance - the pool tasks that take the domain from 11 to 17.
|
|
|
|
3.2 Probes 94 readiness takes a Pod out of the Service 95 a slow starter that liveness keeps killing
|
|
3.3 CLI monitoring 96 why is it Pending? ask the events
|
|
3.4 Logs 97 the logs of Jobs 98 the init container said why
|
|
3.5 Debugging 99 a container with no shell, on the wrong port
|
|
|
|
Probes are judged by what they do - which Pods are Ready, what the Service's endpoints are, whether
|
|
the container was restarted; logs by the lines themselves; the debugging task by the Service
|
|
answering. Every task has a scripted solution in solve.py.
|
|
"""
|
|
from ckad.common import C, NET, ONE, PY, ROLL, NS, ns_pick, name_pick
|
|
|
|
QUESTIONS = [
|
|
# ========================================================================= 3.2 Probes ====
|
|
{
|
|
"id": 94, "domain": "OBS", "title": "Readiness takes a Pod out of the Service",
|
|
"params": lambda r: {"ns": ns_pick(r, "store"), "app": name_pick(r, "catalog")},
|
|
"task": """
|
|
Deployment `%{app}` in Namespace `%{ns}` (3 replicas, Service `%{app}`) serves the directory `/www`
|
|
on port 80. A replica is able to take requests exactly as long as the file `/www/ready` exists in
|
|
it — but nothing tells Kubernetes so.
|
|
|
|
1. Add a **readiness** probe to the Deployment: HTTP GET `/ready` on port 80, every 3 seconds; one
|
|
failed check makes the Pod not ready. Do not add a liveness probe.
|
|
2. When the rollout is finished, make **one** replica unable to serve: delete `/www/ready` in it.
|
|
Write that Pod's name to `{WS}/q94/unready.txt`.
|
|
|
|
The Pod must stay Running and must not be restarted; the Service must be left with 2 endpoints.
|
|
""",
|
|
"setup": NS + ROLL + r"""
|
|
mkns %{ns}; rm -rf "$WS/q94"; mkdir -p "$WS/q94"
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: %{app}, namespace: %{ns}}
|
|
spec:
|
|
replicas: 3
|
|
selector: {matchLabels: {app: %{app}}}
|
|
template:
|
|
metadata: {labels: {app: %{app}}}
|
|
spec:
|
|
containers:
|
|
- name: web
|
|
image: busybox:1.36
|
|
command: ["sh", "-c", "mkdir -p /www && hostname > /www/index.html && touch /www/ready && exec httpd -f -p 80 -h /www"]
|
|
ports: [{containerPort: 80}]
|
|
EOF
|
|
$K -n %{ns} expose deploy %{app} --port=80 >/dev/null
|
|
rolled %{ns} %{app} 3 240
|
|
""",
|
|
"checks": [
|
|
C("the Deployment has the readiness probe, and no liveness probe", 3, PY + r"""
|
|
py -n %{ns} deploy %{app} '
|
|
c = o["spec"]["template"]["spec"]["containers"][0]; p = c.get("readinessProbe", {}); g = p.get("httpGet", {})
|
|
g.get("path") == "/ready" and str(g.get("port")) == "80" and p.get("periodSeconds") == 3 and p.get("failureThreshold") == 1 and "livenessProbe" not in c'"""),
|
|
C("one of its 3 Pods is not ready, none was restarted, and the Service has the other 2 as endpoints", 4, r"""
|
|
[ -n "$(j -n %{ns} deploy %{app} '{.spec.template.spec.containers[0].readinessProbe.httpGet.path}')" ] || exit 1
|
|
bad=$($K -n %{ns} get pod -l app=%{app} -o json | python3 -c '
|
|
import json, sys
|
|
p = [x for x in json.load(sys.stdin)["items"] if "deletionTimestamp" not in x["metadata"]]
|
|
nr = [x for x in p if not all(c["ready"] for c in x["status"].get("containerStatuses", [{"ready": False}]))]
|
|
if len(p) == 3 and len(nr) == 1 and all(x["status"]["phase"] == "Running" and x["status"]["containerStatuses"][0]["restartCount"] == 0 for x in p):
|
|
print(nr[0]["metadata"]["name"], nr[0]["status"].get("podIP", ""))') || exit 1
|
|
[ -n "$bad" ] || exit 1; echo "${bad% *}" > "$PRIV/q94.bad"
|
|
$K -n %{ns} get endpointslice -l kubernetes.io/service-name=%{app} -o json | BAD="${bad#* }" python3 -c '
|
|
import json, os, sys
|
|
e = [a for s in json.load(sys.stdin)["items"] for x in s.get("endpoints", []) if x.get("conditions", {}).get("ready") for a in x["addresses"]]
|
|
sys.exit(0 if len(e) == 2 and os.environ["BAD"] not in e else 1)'"""),
|
|
C("unready.txt names that Pod", 3, ONE + r"""
|
|
bad=$($K -n %{ns} get pod -l app=%{app} -o json | python3 -c '
|
|
import json, sys
|
|
p = [x for x in json.load(sys.stdin)["items"] if "deletionTimestamp" not in x["metadata"]]
|
|
nr = [x["metadata"]["name"] for x in p if not all(c["ready"] for c in x["status"].get("containerStatuses", [{"ready": False}]))]
|
|
print(nr[0] if len(p) == 3 and len(nr) == 1 else "")')
|
|
[ -n "$bad" ] && eq "$(one "$WS/q94/unready.txt" | sed 's#.*/##')" "$bad" """),
|
|
],
|
|
"solution": """
|
|
k -n %{ns} edit deploy %{app}
|
|
# containers:
|
|
# - name: web
|
|
# readinessProbe:
|
|
# httpGet: {path: /ready, port: 80}
|
|
# periodSeconds: 3
|
|
# failureThreshold: 1
|
|
k -n %{ns} rollout status deploy/%{app}
|
|
POD=$(k -n %{ns} get pod -l app=%{app} -o jsonpath='{.items[0].metadata.name}')
|
|
k -n %{ns} exec $POD -- rm /www/ready; echo $POD > {WS}/q94/unready.txt
|
|
k -n %{ns} get pod; k -n %{ns} get endpointslices -l kubernetes.io/service-name=%{app}
|
|
# the Pod: 0/1 Running, RESTARTS 0 - and two addresses left behind the Service
|
|
# Readiness answers "may this Pod receive traffic now?": failing it removes the Pod from the
|
|
# Service's endpoints and nothing else. Liveness answers "is this container beyond repair?":
|
|
# failing it restarts the container. A dependency that is down belongs in readiness - with a
|
|
# liveness probe on it, every replica would be restarted for a problem a restart cannot fix.
|
|
""",
|
|
},
|
|
{
|
|
"id": 95, "domain": "OBS", "title": "A slow starter that liveness keeps killing",
|
|
"params": lambda r: {"ns": ns_pick(r, "reports"), "dep": name_pick(r, "renderer")},
|
|
"task": """
|
|
The container of Deployment `%{dep}` in Namespace `%{ns}` needs about 40 seconds before it answers
|
|
on port 80. Its liveness probe does not wait that long: the container is killed and restarted
|
|
again and again, and never becomes Ready.
|
|
|
|
Fix it **without changing the liveness probe**, the image or the command: add a **startup** probe
|
|
with the same HTTP check (GET `/` on port 80) that checks every 5 seconds and gives the container
|
|
90 seconds in all to come up.
|
|
|
|
The Pod must become Ready without a single restart.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: %{dep}, namespace: %{ns}}
|
|
spec:
|
|
replicas: 1
|
|
selector: {matchLabels: {app: %{dep}}}
|
|
template:
|
|
metadata: {labels: {app: %{dep}}}
|
|
spec:
|
|
terminationGracePeriodSeconds: 2
|
|
containers:
|
|
- name: app
|
|
image: busybox:1.36
|
|
command: ["sh", "-c", "sleep 40; mkdir -p /www; echo ok > /www/index.html; exec httpd -f -p 80 -h /www"]
|
|
livenessProbe:
|
|
httpGet: {path: /, port: 80}
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 5
|
|
failureThreshold: 2
|
|
EOF
|
|
""",
|
|
"checks": [
|
|
C("the liveness probe, the image and the command are as they were", 3, PY + r"""
|
|
py -n %{ns} deploy %{dep} '
|
|
c = o["spec"]["template"]["spec"]["containers"][0]; p = c.get("livenessProbe", {})
|
|
p.get("httpGet", {}).get("path") == "/" and p.get("initialDelaySeconds") == 5 and p.get("periodSeconds") == 5 and p.get("failureThreshold") == 2 and c["image"] == "busybox:1.36" and "sleep 40" in c["command"][-1] and "startupProbe" in c'"""),
|
|
C("a startup probe: GET / on port 80, every 5 seconds, 90 seconds in all", 3, PY + r"""
|
|
py -n %{ns} deploy %{dep} '
|
|
p = o["spec"]["template"]["spec"]["containers"][0].get("startupProbe", {}); g = p.get("httpGet", {})
|
|
g.get("path") == "/" and str(g.get("port")) == "80" and p.get("periodSeconds") == 5 and p.get("failureThreshold") == 18'"""),
|
|
C("the Pod became Ready without a restart", 4, ROLL + r"""
|
|
[ -n "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].startupProbe}')" ] && rolled %{ns} %{dep} 1 110 || exit 1
|
|
p=$(newest %{ns} app=%{dep}); [ -n "$p" ] && eq "$(j -n %{ns} pod "$p" '{.status.containerStatuses[0].restartCount}')" 0 && eq "$(j -n %{ns} pod "$p" '{.status.containerStatuses[0].ready}')" true"""),
|
|
],
|
|
"solution": """
|
|
k -n %{ns} get pod -w # Running, restarted every 20 seconds or so, never READY
|
|
k -n %{ns} describe pod -l app=%{dep} | grep -A3 -i liveness
|
|
k -n %{ns} edit deploy %{dep}
|
|
# startupProbe:
|
|
# httpGet: {path: /, port: 80}
|
|
# periodSeconds: 5
|
|
# failureThreshold: 18 # 18 x 5 s = 90 s
|
|
k -n %{ns} rollout status deploy/%{dep} # about 45 seconds
|
|
k -n %{ns} get pod # 1/1, RESTARTS 0
|
|
# While a startup probe has not succeeded, liveness and readiness probes are not run at all - it is
|
|
# the place for "this takes long once". Raising the liveness probe's initialDelaySeconds would work
|
|
# too, but then a hung container is also discovered that much later for the rest of its life.
|
|
# The time a startup probe allows is failureThreshold x periodSeconds.
|
|
""",
|
|
},
|
|
# ================================================================= 3.3 CLI monitoring ====
|
|
{
|
|
"id": 96, "domain": "OBS", "title": "Why is it Pending? Ask the events",
|
|
"params": lambda r: {"ns": ns_pick(r, "ingest"), "dep": name_pick(r, "parser")},
|
|
"task": """
|
|
The Pod of Deployment `%{dep}` in Namespace `%{ns}` has been `Pending` since it was created.
|
|
|
|
1. Find the event that says why. Write its `reason` to `{WS}/q96/reason.txt` and its message to
|
|
`{WS}/q96/message.txt`.
|
|
2. The cause is a typo in the Deployment: a quantity that should be in `Mi` was written in `Gi`.
|
|
Correct it — nothing else — so that the Pod runs.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}; rm -rf "$WS/q96"; mkdir -p "$WS/q96"
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: %{dep}, namespace: %{ns}}
|
|
spec:
|
|
replicas: 1
|
|
selector: {matchLabels: {app: %{dep}}}
|
|
template:
|
|
metadata: {labels: {app: %{dep}}}
|
|
spec:
|
|
containers:
|
|
- name: parser
|
|
image: busybox:1.36
|
|
command: ["sleep", "86400"]
|
|
resources:
|
|
requests: {cpu: 50m, memory: 640Gi}
|
|
limits: {memory: 640Gi}
|
|
EOF
|
|
for i in $(seq 1 40); do $K -n %{ns} get events --field-selector reason=FailedScheduling -o name 2>/dev/null | grep -q . && exit 0; sleep 2; done
|
|
echo "no FailedScheduling event appeared" >&2; exit 1
|
|
""",
|
|
"checks": [
|
|
C("reason.txt holds the event's reason", 3, ONE + r"""eq "$(one "$WS/q96/reason.txt")" FailedScheduling"""),
|
|
C("message.txt holds its message", 3, r"""grep -qi 'insufficient memory' "$WS/q96/message.txt" 2>/dev/null"""),
|
|
C("the request and limit are 640Mi, and the Pod runs", 4, ROLL + r"""
|
|
eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].resources.requests.memory}')" 640Mi \
|
|
&& eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].resources.limits.memory}')" 640Mi \
|
|
&& eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].resources.requests.cpu}')" 50m && rolled %{ns} %{dep} 1 90"""),
|
|
],
|
|
"solution": """
|
|
k -n %{ns} get events --sort-by=.lastTimestamp | tail -5
|
|
# Warning FailedScheduling pod/%{dep}-... 0/1 nodes are available: 1 Insufficient memory. ...
|
|
k -n %{ns} get events --field-selector reason=FailedScheduling -o jsonpath='{.items[-1:].message}' | tee {WS}/q96/message.txt
|
|
echo FailedScheduling > {WS}/q96/reason.txt
|
|
k -n %{ns} set resources deploy/%{dep} --requests=cpu=50m,memory=640Mi --limits=memory=640Mi
|
|
k -n %{ns} rollout status deploy/%{dep}
|
|
# A Pending Pod has no node, so it has no logs and no container state: the events are all there
|
|
# is - `k describe pod` prints them at the bottom, `k get events` lists them for the Namespace
|
|
# (unsorted unless you ask). The scheduler compares the REQUEST with what is free on each node.
|
|
""",
|
|
},
|
|
# =========================================================================== 3.4 Logs ====
|
|
{
|
|
"id": 97, "domain": "OBS", "title": "The logs of Jobs",
|
|
"params": lambda r: {"ns": ns_pick(r, "nightly"), "job": name_pick(r, "export"), "salt": "".join(r.choice("abcdefghjkmnpqrstuvwxyz") for _ in range(6)),
|
|
"boot": "".join(r.choice("0123456789abcdef") for _ in range(8)), "file": r.choice(["ledger.csv", "customers.db", "orders.json", "prices.xml"])},
|
|
"task": """
|
|
Two Jobs ran in Namespace `%{ns}` last night.
|
|
|
|
1. Job `%{job}` ran three Pods. Each printed one line that starts with `result`. Save those three
|
|
lines to `{WS}/q97/results.txt` (in any order).
|
|
2. Job `%{job}-load` failed. Save the line with which its Pod gave up — it starts with `FATAL` —
|
|
to `{WS}/q97/fatal.txt`, and the **first** line that Pod logged to `{WS}/q97/first.txt`.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}; rm -rf "$WS/q97"; mkdir -p "$WS/q97"
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata: {name: %{job}, namespace: %{ns}}
|
|
spec:
|
|
completions: 3
|
|
parallelism: 3
|
|
completionMode: Indexed
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
containers:
|
|
- name: export
|
|
image: busybox:1.36
|
|
command: ["sh", "-c", "i=1; while [ $i -le 15 ]; do echo \"step $i of 15\"; i=$((i+1)); done; echo \"result index=$JOB_COMPLETION_INDEX checksum=$(echo %{salt}-$JOB_COMPLETION_INDEX | md5sum | cut -c1-8)\""]
|
|
---
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata: {name: %{job}-load, namespace: %{ns}}
|
|
spec:
|
|
backoffLimit: 0
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
containers:
|
|
- name: load
|
|
image: busybox:1.36
|
|
command: ["sh", "-c", "echo \"boot id=%{boot}\"; i=1; while [ $i -le 14 ]; do echo \"loading chunk $i\"; i=$((i+1)); done; echo \"FATAL cannot open /import/%{file}\"; exit 1"]
|
|
EOF
|
|
$K -n %{ns} wait --for=condition=Complete job/%{job} --timeout=240s >/dev/null || exit 1
|
|
$K -n %{ns} wait --for=condition=Failed job/%{job}-load --timeout=240s >/dev/null || exit 1
|
|
for i in 0 1 2; do echo "result index=$i checksum=$(echo %{salt}-$i | md5sum | cut -c1-8)"; done | sort > "$PRIV/q97.results"
|
|
""",
|
|
"checks": [
|
|
C("results.txt holds the three result lines", 4, r"""
|
|
[ -s "$WS/q97/results.txt" ] && diff <(tr -d '\r' < "$WS/q97/results.txt" | sed 's/^.*\(result index=\)/\1/; s/ *$//' | grep . | sort) "$PRIV/q97.results" >/dev/null"""),
|
|
C("fatal.txt holds the line the failed Pod gave up with", 3, r"""
|
|
grep -qx 'FATAL cannot open /import/%{file}' <(tr -d '\r' < "$WS/q97/fatal.txt" 2>/dev/null | sed 's/^.*\(FATAL \)/\1/; s/ *$//')"""),
|
|
C("first.txt holds the first line it logged", 3, r"""
|
|
grep -qx 'boot id=%{boot}' <(tr -d '\r' < "$WS/q97/first.txt" 2>/dev/null | sed 's/^.*\(boot id=\)/\1/; s/ *$//')"""),
|
|
],
|
|
"solution": """
|
|
k -n %{ns} get job,pod # the Pods of a Job carry the label job-name=<job>
|
|
k -n %{ns} logs -l job-name=%{job} --tail=-1 | grep '^result' | tee {WS}/q97/results.txt
|
|
k -n %{ns} logs job/%{job}-load | grep '^FATAL' | tee {WS}/q97/fatal.txt
|
|
k -n %{ns} logs job/%{job}-load | head -1 | tee {WS}/q97/first.txt
|
|
# Two traps: `logs job/NAME` shows ONE Pod of the Job, so three Pods need the label selector; and
|
|
# with a selector kubectl prints only the last 10 lines of each Pod unless told --tail=-1 - the
|
|
# result lines happen to be last, the first line of a 16-line log is not among them.
|
|
# --prefix puts the Pod's name in front of every line when it matters which one said what.
|
|
""",
|
|
},
|
|
{
|
|
"id": 98, "domain": "OBS", "title": "The init container said why",
|
|
"params": lambda r: {"ns": ns_pick(r, "crm"), "dep": name_pick(r, "sync")},
|
|
"task": """
|
|
The Pod of Deployment `%{dep}` in Namespace `%{ns}` never starts.
|
|
|
|
1. Save the log of the container that fails to `{WS}/q98/why.log`.
|
|
2. The log names the cause. It lies in the Deployment's **ConfigMap**, not in the Deployment:
|
|
correct the value there.
|
|
3. Get the Pod running.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}; rm -rf "$WS/q98"; mkdir -p "$WS/q98"
|
|
$K -n %{ns} create configmap %{dep}-config --from-literal=DB_HOST=db.%{ns}.svc.local --from-literal=DB_NAME=crm >/dev/null
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: %{dep}, namespace: %{ns}}
|
|
spec:
|
|
replicas: 1
|
|
selector: {matchLabels: {app: %{dep}}}
|
|
template:
|
|
metadata: {labels: {app: %{dep}}}
|
|
spec:
|
|
initContainers:
|
|
- name: migrate
|
|
image: busybox:1.36
|
|
command: ["sh", "-c", "echo \"migrate: connecting to $DB_HOST\"; [ \"$DB_HOST\" = db.%{ns}.svc ] || { echo \"FATAL: unknown database host $DB_HOST (expected db.%{ns}.svc)\"; exit 1; }; echo \"migrate: schema is current\""]
|
|
envFrom: [{configMapRef: {name: %{dep}-config}}]
|
|
containers:
|
|
- name: sync
|
|
image: busybox:1.36
|
|
command: ["sleep", "86400"]
|
|
envFrom: [{configMapRef: {name: %{dep}-config}}]
|
|
EOF
|
|
for i in $(seq 1 60); do $K -n %{ns} get pod -l app=%{dep} -o jsonpath='{.items[0].status.initContainerStatuses[0].restartCount}' 2>/dev/null | grep -q '[1-9]' && exit 0; sleep 2; done
|
|
echo "the init container did not fail" >&2; exit 1
|
|
""",
|
|
"checks": [
|
|
C("why.log holds the init container's log", 3, r"""grep -q 'FATAL: unknown database host db.%{ns}.svc.local' "$WS/q98/why.log" 2>/dev/null"""),
|
|
C("the ConfigMap has the right host, and the Deployment's containers are as they were", 3, PY + r"""
|
|
eq "$(j -n %{ns} cm %{dep}-config '{.data.DB_HOST}')" db.%{ns}.svc && eq "$(j -n %{ns} cm %{dep}-config '{.data.DB_NAME}')" crm || exit 1
|
|
py -n %{ns} deploy %{dep} '
|
|
t = o["spec"]["template"]["spec"]; i = t["initContainers"][0]
|
|
i["name"] == "migrate" and "FATAL: unknown database host" in i["command"][-1] and i["envFrom"][0]["configMapRef"]["name"] == "%{dep}-config" and t["containers"][0]["command"] == ["sleep", "86400"]'"""),
|
|
C("the Pod runs", 4, ROLL + r"""
|
|
eq "$(j -n %{ns} cm %{dep}-config '{.data.DB_HOST}')" db.%{ns}.svc && rolled %{ns} %{dep} 1 90"""),
|
|
],
|
|
"solution": """
|
|
k -n %{ns} get pod # Init:Error / Init:CrashLoopBackOff
|
|
k -n %{ns} logs deploy/%{dep} -c migrate | tee {WS}/q98/why.log
|
|
# FATAL: unknown database host db.%{ns}.svc.local (expected db.%{ns}.svc)
|
|
k -n %{ns} edit cm %{dep}-config # DB_HOST: db.%{ns}.svc
|
|
k -n %{ns} rollout restart deploy/%{dep} # or delete the Pod: do not wait for the back-off
|
|
k -n %{ns} rollout status deploy/%{dep}
|
|
# "Init:" in the STATUS column means an init container has not finished: `logs` needs -c with its
|
|
# name, since the main container has not started and has nothing to say. A crash-looping container
|
|
# is restarted after a growing delay (up to five minutes) - after fixing the cause, a new Pod is
|
|
# quicker than waiting for the next attempt.
|
|
""",
|
|
},
|
|
# ====================================================================== 3.5 Debugging ====
|
|
{
|
|
"id": 99, "domain": "OBS", "title": "A container with no shell, on the wrong port", "difficulty": "hard",
|
|
"params": lambda r: {"ns": ns_pick(r, "edge"), "pod": name_pick(r, "whoami"), "port": r.choice([8088, 9191, 7070, 8765])},
|
|
"task": """
|
|
Pod `%{pod}` in Namespace `%{ns}` runs an image with no shell and no tools. It answers HTTP — but
|
|
not on port 80, where Service `%{pod}` sends its requests: the port was set through a Secret that
|
|
has since been deleted, and nobody remembers the number.
|
|
|
|
Without restarting or re-creating the Pod:
|
|
|
|
1. Find the port its process listens on, and write the number to `{WS}/q99/port.txt`.
|
|
2. Repair the **Service** so that `http://%{pod}` (the Service's port 80) reaches the Pod.
|
|
|
|
Pod `probe` in the same Namespace has `wget` for your tests.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}; rm -rf "$WS/q99"; mkdir -p "$WS/q99"
|
|
$K -n %{ns} create secret generic %{pod}-boot --from-literal=port=%{port} >/dev/null
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: %{pod}, namespace: %{ns}, labels: {app: %{pod}}}
|
|
spec:
|
|
containers:
|
|
- name: whoami
|
|
image: traefik/whoami:v1.10
|
|
env: [{name: WHOAMI_PORT_NUMBER, valueFrom: {secretKeyRef: {name: %{pod}-boot, key: port}}}]
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata: {name: %{pod}, namespace: %{ns}}
|
|
spec:
|
|
selector: {app: %{pod}}
|
|
ports: [{port: 80, targetPort: 80}]
|
|
EOF
|
|
$K -n %{ns} run probe --image=busybox:1.36 --command -- sleep 86400 >/dev/null
|
|
$K -n %{ns} wait --for=condition=Ready pod/%{pod} pod/probe --timeout=240s >/dev/null || exit 1
|
|
ip=$(j -n %{ns} pod %{pod} '{.status.podIP}')
|
|
for i in $(seq 1 15); do $K -n %{ns} exec probe -- wget -T 2 -qO- "http://$ip:%{port}" 2>/dev/null | grep -q 'Hostname: %{pod}' && break; sleep 2; done
|
|
$K -n %{ns} exec probe -- wget -T 2 -qO- "http://$ip:%{port}" 2>/dev/null | grep -q 'Hostname: %{pod}' || { echo "the Pod does not answer on its port" >&2; exit 1; }
|
|
# once the process runs, the number lives only in its environment
|
|
$K -n %{ns} delete secret %{pod}-boot >/dev/null
|
|
j -n %{ns} pod %{pod} '{.metadata.uid}' > "$PRIV/q99.uid"
|
|
""",
|
|
"checks": [
|
|
C("port.txt holds the port the process listens on", 3, ONE + r"""eq "$(one "$WS/q99/port.txt")" %{port}"""),
|
|
C("found with an ephemeral debug container, in the same Pod, which was not restarted", 2, r"""
|
|
eq "$(j -n %{ns} pod %{pod} '{.metadata.uid}')" "$(cat "$PRIV/q99.uid")" && eq "$(j -n %{ns} pod %{pod} '{.status.containerStatuses[0].restartCount}')" 0 \
|
|
&& [ -n "$(j -n %{ns} pod %{pod} '{.spec.ephemeralContainers[*].name}')" ]"""),
|
|
C("the Service's port 80 now reaches the Pod", 5, NET + r"""
|
|
eq "$(j -n %{ns} pod %{pod} '{.metadata.uid}')" "$(cat "$PRIV/q99.uid")" && eq "$(j -n %{ns} svc %{pod} '{.spec.ports[0].port}')" 80 || exit 1
|
|
for i in 1 2 3 4; do $K -n %{ns} exec probe -- wget -T 3 -qO- http://%{pod} 2>/dev/null | grep -q 'Hostname: %{pod}' && exit 0; sleep 2; done; exit 1"""),
|
|
],
|
|
"solution": """
|
|
k -n %{ns} exec %{pod} -- sh # ... executable file not found: no shell in there
|
|
k -n %{ns} debug -it %{pod} --image=busybox:1.36 --target=whoami -- sh
|
|
# netstat -tln # the debug container shares the Pod's network: :::%{port}
|
|
# tr '\\0' '\\n' < /proc/1/environ | grep PORT # --target also shares its processes: WHOAMI_PORT_NUMBER=%{port}
|
|
# exit
|
|
echo %{port} > {WS}/q99/port.txt
|
|
k -n %{ns} patch svc %{pod} -p '{"spec":{"ports":[{"port":80,"targetPort":%{port}}]}}'
|
|
k -n %{ns} exec probe -- wget -qO- http://%{pod} | head -2
|
|
# An ephemeral container is added to the running Pod - nothing restarts - and brings the tools the
|
|
# image lacks. It always shares the Pod's network; with --target it shares the named container's
|
|
# process namespace too, so that process is PID 1 there and /proc/1 is its environment, its files
|
|
# (/proc/1/root) and its command line. The Pod's spec shows nothing: the value came from a Secret.
|
|
""",
|
|
},
|
|
]
|