f3d0e536b6
kubestronaut-sim runs each exam task on a host of its own - a kind (kubeadm) cluster in rootless podman, or a QEMU/KVM Ubuntu VM for the CKS tasks that need their own kernel (Falco, AppArmor) - and puts a browser desktop in front of them, the way killer.sh and sailor.sh do. - ksim, sim/: the engine (sessions, drills, study mode, grading, coverage, audit), host and VM lifecycle, the desktop - images/: exam host images (common, CKS variant with its tools, VM guest) - cka/ (131 tasks), ckad/ (105), cks/ (110): task banks, curricula with study notes, reference solutions and scripted solutions Every domain of every track has at least 17 pool tasks. `./ksim audit` proves a task three ways: it seeds cleanly on its own host, scores nothing before anyone has worked on it, and reaches full marks when its scripted solution is run on the host as `candidate`. All 346 tasks pass (CKA audited 2026-10-13, CKAD and CKS 2026-10-12). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01JZPd3GGWyxTgAkL6a4gAJz
240 lines
13 KiB
Python
240 lines
13 KiB
Python
"""CKA pool tasks 129-131: Troubleshooting - a node whose container runtime is down, memory usage
|
|
measured against requests, and where a container's two output streams end up on the node.
|
|
|
|
Written for the per-task hosts: 129 stops containerd on the task's own worker node.
|
|
"""
|
|
from cka.common import C, NS, ONE, ns_pick, name_pick
|
|
|
|
QUESTIONS = [
|
|
# ------------------------------------------------------------------ 129 ----
|
|
{
|
|
"id": 129, "domain": "TSHOOT", "difficulty": "hard", "workers": 1, "title": "The node's container runtime",
|
|
"params": lambda r: {},
|
|
"task": """
|
|
Node `{W1}` is `NotReady` since last night's maintenance, and Pods assigned to it never start.
|
|
|
|
1. Find the service on the node that is down and bring the node back to `Ready`. The fix has to
|
|
survive a reboot of the node.
|
|
2. Write the name of that systemd unit to `{WS}/q129/unit.txt`.
|
|
3. `crictl ps` on the node fails as well, for a reason of its own. Make it work without extra
|
|
flags.
|
|
|
|
Reach the node with `{SHELL_W1}`.
|
|
""",
|
|
"setup": r"""
|
|
rm -rf "$WS/q129"; mkdir -p "$WS/q129"
|
|
nx "$W1" '
|
|
printf "runtime-endpoint: unix:///var/run/dockershim.sock\n" > /etc/crictl.yaml
|
|
systemctl disable --now containerd >/dev/null 2>&1
|
|
systemctl is-active -q containerd && exit 1; exit 0' </dev/null || { echo "could not stop containerd on $W1"; exit 1; }
|
|
""",
|
|
"checks": [
|
|
C("containerd on {W1} is running and enabled", 2, r"""nx "$W1" 'systemctl is-active -q containerd && systemctl is-enabled -q containerd' </dev/null"""),
|
|
C("node {W1} is Ready", 3, r"""
|
|
nx "$W1" 'systemctl is-active -q containerd' </dev/null || exit 1
|
|
for i in $(seq 1 30); do
|
|
eq "$(j node "$W1" '{.status.conditions[?(@.type=="Ready")].status}')" True && exit 0; sleep 3
|
|
done; exit 1"""),
|
|
C("a new Pod on {W1} reaches Running", 3, r"""
|
|
nx "$W1" 'systemctl is-active -q containerd' </dev/null || exit 1
|
|
$K -n default delete pod ksim-probe --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1
|
|
$K -n default run ksim-probe --image=busybox:1.36 --restart=Never --overrides="{\"spec\":{\"nodeName\":\"$W1\",\"terminationGracePeriodSeconds\":1}}" -- sleep 300 >/dev/null 2>&1 || exit 1
|
|
rc=1; for i in $(seq 1 30); do eq "$(j -n default pod ksim-probe '{.status.phase}')" Running && { rc=0; break; }; sleep 3; done
|
|
$K -n default delete pod ksim-probe --wait=false --grace-period=1 >/dev/null 2>&1
|
|
exit $rc"""),
|
|
C("crictl ps works on the node without flags", 2, r"""nx "$W1" 'crictl ps >/dev/null 2>&1' </dev/null"""),
|
|
C("unit.txt names containerd", 1, ONE + r"""
|
|
case "$(one "$WS/q129/unit.txt")" in containerd|containerd.service) ;; *) exit 1 ;; esac"""),
|
|
],
|
|
"solution": """
|
|
k get nodes # {W1} NotReady
|
|
k describe node {W1} | grep -A8 Conditions # Ready False/Unknown ... container runtime is down / kubelet stopped posting
|
|
{SHELL_W1}
|
|
sudo -i
|
|
systemctl status kubelet --no-pager | head -5 # running (or restarting) - not the cause
|
|
journalctl -u kubelet --no-pager | tail -5 # ... unix:///run/containerd/containerd.sock: connect: no such file or directory
|
|
systemctl status containerd --no-pager | head -5 # inactive (dead); disabled
|
|
systemctl enable --now containerd # enable = survives a reboot, --now = start it
|
|
crictl ps # ... unix:///var/run/dockershim.sock ... no such file
|
|
cat /etc/crictl.yaml # runtime-endpoint: unix:///var/run/dockershim.sock - a leftover
|
|
vi /etc/crictl.yaml # runtime-endpoint: unix:///run/containerd/containerd.sock
|
|
crictl ps # the node's containers
|
|
exit; exit
|
|
echo containerd > {WS}/q129/unit.txt
|
|
k get nodes -w # Ready within half a minute
|
|
# A node needs two services: the kubelet and the container runtime it talks to. When the kubelet
|
|
# is healthy and the node is still NotReady, its journal names the socket it cannot reach. crictl
|
|
# reads its endpoint from /etc/crictl.yaml; `crictl --runtime-endpoint unix:///run/containerd/containerd.sock ps`
|
|
# is the way around a wrong file while you are still looking.
|
|
""",
|
|
},
|
|
# ------------------------------------------------------------------ 130 ----
|
|
{
|
|
"id": 130, "domain": "TSHOOT", "title": "Memory used against memory requested",
|
|
"params": lambda r: {"ns": ns_pick(r, "search")},
|
|
"task": """
|
|
Namespace `%{ns}` runs three Pods: `ingest`, `index` and `query`. Use the metrics API
|
|
(metrics-server is installed) to answer:
|
|
|
|
1. Which **container** of Pod `query` uses the most memory? Write its name to
|
|
`{WS}/q130/container.txt`.
|
|
2. Which of the three Pods use more memory than they **request**, all containers of a Pod taken
|
|
together? Write their names to `{WS}/q130/over.txt`, one per line.
|
|
3. Write a command into `{WS}/q130/top.sh` that prints the current CPU and memory usage of every
|
|
container of every Pod in `%{ns}`, one line per container.
|
|
""",
|
|
"setup": NS + r"""
|
|
ensure_metrics_server || { echo "could not install metrics-server (no network?) - the task needs it"; exit 1; }
|
|
mkns %{ns}; rm -rf "$WS/q130"; mkdir -p "$WS/q130"
|
|
$K -n %{ns} delete pod ingest index query --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1
|
|
ctr() { # name requestMi megabytes-held
|
|
cat <<EOF
|
|
- name: $1
|
|
image: busybox:1.36
|
|
command: [sh, -c, "dd if=/dev/zero of=/mem/$1.blob bs=1M count=$3 2>/dev/null; sleep 86400"]
|
|
resources: {requests: {memory: $2Mi, cpu: 10m}}
|
|
volumeMounts: [{name: mem, mountPath: /mem}]
|
|
EOF
|
|
}
|
|
pod() { # name
|
|
cat <<EOF
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: $1, namespace: %{ns}, labels: {app: $1}}
|
|
spec:
|
|
terminationGracePeriodSeconds: 1
|
|
volumes: [{name: mem, emptyDir: {medium: Memory}}]
|
|
containers:
|
|
EOF
|
|
}
|
|
{ pod ingest; ctr ingest 16 45
|
|
pod index; ctr index 64 8
|
|
pod query; ctr api 16 6; ctr cache 16 60
|
|
} | $K apply -f - >/dev/null
|
|
$K -n %{ns} wait --for=condition=Ready pod/ingest pod/index pod/query --timeout=240s >/dev/null || exit 1
|
|
""",
|
|
"checks": [
|
|
C("container.txt names the container of query that holds the most memory: cache", 2, ONE + r"""eq "$(one "$WS/q130/container.txt")" cache"""),
|
|
C("over.txt lists the Pods above their memory request: ingest and query", 3, r"""
|
|
eq "$(tr -d ' \t\r' < "$WS/q130/over.txt" 2>/dev/null | sed 's#.*/##' | grep . | sort | tr '\n' ' ')" "ingest query " """),
|
|
C("top.sh prints one line of usage for each of the four containers", 2, r"""
|
|
f="$WS/q130/top.sh"; [ -s "$f" ] || exit 1
|
|
out=""
|
|
for i in 1 2 3 4 5 6; do out=$(cd / && bash "$f" 2>/dev/null) && [ "$(echo "$out" | grep -c Mi)" -ge 4 ] && break; sleep 10; done
|
|
for pc in "ingest ingest" "index index" "query api" "query cache"; do
|
|
set -- $pc
|
|
[ "$(echo "$out" | awk -v p="$1" -v c="$2" '$1 == p && $2 == c && $3 ~ /m$/ && $4 ~ /i$/' | wc -l)" = 1 ] || exit 1
|
|
done"""),
|
|
],
|
|
"solution": """
|
|
k -n %{ns} top pod # ingest ~45Mi, index ~8Mi, query ~66Mi (metrics lag about a minute behind)
|
|
k -n %{ns} top pod query --containers # api ~6Mi, cache ~60Mi
|
|
echo cache > {WS}/q130/container.txt
|
|
k -n %{ns} get pod -o custom-columns='POD:.metadata.name,REQUESTS:.spec.containers[*].resources.requests.memory'
|
|
# ingest 16Mi index 64Mi query 16Mi,16Mi (= 32Mi)
|
|
printf 'ingest\\nquery\\n' > {WS}/q130/over.txt
|
|
echo 'kubectl -n %{ns} top pod --containers' > {WS}/q130/top.sh
|
|
# `top` shows what a container uses NOW, as the kubelet measures it; requests are what the
|
|
# scheduler reserved. A Pod far above its request is the first to be evicted when its node runs
|
|
# short of memory - and a node full of such Pods is overcommitted without the scheduler knowing.
|
|
# Without --containers you see one sum per Pod; --sort-by=memory|cpu orders the list.
|
|
""",
|
|
},
|
|
# ------------------------------------------------------------------ 131 ----
|
|
{
|
|
"id": 131, "domain": "TSHOOT", "difficulty": "hard", "workers": 1, "title": "Where a container's output ends up",
|
|
"params": lambda r: {"ns": ns_pick(r, "edge"), "pod": name_pick(r, "gateway"), "tok": f"cfg-{r.randint(1000, 9999)}"},
|
|
"task": """
|
|
Pod `%{pod}` in Namespace `%{ns}` runs two containers, `api` and `audit`. Container `api` was
|
|
restarted once and now writes to both of its output streams.
|
|
|
|
1. Save the output of the **previous** instance of `api` to `{WS}/q131/previous.log`.
|
|
2. The node keeps each container's output in a file. Write the full path, on the node, of the
|
|
file that receives the output of the **running** instance of `api` to `{WS}/q131/logfile.txt`.
|
|
3. Save only the lines the running `api` wrote to **stderr** - the lines themselves, without the
|
|
timestamp and stream columns the node puts in front - to `{WS}/q131/stderr.log`.
|
|
4. Write a command into `{WS}/q131/all.sh` that prints the logs of all containers of the Pod with
|
|
a timestamp on each line, each line prefixed with the Pod and container it comes from.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}; rm -rf "$WS/q131"; mkdir -p "$WS/q131"
|
|
$K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: %{pod}, namespace: %{ns}}
|
|
spec:
|
|
terminationGracePeriodSeconds: 2
|
|
volumes: [{name: state, emptyDir: {}}]
|
|
containers:
|
|
- name: api
|
|
image: busybox:1.36
|
|
volumeMounts: [{name: state, mountPath: /state}]
|
|
command:
|
|
- sh
|
|
- -c
|
|
- |
|
|
if [ ! -f /state/booted ]; then
|
|
touch /state/booted
|
|
echo "FATAL boot attempt 1 failed: config %{tok} not loaded" >&2
|
|
exit 1
|
|
fi
|
|
i=0
|
|
while true; do
|
|
i=$((i+1))
|
|
echo "INFO request $i served"
|
|
[ $((i % 4)) = 0 ] && echo "ERROR upstream refused request $i (%{tok})" >&2
|
|
sleep 2
|
|
done
|
|
- name: audit
|
|
image: busybox:1.36
|
|
command: [sh, -c, 'while true; do echo "AUDIT heartbeat"; sleep 5; done']
|
|
EOF
|
|
for i in $(seq 1 60); do
|
|
eq "$(j -n %{ns} pod %{pod} '{.status.containerStatuses[?(@.name=="api")].restartCount}/{.status.containerStatuses[?(@.name=="api")].ready}')" 1/true && exit 0
|
|
sleep 3
|
|
done
|
|
echo "container api of %{pod} did not restart once and come up"; exit 1
|
|
""",
|
|
"checks": [
|
|
C("previous.log holds the output of the instance that failed", 2, r"""
|
|
grep -q 'FATAL boot attempt 1 failed: config %{tok} not loaded' "$WS/q131/previous.log" 2>/dev/null \
|
|
&& ! grep -q 'INFO request' "$WS/q131/previous.log" """),
|
|
C("logfile.txt is the node's log file of the running api container", 2, ONE + r"""
|
|
p=$(one "$WS/q131/logfile.txt"); case "$p" in /var/log/*) ;; *) exit 1 ;; esac
|
|
node=$(j -n %{ns} pod %{pod} '{.spec.nodeName}'); uid=$(j -n %{ns} pod %{pod} '{.metadata.uid}')
|
|
n=$(j -n %{ns} pod %{pod} '{.status.containerStatuses[?(@.name=="api")].restartCount}')
|
|
[ -n "$node" ] && [ -n "$uid" ] && [ -n "$n" ] || exit 1
|
|
eq "$(nx "$node" "readlink -f '$p'" </dev/null 2>/dev/null | tr -d '\r')" "/var/log/pods/%{ns}_%{pod}_$uid/api/$n.log" """),
|
|
C("stderr.log holds api's stderr lines and nothing else", 3, r"""
|
|
f="$WS/q131/stderr.log"; [ "$(grep -c . "$f" 2>/dev/null)" -ge 2 ] || exit 1
|
|
[ -z "$(grep . "$f" | grep -Ev '^ERROR upstream refused request [0-9]+ \(%{tok}\)[[:space:]]*$')" ]"""),
|
|
C("all.sh prints both containers' logs, prefixed and with timestamps", 2, r"""
|
|
f="$WS/q131/all.sh"; [ -s "$f" ] || exit 1
|
|
out=$(cd / && bash "$f" 2>/dev/null) || exit 1
|
|
echo "$out" | grep -Eq '^\[pod/%{pod}/api\] 20[0-9]{2}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z INFO request' \
|
|
&& echo "$out" | grep -Eq '^\[pod/%{pod}/audit\] 20[0-9]{2}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z AUDIT heartbeat'"""),
|
|
],
|
|
"solution": """
|
|
k -n %{ns} get pod %{pod} # 2/2 Running, RESTARTS 1
|
|
k -n %{ns} logs %{pod} -c api --previous > {WS}/q131/previous.log # FATAL boot attempt 1 failed ...
|
|
k -n %{ns} get pod %{pod} -o jsonpath='{.spec.nodeName} {.metadata.uid}'; echo
|
|
ssh <that node>
|
|
sudo ls /var/log/pods/ # <namespace>_<pod>_<uid>/
|
|
sudo ls /var/log/pods/%{ns}_%{pod}_*/api/ # 0.log (the failed instance) 1.log (the running one)
|
|
sudo tail -3 /var/log/pods/%{ns}_%{pod}_*/api/1.log
|
|
# 2026-10-12T20:11:22.123456789Z stdout F INFO request 7 served
|
|
# 2026-10-12T20:11:24.125634211Z stderr F ERROR upstream refused request 8 (%{tok})
|
|
exit
|
|
echo /var/log/pods/%{ns}_%{pod}_<uid>/api/1.log > {WS}/q131/logfile.txt
|
|
ssh <that node> "sudo grep ' stderr F ' /var/log/pods/%{ns}_%{pod}_*/api/1.log" | cut -d' ' -f4- > {WS}/q131/stderr.log
|
|
echo 'kubectl -n %{ns} logs %{pod} --all-containers --prefix --timestamps' > {WS}/q131/all.sh
|
|
# The runtime writes each container's stdout and stderr to ONE file per instance,
|
|
# /var/log/pods/<ns>_<pod>_<uid>/<container>/<restart count>.log (linked from /var/log/containers),
|
|
# and tags every line with its stream. `kubectl logs` serves that file with the tags removed,
|
|
# which is why it cannot tell the streams apart; --previous serves the file before the last one.
|
|
""",
|
|
},
|
|
]
|