"""CKA pool tasks 129-131: Troubleshooting - a node whose container runtime is down, memory usage
measured against requests, and where a container's two output streams end up on the node.

Written for the per-task hosts: 129 stops containerd on the task's own worker node.
"""
from cka.common import C, NS, ONE, ns_pick, name_pick

QUESTIONS = [
# ------------------------------------------------------------------ 129 ----
{
 "id": 129, "domain": "TSHOOT", "difficulty": "hard", "workers": 1, "title": "The node's container runtime",
 "params": lambda r: {},
 "task": """
Node `{W1}` is `NotReady` since last night's maintenance, and Pods assigned to it never start.

1. Find the service on the node that is down and bring the node back to `Ready`. The fix has to
   survive a reboot of the node.
2. Write the name of that systemd unit to `{WS}/q129/unit.txt`.
3. `crictl ps` on the node fails as well, for a reason of its own. Make it work without extra
   flags.

Reach the node with `{SHELL_W1}`.
""",
 "setup": r"""
rm -rf "$WS/q129"; mkdir -p "$WS/q129"
nx "$W1" '
printf "runtime-endpoint: unix:///var/run/dockershim.sock\n" > /etc/crictl.yaml
systemctl disable --now containerd >/dev/null 2>&1
systemctl is-active -q containerd && exit 1; exit 0' </dev/null || { echo "could not stop containerd on $W1"; exit 1; }
""",
 "checks": [
  C("containerd on {W1} is running and enabled", 2, r"""nx "$W1" 'systemctl is-active -q containerd && systemctl is-enabled -q containerd' </dev/null"""),
  C("node {W1} is Ready", 3, r"""
nx "$W1" 'systemctl is-active -q containerd' </dev/null || exit 1
for i in $(seq 1 30); do
  eq "$(j node "$W1" '{.status.conditions[?(@.type=="Ready")].status}')" True && exit 0; sleep 3
done; exit 1"""),
  C("a new Pod on {W1} reaches Running", 3, r"""
nx "$W1" 'systemctl is-active -q containerd' </dev/null || exit 1
$K -n default delete pod ksim-probe --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1
$K -n default run ksim-probe --image=busybox:1.36 --restart=Never --overrides="{\"spec\":{\"nodeName\":\"$W1\",\"terminationGracePeriodSeconds\":1}}" -- sleep 300 >/dev/null 2>&1 || exit 1
rc=1; for i in $(seq 1 30); do eq "$(j -n default pod ksim-probe '{.status.phase}')" Running && { rc=0; break; }; sleep 3; done
$K -n default delete pod ksim-probe --wait=false --grace-period=1 >/dev/null 2>&1
exit $rc"""),
  C("crictl ps works on the node without flags", 2, r"""nx "$W1" 'crictl ps >/dev/null 2>&1' </dev/null"""),
  C("unit.txt names containerd", 1, ONE + r"""
case "$(one "$WS/q129/unit.txt")" in containerd|containerd.service) ;; *) exit 1 ;; esac"""),
 ],
 "solution": """
k get nodes                                # {W1} NotReady
k describe node {W1} | grep -A8 Conditions # Ready False/Unknown ... container runtime is down / kubelet stopped posting
{SHELL_W1}
  sudo -i
  systemctl status kubelet --no-pager | head -5        # running (or restarting) - not the cause
  journalctl -u kubelet --no-pager | tail -5           # ... unix:///run/containerd/containerd.sock: connect: no such file or directory
  systemctl status containerd --no-pager | head -5     # inactive (dead); disabled
  systemctl enable --now containerd                    # enable = survives a reboot, --now = start it
  crictl ps                                            # ... unix:///var/run/dockershim.sock ... no such file
  cat /etc/crictl.yaml                                 # runtime-endpoint: unix:///var/run/dockershim.sock - a leftover
  vi /etc/crictl.yaml                                  # runtime-endpoint: unix:///run/containerd/containerd.sock
  crictl ps                                            # the node's containers
  exit; exit
echo containerd > {WS}/q129/unit.txt
k get nodes -w                             # Ready within half a minute
# A node needs two services: the kubelet and the container runtime it talks to. When the kubelet
# is healthy and the node is still NotReady, its journal names the socket it cannot reach. crictl
# reads its endpoint from /etc/crictl.yaml; `crictl --runtime-endpoint unix:///run/containerd/containerd.sock ps`
# is the way around a wrong file while you are still looking.
""",
},
# ------------------------------------------------------------------ 130 ----
{
 "id": 130, "domain": "TSHOOT", "title": "Memory used against memory requested",
 "params": lambda r: {"ns": ns_pick(r, "search")},
 "task": """
Namespace `%{ns}` runs three Pods: `ingest`, `index` and `query`. Use the metrics API
(metrics-server is installed) to answer:

1. Which **container** of Pod `query` uses the most memory? Write its name to
   `{WS}/q130/container.txt`.
2. Which of the three Pods use more memory than they **request**, all containers of a Pod taken
   together? Write their names to `{WS}/q130/over.txt`, one per line.
3. Write a command into `{WS}/q130/top.sh` that prints the current CPU and memory usage of every
   container of every Pod in `%{ns}`, one line per container.
""",
 "setup": NS + r"""
ensure_metrics_server || { echo "could not install metrics-server (no network?) - the task needs it"; exit 1; }
mkns %{ns}; rm -rf "$WS/q130"; mkdir -p "$WS/q130"
$K -n %{ns} delete pod ingest index query --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1
ctr() {  # name requestMi megabytes-held
  cat <<EOF
    - name: $1
      image: busybox:1.36
      command: [sh, -c, "dd if=/dev/zero of=/mem/$1.blob bs=1M count=$3 2>/dev/null; sleep 86400"]
      resources: {requests: {memory: $2Mi, cpu: 10m}}
      volumeMounts: [{name: mem, mountPath: /mem}]
EOF
}
pod() {  # name
  cat <<EOF
---
apiVersion: v1
kind: Pod
metadata: {name: $1, namespace: %{ns}, labels: {app: $1}}
spec:
  terminationGracePeriodSeconds: 1
  volumes: [{name: mem, emptyDir: {medium: Memory}}]
  containers:
EOF
}
{ pod ingest; ctr ingest 16 45
  pod index;  ctr index 64 8
  pod query;  ctr api 16 6; ctr cache 16 60
} | $K apply -f - >/dev/null
$K -n %{ns} wait --for=condition=Ready pod/ingest pod/index pod/query --timeout=240s >/dev/null || exit 1
""",
 "checks": [
  C("container.txt names the container of query that holds the most memory: cache", 2, ONE + r"""eq "$(one "$WS/q130/container.txt")" cache"""),
  C("over.txt lists the Pods above their memory request: ingest and query", 3, r"""
eq "$(tr -d ' \t\r' < "$WS/q130/over.txt" 2>/dev/null | sed 's#.*/##' | grep . | sort | tr '\n' ' ')" "ingest query " """),
  C("top.sh prints one line of usage for each of the four containers", 2, r"""
f="$WS/q130/top.sh"; [ -s "$f" ] || exit 1
out=""
for i in 1 2 3 4 5 6; do out=$(cd / && bash "$f" 2>/dev/null) && [ "$(echo "$out" | grep -c Mi)" -ge 4 ] && break; sleep 10; done
for pc in "ingest ingest" "index index" "query api" "query cache"; do
  set -- $pc
  [ "$(echo "$out" | awk -v p="$1" -v c="$2" '$1 == p && $2 == c && $3 ~ /m$/ && $4 ~ /i$/' | wc -l)" = 1 ] || exit 1
done"""),
 ],
 "solution": """
k -n %{ns} top pod                         # ingest ~45Mi, index ~8Mi, query ~66Mi (metrics lag about a minute behind)
k -n %{ns} top pod query --containers      # api ~6Mi, cache ~60Mi
echo cache > {WS}/q130/container.txt
k -n %{ns} get pod -o custom-columns='POD:.metadata.name,REQUESTS:.spec.containers[*].resources.requests.memory'
#   ingest 16Mi   index 64Mi   query 16Mi,16Mi (= 32Mi)
printf 'ingest\\nquery\\n' > {WS}/q130/over.txt
echo 'kubectl -n %{ns} top pod --containers' > {WS}/q130/top.sh
# `top` shows what a container uses NOW, as the kubelet measures it; requests are what the
# scheduler reserved. A Pod far above its request is the first to be evicted when its node runs
# short of memory - and a node full of such Pods is overcommitted without the scheduler knowing.
# Without --containers you see one sum per Pod; --sort-by=memory|cpu orders the list.
""",
},
# ------------------------------------------------------------------ 131 ----
{
 "id": 131, "domain": "TSHOOT", "difficulty": "hard", "workers": 1, "title": "Where a container's output ends up",
 "params": lambda r: {"ns": ns_pick(r, "edge"), "pod": name_pick(r, "gateway"), "tok": f"cfg-{r.randint(1000, 9999)}"},
 "task": """
Pod `%{pod}` in Namespace `%{ns}` runs two containers, `api` and `audit`. Container `api` was
restarted once and now writes to both of its output streams.

1. Save the output of the **previous** instance of `api` to `{WS}/q131/previous.log`.
2. The node keeps each container's output in a file. Write the full path, on the node, of the
   file that receives the output of the **running** instance of `api` to `{WS}/q131/logfile.txt`.
3. Save only the lines the running `api` wrote to **stderr** - the lines themselves, without the
   timestamp and stream columns the node puts in front - to `{WS}/q131/stderr.log`.
4. Write a command into `{WS}/q131/all.sh` that prints the logs of all containers of the Pod with
   a timestamp on each line, each line prefixed with the Pod and container it comes from.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q131"; mkdir -p "$WS/q131"
$K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1
$K apply -f - >/dev/null <<'EOF'
apiVersion: v1
kind: Pod
metadata: {name: %{pod}, namespace: %{ns}}
spec:
  terminationGracePeriodSeconds: 2
  volumes: [{name: state, emptyDir: {}}]
  containers:
    - name: api
      image: busybox:1.36
      volumeMounts: [{name: state, mountPath: /state}]
      command:
        - sh
        - -c
        - |
          if [ ! -f /state/booted ]; then
            touch /state/booted
            echo "FATAL boot attempt 1 failed: config %{tok} not loaded" >&2
            exit 1
          fi
          i=0
          while true; do
            i=$((i+1))
            echo "INFO request $i served"
            [ $((i % 4)) = 0 ] && echo "ERROR upstream refused request $i (%{tok})" >&2
            sleep 2
          done
    - name: audit
      image: busybox:1.36
      command: [sh, -c, 'while true; do echo "AUDIT heartbeat"; sleep 5; done']
EOF
for i in $(seq 1 60); do
  eq "$(j -n %{ns} pod %{pod} '{.status.containerStatuses[?(@.name=="api")].restartCount}/{.status.containerStatuses[?(@.name=="api")].ready}')" 1/true && exit 0
  sleep 3
done
echo "container api of %{pod} did not restart once and come up"; exit 1
""",
 "checks": [
  C("previous.log holds the output of the instance that failed", 2, r"""
grep -q 'FATAL boot attempt 1 failed: config %{tok} not loaded' "$WS/q131/previous.log" 2>/dev/null \
 && ! grep -q 'INFO request' "$WS/q131/previous.log" """),
  C("logfile.txt is the node's log file of the running api container", 2, ONE + r"""
p=$(one "$WS/q131/logfile.txt"); case "$p" in /var/log/*) ;; *) exit 1 ;; esac
node=$(j -n %{ns} pod %{pod} '{.spec.nodeName}'); uid=$(j -n %{ns} pod %{pod} '{.metadata.uid}')
n=$(j -n %{ns} pod %{pod} '{.status.containerStatuses[?(@.name=="api")].restartCount}')
[ -n "$node" ] && [ -n "$uid" ] && [ -n "$n" ] || exit 1
eq "$(nx "$node" "readlink -f '$p'" </dev/null 2>/dev/null | tr -d '\r')" "/var/log/pods/%{ns}_%{pod}_$uid/api/$n.log" """),
  C("stderr.log holds api's stderr lines and nothing else", 3, r"""
f="$WS/q131/stderr.log"; [ "$(grep -c . "$f" 2>/dev/null)" -ge 2 ] || exit 1
[ -z "$(grep . "$f" | grep -Ev '^ERROR upstream refused request [0-9]+ \(%{tok}\)[[:space:]]*$')" ]"""),
  C("all.sh prints both containers' logs, prefixed and with timestamps", 2, r"""
f="$WS/q131/all.sh"; [ -s "$f" ] || exit 1
out=$(cd / && bash "$f" 2>/dev/null) || exit 1
echo "$out" | grep -Eq '^\[pod/%{pod}/api\] 20[0-9]{2}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z INFO request' \
 && echo "$out" | grep -Eq '^\[pod/%{pod}/audit\] 20[0-9]{2}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z AUDIT heartbeat'"""),
 ],
 "solution": """
k -n %{ns} get pod %{pod}                  # 2/2 Running, RESTARTS 1
k -n %{ns} logs %{pod} -c api --previous > {WS}/q131/previous.log        # FATAL boot attempt 1 failed ...
k -n %{ns} get pod %{pod} -o jsonpath='{.spec.nodeName} {.metadata.uid}'; echo
ssh <that node>
  sudo ls /var/log/pods/                                     # <namespace>_<pod>_<uid>/
  sudo ls /var/log/pods/%{ns}_%{pod}_*/api/                  # 0.log (the failed instance)  1.log (the running one)
  sudo tail -3 /var/log/pods/%{ns}_%{pod}_*/api/1.log
  #   2026-10-12T20:11:22.123456789Z stdout F INFO request 7 served
  #   2026-10-12T20:11:24.125634211Z stderr F ERROR upstream refused request 8 (%{tok})
  exit
echo /var/log/pods/%{ns}_%{pod}_<uid>/api/1.log > {WS}/q131/logfile.txt
ssh <that node> "sudo grep ' stderr F ' /var/log/pods/%{ns}_%{pod}_*/api/1.log" | cut -d' ' -f4- > {WS}/q131/stderr.log
echo 'kubectl -n %{ns} logs %{pod} --all-containers --prefix --timestamps' > {WS}/q131/all.sh
# The runtime writes each container's stdout and stderr to ONE file per instance,
# /var/log/pods/<ns>_<pod>_<uid>/<container>/<restart count>.log (linked from /var/log/containers),
# and tags every line with its stream. `kubectl logs` serves that file with the tags removed,
# which is why it cannot tell the streams apart; --previous serves the file before the last one.
""",
},
]