kubestronaut-sim/cks/bank_vm.py

"""CKS tasks 103-110: what needs a machine of its own - Falco and AppArmor.

  6.1 Behavioural       103 let Falco tell you which Pod         104 a rule of your own
      analytics         105 the output of a shipped rule          106 alerts as JSON, to a file
                        107 a list, and one Namespace only
  3.4 Kernel hardening  108 load a profile, confine a Deployment  109 a Pod that waits for its profile
                        110 write a profile

These tasks have "backend": "vm". Their host is not a container but an Ubuntu virtual machine
(images/vm) with a single-node kubeadm cluster: Falco's eBPF probe and AppArmor both need a kernel
that the host owns, which a container under rootless podman never does. The VM host is built with
`./ksim build cks --vm`; until it is, these tasks stay out of every session.

Falco is tested by doing the thing a rule watches for and reading the alert it writes; AppArmor by
trying, inside the container, what the profile forbids. Every task has a scripted solution in
solve.py.
"""
from cks.common import C, PY, NS, ns_pick, name_pick

# Falco on a VM host: the systemd unit falco-modern-bpf, alerts in the journal.
FALCO = r"""
# falco_log [SINCE]: Falco's alerts since then (default: the last 5 minutes), one per line
falco_log() { nx "$CP" "journalctl -u falco-modern-bpf --no-pager -o cat --since '${1:--5min}'" </dev/null; }
falco_up() { nx "$CP" 'systemctl is-active --quiet falco-modern-bpf' </dev/null; }
# alerted PATTERN [SECONDS]: an alert matching the extended regular expression shows up (within SECONDS, default 15)
alerted() { local i; for i in $(seq 1 $(( ${2:-15} / 3 ))); do falco_log -40s | grep -Eq -- "$1" && return 0; sleep 3; done; return 1; }
"""

AA = r"""
# loaded PROFILE: the AppArmor profile is loaded on the node, in enforce mode
loaded() { nx "$CP" "grep -qx '$1 (enforce)' /sys/kernel/security/apparmor/profiles" </dev/null; }
# confined NS POD [CONTAINER]: the profile the container's first process runs under
confined() { $K -n "$1" exec "$2" ${3:+-c "$3"} -- cat /proc/1/attr/current 2>/dev/null | tr -d '\0\n'; }
# says NS POD 'COMMAND' TEXT: the command's output (stdout and stderr) contains TEXT
says() { local out; out=$($K -n "$1" exec "$2" -- sh -c "$3" 2>&1); echo "$out" | grep -q -- "$4"; }
"""

QUESTIONS = [
# ======================================================== 6.1 Behavioural analytics ====
{
 "id": 103, "domain": "RUNTIME", "backend": "vm", "title": "Let Falco tell you which Pod",
 "params": lambda r: {"ns": ns_pick(r, "shop"), "bad": r.choice(["frontend", "worker", "cache", "reports"])},
 "task": """
Falco runs on this host as the systemd unit `falco-modern-bpf` (also known as `falco`) and writes
its alerts to the journal. Something in Namespace `%{ns}` keeps reading `/etc/shadow`.

1. From Falco's alerts, find the Pod that does it and write `<namespace>/<pod name>` to
   `{WS}/q103/pod.txt`.
2. Save one of the alert lines to `{WS}/q103/alert.txt`.
3. Which Falco **rule** raised it? Write the rule's name to `{WS}/q103/rule.txt`.
4. Scale the Deployment that owns the Pod to 0 replicas. Leave the others running.
""",
 "setup": NS + FALCO + r"""
mkns %{ns}; rm -rf "$WS/q103"; mkdir -p "$WS/q103"
for d in frontend worker cache reports; do
  cmd='exec sleep 2147483647'; [ $d = %{bad} ] && cmd='while true; do cat /etc/shadow >/dev/null 2>&1; sleep 6; done'
  $K -n %{ns} create deploy $d --image=busybox:1.36 -- sh -c "$cmd" >/dev/null
done
for d in frontend worker cache reports; do $K -n %{ns} rollout status deploy $d --timeout=240s >/dev/null || exit 1; done
# Once, in an audit of 19 tasks six hosts at a time, a minute passed without an alert naming the
# Pod. A new Pod is a new container, which Falco's container plugin looks up afresh: the setup
# tries one before it gives up.
named=
for try in 1 2; do
  pod=$($K -n %{ns} get pod -l app=%{bad} -o jsonpath='{.items[0].metadata.name}'); echo "%{ns}/$pod" > "$PRIV/q103"
  falco_up && alerted "Sensitive file opened.*k8s_pod_name=$pod " 60 && { named=1; break; }
  [ $try = 2 ] && break
  $K -n %{ns} delete pod "$pod" --grace-period=1 >/dev/null 2>&1
  $K -n %{ns} wait --for=condition=Ready pod -l app=%{bad} --timeout=240s >/dev/null || exit 1
done
[ -n "$named" ] || { echo "Falco raised no alert for the Pod" >&2; exit 1; }
""",
 "checks": [
  C("pod.txt names the Pod that reads /etc/shadow", 3, r"""eq "$(tr -d ' \n\r' < "$WS/q103/pod.txt" 2>/dev/null)" "$(cat "$PRIV/q103")" """),
  C("alert.txt holds a Falco alert about it", 2, r"""
p=$(cat "$PRIV/q103"); grep -q 'Sensitive file opened' "$WS/q103/alert.txt" 2>/dev/null && grep -q "k8s_pod_name=${p#*/}" "$WS/q103/alert.txt" """),
  C("rule.txt names the rule", 2, r"""eq "$(tr -s ' ' < "$WS/q103/rule.txt" 2>/dev/null | sed 's/^ *//; s/ *$//' | tr -d '\r\n')" "Read sensitive file untrusted" """),
  C("the Pod's Deployment is scaled to 0, the others still run", 3, r"""
eq "$(j -n %{ns} deploy %{bad} '{.spec.replicas}')" 0 || exit 1
for d in frontend worker cache reports; do [ $d = %{bad} ] || [ "$(j -n %{ns} deploy $d '{.status.availableReplicas}')" = 1 ] || exit 1; done"""),
 ],
 "solution": """
sudo journalctl -u falco-modern-bpf --since '5 min ago' | grep -i shadow | tail -3
#   ... Warning Sensitive file opened for reading by non-trusted program | file=/etc/shadow ... command=cat /etc/shadow
#       container_name=busybox ... k8s_pod_name=%{bad}-xxxx k8s_ns_name=%{ns}
#   -> one line to {WS}/q103/alert.txt, %{ns}/<pod> to {WS}/q103/pod.txt
sudo grep -n -B40 'Sensitive file opened for reading by non-trusted program' /etc/falco/falco_rules.yaml | grep -- '- rule:' | tail -1
#   - rule: Read sensitive file untrusted                         -> {WS}/q103/rule.txt
k -n %{ns} scale deploy %{bad} --replicas=0
# An alert prints the rule's `output`, not its name: search the rules file for the text to find the
# rule (with json_output the name is a field of its own). The fields after the bar are the rule's
# output fields; container and Pod names come from Falco's container plugin.
# docs: falco.org/docs -> Concepts -> Rules.
""",
},
{
 "id": 104, "domain": "RUNTIME", "backend": "vm", "title": "A Falco rule of your own",
 "params": lambda r: {"ns": ns_pick(r, "apps"), "pod": name_pick(r, "billing")},
 "task": """
Falco runs on this host (systemd unit `falco-modern-bpf`, alerts in the journal). Add a rule of
your own, in `/etc/falco/falco_rules.local.yaml`:

- name `Shell in container`, priority `WARNING`;
- it fires whenever a shell — `sh` or `bash` — is started inside a container;
- its output is exactly

      Shell in container: user=%user.name container=%container.name pod=%k8s.pod.name cmdline=%proc.cmdline

Falco must be running with the rule. Then set it off — `kubectl -n %{ns} exec %{pod} -- sh -c 'echo hello'`
— and save the alert line to `{WS}/q104/alert.txt`.
""",
 "setup": NS + FALCO + r"""
mkns %{ns}; rm -rf "$WS/q104"; mkdir -p "$WS/q104"
$K -n %{ns} run %{pod} --image=busybox:1.36 --command -- sleep 86400 >/dev/null
$K -n %{ns} wait --for=condition=Ready pod/%{pod} --timeout=240s >/dev/null && falco_up
""",
 "checks": [
  C("falco_rules.local.yaml has the rule Shell in container, and Falco accepts the file", 3, r"""
f=/etc/falco/falco_rules.local.yaml
eq "$(nx "$CP" "yq '.[] | select(.rule == \"Shell in container\") | .priority' $f" </dev/null | tr 'a-z' 'A-Z')" WARNING \
 && nx "$CP" "falco -V /etc/falco/falco_rules.yaml -V $f" </dev/null >/dev/null 2>&1"""),
  C("Falco is running", 2, FALCO + r"""falco_up && nx "$CP" "grep -q 'Shell in container' /etc/falco/falco_rules.local.yaml" </dev/null"""),
  C("a shell started in a container raises the alert, in the given format", 4, FALCO + r"""
falco_up || exit 1; m=ksim$RANDOM$RANDOM
$K -n %{ns} exec %{pod} -- sh -c "echo $m" >/dev/null 2>&1
alerted "Warning Shell in container: user=root container=[^ ]+ pod=%{pod} cmdline=sh -c echo $m" 18"""),
  C("alert.txt holds such an alert", 1, r"""grep -q 'Shell in container: user=.* container=.* pod=%{pod} cmdline=' "$WS/q104/alert.txt" 2>/dev/null"""),
 ],
 "solution": """
sudo -i
cat >> /etc/falco/falco_rules.local.yaml <<'EOF'
- rule: Shell in container
  desc: A shell was started inside a container
  condition: spawned_process and container and proc.name in (sh, bash)
  output: "Shell in container: user=%user.name container=%container.name pod=%k8s.pod.name cmdline=%proc.cmdline"
  priority: WARNING
EOF
falco -V /etc/falco/falco_rules.yaml -V /etc/falco/falco_rules.local.yaml      # both: Ok
systemctl restart falco-modern-bpf && systemctl is-active falco-modern-bpf     # (Falco also reloads changed rules by itself)
exit
k -n %{ns} exec %{pod} -- sh -c 'echo hello'
sudo journalctl -u falco-modern-bpf --since '1 min ago' | grep 'Shell in container' | tail -1 | tee {WS}/q104/alert.txt
# `spawned_process` and `container` are macros from the shipped rules file (a successful execve;
# any container id but the host's): the local file is loaded after it and may use them.
# `falco --list` prints every field a condition or an output may use. A rule needs rule, desc,
# condition, output and priority - leave one out and Falco refuses to start.
""",
},
{
 "id": 105, "domain": "RUNTIME", "backend": "vm", "title": "The output of a shipped rule",
 "params": lambda r: {"ns": ns_pick(r, "data"), "pod": name_pick(r, "exporter")},
 "task": """
The alerts of Falco's rule `Read sensitive file untrusted` are to be read by a script. Change the
rule's **output** to exactly

    %evt.time,%container.id,%container.name,%k8s.ns.name,%k8s.pod.name,%user.name,%fd.name

without editing the shipped rules file `/etc/falco/falco_rules.yaml`: override it in
`/etc/falco/falco_rules.local.yaml`. The rule's condition and priority stay as they are.

Pod `%{pod}` in Namespace `%{ns}` reads `/etc/shadow` every few seconds. When Falco runs with your
change, save at least two of its alerts in the new format to `{WS}/q105/alerts.txt`.
""",
 "setup": NS + FALCO + r"""
mkns %{ns}; rm -rf "$WS/q105"; mkdir -p "$WS/q105"
$K -n %{ns} run %{pod} --image=busybox:1.36 --command -- sh -c 'while true; do cat /etc/shadow >/dev/null 2>&1; sleep 5; done' >/dev/null
$K -n %{ns} wait --for=condition=Ready pod/%{pod} --timeout=240s >/dev/null || exit 1
nx "$CP" 'sha256sum /etc/falco/falco_rules.yaml' </dev/null | cut -d' ' -f1 > "$PRIV/q105.sha"
falco_up && alerted "Sensitive file opened.*k8s_pod_name=%{pod} " 60 && [ -s "$PRIV/q105.sha" ]
""",
 "checks": [
  C("the shipped rules file is untouched; the local file overrides the rule's output", 3, r"""
eq "$(nx "$CP" 'sha256sum /etc/falco/falco_rules.yaml' </dev/null | cut -d' ' -f1)" "$(cat "$PRIV/q105.sha")" || exit 1
f=/etc/falco/falco_rules.local.yaml
eq "$(nx "$CP" "yq '.[] | select(.rule == \"Read sensitive file untrusted\") | .output' $f" </dev/null)" '%evt.time,%container.id,%container.name,%k8s.ns.name,%k8s.pod.name,%user.name,%fd.name'"""),
  C("Falco now reports the Pod's reads in the new format", 4, FALCO + r"""
falco_up && alerted "Warning [0-9:.]+,[0-9a-f]{12},[^,]+,%{ns},%{pod},root,/etc/shadow( |$)" 24"""),
  C("alerts.txt holds at least two alerts in that format", 3, r"""
[ "$(grep -Ec '[0-9:.]+,[0-9a-f]{12},[^,]+,%{ns},%{pod},root,/etc/shadow' "$WS/q105/alerts.txt" 2>/dev/null)" -ge 2 ]"""),
 ],
 "solution": """
sudo -i
cat >> /etc/falco/falco_rules.local.yaml <<'EOF'
- rule: Read sensitive file untrusted
  output: "%evt.time,%container.id,%container.name,%k8s.ns.name,%k8s.pod.name,%user.name,%fd.name"
  override:
    output: replace
EOF
falco -V /etc/falco/falco_rules.yaml -V /etc/falco/falco_rules.local.yaml
systemctl restart falco-modern-bpf
sleep 15; journalctl -u falco-modern-bpf --since '30 sec ago' -o cat | grep ',%{pod},' | tee {WS}/q105/alerts.txt
# A rule of the same name in a later file changes the earlier one: `override` says which keys and
# how (replace, or append for conditions and lists). Without it Falco takes the entry for a whole
# new rule and complains about the keys that are missing. The shipped file is replaced by every
# package update - which is why changes go into falco_rules.local.yaml.
# docs: falco.org/docs -> Concepts -> Rules -> Overriding rules.
""",
},
{
 "id": 106, "domain": "RUNTIME", "backend": "vm", "title": "Alerts as JSON, to a file",
 "params": lambda r: {"ns": ns_pick(r, "ops"), "pod": name_pick(r, "collector")},
 "task": """
A log shipper is to pick up Falco's alerts on this host. Change Falco's **configuration** (not its
rules) so that

- alerts are written as **JSON**, and
- besides the journal they also go to the file `/var/log/falco/events.json`.

Falco must be running with the change. A file of your own in `/etc/falco/config.d/` is the clean
way to do it; `/etc/falco/falco.yaml` documents every setting.

Pod `%{pod}` in Namespace `%{ns}` produces an alert every few seconds.
""",
 "setup": NS + FALCO + r"""
mkns %{ns}
$K -n %{ns} run %{pod} --image=busybox:1.36 --command -- sh -c 'while true; do cat /etc/shadow >/dev/null 2>&1; sleep 5; done' >/dev/null
$K -n %{ns} wait --for=condition=Ready pod/%{pod} --timeout=240s >/dev/null || exit 1
nx "$CP" 'rm -rf /var/log/falco' </dev/null
falco_up && alerted "Sensitive file opened.*k8s_pod_name=%{pod} " 60
""",
 "checks": [
  C("Falco is running and /var/log/falco/events.json is being written", 3, FALCO + r"""
falco_up || exit 1
a=$(nx "$CP" 'wc -l < /var/log/falco/events.json' </dev/null 2>/dev/null); [ -n "$a" ] || exit 1
for i in 1 2 3 4 5; do sleep 3; b=$(nx "$CP" 'wc -l < /var/log/falco/events.json' </dev/null 2>/dev/null); [ "${b:-0}" -gt "$a" ] && exit 0; done; exit 1"""),
  C("its lines are JSON alerts: the newest is the Pod reading /etc/shadow", 5, r"""
nx "$CP" 'tail -n 1 /var/log/falco/events.json' </dev/null 2>/dev/null | python3 -c '
import json, sys
e = json.loads(sys.stdin.read())
f = e.get("output_fields", {})
sys.exit(0 if e.get("rule") == "Read sensitive file untrusted" and e.get("priority") == "Warning" and f.get("k8s.pod.name") == "%{pod}" and f.get("fd.name") == "/etc/shadow" else 1)'"""),
  C("the journal still gets the alerts", 2, FALCO + r"""
nx "$CP" 'test -s /var/log/falco/events.json' </dev/null && alerted '"rule": ?"Read sensitive file untrusted"' 18"""),
 ],
 "solution": """
sudo -i
grep -n -A12 '^json_output\\|^file_output' /etc/falco/falco.yaml        # the settings and their defaults
mkdir -p /var/log/falco
cat > /etc/falco/config.d/90-json-file.yaml <<EOF
json_output: true
file_output:
  enabled: true
  keep_alive: false
  filename: /var/log/falco/events.json
EOF
systemctl restart falco-modern-bpf && systemctl is-active falco-modern-bpf
sleep 10; tail -1 /var/log/falco/events.json | jq '{rule, priority, pod: .output_fields["k8s.pod.name"]}'
# Outputs are channels and they add up: stdout (which the journal collects), syslog, a file, a
# program, an HTTP endpoint. json_output changes the format on all of them, and turns the fields
# of the output line into output_fields - the rule's name is then a field of its own.
# Files in config.d are merged over falco.yaml and survive a package update.
""",
},
{
 "id": 107, "domain": "RUNTIME", "backend": "vm", "title": "A list, and one Namespace only", "difficulty": "hard",
 "params": lambda r: {"ns": ns_pick(r, "vault")},
 "task": """
Workloads in Namespace `%{ns}` have no reason to run network tools. Make Falco say so when one
does. In `/etc/falco/falco_rules.local.yaml`:

- a **list** named `network_tools` with the programs `nc`, `ncat`, `wget`, `curl` and `nslookup`;
- a rule named `Network tool in restricted namespace`, priority `ERROR`, that fires when one of
  the programs of that list is started inside a container of Namespace `%{ns}` — and not for any
  other Namespace;
- its output, exactly:

      Network tool: ns=%k8s.ns.name pod=%k8s.pod.name tool=%proc.name cmdline=%proc.cmdline

Falco must be running with it. Pod `probe` exists in `%{ns}` and in `%{ns}-open` for your tests.
""",
 "setup": NS + FALCO + r"""
mkns %{ns}; mkns %{ns}-open
for n in %{ns} %{ns}-open; do $K -n $n run probe --image=busybox:1.36 --command -- sleep 86400 >/dev/null; done
for n in %{ns} %{ns}-open; do $K -n $n wait --for=condition=Ready pod/probe --timeout=240s >/dev/null || exit 1; done
falco_up
""",
 "checks": [
  C("the local rules file has the list network_tools and the rule, and Falco accepts it", 2, r"""
f=/etc/falco/falco_rules.local.yaml
l=$(nx "$CP" "yq '.[] | select(.list == \"network_tools\") | .items | join(\",\")' $f" </dev/null)
for t in nc ncat wget curl nslookup; do echo ",$l," | grep -q ",$t," || exit 1; done
eq "$(nx "$CP" "yq '.[] | select(.rule == \"Network tool in restricted namespace\") | .priority' $f" </dev/null | tr 'a-z' 'A-Z')" ERROR \
 && nx "$CP" "yq '.[] | select(.rule == \"Network tool in restricted namespace\") | .condition' $f" </dev/null | grep -q network_tools \
 && nx "$CP" "falco -V /etc/falco/falco_rules.yaml -V $f" </dev/null >/dev/null 2>&1"""),
  C("a network tool started in %{ns} raises the alert, in the given format", 5, FALCO + r"""
falco_up || exit 1; m=ksim$RANDOM$RANDOM
$K -n %{ns} exec probe -- wget -T 1 -qO- "http://127.0.0.1:9/$m" >/dev/null 2>&1
alerted "Error Network tool: ns=%{ns} pod=probe tool=wget cmdline=wget .*$m" 18"""),
  C("the same tool in another Namespace does not, nor does another program in %{ns}", 3, FALCO + r"""
falco_up || exit 1; m=ksim$RANDOM$RANDOM
$K -n %{ns} exec probe -- nslookup "$m.invalid" >/dev/null 2>&1
$K -n %{ns}-open exec probe -- wget -T 1 -qO- "http://127.0.0.1:9/$m" >/dev/null 2>&1
$K -n %{ns} exec probe -- ls "/$m" >/dev/null 2>&1
alerted "Network tool: ns=%{ns} pod=probe tool=nslookup cmdline=nslookup $m" 18 || exit 1
! falco_log -40s | grep "Network tool:" | grep "$m" | grep -q -e "ns=%{ns}-open " -e "tool=ls " """),
 ],
 "solution": """
sudo -i
cat >> /etc/falco/falco_rules.local.yaml <<'EOF'
- list: network_tools
  items: [nc, ncat, wget, curl, nslookup]

- rule: Network tool in restricted namespace
  desc: A network tool was started in a container of a Namespace that should not need one
  condition: spawned_process and container and proc.name in (network_tools) and k8s.ns.name = "%{ns}"
  output: "Network tool: ns=%k8s.ns.name pod=%k8s.pod.name tool=%proc.name cmdline=%proc.cmdline"
  priority: ERROR
EOF
falco -V /etc/falco/falco_rules.yaml -V /etc/falco/falco_rules.local.yaml
systemctl restart falco-modern-bpf
exit
k -n %{ns} exec probe -- wget -T 1 -qO- http://127.0.0.1:9/x; sudo journalctl -u falco-modern-bpf --since '30 sec ago' -o cat | grep 'Network tool'
k -n %{ns}-open exec probe -- wget -T 1 -qO- http://127.0.0.1:9/x      # no alert for this one
# A list is a named set of values for `in (...)`; a macro is a named piece of condition. Both must
# be defined before the rule that uses them. `=` compares exactly; `k8s.ns.name` comes from the
# container plugin, like k8s.pod.name. In busybox every tool is the same binary - proc.name is the
# name it was started as, which is what you want here.
""",
},
# =========================================================== 3.4 Kernel hardening ====
{
 "id": 108, "domain": "SYS", "backend": "vm", "title": "Load an AppArmor profile, confine a Deployment",
 "params": lambda r: {"ns": ns_pick(r, "render"), "dep": name_pick(r, "converter")},
 "task": """
`{WS}/q108/k8s-deny-write` is an AppArmor profile that forbids every write to a file.

1. Load the profile on this host's node, in enforce mode.
2. Make the container of Deployment `%{dep}` in Namespace `%{ns}` run under it.
3. In the new Pod, run `touch /tmp/test` and save the error message to `{WS}/q108/denied.txt`.
4. Write the profile the container's first process runs under — the content of
   `/proc/1/attr/current` inside the container — to `{WS}/q108/current.txt`.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q108"; mkdir -p "$WS/q108"
cat > "$WS/q108/k8s-deny-write" <<'EOF'
#include <tunables/global>

profile k8s-deny-write flags=(attach_disconnected) {
  #include <abstractions/base>

  file,

  # Deny all file writes.
  deny /** w,
}
EOF
$K -n %{ns} create deploy %{dep} --image=busybox:1.36 -- sleep 86400 >/dev/null
$K -n %{ns} rollout status deploy %{dep} --timeout=240s >/dev/null
""",
 "checks": [
  C("the profile k8s-deny-write is loaded on the node, in enforce mode", 3, AA + r"""loaded k8s-deny-write"""),
  C("the Deployment's Pod template asks for it, as a Localhost profile", 3, PY + r"""
py -n %{ns} deploy %{dep} '
t = o["spec"]["template"]["spec"]
p = [t.get("securityContext", {}).get("appArmorProfile")] + [c.get("securityContext", {}).get("appArmorProfile") for c in t["containers"]]
any(x and x.get("type") == "Localhost" and x.get("localhostProfile") == "k8s-deny-write" for x in p)'"""),
  C("in the running Pod the profile is in force: nothing can be written", 3, AA + r"""
for i in $(seq 1 20); do [ "$(j -n %{ns} deploy %{dep} '{.status.updatedReplicas}')" = 1 ] && [ "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" = 1 ] && break; sleep 3; done
p=$($K -n %{ns} get pod -l app=%{dep} --field-selector status.phase=Running --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}'); [ -n "$p" ] || exit 1
eq "$(confined %{ns} "$p")" "k8s-deny-write (enforce)" && says %{ns} "$p" 'touch /tmp/ksim-probe' 'Permission denied' && says %{ns} "$p" 'cat /etc/hostname' "$p" """),
  C("denied.txt and current.txt say so", 1, r"""
grep -qi 'permission denied' "$WS/q108/denied.txt" 2>/dev/null && grep -q 'k8s-deny-write (enforce)' "$WS/q108/current.txt" 2>/dev/null"""),
 ],
 "solution": """
sudo apparmor_parser -q {WS}/q108/k8s-deny-write            # -q: load quietly; -r to replace one that is loaded
sudo aa-status | grep k8s-deny-write                          # listed under "profiles are in enforce mode"
k -n %{ns} patch deploy %{dep} -p '{"spec":{"template":{"spec":{"containers":[{"name":"busybox","securityContext":{"appArmorProfile":{"type":"Localhost","localhostProfile":"k8s-deny-write"}}}]}}}}'
k -n %{ns} rollout status deploy %{dep}
k -n %{ns} exec deploy/%{dep} -- touch /tmp/test 2>&1 | tee {WS}/q108/denied.txt        # Permission denied
k -n %{ns} exec deploy/%{dep} -- cat /proc/1/attr/current | tee {WS}/q108/current.txt   # k8s-deny-write (enforce)
# localhostProfile is the NAME of the profile (the word after `profile` in the file), not the file
# name - here they happen to be the same. The profile must be loaded on every node the Pod can run
# on, before the Pod starts. apparmor_parser loads it until the next reboot; a file in
# /etc/apparmor.d/ is loaded at every boot. docs: Tutorials -> "Restrict a Container's Access to
# Resources with AppArmor".
""",
},
{
 "id": 109, "domain": "SYS", "backend": "vm", "title": "A Pod that waits for its profile",
 "params": lambda r: {"ns": ns_pick(r, "audit"), "pod": name_pick(r, "agent")},
 "task": """
Pod `%{pod}` in Namespace `%{ns}` does not start. It is meant to run under the AppArmor profile
that the security team wrote and put in `/etc/apparmor.d/k8s-audit-agent` on the node.

Get the Pod running under that profile. Keep its image and command, and keep the profile's
content as it is.
""",
 "setup": NS + r"""
mkns %{ns}
nx "$CP" 'cat > /etc/apparmor.d/k8s-audit-agent' <<'EOF' || exit 1
abi <abi/4.0>,
#include <tunables/global>

profile audit-agent-restricted flags=(attach_disconnected,mediate_deleted) {
  #include <abstractions/base>

  file,
  capability,

  # an audit agent reads files; it has no business on the network
  deny network inet,
  deny network inet6,
}
EOF
nx "$CP" 'apparmor_parser -R /etc/apparmor.d/k8s-audit-agent 2>/dev/null; sha256sum /etc/apparmor.d/k8s-audit-agent' </dev/null | cut -d' ' -f1 > "$PRIV/q109.sha"
$K apply -f - >/dev/null <<EOF
apiVersion: v1
kind: Pod
metadata: {name: %{pod}, namespace: %{ns}}
spec:
  containers:
    - name: agent
      image: busybox:1.36
      command: [sleep, "86400"]
      securityContext:
        appArmorProfile: {type: Localhost, localhostProfile: k8s-audit-agent}
EOF
[ -s "$PRIV/q109.sha" ]
""",
 "checks": [
  C("the security team's profile is loaded on the node, unchanged", 3, AA + r"""
loaded audit-agent-restricted && eq "$(nx "$CP" 'sha256sum /etc/apparmor.d/k8s-audit-agent' </dev/null | cut -d' ' -f1)" "$(cat "$PRIV/q109.sha")" """),
  C("the Pod is Running with its image and command, under a Localhost profile", 3, r"""
eq "$(j -n %{ns} pod %{pod} '{.status.phase}')" Running && eq "$(j -n %{ns} pod %{pod} '{.spec.containers[0].image}')" busybox:1.36 \
 && eq "$(j -n %{ns} pod %{pod} '{.spec.containers[0].command}')" '["sleep","86400"]' \
 && eq "$(j -n %{ns} pod %{pod} '{.spec.containers[0].securityContext.appArmorProfile.type}')" Localhost"""),
  C("the profile is in force: files can be read, the network cannot be used", 4, AA + r"""
eq "$(confined %{ns} %{pod})" "audit-agent-restricted (enforce)" && says %{ns} %{pod} 'cat /etc/hostname' %{pod} \
 && says %{ns} %{pod} 'wget -T 2 -qO- http://10.96.0.1:443/' 'Permission denied'"""),
 ],
 "solution": """
k -n %{ns} describe pod %{pod} | tail -3          # ... apparmor profile not found k8s-audit-agent
sudo head -5 /etc/apparmor.d/k8s-audit-agent      # profile audit-agent-restricted ...   <- the profile's NAME
sudo aa-status | grep audit-agent                 # nothing: it is not loaded either
sudo apparmor_parser -q /etc/apparmor.d/k8s-audit-agent
sudo aa-status | grep audit-agent                 # audit-agent-restricted
k -n %{ns} get pod %{pod} -o yaml > /tmp/pod.yaml
vi /tmp/pod.yaml                                  # localhostProfile: audit-agent-restricted
k replace --force -f /tmp/pod.yaml
k -n %{ns} exec %{pod} -- cat /proc/1/attr/current        # audit-agent-restricted (enforce)
k -n %{ns} exec %{pod} -- wget -T 2 -qO- http://10.96.0.1:443/     # Permission denied: no sockets for this container
# Two mistakes in one, and the usual two: the profile was on disk but never loaded, and the Pod
# named the file instead of the profile. The kubelet refuses to start a container whose profile
# the node does not have (CreateContainerError) rather than run it unconfined.
# The `abi <abi/4.0>,` line matters on Ubuntu 24.04: without it the profile is compiled for an older
# kernel interface in which `network` rules are not enforced - and `deny network` does nothing.
""",
},
{
 "id": 110, "domain": "SYS", "backend": "vm", "title": "Write an AppArmor profile", "difficulty": "hard",
 "params": lambda r: {"ns": ns_pick(r, "ledger"), "pod": name_pick(r, "report"), "name": r.choice(["k8s-no-shadow", "k8s-app-guard", "k8s-reader"])},
 "task": """
Write an AppArmor profile named `%{name}` for an ordinary application container. It allows what
such a container normally does — file access, network, capabilities — with two exceptions:

- `/etc/shadow` cannot be read;
- nothing under `/usr` can be written.

1. Save the profile as `/etc/apparmor.d/%{name}` on this host's node, so that it is loaded at
   every boot, and load it now.
2. Create Pod `%{pod}` in Namespace `%{ns}` (image `busybox:1.36`, command `sleep 86400`) whose
   container runs under it.
""",
 "setup": NS + r"""
mkns %{ns}
nx "$CP" 'apparmor_parser -R /etc/apparmor.d/%{name} 2>/dev/null; rm -f /etc/apparmor.d/%{name}' </dev/null; true
""",
 "checks": [
  C("/etc/apparmor.d/%{name} is a valid profile of that name, loaded in enforce mode", 3, AA + r"""
nx "$CP" 'test -s /etc/apparmor.d/%{name} && apparmor_parser -Q -K /etc/apparmor.d/%{name}' </dev/null >/dev/null 2>&1 \
 && nx "$CP" "grep -Eq '^[[:space:]]*profile[[:space:]]+%{name}[[:space:]]' /etc/apparmor.d/%{name}" </dev/null && loaded %{name}"""),
  C("Pod %{pod} runs under it", 3, AA + r"""
eq "$(j -n %{ns} pod %{pod} '{.status.phase}')" Running && eq "$(j -n %{ns} pod %{pod} '{.spec.containers[0].image}')" busybox:1.36 && eq "$(confined %{ns} %{pod})" "%{name} (enforce)" """),
  C("/etc/shadow cannot be read and /usr not written; everything else still works", 4, AA + r"""
eq "$(confined %{ns} %{pod})" "%{name} (enforce)" || exit 1
says %{ns} %{pod} 'cat /etc/shadow' 'Permission denied' && says %{ns} %{pod} 'touch /usr/ksim-probe' 'Permission denied' \
 && says %{ns} %{pod} 'cat /etc/hostname' %{pod} && says %{ns} %{pod} 'touch /tmp/ok && echo written' written \
 && says %{ns} %{pod} 'nslookup kubernetes.default.svc.cluster.local' '10.96.0.1'"""),
 ],
 "solution": """
sudo -i
cat > /etc/apparmor.d/%{name} <<'EOF'
#include <tunables/global>

profile %{name} flags=(attach_disconnected,mediate_deleted) {
  #include <abstractions/base>

  file,
  network,
  capability,

  deny /etc/shadow r,
  deny /usr/** w,
}
EOF
apparmor_parser -q /etc/apparmor.d/%{name} && aa-status | grep %{name}
exit
cat <<EOF | k apply -f -
apiVersion: v1
kind: Pod
metadata: {name: %{pod}, namespace: %{ns}}
spec:
  containers:
    - name: app
      image: busybox:1.36
      command: [sleep, "86400"]
      securityContext:
        appArmorProfile: {type: Localhost, localhostProfile: %{name}}
EOF
k -n %{ns} exec %{pod} -- cat /etc/shadow            # Permission denied
k -n %{ns} exec %{pod} -- touch /usr/x               # Permission denied
# A profile allows nothing it does not name: `file,` `network,` and `capability,` hand back the
# three big groups, and `deny` rules take precedence over any allow. `/usr/**` is everything
# below /usr at any depth (`/usr/*` would be one level). If a container does not start under a new
# profile, `aa-complain <profile>` logs what would have been denied (dmesg, journalctl -k) instead
# of denying it.
""",
},
]