kubestronaut-sim/cks/bank_micro.py

"""CKS tasks 56-70: Minimize Microservice Vulnerabilities - the pool tasks that take the domain from 2 to 17.

  4.1 Pod Security      56 enforce baseline, and who breaks it     57 a default for the whole cluster
      Standards         58 a Deployment that restricted refuses    59 the strictest level each Namespace passes
  4.2 Secrets           60 encrypt Secrets at rest                 61 read a Secret straight from etcd
                        62 from environment variables to files     63 secrets in a ConfigMap
                        64 rotate the encryption key
  4.3 Isolation         65 run a workload in gVisor                66 fence a tenant in
                        67 workloads that reach into the node
  4.4 Pod-to-Pod        68 mutual TLS for a Namespace (Istio)      69 strict for the mesh, one port excepted
      encryption        70 mutual authentication in a CiliumNetworkPolicy

Admission is tested by asking the API server to admit Pods (server-side dry run), encryption by
reading etcd, gVisor by the sandbox's own kernel log, and Istio's mutual TLS with plain-text
requests from outside the mesh: CKS hosts carry gVisor (runsc) and istioctl, and 68-69 install
Istio on the task's cluster. No Cilium agent runs here - kindnet is the CNI - so 70 is graded on
the object. Every task has a scripted solution in solve.py.
"""
from cks.common import API, C, NET, PODS, PY, NS, ns_pick, name_pick

# Would the API server admit a Pod to a Namespace? (server-side dry run - nothing is created)
PSA = r"""
admits() { local ns=$1; shift; $K -n "$ns" run ksim-psa-probe --image=busybox:1.36 --dry-run=server "$@" -- sleep 1 >/dev/null 2>&1; }
# (the refusal is kubectl failing: its output is looked at afterwards, not through a pipe, which
# `pipefail` would report as a failure of the whole check)
refuses() { local ns=$1 out; shift; out=$($K -n "$ns" run ksim-psa-probe --image=busybox:1.36 --dry-run=server "$@" -- sleep 1 2>&1) && return 1; echo "$out" | grep -q 'violates PodSecurity'; }
"""

ETCD = r"""
# shows FILE MARKER: a saved etcd value - raw, as text, or as a `hexdump -C` - contains MARKER
shows() { grep -qa -- "$2" "$1" 2>/dev/null || sed -n 's/^[0-9a-f]\{8\}  .*|\(.*\)|$/\1/p' "$1" 2>/dev/null | tr -d '\n' | grep -q -- "$2"; }
# etcdval KEY: what etcd stores under KEY, with the unprintable bytes dropped
etcdval() {
  nx "$CP" "ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key get '$1' --print-value-only" </dev/null 2>/dev/null | tr -dc '[:print:]\n'
}
# enc_setup: encrypt Secrets at rest with aescbc, key "key1" (/etc/kubernetes/enc/enc.yaml)
enc_setup() {
  local key; key=$(head -c 32 /dev/urandom | base64)
  nx "$CP" 'mkdir -p /etc/kubernetes/enc && cat > /etc/kubernetes/enc/enc.yaml && chmod 600 /etc/kubernetes/enc/enc.yaml' <<EOF || return 1
apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
  - resources:
      - secrets
    providers:
      - aescbc:
          keys:
            - name: key1
              secret: $key
      - identity: {}
EOF
  medit kube-apiserver.yaml '.spec.containers[0].command += ["--encryption-provider-config=/etc/kubernetes/enc/enc.yaml"] | .spec.containers[0].volumeMounts += [{"name": "enc", "mountPath": "/etc/kubernetes/enc", "readOnly": true}] | .spec.volumes += [{"name": "enc", "hostPath": {"path": "/etc/kubernetes/enc", "type": "DirectoryOrCreate"}}]' || return 1
  wait_proc kube-apiserver '--encryption-provider-config=' && wait_api
}
"""

ISTIO = r"""
# ensure_istio: istiod in istio-system (the minimal profile), installed with the host's istioctl
ensure_istio() {
  $K -n istio-system get deploy istiod >/dev/null 2>&1 || nx "$CP" 'istioctl install --set profile=minimal -y >/dev/null 2>&1' </dev/null || return 1
  $K -n istio-system rollout status deploy istiod --timeout=300s >/dev/null
}
# web NS NAME [PORTS..]: Deployment NAME (label app=NAME) answering on the ports (default 80), with a Service
web() {
  local ns=$1 n=$2; shift 2; local ports="${*:-80}" p cmd="mkdir -p /www && echo $n > /www/index.html" sp="" cp=""
  for p in $ports; do cmd="$cmd && httpd -p $p -h /www"; sp="$sp{name: http-$p, port: $p, targetPort: $p}, "; cp="$cp{containerPort: $p}, "; done
  $K apply -f - >/dev/null <<EOF
apiVersion: apps/v1
kind: Deployment
metadata: {name: $n, namespace: $ns}
spec:
  replicas: 1
  selector: {matchLabels: {app: $n}}
  template:
    metadata: {labels: {app: $n}}
    spec:
      containers:
        - name: $n
          image: busybox:1.36
          command: ["sh", "-c", "$cmd && exec sleep 2147483647"]
          ports: [${cp%, }]
---
apiVersion: v1
kind: Service
metadata: {name: $n, namespace: $ns}
spec:
  selector: {app: $n}
  ports: [${sp%, }]
EOF
}
# meshed NS: every Pod of NS that is not going away has an istio-proxy and is Ready
meshed() {
  $K -n "$1" get pod -o json | python3 -c '
import json, sys
p = [x for x in json.load(sys.stdin)["items"] if "deletionTimestamp" not in x["metadata"]]
ok = bool(p) and all(any(c["name"] == "istio-proxy" for c in x["spec"]["containers"] + x["spec"].get("initContainers", []))
                     and any(c["type"] == "Ready" and c["status"] == "True" for c in x["status"].get("conditions", [])) for x in p)
sys.exit(0 if ok else 1)'
}
"""

def _levels(r):
    lv = r.sample(["restricted", "baseline", "privileged"], 3)
    return {"ns": ns_pick(r, "sec"), "la": lv[0], "lb": lv[1], "lc": lv[2]}


QUESTIONS = [
# ================================================== 4.1 Pod Security Standards ====
{
 "id": 56, "domain": "MICRO", "title": "Enforce baseline, and find who breaks it",
 "params": lambda r: {"ns": ns_pick(r, "legacy")},
 "task": """
Namespace `%{ns}` is to enforce the **baseline** Pod Security Standard. It runs five Pods: `web`,
`cache`, `agent`, `tools` and `collector`.

1. Before you change anything, find out which of the running Pods violate `baseline`, and write
   their names to `{WS}/q56/violators.txt`, one per line.
2. Label the Namespace: **enforce** `baseline` at version `latest`; **warn** and **audit** at
   `restricted`.
3. Enforcement only applies to Pods that are created from now on. Delete the violating Pods
   yourself; the others must keep running untouched.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q56"; mkdir -p "$WS/q56"
$K apply -f - >/dev/null <<EOF
apiVersion: v1
kind: Pod
metadata: {name: web, namespace: %{ns}}
spec: {containers: [{name: c, image: "busybox:1.36", command: [sleep, "86400"]}]}
---
apiVersion: v1
kind: Pod
metadata: {name: cache, namespace: %{ns}}
spec: {containers: [{name: c, image: "busybox:1.36", command: [sleep, "86400"], securityContext: {runAsUser: 1000}}]}
---
apiVersion: v1
kind: Pod
metadata: {name: agent, namespace: %{ns}}
spec: {hostNetwork: true, containers: [{name: c, image: "busybox:1.36", command: [sleep, "86400"]}]}
---
apiVersion: v1
kind: Pod
metadata: {name: tools, namespace: %{ns}}
spec: {containers: [{name: c, image: "busybox:1.36", command: [sleep, "86400"], securityContext: {privileged: true}}]}
---
apiVersion: v1
kind: Pod
metadata: {name: collector, namespace: %{ns}}
spec:
  volumes: [{name: logs, hostPath: {path: /var/log}}]
  containers: [{name: c, image: "busybox:1.36", command: [sleep, "86400"], volumeMounts: [{name: logs, mountPath: /host-logs, readOnly: true}]}]
EOF
$K -n %{ns} wait --for=condition=Ready pod --all --timeout=240s >/dev/null || exit 1
echo "$(j -n %{ns} pod web '{.metadata.uid}') $(j -n %{ns} pod cache '{.metadata.uid}')" > "$PRIV/q56.uids"
""",
 "checks": [
  C("violators.txt names the three Pods that violate baseline", 3, r"""
[ -s "$WS/q56/violators.txt" ] && diff <(tr -d ' \r' < "$WS/q56/violators.txt" | sed 's#.*/##' | grep . | sort -u) <(printf 'agent\ncollector\ntools\n') >/dev/null"""),
  C("the Namespace enforces baseline (latest) and warns and audits at restricted", 3, r"""
l() { j ns %{ns} "{.metadata.labels.pod-security\.kubernetes\.io/$1}"; }
eq "$(l enforce)" baseline && eq "$(l enforce-version)" latest && eq "$(l warn)" restricted && eq "$(l audit)" restricted"""),
  C("the violators are gone; web and cache are the same Pods as before", 2, r"""
read -r u1 u2 < "$PRIV/q56.uids"
eq "$(j ns %{ns} '{.metadata.labels.pod-security\.kubernetes\.io/enforce}')" baseline || exit 1
for p in agent tools collector; do [ -z "$(j -n %{ns} pod $p '{.metadata.name}')" ] || exit 1; done
eq "$(j -n %{ns} pod web '{.metadata.uid}')" "$u1" && eq "$(j -n %{ns} pod cache '{.metadata.uid}')" "$u2" """),
  C("a privileged Pod is now refused there, an ordinary one admitted", 2, PSA + r"""refuses %{ns} --privileged && admits %{ns}"""),
 ],
 "solution": """
k label --dry-run=server --overwrite ns %{ns} pod-security.kubernetes.io/enforce=baseline
#   Warning: existing pods in namespace "%{ns}" violate the new PodSecurity enforce level "baseline:latest"
#   Warning: agent: host namespaces / collector: hostPath volumes / tools: privileged
printf 'agent\\ncollector\\ntools\\n' > {WS}/q56/violators.txt
k label ns %{ns} pod-security.kubernetes.io/enforce=baseline pod-security.kubernetes.io/enforce-version=latest \\
  pod-security.kubernetes.io/warn=restricted pod-security.kubernetes.io/audit=restricted
k -n %{ns} delete pod agent collector tools
k -n %{ns} run probe --image=busybox:1.36 --privileged --dry-run=server -- sleep 1      # Forbidden: violates PodSecurity
# The server-side dry run of the label is the quickest audit there is: the API server evaluates
# every existing Pod and answers with warnings, and nothing is changed.
# warn and audit at a stricter level than enforce show what the next step would break.
""",
},
{
 "id": 57, "domain": "MICRO", "title": "A Pod Security default for the whole cluster", "difficulty": "hard",
 "task": """
Namespaces of this cluster that carry no Pod Security labels admit anything. Give the cluster a
default: every Namespace without labels of its own enforces **baseline** — except `kube-system`,
which is exempt.

On `{CP}`:

1. Write an `AdmissionConfiguration` to `/etc/kubernetes/psa/admission.yaml` that configures the
   `PodSecurity` plugin with a `PodSecurityConfiguration`: defaults `enforce: baseline`,
   `audit: restricted`, `warn: restricted` (versions `latest`), and Namespace `kube-system` exempt.
2. Make the API server use it (`--admission-control-config-file`); the file has to be visible
   inside the static Pod.
3. The API server must be back and healthy.
""",
 "setup": r"""
nx "$CP" 'rm -rf /etc/kubernetes/psa' </dev/null
""",
 "checks": [
  C("admission.yaml configures PodSecurity: enforce baseline by default, kube-system exempt", 3, r"""
f=/etc/kubernetes/psa/admission.yaml; y() { nx "$CP" "yq '$1' $f" </dev/null; }
eq "$(y .kind)" AdmissionConfiguration && eq "$(y '.plugins[0].name')" PodSecurity && eq "$(y '.plugins[0].configuration.kind')" PodSecurityConfiguration \
 && eq "$(y '.plugins[0].configuration.defaults.enforce')" baseline && eq "$(y '.plugins[0].configuration.defaults.warn')" restricted \
 && eq "$(y '.plugins[0].configuration.defaults.audit')" restricted && y '.plugins[0].configuration.exemptions.namespaces[]' | grep -qx kube-system"""),
  C("the API server runs with --admission-control-config-file and is healthy", 3, API + r"""
runs kube-apiserver '--admission-control-config-file=/etc/kubernetes/psa/admission.yaml' && wait_api 90 && pod_ready kube-system "kube-apiserver-$CP" 90"""),
  C("a Namespace without labels refuses a privileged Pod; kube-system does not", 4, API + PSA + r"""
runs kube-apiserver '--admission-control-config-file=' && wait_api 60 || exit 1
$K create ns ksim-psa-check >/dev/null 2>&1; for i in $(seq 1 30); do $K -n ksim-psa-check get sa default >/dev/null 2>&1 && break; sleep 1; done
refuses ksim-psa-check --privileged; a=$?; admits ksim-psa-check; b=$?; admits kube-system --privileged; c=$?
$K delete ns ksim-psa-check --wait=false >/dev/null 2>&1
[ $a = 0 ] && [ $b = 0 ] && [ $c = 0 ]"""),
 ],
 "solution": """
sudo -i
mkdir -p /etc/kubernetes/psa
cat > /etc/kubernetes/psa/admission.yaml <<EOF
apiVersion: apiserver.config.k8s.io/v1
kind: AdmissionConfiguration
plugins:
  - name: PodSecurity
    configuration:
      apiVersion: pod-security.admission.config.k8s.io/v1
      kind: PodSecurityConfiguration
      defaults:
        enforce: baseline
        enforce-version: latest
        audit: restricted
        audit-version: latest
        warn: restricted
        warn-version: latest
      exemptions:
        usernames: []
        runtimeClasses: []
        namespaces: [kube-system]
EOF
cp /etc/kubernetes/manifests/kube-apiserver.yaml /root/
vi /etc/kubernetes/manifests/kube-apiserver.yaml
#   command:       - --admission-control-config-file=/etc/kubernetes/psa/admission.yaml
#   volumeMounts:  - {name: psa, mountPath: /etc/kubernetes/psa, readOnly: true}
#   volumes:       - {name: psa, hostPath: {path: /etc/kubernetes/psa, type: DirectoryOrCreate}}
watch crictl ps                                    # until kube-apiserver is back
kubectl create ns demo && kubectl -n demo run p --image=busybox:1.36 --privileged --dry-run=server -- sleep 1     # Forbidden
# docs: "Enforce Pod Security Standards by Configuring the Built-in Admission Controller" has this
# file to copy. Labels on a Namespace still override the defaults. An exemption switches the
# checks off for that Namespace altogether - the system Pods need it.
""",
},
{
 "id": 58, "domain": "MICRO", "title": "A Deployment that restricted refuses",
 "params": lambda r: {"ns": ns_pick(r, "strict"), "dep": name_pick(r, "uploader")},
 "task": """
Deployment `%{dep}` in Namespace `%{ns}` has no Pods: the Namespace enforces the **restricted**
Pod Security Standard, and the Deployment's Pod template violates it in several ways.

Fix the Deployment — not the Namespace — so that both replicas run:

- keep the image (`busybox:1.36`), the command and the 2 replicas;
- the container only needs a scratch directory at `/data`: it has no use for the node's
  filesystem.

`k -n %{ns} describe rs` (or the Namespace's events) names everything that is refused.
""",
 "setup": NS + r"""
mkns %{ns}
$K label ns %{ns} pod-security.kubernetes.io/enforce=restricted pod-security.kubernetes.io/enforce-version=latest --overwrite >/dev/null
$K apply -f - >/dev/null 2>&1 <<EOF
apiVersion: apps/v1
kind: Deployment
metadata: {name: %{dep}, namespace: %{ns}}
spec:
  replicas: 2
  selector: {matchLabels: {app: %{dep}}}
  template:
    metadata: {labels: {app: %{dep}}}
    spec:
      volumes:
        - {name: data, hostPath: {path: /var/lib/uploads, type: DirectoryOrCreate}}
      containers:
        - name: uploader
          image: busybox:1.36
          command: ["sh", "-c", "while true; do date > /data/heartbeat; sleep 5; done"]
          securityContext: {runAsUser: 0, capabilities: {add: [NET_ADMIN, SYS_TIME]}}
          volumeMounts: [{name: data, mountPath: /data}]
EOF
[ -n "$(j -n %{ns} deploy %{dep} '{.metadata.name}')" ]
""",
 "checks": [
  C("both replicas are available, under the unchanged restricted enforcement, image and command kept", 4, r"""
eq "$(j ns %{ns} '{.metadata.labels.pod-security\.kubernetes\.io/enforce}')" restricted || exit 1
eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].image}')" busybox:1.36 && eq "$(j -n %{ns} deploy %{dep} '{.spec.replicas}')" 2 \
 && j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].command}' | grep -q heartbeat || exit 1
for i in $(seq 1 20); do [ "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" = 2 ] && exit 0; sleep 3; done; exit 1"""),
  C("the Pod template meets restricted: non-root, no escalation, all capabilities dropped, seccomp, no hostPath", 4, PY + r"""
py -n %{ns} deploy %{dep} '
t = o["spec"]["template"]["spec"]; ps = t.get("securityContext", {}); c = t["containers"][0]; cs = c.get("securityContext", {})
nonroot = cs.get("runAsNonRoot", ps.get("runAsNonRoot")) is True and cs.get("runAsUser", ps.get("runAsUser")) not in (0,)
sec = (cs.get("seccompProfile") or ps.get("seccompProfile") or {}).get("type") in ("RuntimeDefault", "Localhost")
caps = cs.get("capabilities", {})
nonroot and sec and cs.get("allowPrivilegeEscalation") is False and caps.get("drop") == ["ALL"] and set(caps.get("add", [])) <= {"NET_BIND_SERVICE"} and not any("hostPath" in v for v in t.get("volumes", []))'"""),
  C("/data is an emptyDir the container writes to, and it does not run as root", 2, PY + r"""
p=$($K -n %{ns} get pod -l app=%{dep} --field-selector status.phase=Running -o jsonpath='{.items[0].metadata.name}'); [ -n "$p" ] || exit 1
u=$($K -n %{ns} exec "$p" -- id -u 2>/dev/null); [ -n "$u" ] && [ "$u" != 0 ] && $K -n %{ns} exec "$p" -- test -s /data/heartbeat || exit 1
py -n %{ns} pod "$p" '
m = [x["name"] for c in o["spec"]["containers"] for x in c.get("volumeMounts", []) if x["mountPath"].rstrip("/") == "/data"]
len(m) == 1 and any(v["name"] == m[0] and "emptyDir" in v for v in o["spec"]["volumes"])'"""),
 ],
 "solution": """
k -n %{ns} describe rs | grep -A6 Warning        # FailedCreate ... violates PodSecurity "restricted:latest":
#   allowPrivilegeEscalation != false, unrestricted capabilities, restricted volume types (hostPath),
#   runAsNonRoot != true, runAsUser=0, seccompProfile
k -n %{ns} edit deploy %{dep}
#   spec.template.spec:
#     securityContext: {runAsNonRoot: true, runAsUser: 1000, seccompProfile: {type: RuntimeDefault}}
#     volumes: [{name: data, emptyDir: {}}]
#     containers[0].securityContext: {allowPrivilegeEscalation: false, capabilities: {drop: [ALL]}}
k -n %{ns} rollout status deploy %{dep}
# The Deployment itself was accepted (with a warning): Pod Security judges Pods, so the failure only
# shows on the ReplicaSet. restricted wants four things on every container - runAsNonRoot,
# allowPrivilegeEscalation: false, capabilities drop ALL, a seccomp profile - and only "safe" volume
# types: no hostPath.
""",
},
{
 "id": 59, "domain": "MICRO", "title": "The strictest level each Namespace passes",
 "params": _levels,
 "task": """
Three Namespaces are to get the **strictest** Pod Security Standard that the workloads running in
them already satisfy: `%{ns}-a`, `%{ns}-b` and `%{ns}-c`.

1. For each of them set the label `pod-security.kubernetes.io/enforce` to `restricted`, `baseline`
   or `privileged` — the strictest level under which its Pods would still be admitted as they are.
2. Write the result to `{WS}/q59/levels.txt`, one line per Namespace: `<namespace> <level>`.

Do not change, restart or delete any workload.
""",
 "setup": NS + r"""
rm -rf "$WS/q59"; mkdir -p "$WS/q59"; : > "$PRIV/q59"
mk() {  # mk NAMESPACE LEVEL: a Deployment that satisfies LEVEL and nothing stricter
  mkns $1
  case $2 in
    restricted) spec='securityContext: {runAsNonRoot: true, runAsUser: 1000, seccompProfile: {type: RuntimeDefault}}
      containers:
        - {name: app, image: "busybox:1.36", command: [sleep, "86400"], securityContext: {allowPrivilegeEscalation: false, capabilities: {drop: [ALL]}}}' ;;
    baseline) spec='containers:
        - {name: app, image: "busybox:1.36", command: [sleep, "86400"]}' ;;
    privileged) spec='hostPID: true
      containers:
        - {name: app, image: "busybox:1.36", command: [sleep, "86400"]}' ;;
  esac
  $K apply -f - >/dev/null <<EOF
apiVersion: apps/v1
kind: Deployment
metadata: {name: app, namespace: $1}
spec:
  replicas: 1
  selector: {matchLabels: {app: app}}
  template:
    metadata: {labels: {app: app}}
    spec:
      $spec
EOF
}
mk %{ns}-a %{la}; mk %{ns}-b %{lb}; mk %{ns}-c %{lc}
for s in a b c; do $K -n %{ns}-$s rollout status deploy app --timeout=240s >/dev/null || exit 1; done
for s in a b c; do echo "%{ns}-$s $($K -n %{ns}-$s get pod -l app=app -o jsonpath='{.items[0].metadata.uid}')" >> "$PRIV/q59"; done
""",
 "checks": [
  C("%{ns}-a enforces %{la}", 3, r"""eq "$(j ns %{ns}-a '{.metadata.labels.pod-security\.kubernetes\.io/enforce}')" %{la}"""),
  C("%{ns}-b enforces %{lb}", 3, r"""eq "$(j ns %{ns}-b '{.metadata.labels.pod-security\.kubernetes\.io/enforce}')" %{lb}"""),
  C("%{ns}-c enforces %{lc}", 2, r"""eq "$(j ns %{ns}-c '{.metadata.labels.pod-security\.kubernetes\.io/enforce}')" %{lc}"""),
  C("levels.txt says the same, and no Pod was touched", 2, r"""
diff <(tr -s ' \t' ' ' < "$WS/q59/levels.txt" 2>/dev/null | sed 's/ *$//' | grep . | sort) <(printf '%{ns}-a %{la}\n%{ns}-b %{lb}\n%{ns}-c %{lc}\n' | sort) >/dev/null || exit 1
while read -r n uid; do eq "$($K -n $n get pod -l app=app -o jsonpath='{.items[0].metadata.uid}')" "$uid" || exit 1; done < "$PRIV/q59" """),
 ],
 "solution": """
for n in %{ns}-a %{ns}-b %{ns}-c; do for l in restricted baseline; do
  echo "== $n $l"; k label --dry-run=server --overwrite ns $n pod-security.kubernetes.io/enforce=$l 2>&1 | grep -i warning
done; done
#   no warning at restricted          -> restricted
#   warning at restricted only        -> baseline
#   warning at baseline as well       -> privileged
k label ns %{ns}-a pod-security.kubernetes.io/enforce=%{la}
k label ns %{ns}-b pod-security.kubernetes.io/enforce=%{lb}
k label ns %{ns}-c pod-security.kubernetes.io/enforce=%{lc}
printf '%{ns}-a %{la}\\n%{ns}-b %{lb}\\n%{ns}-c %{lc}\\n' > {WS}/q59/levels.txt
# The dry run evaluates the Pods that exist now against the level and warns for each violation -
# without it you would be reading Pod specs against the standard's tables by hand.
""",
},
# ===================================================================== 4.2 Secrets ====
{
 "id": 60, "domain": "MICRO", "title": "Encrypt Secrets at rest", "difficulty": "hard",
 "params": lambda r: {"ns": ns_pick(r, "bank"), "pw": "pw-" + "".join(r.choice("abcdefghjkmnpqrstuvwxyz23456789") for _ in range(14))},
 "task": """
This cluster stores Secrets in etcd as plain text. Encrypt them at rest. On `{CP}`:

1. Write an `EncryptionConfiguration` to `/etc/kubernetes/enc/enc.yaml` for the resource
   `secrets`: provider `aescbc` with one key named `key1` (32 random bytes, base64-encoded),
   followed by the `identity` provider.
2. Make the API server use it (`--encryption-provider-config`); the file has to be visible inside
   the static Pod.
3. Existing Secrets stay as they are until they are written again: rewrite every Secret in
   Namespace `%{ns}`.
4. Read Secret `db-credentials` of `%{ns}` directly from etcd with `etcdctl` and save what etcd
   returns to `{WS}/q60/etcd.txt` (as text or as a hex dump).
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q60"; mkdir -p "$WS/q60"
nx "$CP" 'rm -rf /etc/kubernetes/enc' </dev/null
$K -n %{ns} create secret generic db-credentials --from-literal=user=ledger --from-literal=password=%{pw} >/dev/null
$K -n %{ns} create secret generic api-token --from-literal=token=t-%{pw} >/dev/null
""",
 "checks": [
  C("enc.yaml encrypts secrets with aescbc key1 first, identity after it", 2, r"""
f=/etc/kubernetes/enc/enc.yaml; y() { nx "$CP" "yq '$1' $f" </dev/null; }
eq "$(y .kind)" EncryptionConfiguration && y '.resources[0].resources[]' | grep -qx secrets && eq "$(y '.resources[0].providers[0].aescbc.keys[0].name')" key1 \
 && eq "$(y '.resources[0].providers[0].aescbc.keys[0].secret' | base64 -d 2>/dev/null | wc -c)" 32 && [ "$(y '.resources[0].providers[1] | has("identity")')" = true ]"""),
  C("the API server runs with --encryption-provider-config and is healthy", 3, API + r"""
runs kube-apiserver '--encryption-provider-config=/etc/kubernetes/enc/enc.yaml' && wait_api 90 && pod_ready kube-system "kube-apiserver-$CP" 90"""),
  C("in etcd, both Secrets of %{ns} are encrypted with key1 - and the API still returns the password", 4, API + ETCD + r"""
wait_api 30 || exit 1
etcdval /registry/secrets/%{ns}/db-credentials | grep -q '^k8s:enc:aescbc:v1:key1:' && etcdval /registry/secrets/%{ns}/api-token | grep -q '^k8s:enc:aescbc:v1:key1:' \
 && ! etcdval /registry/secrets/%{ns}/db-credentials | grep -q '%{pw}' && eq "$(j -n %{ns} secret db-credentials '{.data.password}' | base64 -d)" %{pw}"""),
  C("etcd.txt shows the encrypted value", 1, ETCD + r"""shows "$WS/q60/etcd.txt" 'k8s:enc:aescbc:v1:key1'"""),
 ],
 "solution": """
sudo -i
mkdir -p /etc/kubernetes/enc
cat > /etc/kubernetes/enc/enc.yaml <<EOF
apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
  - resources: [secrets]
    providers:
      - aescbc:
          keys:
            - name: key1
              secret: $(head -c 32 /dev/urandom | base64)
      - identity: {}
EOF
cp /etc/kubernetes/manifests/kube-apiserver.yaml /root/
vi /etc/kubernetes/manifests/kube-apiserver.yaml
#   command:       - --encryption-provider-config=/etc/kubernetes/enc/enc.yaml
#   volumeMounts:  - {name: enc, mountPath: /etc/kubernetes/enc, readOnly: true}
#   volumes:       - {name: enc, hostPath: {path: /etc/kubernetes/enc, type: DirectoryOrCreate}}
watch crictl ps                                         # until kube-apiserver is back
kubectl -n %{ns} get secrets -o json | kubectl replace -f -
ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt \\
  --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key \\
  get /registry/secrets/%{ns}/db-credentials | hexdump -C | tee {WS}/q60/etcd.txt | head -4     # k8s:enc:aescbc:v1:key1:...
# docs: "Encrypting Confidential Data at Rest". The first provider encrypts, all of them can
# decrypt: `identity` last keeps the old plain-text Secrets readable until they are rewritten.
# With identity FIRST nothing would be encrypted at all.
""",
},
{
 "id": 61, "domain": "MICRO", "title": "Read a Secret straight from etcd",
 "params": lambda r: {"ns": ns_pick(r, "treasury"), "name": name_pick(r, "signing-key"),
                      "tok": "tok-" + "".join(r.choice("ABCDEFGHJKLMNPQRSTUVWXYZ23456789") for _ in range(20))},
 "task": """
The team believes that Secrets are safe because RBAC lets almost nobody read them. Show what
access to etcd means. On `{CP}`, **without asking the Kubernetes API** for it:

1. Read Secret `%{name}` of Namespace `%{ns}` directly from etcd with `etcdctl`. etcd's client
   certificates are in `/etc/kubernetes/pki/etcd/`.
2. Write the value of the Secret's key `token`, as plain text, to `{WS}/q61/token.txt`.
3. Write the etcd key under which the Secret is stored (it starts with `/registry/`) to
   `{WS}/q61/key.txt`.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q61"; mkdir -p "$WS/q61"
$K -n %{ns} create secret generic %{name} --from-literal=token=%{tok} --from-literal=owner=treasury >/dev/null
""",
 "checks": [
  C("token.txt holds the Secret's token", 6, r"""eq "$(tr -d ' \n\r' < "$WS/q61/token.txt" 2>/dev/null)" %{tok}"""),
  C("key.txt holds the etcd key of the Secret", 4, r"""eq "$(tr -d ' \n\r' < "$WS/q61/key.txt" 2>/dev/null)" /registry/secrets/%{ns}/%{name}"""),
 ],
 "solution": """
sudo -i
export ETCDCTL_API=3
E="etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key"
$E get /registry/secrets/%{ns} --prefix --keys-only         # /registry/secrets/%{ns}/%{name}   -> {WS}/q61/key.txt
$E get /registry/secrets/%{ns}/%{name} | hexdump -C | less   # the value is in there, in the clear: token ... %{tok}
$E get /registry/secrets/%{ns}/%{name} --print-value-only | strings | grep tok-
# Objects are stored as protobuf under /registry/<resource>/<namespace>/<name>; a Secret's data is
# in it as raw bytes - not even base64. Whoever reads etcd, or a backup of it, reads every Secret,
# whatever RBAC says: hence client certificates on port 2379, and encryption at rest.
""",
},
{
 "id": 62, "domain": "MICRO", "title": "From environment variables to files",
 "params": lambda r: {"ns": ns_pick(r, "crm"), "dep": name_pick(r, "sync"),
                      "pw": "s3c-" + "".join(r.choice("abcdefghjkmnpqrstuvwxyz23456789") for _ in range(12))},
 "task": """
Deployment `%{dep}` in Namespace `%{ns}` gets the credentials of Secret `api-creds` as the
environment variables `API_USER` and `API_PASSWORD` — where every child process, crash report and
`kubectl describe` of a careless tool can see them.

Change the Deployment so that it gets them as files instead:

- Secret `api-creds` mounted read-only at `/etc/creds`, the files with mode `0400`;
- no trace of the two variables in the container's environment.

Leave the image and the command as they are. The Deployment must be available.
""",
 "setup": NS + r"""
mkns %{ns}
$K -n %{ns} create secret generic api-creds --from-literal=user=svc-sync --from-literal=password=%{pw} >/dev/null
$K apply -f - >/dev/null <<EOF
apiVersion: apps/v1
kind: Deployment
metadata: {name: %{dep}, namespace: %{ns}}
spec:
  replicas: 1
  selector: {matchLabels: {app: %{dep}}}
  template:
    metadata: {labels: {app: %{dep}}}
    spec:
      containers:
        - name: sync
          image: busybox:1.36
          command: ["sh", "-c", "while true; do echo syncing; sleep 30; done"]
          env:
            - {name: SYNC_INTERVAL, value: "30"}
            - {name: API_USER, valueFrom: {secretKeyRef: {name: api-creds, key: user}}}
            - {name: API_PASSWORD, valueFrom: {secretKeyRef: {name: api-creds, key: password}}}
EOF
$K -n %{ns} rollout status deploy %{dep} --timeout=180s >/dev/null
""",
 "checks": [
  C("the Pod template no longer takes the Secret into the environment", 3, PY + r"""
py -n %{ns} deploy %{dep} '
c = o["spec"]["template"]["spec"]["containers"][0]
not any(e["name"] in ("API_USER", "API_PASSWORD") or "secretKeyRef" in e.get("valueFrom", {}) for e in c.get("env", [])) and not any("secretRef" in e for e in c.get("envFrom", [])) and any(e["name"] == "SYNC_INTERVAL" for e in c.get("env", []))'"""),
  C("Secret api-creds is mounted read-only at /etc/creds with mode 0400", 3, PY + r"""
py -n %{ns} deploy %{dep} '
t = o["spec"]["template"]["spec"]; c = t["containers"][0]
v = {x["name"]: x["secret"] for x in t.get("volumes", []) if "secret" in x}
m = [x for x in c.get("volumeMounts", []) if x["mountPath"].rstrip("/") == "/etc/creds"]
bool(m) and m[0].get("readOnly") is True and v.get(m[0]["name"], {}).get("secretName") == "api-creds" and v[m[0]["name"]].get("defaultMode") == 0o400'"""),
  C("in the running Pod the password is a file of mode 400 and not in the environment", 4, r"""
for i in $(seq 1 20); do [ "$(j -n %{ns} deploy %{dep} '{.status.updatedReplicas}')" = 1 ] && [ "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" = 1 ] && break; sleep 3; done
p=$($K -n %{ns} get pod -l app=%{dep} --field-selector status.phase=Running --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}'); [ -n "$p" ] || exit 1
eq "$($K -n %{ns} exec "$p" -- cat /etc/creds/password 2>/dev/null)" %{pw} && eq "$($K -n %{ns} exec "$p" -- stat -L -c %a /etc/creds/password 2>/dev/null)" 400 \
 && ! $K -n %{ns} exec "$p" -- env 2>/dev/null | grep -q -e '%{pw}' -e '^API_'"""),
 ],
 "solution": """
k -n %{ns} edit deploy %{dep}
#   containers[0]:
#     env:                      # delete API_USER and API_PASSWORD, keep SYNC_INTERVAL
#     volumeMounts:
#       - {name: creds, mountPath: /etc/creds, readOnly: true}
#   volumes:
#     - name: creds
#       secret: {secretName: api-creds, defaultMode: 0400}
k -n %{ns} rollout status deploy %{dep}
k -n %{ns} exec deploy/%{dep} -- sh -c 'ls -lL /etc/creds; env | grep API'
# Each key of the Secret becomes a file. A mounted Secret is updated in place when the Secret
# changes; an environment variable never is. defaultMode is octal in YAML (0400) and the same
# number in decimal (256) in JSON - `kubectl get -o json` shows 256.
""",
},
{
 "id": 63, "domain": "MICRO", "title": "Secrets in a ConfigMap",
 "params": lambda r: {"ns": ns_pick(r, "shop"), "dep": name_pick(r, "checkout"),
                      "pw": "db-" + "".join(r.choice("abcdefghjkmnpqrstuvwxyz23456789") for _ in range(12)),
                      "tok": "at-" + "".join(r.choice("ABCDEFGHJKLMNPQRSTUVWXYZ23456789") for _ in range(16))},
 "task": """
ConfigMap `app-config` in Namespace `%{ns}` holds four settings of Deployment `%{dep}`. Two of
them are secrets and have no business in a ConfigMap: `DB_PASSWORD` and `API_TOKEN`.

1. Create Secret `app-secrets` in `%{ns}` with these two keys and their present values.
2. Remove the two keys from the ConfigMap. `LOG_LEVEL` and `DB_HOST` stay.
3. The Deployment's container must still find all four in its environment, under the same names:
   two from the ConfigMap, two from the Secret.
""",
 "setup": NS + r"""
mkns %{ns}
$K -n %{ns} create configmap app-config --from-literal=LOG_LEVEL=info --from-literal=DB_HOST=db.%{ns}.svc --from-literal=DB_PASSWORD=%{pw} --from-literal=API_TOKEN=%{tok} >/dev/null
$K apply -f - >/dev/null <<EOF
apiVersion: apps/v1
kind: Deployment
metadata: {name: %{dep}, namespace: %{ns}}
spec:
  replicas: 1
  selector: {matchLabels: {app: %{dep}}}
  template:
    metadata: {labels: {app: %{dep}}}
    spec:
      containers:
        - name: app
          image: busybox:1.36
          command: [sleep, "86400"]
          envFrom: [{configMapRef: {name: app-config}}]
EOF
$K -n %{ns} rollout status deploy %{dep} --timeout=180s >/dev/null
""",
 "checks": [
  C("Secret app-secrets holds DB_PASSWORD and API_TOKEN with their values", 3, r"""
eq "$(j -n %{ns} secret app-secrets '{.data.DB_PASSWORD}' | base64 -d)" %{pw} && eq "$(j -n %{ns} secret app-secrets '{.data.API_TOKEN}' | base64 -d)" %{tok}"""),
  C("the ConfigMap no longer has them, and keeps LOG_LEVEL and DB_HOST", 3, PY + r"""
py -n %{ns} cm app-config 'sorted(o.get("data", {})) == ["DB_HOST", "LOG_LEVEL"]'"""),
  C("the running container still has all four variables, the two secrets from the Secret", 4, PY + r"""
py -n %{ns} deploy %{dep} '
c = o["spec"]["template"]["spec"]["containers"][0]
any(e.get("secretRef", {}).get("name") == "app-secrets" for e in c.get("envFrom", [])) or sum(1 for e in c.get("env", []) if e.get("valueFrom", {}).get("secretKeyRef", {}).get("name") == "app-secrets") >= 2' || exit 1
for i in $(seq 1 20); do [ "$(j -n %{ns} deploy %{dep} '{.status.updatedReplicas}')" = 1 ] && [ "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" = 1 ] && break; sleep 3; done
p=$($K -n %{ns} get pod -l app=%{dep} --field-selector status.phase=Running --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}'); [ -n "$p" ] || exit 1
e=$($K -n %{ns} exec "$p" -- env 2>/dev/null)
echo "$e" | grep -qx 'DB_PASSWORD=%{pw}' && echo "$e" | grep -qx 'API_TOKEN=%{tok}' && echo "$e" | grep -qx 'LOG_LEVEL=info' && echo "$e" | grep -q '^DB_HOST=db\.'"""),
 ],
 "solution": """
k -n %{ns} get cm app-config -o yaml
k -n %{ns} create secret generic app-secrets \\
  --from-literal=DB_PASSWORD="$(k -n %{ns} get cm app-config -o jsonpath='{.data.DB_PASSWORD}')" \\
  --from-literal=API_TOKEN="$(k -n %{ns} get cm app-config -o jsonpath='{.data.API_TOKEN}')"
k -n %{ns} patch cm app-config --type=json -p '[{"op":"remove","path":"/data/DB_PASSWORD"},{"op":"remove","path":"/data/API_TOKEN"}]'
k -n %{ns} edit deploy %{dep}
#   envFrom:
#     - configMapRef: {name: app-config}
#     - secretRef: {name: app-secrets}
k -n %{ns} rollout status deploy %{dep}
k -n %{ns} exec deploy/%{dep} -- env | grep -E 'DB_|API_|LOG_'
# A ConfigMap is readable by everyone with `view`; a Secret is not, can be encrypted at rest and is
# only sent to nodes that run a Pod using it. envFrom takes every key of the object as a variable.
# Changing a ConfigMap does not restart Pods: without the rollout the old values would stay in
# the running container's environment.
""",
},
{
 "id": 64, "domain": "MICRO", "title": "Rotate the encryption key", "difficulty": "hard",
 "params": lambda r: {"ns": ns_pick(r, "vault"), "pw": "vp-" + "".join(r.choice("abcdefghjkmnpqrstuvwxyz23456789") for _ in range(14))},
 "task": """
This cluster encrypts Secrets at rest with `aescbc` and a key named `key1`
(`/etc/kubernetes/enc/enc.yaml` on `{CP}`). That key may have leaked. Replace it:

1. Add a new key named `key2` (32 random bytes, base64) to the `aescbc` provider so that **it** is
   used for writing, while `key1` can still decrypt. Restart the API server — it reads the file
   only when it starts.
2. Rewrite every Secret of the cluster, so that all of them are encrypted with `key2`.
3. Remove `key1` from the configuration and restart the API server again.
4. Save what etcd stores for Secret `vault-pass` of Namespace `%{ns}` to `{WS}/q64/etcd.txt`.

All Secrets must stay readable through the API.
""",
 "setup": NS + API + ETCD + r"""
mkns %{ns}; rm -rf "$WS/q64"; mkdir -p "$WS/q64"
enc_setup || { echo "could not switch on encryption at rest" >&2; exit 1; }
$K -n %{ns} create secret generic vault-pass --from-literal=password=%{pw} >/dev/null
$K -n kube-system create secret generic ksim-canary --from-literal=k=v >/dev/null
etcdval /registry/secrets/%{ns}/vault-pass | grep -q '^k8s:enc:aescbc:v1:key1:'
""",
 "checks": [
  C("enc.yaml: aescbc has key2 and no key1", 3, r"""
f=/etc/kubernetes/enc/enc.yaml; y() { nx "$CP" "yq '$1' $f" </dev/null; }
eq "$(y '[.resources[0].providers[] | select(has("aescbc")) | .aescbc.keys[].name] | join(",")')" key2 \
 && eq "$(y '.resources[0].providers[] | select(has("aescbc")) | .aescbc.keys[0].secret' | base64 -d 2>/dev/null | wc -c)" 32"""),
  C("Secret vault-pass is stored encrypted with key2 and still readable through the API", 4, API + ETCD + r"""
wait_api 90 || exit 1
etcdval /registry/secrets/%{ns}/vault-pass | grep -q '^k8s:enc:aescbc:v1:key2:' && eq "$(j -n %{ns} secret vault-pass '{.data.password}' | base64 -d)" %{pw}"""),
  C("so are the Secrets of other Namespaces", 2, API + ETCD + r"""
wait_api 30 || exit 1
etcdval /registry/secrets/kube-system/ksim-canary | grep -q '^k8s:enc:aescbc:v1:key2:' && eq "$(j -n kube-system secret ksim-canary '{.data.k}' | base64 -d)" v \
 && $K get secrets -A >/dev/null 2>&1"""),
  C("etcd.txt shows key2", 1, ETCD + r"""shows "$WS/q64/etcd.txt" 'k8s:enc:aescbc:v1:key2'"""),
 ],
 "solution": """
sudo -i
vi /etc/kubernetes/enc/enc.yaml                 # key2 FIRST, key1 second:
#       - aescbc:
#           keys:
#             - {name: key2, secret: <head -c 32 /dev/urandom | base64>}
#             - {name: key1, secret: <as it was>}
# restart the API server: move its manifest out of the directory and back
mv /etc/kubernetes/manifests/kube-apiserver.yaml /root/ && sleep 20 && mv /root/kube-apiserver.yaml /etc/kubernetes/manifests/
watch crictl ps                                 # until kube-apiserver is back
kubectl get secrets -A -o json | kubectl replace -f -          # every Secret is written again, with key2
vi /etc/kubernetes/enc/enc.yaml                 # now delete key1
mv /etc/kubernetes/manifests/kube-apiserver.yaml /root/ && sleep 20 && mv /root/kube-apiserver.yaml /etc/kubernetes/manifests/
ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt \\
  --key=/etc/kubernetes/pki/etcd/server.key get /registry/secrets/%{ns}/vault-pass | strings | tee {WS}/q64/etcd.txt     # k8s:enc:aescbc:v1:key2:
kubectl get secrets -A >/dev/null && echo all readable
# The order of the three steps is the whole task: new key first (writes use the first key, reads
# try them all), rewrite, and only then drop the old key. Removing key1 before the rewrite leaves
# every Secret unreadable - and `kubectl get secrets -A` failing for the whole cluster.
""",
},
# =================================================================== 4.3 Isolation ====
{
 "id": 65, "domain": "MICRO", "title": "Run a workload in gVisor", "workers": 1,
 "params": lambda r: {"ns": ns_pick(r, "untrusted"), "dep": name_pick(r, "converter")},
 "task": """
Deployment `%{dep}` in Namespace `%{ns}` processes files uploaded by strangers. It should not
share the node's kernel. The nodes of this cluster have gVisor installed: containerd knows a
runtime handler named `runsc`.

1. Create a RuntimeClass named `gvisor` for that handler.
2. Make the Pods of `%{dep}` run with it.
3. Save the output of `dmesg` from inside one of the new Pods to `{WS}/q65/dmesg.txt`.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q65"; mkdir -p "$WS/q65"
$K delete runtimeclass gvisor --ignore-not-found >/dev/null 2>&1
$K -n %{ns} create deploy %{dep} --image=busybox:1.36 -- sleep 86400 >/dev/null
$K -n %{ns} rollout status deploy %{dep} --timeout=180s >/dev/null
""",
 "checks": [
  C("RuntimeClass gvisor uses the handler runsc", 3, r"""eq "$(j runtimeclass gvisor '{.handler}')" runsc"""),
  C("the Deployment's Pods ask for it, and the Deployment is available", 3, r"""
eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.runtimeClassName}')" gvisor || exit 1
for i in $(seq 1 30); do [ "$(j -n %{ns} deploy %{dep} '{.status.updatedReplicas}')" = 1 ] && [ "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" = 1 ] && exit 0; sleep 3; done; exit 1"""),
  C("the running Pod's kernel is gVisor's", 3, r"""
p=$($K -n %{ns} get pod -l app=%{dep} --field-selector status.phase=Running --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}'); [ -n "$p" ] || exit 1
eq "$(j -n %{ns} pod "$p" '{.spec.runtimeClassName}')" gvisor && $K -n %{ns} exec "$p" -- dmesg 2>/dev/null | grep -qi 'gvisor'"""),
  C("dmesg.txt shows it", 1, r"""grep -qi 'gvisor' "$WS/q65/dmesg.txt" 2>/dev/null"""),
 ],
 "solution": """
cat <<EOF | k apply -f -
apiVersion: node.k8s.io/v1
kind: RuntimeClass
metadata: {name: gvisor}
handler: runsc
EOF
k -n %{ns} patch deploy %{dep} -p '{"spec":{"template":{"spec":{"runtimeClassName":"gvisor"}}}}'
k -n %{ns} rollout status deploy %{dep}
k -n %{ns} exec deploy/%{dep} -- dmesg | tee {WS}/q65/dmesg.txt        # [   0.000000] Starting gVisor...
# RuntimeClass is cluster-scoped and `handler` sits at the top level, not under spec. The handler
# name is whatever the node's containerd configuration calls the runtime:
#   sudo grep -A3 'runtimes.runsc' /etc/containerd/config.toml
# In the sandbox, system calls are answered by gVisor's own kernel in user space - `dmesg` prints
# its boot messages instead of the node's.
""",
},
{
 "id": 66, "domain": "MICRO", "title": "Fence a tenant in",
 "params": lambda r: {"t": r.choice(["acme", "globex", "initech", "umbrella", "hooli"])},
 "task": """
Tenant `%{t}` has moved into Namespace `tenant-%{t}`. Give the Namespace its fences:

1. ResourceQuota `tenant-quota`: at most 5 Pods, `requests.cpu` 1, `requests.memory` 1Gi,
   `limits.cpu` 2, `limits.memory` 2Gi.
2. LimitRange `tenant-defaults`: containers that say nothing get a request of 100m CPU / 64Mi
   memory and a limit of 200m CPU / 128Mi memory.
3. NetworkPolicy `tenant-isolation`: every Pod of the Namespace accepts connections only from
   Pods of the same Namespace.
4. RoleBinding `tenant-admins`: group `%{t}-admins` gets the built-in ClusterRole `admin` — in
   this Namespace only.

Pods `api` and `client` already run in `tenant-%{t}`; Pod `stranger` runs in Namespace `shared`.
""",
 "setup": PODS + r"""
mkns tenant-%{t}; mkns shared
srv tenant-%{t} api; cli tenant-%{t} client; cli shared stranger
ready tenant-%{t} && ready shared
""",
 "checks": [
  C("ResourceQuota tenant-quota has the five limits", 2, PY + r"""
py -n tenant-%{t} resourcequota tenant-quota '
h = o["spec"]["hard"]
h.get("pods") == "5" and h.get("requests.cpu") == "1" and h.get("requests.memory") == "1Gi" and h.get("limits.cpu") == "2" and h.get("limits.memory") == "2Gi"'"""),
  C("a container that says nothing gets the LimitRange's requests and limits", 2, r"""
[ -n "$(j -n tenant-%{t} limitrange tenant-defaults '{.metadata.name}')" ] || exit 1
r=$($K -n tenant-%{t} run ksim-lr-probe --image=busybox:1.36 --dry-run=server -o jsonpath='{.spec.containers[0].resources}' -- sleep 1 2>/dev/null)
echo "$r" | python3 -c '
import json, sys
r = json.load(sys.stdin)
sys.exit(0 if r.get("requests") == {"cpu": "100m", "memory": "64Mi"} and r.get("limits") == {"cpu": "200m", "memory": "128Mi"} else 1)'"""),
  C("client reaches api; stranger in another Namespace does not", 3, NET + r"""
has -n tenant-%{t} netpol tenant-isolation && okr tenant-%{t} client http://api && blocked shared stranger http://api.tenant-%{t}"""),
  C("group %{t}-admins is admin in tenant-%{t} and nowhere else", 3, r"""
A="--as=someone --as-group=%{t}-admins"
eq "$(j -n tenant-%{t} rolebinding tenant-admins '{.roleRef.kind}/{.roleRef.name}')" ClusterRole/admin || exit 1
[ "$($K auth can-i create deployments -n tenant-%{t} $A 2>/dev/null)" = yes ] && [ "$($K auth can-i get secrets -n tenant-%{t} $A 2>/dev/null)" = yes ] \
 && [ "$($K auth can-i list pods -n shared $A 2>/dev/null | head -1)" = no ] && [ "$($K auth can-i list nodes $A 2>/dev/null | head -1)" = no ]"""),
 ],
 "solution": """
k -n tenant-%{t} create quota tenant-quota --hard=pods=5,requests.cpu=1,requests.memory=1Gi,limits.cpu=2,limits.memory=2Gi
cat <<EOF | k apply -f -
apiVersion: v1
kind: LimitRange
metadata: {name: tenant-defaults, namespace: tenant-%{t}}
spec:
  limits:
    - type: Container
      defaultRequest: {cpu: 100m, memory: 64Mi}
      default: {cpu: 200m, memory: 128Mi}
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: tenant-isolation, namespace: tenant-%{t}}
spec:
  podSelector: {}
  policyTypes: [Ingress]
  ingress:
    - from: [{podSelector: {}}]
EOF
k -n tenant-%{t} create rolebinding tenant-admins --clusterrole=admin --group=%{t}-admins
# The four fences of a tenant: what it may use (quota), what a careless Pod gets (LimitRange - and
# without it a quota on cpu or memory rejects every Pod that sets no requests), who may talk to it
# (NetworkPolicy), and who may change it (RBAC, bound in the Namespace only).
# In a LimitRange `default` is the limit and `defaultRequest` the request.
""",
},
{
 "id": 67, "domain": "MICRO", "title": "Workloads that reach into the node",
 "params": lambda r: {"ns": ns_pick(r, "platform")},
 "task": """
Namespace `%{ns}` runs six Deployments. Some of them reach out of their container into the node:
a Deployment counts if its Pod template shares one of the node's namespaces (`hostPID`, `hostIPC`,
`hostNetwork`), runs a privileged container, or mounts a `hostPath` volume.

1. Write the names of those Deployments to `{WS}/q67/breakers.txt`, one per line.
2. `log-shipper` has a reason: it reads the node's logs. Give it the least it needs — its hostPath
   volume limited to `/var/log` and mounted **read-only**. It must stay available.
3. Scale every other Deployment on your list to 0 replicas until its owners have fixed it. Leave
   the rest of the Namespace alone.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q67"; mkdir -p "$WS/q67"
dep() {  # dep NAME 'pod spec lines' 'container lines'
  $K apply -f - >/dev/null <<EOF
apiVersion: apps/v1
kind: Deployment
metadata: {name: $1, namespace: %{ns}}
spec:
  replicas: 1
  selector: {matchLabels: {app: $1}}
  template:
    metadata: {labels: {app: $1}}
    spec:
      $2
      containers:
        - name: main
          image: busybox:1.36
          command: [sleep, "86400"]
          $3
EOF
}
dep frontend 'terminationGracePeriodSeconds: 5' 'resources: {}'
dep worker 'volumes: [{name: scratch, emptyDir: {}}]' 'volumeMounts: [{name: scratch, mountPath: /scratch}]'
dep metrics-agent 'hostPID: true' 'resources: {}'
dep log-shipper 'volumes: [{name: host, hostPath: {path: /}}]' 'volumeMounts: [{name: host, mountPath: /host}]'
dep ci-runner 'volumes: [{name: sock, hostPath: {path: /run/containerd/containerd.sock}}]' 'volumeMounts: [{name: sock, mountPath: /run/containerd/containerd.sock}]'
dep net-tool 'terminationGracePeriodSeconds: 5' 'securityContext: {privileged: true}'
for d in frontend worker metrics-agent log-shipper ci-runner net-tool; do $K -n %{ns} rollout status deploy $d --timeout=240s >/dev/null || exit 1; done
""",
 "checks": [
  C("breakers.txt names the four Deployments", 4, r"""
[ -s "$WS/q67/breakers.txt" ] && diff <(tr -d ' \r' < "$WS/q67/breakers.txt" | sed 's#.*/##' | grep . | sort -u) <(printf 'ci-runner\nlog-shipper\nmetrics-agent\nnet-tool\n') >/dev/null"""),
  C("log-shipper mounts only /var/log of the node, read-only, and is available", 3, PY + r"""
py -n %{ns} deploy log-shipper '
t = o["spec"]["template"]["spec"]
hp = {v["name"]: v["hostPath"]["path"].rstrip("/") for v in t.get("volumes", []) if "hostPath" in v}
m = [x for c in t["containers"] for x in c.get("volumeMounts", []) if x["name"] in hp]
list(hp.values()) == ["/var/log"] and len(m) == 1 and m[0].get("readOnly") is True and o.get("status", {}).get("availableReplicas") == 1 and o["status"].get("updatedReplicas") == 1'"""),
  C("the three others are scaled to 0; frontend and worker still run", 3, r"""
for d in metrics-agent ci-runner net-tool; do eq "$(j -n %{ns} deploy $d '{.spec.replicas}')" 0 || exit 1; done
eq "$(j -n %{ns} deploy frontend '{.status.availableReplicas}')" 1 && eq "$(j -n %{ns} deploy worker '{.status.availableReplicas}')" 1"""),
 ],
 "solution": """
k -n %{ns} get deploy -o json | jq -r '.items[] | select(.spec.template.spec as $s |
    $s.hostPID or $s.hostIPC or $s.hostNetwork or any($s.volumes[]?; .hostPath) or any($s.containers[]; .securityContext.privileged))
  | .metadata.name' | tee {WS}/q67/breakers.txt
#   ci-runner (the runtime socket), log-shipper (hostPath /), metrics-agent (hostPID), net-tool (privileged)
k -n %{ns} edit deploy log-shipper
#   volumes:       - {name: host, hostPath: {path: /var/log}}
#   volumeMounts:  - {name: host, mountPath: /host, readOnly: true}
k -n %{ns} scale deploy metrics-agent ci-runner net-tool --replicas=0
# Each of these is a way to the node: hostPID shows (and lets root signal) every process, a
# hostPath of / is the node's whole disk, the containerd socket starts containers with any
# privilege, and a privileged container has every capability and every device.
# worker's emptyDir is not a hostPath: it lives and dies with the Pod.
""",
},
# ======================================================= 4.4 Pod-to-Pod encryption ====
{
 "id": 68, "domain": "MICRO", "title": "Mutual TLS for a Namespace (Istio)", "workers": 1,
 "params": lambda r: {"ns": ns_pick(r, "orders")},
 "task": """
Istio is installed in this cluster (`istiod` in Namespace `istio-system`), but nothing uses it yet:
Deployments `orders` and `payments` in Namespace `%{ns}` talk plain HTTP, and so can anyone else.

1. Switch on automatic sidecar injection for `%{ns}` and get every Pod of the Namespace a
   sidecar.
2. Require mutual TLS for every workload of `%{ns}`: PeerAuthentication `default` in `%{ns}`,
   mode `STRICT`.

Afterwards `orders` must still reach `http://payments` — the sidecars encrypt that for them —
while a plain-text request from outside the mesh (Pod `legacy-client` in Namespace
`%{ns}-legacy`: `wget -qO- http://payments.%{ns}`) is refused.
""",
 "setup": PODS + ISTIO + r"""
ensure_istio || { echo "Istio could not be installed" >&2; exit 1; }
mkns %{ns}; mkns %{ns}-legacy
web %{ns} orders; web %{ns} payments; cli %{ns}-legacy legacy-client
for d in orders payments; do $K -n %{ns} rollout status deploy $d --timeout=240s >/dev/null || exit 1; done
ready %{ns}-legacy
""",
 "checks": [
  C("Namespace %{ns} has sidecar injection switched on", 2, r"""
eq "$(j ns %{ns} '{.metadata.labels.istio-injection}')" enabled || [ -n "$(j ns %{ns} '{.metadata.labels.istio\.io/rev}')" ]"""),
  C("every Pod of the Namespace has an istio-proxy and is Ready", 3, ISTIO + r"""
for i in $(seq 1 20); do meshed %{ns} && exit 0; sleep 3; done; exit 1"""),
  C("PeerAuthentication default requires STRICT mutual TLS for the whole Namespace", 2, PY + r"""
py -n %{ns} peerauthentication default 'o["spec"].get("mtls", {}).get("mode") == "STRICT" and not o["spec"].get("selector")'"""),
  C("orders still reaches payments; plain text from outside the mesh is refused", 3, NET + ISTIO + r"""
meshed %{ns} && has -n %{ns} peerauthentication default && okr %{ns} deploy/orders http://payments && blocked %{ns}-legacy legacy-client http://payments.%{ns}"""),
 ],
 "solution": """
k label ns %{ns} istio-injection=enabled
k -n %{ns} rollout restart deploy                 # injection happens when a Pod is created
k -n %{ns} get pod                                # 2/2: the application and istio-proxy
cat <<EOF | k apply -f -
apiVersion: security.istio.io/v1
kind: PeerAuthentication
metadata: {name: default, namespace: %{ns}}
spec:
  mtls: {mode: STRICT}
EOF
k -n %{ns} exec deploy/orders -- wget -T 3 -qO- http://payments                            # answers
k -n %{ns}-legacy exec legacy-client -- wget -T 3 -qO- http://payments.%{ns}               # connection reset by peer
# Without a selector, a PeerAuthentication named `default` covers its whole Namespace (in
# istio-system: the whole mesh). The default mode is PERMISSIVE - mutual TLS and plain text both
# accepted - which is how a mesh is migrated; STRICT refuses whatever does not come through a
# sidecar with a certificate of the mesh. docs: istio.io -> Tasks -> Security -> Mutual TLS Migration.
""",
},
{
 "id": 69, "domain": "MICRO", "title": "Strict for the mesh, one port excepted", "workers": 1, "difficulty": "hard",
 "params": lambda r: {"ns": ns_pick(r, "fleet")},
 "task": """
Istio runs in this cluster and Namespace `%{ns}` is in the mesh (its Pods have sidecars), but
mutual TLS is only optional (`PERMISSIVE`, the default).

1. Require mutual TLS for the **whole mesh**: PeerAuthentication `default` in the mesh's root
   Namespace `istio-system`, mode `STRICT`.
2. One exception: the workload `legacy-metrics` (label `app=legacy-metrics`) in `%{ns}` is scraped
   on port **9090** by a system outside the mesh. Create PeerAuthentication `legacy-metrics` in
   `%{ns}` that selects that workload: `STRICT` in general, `PERMISSIVE` for port 9090 only.

Test from outside the mesh with Pod `scraper` in Namespace `%{ns}-ext`:
`wget -qO- http://legacy-metrics.%{ns}:9090` must answer; port 80 of the same workload, and
`http://api.%{ns}`, must not.
""",
 "setup": PODS + ISTIO + r"""
ensure_istio || { echo "Istio could not be installed" >&2; exit 1; }
mkns %{ns}; mkns %{ns}-ext
$K label ns %{ns} istio-injection=enabled --overwrite >/dev/null
web %{ns} api; web %{ns} legacy-metrics 80 9090; cli %{ns}-ext scraper
for d in api legacy-metrics; do $K -n %{ns} rollout status deploy $d --timeout=240s >/dev/null || exit 1; done
ready %{ns}-ext
for i in $(seq 1 20); do meshed %{ns} && exit 0; sleep 3; done
echo "the Pods of %{ns} did not get a sidecar" >&2; exit 1
""",
 "checks": [
  C("PeerAuthentication default in istio-system makes the mesh STRICT", 3, PY + r"""
py -n istio-system peerauthentication default 'o["spec"].get("mtls", {}).get("mode") == "STRICT" and not o["spec"].get("selector")'"""),
  C("PeerAuthentication legacy-metrics: selects the workload, STRICT, port 9090 PERMISSIVE", 3, PY + r"""
py -n %{ns} peerauthentication legacy-metrics '
s = o["spec"]
s.get("selector", {}).get("matchLabels") == {"app": "legacy-metrics"} and s.get("mtls", {}).get("mode") == "STRICT" and {str(k): v.get("mode") for k, v in s.get("portLevelMtls", {}).items()} == {"9090": "PERMISSIVE"}'"""),
  C("from outside the mesh only port 9090 of legacy-metrics answers; inside it, everything still does", 4, NET + r"""
has -n istio-system peerauthentication default && has -n %{ns} peerauthentication legacy-metrics || exit 1
okr %{ns}-ext scraper http://legacy-metrics.%{ns}:9090 && okr %{ns} deploy/api http://legacy-metrics && okr %{ns} deploy/legacy-metrics http://api \
 && blocked %{ns}-ext scraper http://legacy-metrics.%{ns} && blocked %{ns}-ext scraper http://api.%{ns}"""),
 ],
 "solution": """
cat <<EOF | k apply -f -
apiVersion: security.istio.io/v1
kind: PeerAuthentication
metadata: {name: default, namespace: istio-system}
spec:
  mtls: {mode: STRICT}
---
apiVersion: security.istio.io/v1
kind: PeerAuthentication
metadata: {name: legacy-metrics, namespace: %{ns}}
spec:
  selector:
    matchLabels: {app: legacy-metrics}
  mtls: {mode: STRICT}
  portLevelMtls:
    "9090": {mode: PERMISSIVE}
EOF
k -n %{ns}-ext exec scraper -- wget -T 3 -qO- http://legacy-metrics.%{ns}:9090      # answers
k -n %{ns}-ext exec scraper -- wget -T 3 -qO- http://legacy-metrics.%{ns}           # reset
k -n %{ns}-ext exec scraper -- wget -T 3 -qO- http://api.%{ns}                      # reset
# The most specific policy wins: workload (with a selector) over Namespace over mesh. Port-level
# settings need the selector and name the workload's own port, not the Service's.
# docs: istio.io -> Reference -> Configuration -> Security -> PeerAuthentication.
""",
},
{
 "id": 70, "domain": "MICRO", "title": "Mutual authentication in a CiliumNetworkPolicy",
 "params": lambda r: {"ns": ns_pick(r, "ledger")},
 "task": """
This cluster is being moved to Cilium, with mutual authentication between its services. The
CiliumNetworkPolicy API is already installed; no Cilium agent enforces it yet, so you are graded
on the object.

In Namespace `%{ns}`, create CiliumNetworkPolicy `db-mutual-auth`:

- it applies to endpoints labelled `app=db`;
- it allows incoming traffic only from endpoints labelled `app=api`, to TCP port 5432;
- and for that traffic it **requires mutual authentication**.

`k explain ciliumnetworkpolicy.spec.ingress` shows the fields.
""",
 "setup": NS + r"""
mkns %{ns}
if ! $K get crd ciliumnetworkpolicies.cilium.io >/dev/null 2>&1; then
  curl -sfL --max-time 60 https://raw.githubusercontent.com/cilium/cilium/v1.16.5/pkg/k8s/apis/cilium.io/client/crds/v2/ciliumnetworkpolicies.yaml | $K apply -f - >/dev/null 2>&1 \
    || { echo "could not install the CiliumNetworkPolicy CRD (needs github)" >&2; exit 1; }
fi
$K wait --for=condition=Established --timeout=60s crd/ciliumnetworkpolicies.cilium.io >/dev/null
""",
 "checks": [
  C("db-mutual-auth applies to app=db", 3, PY + r"""
py -n %{ns} ciliumnetworkpolicy db-mutual-auth 'o["spec"].get("endpointSelector", {}).get("matchLabels") == {"app": "db"}'"""),
  C("it lets app=api in on TCP 5432, and nothing else", 3, PY + r"""
py -n %{ns} ciliumnetworkpolicy db-mutual-auth '
i = o["spec"].get("ingress", [])
ports = [(p.get("port"), p.get("protocol", "TCP").upper()) for r in i for tp in r.get("toPorts", []) for p in tp.get("ports", [])]
len(i) == 1 and [e.get("matchLabels") for e in i[0].get("fromEndpoints", [])] == [{"app": "api"}] and ports in ([("5432", "TCP")], [("5432", "ANY")])'"""),
  C("and requires mutual authentication for it", 4, PY + r"""
py -n %{ns} ciliumnetworkpolicy db-mutual-auth 'len(o["spec"].get("ingress", [])) == 1 and o["spec"]["ingress"][0].get("authentication", {}).get("mode") == "required"'"""),
 ],
 "solution": """
cat <<EOF | k apply -f -
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata: {name: db-mutual-auth, namespace: %{ns}}
spec:
  endpointSelector:
    matchLabels: {app: db}
  ingress:
    - fromEndpoints:
        - matchLabels: {app: api}
      authentication:
        mode: required
      toPorts:
        - ports:
            - {port: "5432", protocol: TCP}
EOF
# `authentication.mode: required` makes Cilium demand a successful mutual TLS handshake between
# the two workloads' identities (SPIFFE certificates) before the connection is allowed - on top of
# the label match. Encrypting the traffic itself is a separate, cluster-wide switch (WireGuard or
# IPsec: `cilium encrypt status`). docs.cilium.io -> Security -> Mutual Authentication.
""",
},
]