"""CKS tasks 10-24: Cluster Setup - the pool tasks that take the domain from 2 to 17. 1.1 NetworkPolicies 10 only the backend reaches the database 11 a runner that may only call the API 12 quarantine a compromised Pod 1.2 CIS benchmark 13 kubelet findings 14 etcd findings 15 controller manager and scheduler, with a report 1.3 Ingress with TLS 16 a certificate for a host 17 the wrong certificate 18 two hosts, two certificates, HTTPS only 1.4 Metadata and 19 the cloud metadata address 20 a kubelet that answers anyone endpoints 21 metrics endpoints open to the network 1.5 Verify binaries 22 a release directory 23 binaries replaced on a node 24 the binary of a running process NetworkPolicies are tested with real requests (kindnet enforces them), the kubelet and the metrics endpoints with real unauthenticated calls from the other node, and the benchmark findings with kube-bench itself. No Ingress controller runs here: 16-18 are graded on the objects, with the certificates checked by openssl. Every task has a scripted solution in solve.py. """ from cks.common import API, C, EXT, NET, PODS, PY, TLS, NS, ns_pick DOMS = ["acme", "globex", "initech", "umbrella", "hooli", "stark"] def _hex(r, n): return "".join(r.choice("0123456789ABCDEF") for _ in range(n)) QUESTIONS = [ # ========================================================== 1.1 NetworkPolicies ==== { "id": 10, "domain": "SETUP", "title": "Only the backend reaches the database", "params": lambda r: {"ns": ns_pick(r, "vault")}, "task": """ Namespace `%{ns}` holds Pod `db` (label `app=db`) and Pod `admin`. Namespace `%{ns}-app` holds Pods `backend` (label `role=backend`) and `frontend` (label `role=frontend`). Every Pod serves on TCP port 80 and has a Service of its own name. Create **one** NetworkPolicy `db-access` in `%{ns}` for the Pods labelled `app=db`: 1. **Incoming:** TCP 80 only from Pods labelled `role=backend` that run in Namespace `%{ns}-app` — both conditions on the same Pod. Nothing else gets in, not even from `%{ns}` itself. 2. **Outgoing:** DNS only (UDP and TCP port 53). A database has no reason to open connections. kindnet enforces NetworkPolicy here, so this is tested with real requests. """, "setup": PODS + r""" mkns %{ns}; mkns %{ns}-app srv %{ns} db; srv %{ns} admin srv %{ns}-app backend role=backend; srv %{ns}-app frontend role=frontend ready %{ns} && ready %{ns}-app """, "checks": [ C("db-access selects app=db, for Ingress and Egress", 2, r""" eq "$(j -n %{ns} netpol db-access '{.spec.podSelector.matchLabels.app}')" db || exit 1 t=$(j -n %{ns} netpol db-access '{.spec.policyTypes}'); echo "$t" | grep -q Ingress && echo "$t" | grep -q Egress"""), C("backend reaches db; frontend and admin do not", 4, NET + r""" has -n %{ns} netpol db-access && okr %{ns}-app backend http://db.%{ns} \ && blocked %{ns}-app frontend http://db.%{ns} && blocked %{ns} admin http://db"""), C("db still resolves names, but cannot open a connection", 4, NET + r""" has -n %{ns} netpol db-access && okr %{ns}-app backend http://db.%{ns} || exit 1 # "timed out" = the name resolved and the connection was dropped; "bad address" = DNS is cut too out=$($K -n %{ns} exec db -- wget -T 3 -qO- http://backend.%{ns}-app 2>&1); echo "$out" | grep -q 'timed out'"""), ], "solution": """ cat </dev/null cli %{ns} other; srv %{ns}-tools web ready %{ns} && ready %{ns}-tools """, "checks": [ C("runner-egress selects app=runner, for Egress only", 2, r""" eq "$(j -n %{ns} netpol runner-egress '{.spec.podSelector.matchLabels.app}')" runner && eq "$(j -n %{ns} netpol runner-egress '{.spec.policyTypes}')" '["Egress"]'"""), C("runner still gets an answer from https://kubernetes.default.svc", 3, NET + r""" has -n %{ns} netpol runner-egress || exit 1 for i in 1 2 3 4; do $K -n %{ns} exec runner -- curl -sk -m 5 https://kubernetes.default.svc/version 2>/dev/null | grep -q gitVersion && exit 0; sleep 2; done; exit 1"""), C("runner reaches nothing else; other Pods are not affected", 3, NET + r""" has -n %{ns} netpol runner-egress || exit 1 $K -n %{ns} exec runner -- curl -sk -m 5 https://kubernetes.default.svc/version 2>/dev/null | grep -q gitVersion || exit 1 okr %{ns} other http://web.%{ns}-tools || exit 1 for i in 1 2 3 4 5 6; do $K -n %{ns} exec runner -- curl -s -m 3 -o /dev/null http://web.%{ns}-tools >/dev/null 2>&1 || exit 0; sleep 3; done; exit 1"""), C("the API server is allowed by its own address and port 6443, nothing wider", 2, r""" ep=$($K -n default get endpointslice -l kubernetes.io/service-name=kubernetes -o jsonpath='{.items[0].endpoints[0].addresses[0]}') [ -n "$ep" ] || exit 1 $K -n %{ns} get netpol runner-egress -o json 2>/dev/null | EP="$ep/32" python3 -c ' import json, os, sys api = False for r in json.load(sys.stdin)["spec"].get("egress", []): ports = {str(p.get("port")) for p in r.get("ports", [])} blocks = [t.get("ipBlock", {}).get("cidr", "") for t in r.get("to", [])] if any(b.endswith("/0") for b in blocks): sys.exit(1) if not r.get("to") and not (ports and ports <= {"53"}): # "anywhere" is acceptable for DNS only sys.exit(1) if os.environ["EP"] in blocks and "6443" in ports: api = True sys.exit(0 if api else 1)'"""), ], "solution": """ k get endpointslice -l kubernetes.io/service-name=kubernetes # ADDRESS and port 6443: the API server itself # (k get svc kubernetes shows 10.96.0.1:443 - that is NOT what the policy sees) API=$(k get endpointslice -l kubernetes.io/service-name=kubernetes -o jsonpath='{.items[0].endpoints[0].addresses[0]}') cat <:6443 before the policy is evaluated, so a rule # for the ClusterIP or for port 443 never matches. The API server runs on the host network, which is # why it needs an ipBlock: a podSelector cannot select it. """, }, { "id": 12, "domain": "SETUP", "title": "Quarantine a compromised Pod", "params": lambda r: {"ns": ns_pick(r, "store")}, "task": """ Deployment `shop` (3 replicas, label `app=shop`, Service `shop`) runs in Namespace `%{ns}`. One of its Pods is compromised: the one whose IP address is in `{WS}/q12/suspect-ip.txt`. Take it out of service without destroying the evidence: 1. Create NetworkPolicy `quarantine` in `%{ns}` that selects Pods labelled `quarantine=true` and allows them **no** incoming and **no** outgoing traffic at all. 2. Put the compromised Pod under that policy, and take it out of the Deployment and the Service by removing its `app` label. Do **not** delete the Pod. 3. The Deployment must be back at 3 healthy replicas. """, "setup": PODS + r""" mkns %{ns}; rm -rf "$WS/q12"; mkdir -p "$WS/q12" $K apply -f - >/dev/null < /www/index.html && exec httpd -f -p 80 -h /www"] ports: [{containerPort: 80}] EOF $K -n %{ns} expose deploy shop --port=80 >/dev/null srv %{ns} db; cli %{ns} probe $K -n %{ns} rollout status deploy shop --timeout=240s >/dev/null && ready %{ns} || exit 1 pod=$($K -n %{ns} get pod -l app=shop -o jsonpath='{.items[1].metadata.name}') ip=$(j -n %{ns} pod "$pod" '{.status.podIP}'); uid=$(j -n %{ns} pod "$pod" '{.metadata.uid}') [ -n "$ip" ] && [ -n "$uid" ] || exit 1 echo "$ip" > "$WS/q12/suspect-ip.txt"; echo "$pod $uid $ip" > "$PRIV/q12" """, "checks": [ C("quarantine selects quarantine=true and allows nothing in either direction", 2, r""" n="$K -n %{ns} get netpol quarantine" eq "$($n -o jsonpath='{.spec.podSelector.matchLabels.quarantine}' 2>/dev/null)" true || exit 1 t=$($n -o jsonpath='{.spec.policyTypes}'); echo "$t" | grep -q Ingress && echo "$t" | grep -q Egress \ && [ -z "$($n -o jsonpath='{.spec.ingress}')" ] && [ -z "$($n -o jsonpath='{.spec.egress}')" ]"""), C("the compromised Pod is still there, labelled quarantine=true, without its app label", 3, r""" read -r pod uid ip < "$PRIV/q12" eq "$(j -n %{ns} pod "$pod" '{.metadata.uid}')" "$uid" && eq "$(j -n %{ns} pod "$pod" '{.metadata.labels.quarantine}')" true \ && [ -z "$(j -n %{ns} pod "$pod" '{.metadata.labels.app}')" ]"""), C("it can neither reach db nor be reached, while the shop keeps answering", 3, NET + r""" read -r pod uid ip < "$PRIV/q12" has -n %{ns} netpol quarantine && eq "$(j -n %{ns} pod "$pod" '{.metadata.uid}')" "$uid" && okr %{ns} probe http://shop \ && blocked %{ns} "$pod" http://db && blocked %{ns} probe "http://$ip" """), C("shop has 3 ready replicas, none of them the compromised Pod", 2, r""" read -r pod uid ip < "$PRIV/q12" for i in $(seq 1 10); do [ "$(j -n %{ns} deploy shop '{.status.readyReplicas}')" = 3 ] && break; sleep 3; done eq "$(j -n %{ns} deploy shop '{.status.readyReplicas}')" 3 && ! $K -n %{ns} get pod -l app=shop -o name | grep -qx "pod/$pod" """), ], "solution": """ IP=$(cat {WS}/q12/suspect-ip.txt) k -n %{ns} get pod -o wide | grep -w "$IP" # its name POD=$(k -n %{ns} get pod -o jsonpath="{.items[?(@.status.podIP=='$IP')].metadata.name}") cat <&2; exit 1 """, "checks": [ C("kubelet configuration: anonymous access off, authorisation by Webhook", 3, API + r""" eq "$(kcfg "$CP" .authentication.anonymous.enabled)" false && eq "$(kcfg "$CP" .authorization.mode)" Webhook"""), C("the kubelet runs, the node is Ready, and an unauthenticated request gets 401", 4, API + r""" nx "$CP" 'systemctl is-active --quiet kubelet' /dev/null && grep -q '^\[PASS\] 4\.2\.2 ' "$WS/q13/after.txt" """), ], "solution": """ sudo -i kube-bench run --targets node --check 4.2.1,4.2.2 # both FAIL, each with its remediation vi /var/lib/kubelet/config.yaml # authentication: # anonymous: # enabled: false # authorization: # mode: Webhook systemctl restart kubelet && systemctl is-active kubelet kube-bench run --targets node --check 4.2.1,4.2.2 | grep '^\\[' > {WS}/q13/after.txt curl -sk https://localhost:10250/pods # Unauthorized - before, it listed every Pod # kube-bench names the flags (--anonymous-auth, --authorization-mode), but a kubeadm node sets them # in the kubelet's configuration file, which the remediation text mentions second. A flag in # /var/lib/kubelet/kubeadm-flags.env would work too and be gone after the next kubeadm upgrade. """, }, { "id": 14, "domain": "SETUP", "title": "kube-bench: etcd findings", "difficulty": "hard", "task": """ kube-bench on `{CP}` reports four failures that concern etcd: - `2.2 Ensure that the --client-cert-auth argument is set to true` - `2.5 Ensure that the --peer-client-cert-auth argument is set to true` - `1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive` - `1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd` Fix all four. The node has no `etcd` user yet: create one as a system account. etcd and the API server must be running afterwards. """, "setup": API + r""" medit etcd.yaml '(.spec.containers[0].command[] | select(. == "--client-cert-auth=true")) = "--client-cert-auth=false" | (.spec.containers[0].command[] | select(. == "--peer-client-cert-auth=true")) = "--peer-client-cert-auth=false"' || exit 1 nx "$CP" 'chmod 755 /var/lib/etcd' /dev/null' "$PRIV/q15.failed" [ "$(wc -l < "$PRIV/q15.failed")" -ge 4 ] """, "checks": [ C("failed.txt lists exactly the failing checks of sections 1.3 and 1.4", 3, r""" [ -s "$WS/q15/failed.txt" ] && diff <(tr -d ' \r' < "$WS/q15/failed.txt" | grep . | sort -u) "$PRIV/q15.failed" >/dev/null"""), C("kube-controller-manager runs with --profiling=false on 127.0.0.1", 3, API + r""" runs kube-controller-manager '--profiling=false' && runs kube-controller-manager '--bind-address=127.0.0.1' \ && pod_ready kube-system "kube-controller-manager-$CP" 60"""), C("kube-scheduler runs with --profiling=false on 127.0.0.1", 3, API + r""" runs kube-scheduler '--profiling=false' && runs kube-scheduler '--bind-address=127.0.0.1' && pod_ready kube-system "kube-scheduler-$CP" 60"""), C("kube-bench reports no FAIL in sections 1.3 and 1.4", 1, API + r""" runs kube-controller-manager '--profiling=false' && runs kube-scheduler '--profiling=false' || exit 1 ! nx "$CP" 'kube-bench run --targets master 2>/dev/null' {WS}/q15/failed.txt kube-bench run --targets master --check 1.3.2,1.3.7 # the remediation of each vi /etc/kubernetes/manifests/kube-controller-manager.yaml # - --bind-address=127.0.0.1 # - --profiling=false vi /etc/kubernetes/manifests/kube-scheduler.yaml # the same two flags watch crictl ps # both restart by themselves kube-bench run --targets master | grep -E '^\\[(FAIL|PASS)\\] 1\\.[34]\\.' # The probes of both static Pods call 127.0.0.1, so binding to the loopback address breaks nothing. # On 0.0.0.0 the metrics and profiling endpoints of both were reachable from the whole network. """, }, # ========================================================= 1.3 Ingress with TLS ==== { "id": 16, "domain": "SETUP", "title": "An Ingress with a certificate", "params": lambda r: {"ns": ns_pick(r, "web"), "dom": r.choice(DOMS)}, "task": """ Service `shop` (port 80) in Namespace `%{ns}` is to be published at `https://shop.%{dom}.test`. The certificate and its key are in `{WS}/q16/tls.crt` and `{WS}/q16/tls.key`. 1. Create Secret `shop-tls` in `%{ns}` from them, of the type made for TLS. 2. Create Ingress `shop` in `%{ns}` with IngressClass `nginx`: host `shop.%{dom}.test`, every path (`/`, type `Prefix`) to Service `shop` port 80, and TLS for that host with `shop-tls`. No Ingress controller runs in this cluster: you are graded on the objects. """, "setup": NS + TLS + r""" mkns %{ns}; rm -rf "$WS/q16"; mkdir -p "$WS/q16" selfsigned shop.%{dom}.test "$WS/q16/tls.key" "$WS/q16/tls.crt" || exit 1 $K -n %{ns} create deploy shop --image=nginx:1.27 >/dev/null; $K -n %{ns} expose deploy shop --port=80 >/dev/null $K apply -f - >/dev/null </dev/null $K -n %{ns} create deploy portal --image=nginx:1.27 >/dev/null; $K -n %{ns} expose deploy portal --port=80 >/dev/null $K apply -f - >/dev/null </dev/null' /dev/null | cut -d= -f2)" +%s 2>/dev/null) || exit 1 e=$(date -d "$(openssl x509 -in "$WS/q17/tls.crt" -noout -enddate | cut -d= -f2)" +%s) || exit 1 d=$(( (e - s) / 86400 )); [ "$d" -ge 30 ] && [ "$d" -le 365 ]"""), C("Secret portal-tls holds that certificate with its key", 3, TLS + r""" tls_secret %{ns} portal-tls portal.%{dom}.test && [ "$(crt_of %{ns} portal-tls | fp)" = "$(fp < "$WS/q17/tls.crt")" ]"""), C("the Ingress's TLS section covers portal.%{dom}.test, and only that, with portal-tls", 3, PY + r""" py -n %{ns} ingress portal ' t = o["spec"].get("tls", []) len(t) == 1 and t[0].get("hosts") == ["portal.%{dom}.test"] and t[0].get("secretName") == "portal-tls" and o["spec"]["rules"][0]["host"] == "portal.%{dom}.test"'"""), ], "solution": """ k -n %{ns} get secret portal-tls -o jsonpath='{.data.tls\\.crt}' | base64 -d | openssl x509 -noout -subject -ext subjectAltName # -> old-portal.%{dom}.test: not the host the Ingress serves cd {WS}/q17 openssl req -x509 -newkey rsa:2048 -nodes -days 365 -keyout tls.key -out tls.crt \\ -subj "/CN=portal.%{dom}.test" -addext "subjectAltName=DNS:portal.%{dom}.test" k -n %{ns} create secret tls portal-tls --cert=tls.crt --key=tls.key --dry-run=client -o yaml | k apply -f - k -n %{ns} edit ingress portal # spec.tls[0].hosts: [portal.%{dom}.test] # Clients ignore the CN and read the subjectAltName: a certificate without it is rejected even when # the CN is right. `--dry-run=client -o yaml | k apply -f -` replaces a Secret in place; deleting # and re-creating it works as well. A controller serves its default (fake) certificate whenever # the host asked for is not listed under spec.tls or the Secret does not fit. """, }, { "id": 18, "domain": "SETUP", "title": "Two hosts, two certificates, HTTPS only", "params": lambda r: (lambda c, k: {"ns": ns_pick(r, "edge"), "dom": r.choice(DOMS), "c1": c[0], "c2": c[1], "k1": k[0], "k2": k[1]})( r.sample(["api", "www"], 2), r.sample(["api", "www"], 2)), "task": """ Ingress `edge` in Namespace `%{ns}` serves two hosts over plain HTTP: `api.%{dom}.test` and `www.%{dom}.test`. Each host has its own certificate. The files were handed over without a note: `cert-1.pem`, `cert-2.pem`, `key-1.pem` and `key-2.pem` in `{WS}/q18/` — find out which certificate is for which host, and which key belongs to which certificate. 1. Create the TLS Secrets `api-tls` and `www-tls` in `%{ns}`, each with the certificate of its host and the key that belongs to it. 2. Add TLS to the **existing** Ingress `edge` (do not delete and re-create it): each host with its own Secret. 3. Plain HTTP must be redirected to HTTPS: set the annotation `nginx.ingress.kubernetes.io/force-ssl-redirect` to `"true"`. """, "setup": NS + TLS + r""" mkns %{ns}; d="$WS/q18"; rm -rf "$d"; mkdir -p "$d" for h in api www; do selfsigned $h.%{dom}.test "$PRIV/q18-$h.key" "$PRIV/q18-$h.crt" || exit 1; done cp "$PRIV/q18-%{c1}.crt" "$d/cert-1.pem"; cp "$PRIV/q18-%{c2}.crt" "$d/cert-2.pem" cp "$PRIV/q18-%{k1}.key" "$d/key-1.pem"; cp "$PRIV/q18-%{k2}.key" "$d/key-2.pem" for s in api www; do $K -n %{ns} create deploy $s --image=nginx:1.27 >/dev/null; $K -n %{ns} expose deploy $s --port=80 >/dev/null; done $K apply -f - >/dev/null < "$PRIV/q18.uid"; [ -s "$PRIV/q18.uid" ] """, "checks": [ C("api-tls holds the certificate for api.%{dom}.test and the key that fits it", 3, TLS + r"""tls_secret %{ns} api-tls api.%{dom}.test"""), C("www-tls holds the certificate for www.%{dom}.test and the key that fits it", 3, TLS + r"""tls_secret %{ns} www-tls www.%{dom}.test"""), C("the same Ingress now has TLS for each host with its own Secret, rules unchanged", 3, PY + r""" eq "$(j -n %{ns} ingress edge '{.metadata.uid}')" "$(cat "$PRIV/q18.uid")" || exit 1 py -n %{ns} ingress edge ' m = {h: t.get("secretName") for t in o["spec"].get("tls", []) for h in t.get("hosts", [])} m == {"api.%{dom}.test": "api-tls", "www.%{dom}.test": "www-tls"} and sorted(r["host"] for r in o["spec"]["rules"]) == ["api.%{dom}.test", "www.%{dom}.test"]'"""), C("force-ssl-redirect is \"true\"", 1, r""" eq "$(j -n %{ns} ingress edge '{.metadata.annotations.nginx\.ingress\.kubernetes\.io/force-ssl-redirect}')" true"""), ], "solution": """ cd {WS}/q18 for c in cert-1.pem cert-2.pem; do echo "$c: $(openssl x509 -in $c -noout -ext subjectAltName | tail -1)"; done # a key belongs to a certificate when both have the same public key: for f in cert-1.pem cert-2.pem; do echo "$f $(openssl x509 -in $f -noout -pubkey | sha256sum)"; done for f in key-1.pem key-2.pem; do echo "$f $(openssl pkey -in $f -pubout | sha256sum)"; done k -n %{ns} create secret tls api-tls --cert=cert-N.pem --key=key-M.pem # the pair for api.%{dom}.test k -n %{ns} create secret tls www-tls --cert=cert-N.pem --key=key-M.pem # the pair for www.%{dom}.test k -n %{ns} annotate ingress edge nginx.ingress.kubernetes.io/force-ssl-redirect=true k -n %{ns} edit ingress edge # spec: # tls: # - {hosts: [api.%{dom}.test], secretName: api-tls} # - {hosts: [www.%{dom}.test], secretName: www-tls} # `kubectl create secret tls` refuses a key that does not match the certificate - a quick test, but # the hash comparison tells you which of the two keys is the right one without guessing. # Two entries under spec.tls let the controller pick the certificate by the host the client asks # for (SNI). An annotation value is always a string: "true", in quotes when you write YAML. """, }, # ================================================ 1.4 Node metadata and endpoints ==== { "id": 19, "domain": "SETUP", "title": "Keep Pods away from the cloud metadata address", "workers": 1, "params": lambda r: {"ns": ns_pick(r, "tenant"), "key": _hex(r, 12), "secret": _hex(r, 24).lower()}, "task": """ This cluster's nodes have a cloud metadata service at `169.254.169.254`, and it hands the node's cloud credentials to anything that asks: http://169.254.169.254/latest/meta-data/iam/security-credentials/node-role Namespace `%{ns}` runs Pods `web`, `batch` and `backup` (labels `app=`). Only `backup` has any business with the metadata service. 1. Show the problem: from Pod `web`, fetch the URL above and save the response to `{WS}/q19/leak.txt`. 2. Create NetworkPolicy `deny-metadata` in `%{ns}`: **every** Pod of the Namespace may send traffic anywhere **except** to `169.254.169.254`. 3. Create NetworkPolicy `allow-metadata` in `%{ns}`: Pods labelled `app=backup` may reach `169.254.169.254`. Do not restrict incoming traffic. DNS and traffic inside the cluster must keep working. """, "setup": EXT + PODS + r""" mkns %{ns}; rm -rf "$WS/q19"; mkdir -p "$WS/q19" extsvc cloud-metadata 169.254.169.254 latest/meta-data/iam/security-credentials/node-role 'AccessKeyId=AKIA%{key} SecretAccessKey=%{secret}' || exit 1 srv %{ns} web; srv %{ns} batch; srv %{ns} backup ready %{ns} || exit 1 for i in $(seq 1 20); do $K -n %{ns} exec web -- wget -T 3 -qO- http://169.254.169.254/latest/meta-data/iam/security-credentials/node-role 2>/dev/null | grep -q %{secret} && exit 0; sleep 2; done echo "the metadata address does not answer from a Pod" >&2; exit 1 """, "checks": [ C("leak.txt holds the credentials web could read", 2, r"""grep -q 'SecretAccessKey=%{secret}' "$WS/q19/leak.txt" 2>/dev/null"""), C("web and batch no longer reach 169.254.169.254; backup still does", 3, NET + r""" U=http://169.254.169.254/latest/meta-data/iam/security-credentials/node-role has -n %{ns} netpol deny-metadata && okr %{ns} backup $U && blocked %{ns} web $U && blocked %{ns} batch $U"""), C("DNS and traffic inside the cluster still work for web", 2, NET + r""" has -n %{ns} netpol deny-metadata && okr %{ns} web http://batch.%{ns}.svc.cluster.local"""), C("deny-metadata covers every Pod with an ipBlock exception; allow-metadata selects app=backup", 3, PY + r""" py -n %{ns} netpol deny-metadata ' s = o["spec"] b = [t.get("ipBlock", {}) for r in s.get("egress", []) for t in r.get("to", [])] s["podSelector"] == {} and s.get("policyTypes") == ["Egress"] and any(x.get("cidr") == "0.0.0.0/0" and x.get("except") == ["169.254.169.254/32"] for x in b)' || exit 1 py -n %{ns} netpol allow-metadata ' s = o["spec"] b = [t.get("ipBlock", {}).get("cidr") for r in s.get("egress", []) for t in r.get("to", [])] s["podSelector"].get("matchLabels") == {"app": "backup"} and "Ingress" not in s.get("policyTypes", []) and "169.254.169.254/32" in b'"""), ], "solution": """ k -n %{ns} exec web -- wget -qO- http://169.254.169.254/latest/meta-data/iam/security-credentials/node-role > {WS}/q19/leak.txt cat <&2; exit 1 """, "checks": [ C("port 10255 on {W1} is closed, with the kubelet running and the node Ready", 3, API + r""" nx "$W1" 'systemctl is-active --quiet kubelet' /dev/null)" 401"""), ], "solution": """ curl -s http://{W1}:10255/pods | head -c 300 # from {CP}: everything, no questions asked {SHELL_W1} sudo -i vi /var/lib/kubelet/config.yaml # readOnlyPort: 0 (or delete the line) # authentication: # anonymous: # enabled: false # authorization: # mode: Webhook systemctl restart kubelet && systemctl is-active kubelet exit; exit curl -s -m 3 http://{W1}:10255/pods # connection refused curl -sk -o /dev/null -w '%{http_code}\\n' https://{W1}:10250/pods | tee {WS}/q20/status.txt # 401 # The kubelet API can exec into every container of its node: with anonymous access and AlwaysAllow # it is a remote shell for whoever reaches port 10250. The API server keeps working, because it # presents a client certificate (apiserver-kubelet-client) that Webhook authorisation accepts. """, }, { "id": 21, "domain": "SETUP", "title": "Metrics endpoints open to the network", "workers": 1, "difficulty": "hard", "task": """ Two components of this cluster expose an unauthenticated HTTP endpoint to the whole network instead of to their own node only: - **etcd** serves its metrics and health on port **2381** of `{CP}`: `curl http://{CP}:2381/health` answers from `{W1}`. - **kube-proxy** serves its metrics and its configuration on port **10249** of every node: `curl http://{W1}:10249/configz` answers from `{CP}`. Restrict both to the loopback address (`127.0.0.1`) of their node: 1. etcd: through its static Pod manifest. 2. kube-proxy: through its configuration in ConfigMap `kube-proxy` (Namespace `kube-system`); the DaemonSet's Pods must be running with the new configuration. Both must keep working: etcd healthy, the API server answering, kube-proxy Pods Ready. """, "setup": API + r""" medit etcd.yaml '(.spec.containers[0].command[] | select(. == "--listen-metrics-urls=http://127.0.0.1:2381")) = "--listen-metrics-urls=http://0.0.0.0:2381"' || exit 1 wait_proc etcd '--listen-metrics-urls=http://0.0.0.0:2381' && wait_api || exit 1 $K -n kube-system get cm kube-proxy -o json | python3 -c ' import json, sys o = json.load(sys.stdin) c = o["data"]["config.conf"] assert "metricsBindAddress: \"\"" in c o["data"]["config.conf"] = c.replace("metricsBindAddress: \"\"", "metricsBindAddress: 0.0.0.0:10249") json.dump(o, sys.stdout)' | $K apply -f - >/dev/null || exit 1 $K -n kube-system rollout restart ds kube-proxy >/dev/null && $K -n kube-system rollout status ds kube-proxy --timeout=180s >/dev/null || exit 1 for i in $(seq 1 30); do nx "$W1" "curl -s -m 3 -o /dev/null http://$CP:2381/health" &2; exit 1 """, "checks": [ C("etcd serves metrics on 127.0.0.1:2381 only, and is healthy", 4, API + r""" eq "$(flag etcd listen-metrics-urls)" http://127.0.0.1:2381 && runs etcd '--listen-metrics-urls=http://127.0.0.1:2381' && wait_api 60 || exit 1 nx "$CP" 'curl -s -m 3 http://127.0.0.1:2381/health' "$d/release/$b" && [ -s "$d/release/$b" ] || exit 1 chmod 755 "$d/release/$b" done (cd "$d/release" && sha512sum kubelet kubectl crictl etcdctl runc > ../SHA512SUMS) || exit 1 # one byte changed in the middle: same size, same name, same date for b in %{bad}; do python3 -c 'import sys; p = sys.argv[1]; b = bytearray(open(p, "rb").read()); b[len(b) // 2] ^= 0xFF; open(p, "wb").write(b)' "$d/release/$b" || exit 1 touch -r "$d/SHA512SUMS" "$d/release/$b" done echo "%{bad}" | tr ' ' '\n' | sort > "$PRIV/q22.bad" """, "checks": [ C("tampered.txt names exactly the files that fail verification", 5, r""" [ -s "$WS/q22/tampered.txt" ] && diff <(tr -d ' \r' < "$WS/q22/tampered.txt" | sed 's#.*/##' | grep . | sort -u) "$PRIV/q22.bad" >/dev/null"""), C("those files are gone from release/, the genuine ones are intact", 5, r""" cd "$WS/q22/release" 2>/dev/null || exit 1 for b in $(cat "$PRIV/q22.bad"); do [ ! -e "$b" ] || exit 1; done [ "$(ls | wc -l)" = 3 ] && grep -v -w -F -f "$PRIV/q22.bad" ../SHA512SUMS | sha512sum -c --quiet - >/dev/null 2>&1"""), ], "solution": """ cd {WS}/q22/release sha512sum -c ../SHA512SUMS # OK or FAILED per file sha512sum -c ../SHA512SUMS 2>/dev/null | grep FAILED | cut -d: -f1 > ../tampered.txt xargs rm -f < ../tampered.txt ls -l # The tampered files have the size and date of the originals: only the hash tells. `--check` reads # " " lines and compares for you - never compare 128 hex digits by eye. # For one file with a bare hash: echo "$(cat kubectl.sha512) kubectl" | sha512sum --check """, }, { "id": 23, "domain": "SETUP", "title": "Binaries replaced on a node", "workers": 1, "params": lambda r: {"bad": " ".join(sorted(r.sample(["kubectl", "crictl", "helm", "etcdctl"], 2)))}, "task": """ The security team suspects that binaries on node `{W1}` were replaced. `{WS}/q23/reference.sha512` lists the SHA-512 checksums of the genuine files, with their full paths. The control-plane node `{CP}` has been verified and is clean. 1. Verify the listed files **on `{W1}`**. 2. Write the full paths of the files that fail there to `{WS}/q23/tampered.txt`, one per line. 3. Replace each of them on `{W1}` with the genuine file from `{CP}` — owner `root:root`, mode `755`. """, "setup": r""" d="$WS/q23"; rm -rf "$d"; mkdir -p "$d" nx "$CP" 'sha512sum /usr/bin/kubelet /usr/bin/kubectl "$(command -v crictl)" "$(command -v helm)" "$(command -v etcdctl)" "$(command -v runc)"' "$d/reference.sha512" [ "$(wc -l < "$d/reference.sha512")" = 6 ] || exit 1 : > "$PRIV/q23.bad" for b in %{bad}; do p=$(nx "$W1" "command -v $b" > "$PRIV/q23.bad" done sort -o "$PRIV/q23.bad" "$PRIV/q23.bad" ! nx "$W1" 'sha512sum -c --quiet -' < "$d/reference.sha512" >/dev/null 2>&1 """, "checks": [ C("tampered.txt names exactly the paths that fail on {W1}", 5, r""" [ -s "$WS/q23/tampered.txt" ] && diff <(tr -d ' \r' < "$WS/q23/tampered.txt" | grep . | sort -u) "$PRIV/q23.bad" >/dev/null"""), C("every listed file on {W1} matches its checksum again, root:root and mode 755", 5, r""" nx "$W1" 'sha512sum -c --quiet -' < "$WS/q23/reference.sha512" >/dev/null 2>&1 || exit 1 for p in $(cat "$PRIV/q23.bad"); do eq "$(nx "$W1" "stat -c '%a %U:%G' $p" /dev/null | grep FAILED | cut -d: -f1 > tampered.txt for f in $(cat tampered.txt); do scp $f {W1}:/tmp/genuine && ssh {W1} "sudo install -o root -g root -m 755 /tmp/genuine $f && rm /tmp/genuine" done ssh {W1} sudo sha512sum -c - < reference.sha512 # all OK # A checksum file with full paths can be checked anywhere the paths exist: here the reference stays # on the clean node and only travels over ssh. `install` copies and sets owner and mode in one step. """, }, { "id": 24, "domain": "SETUP", "title": "The binary of a running process", "params": lambda r: {"verdict": r.choice(["match", "mismatch"])}, "task": """ The kube-apiserver of this cluster runs from an image without a shell: there is nothing to `exec` into. Verify the binary the **running process** was started from, on `{CP}`: 1. Write its SHA-512 checksum (the 128 hex digits, nothing else) to `{WS}/q24/apiserver.sha512`. 2. `{WS}/q24/reference.sha512` holds the checksum the release publishes for this binary. Compare, and write `match` or `mismatch` to `{WS}/q24/verdict.txt`. """, "setup": r""" d="$WS/q24"; rm -rf "$d"; mkdir -p "$d" h=$(nx "$CP" 'sha512sum /proc/$(pgrep -o -x kube-apiserver)/exe' "$d/reference.sha512" """, "checks": [ C("apiserver.sha512 is the checksum of the running kube-apiserver's binary", 5, r""" h=$(nx "$CP" 'sha512sum /proc/$(pgrep -o -x kube-apiserver)/exe' /dev/null | cut -c1-128)" "$h" """), C("verdict.txt says %{verdict}", 5, r""" h=$(nx "$CP" 'sha512sum /proc/$(pgrep -o -x kube-apiserver)/exe' /dev/null | cut -c1-128)" "$h" && eq "$(tr -d ' \n\r' < "$WS/q24/verdict.txt" 2>/dev/null)" %{verdict}"""), ], "solution": """ pgrep -a -x kube-apiserver | cut -c1-60 # its pid on the node sudo sha512sum /proc/$(pgrep -o -x kube-apiserver)/exe | cut -d' ' -f1 > {WS}/q24/apiserver.sha512 diff {WS}/q24/apiserver.sha512 {WS}/q24/reference.sha512 && echo match || echo mismatch # -> write that word to {WS}/q24/verdict.txt # /proc//exe is the file the process was started from, whatever the container image has or # lacks - it works for distroless images, and it shows the binary in use even after the file on # disk has been replaced. /proc//root is the container's whole filesystem, seen from the node. """, }, ]