e761d53a96
105 of the 147 CKA reference solutions had you type what nobody types in the exam: heredoc manifests, sed, JSON patches, kubectl wait, loops, command substitution, jsonpath into files. Each is rewritten as the lessons are: a generator and the lines to add in vim, a manifest pasted from the docs page it names, kubectl edit, ssh NODE and sudo -i, or a value read off the screen and echoed. The other 42 were commands typed as they stand already. A solution can now be parts (sim/bank.py cmd, hand) as a lesson step's show is. The page and ./ksim solution N render it as Markdown, each thing to type a block of its own that a click copies, with the edit beneath it. A solution in parts is its own audit script: load_track runs its parts, and for each edit by hand the script beside it, so the solution you read is the one proven. Their separate scripts in cka/solve.py are dropped, and a task that has both is refused. ./ksim solution lint [N|DOMAIN..] reads what the solutions have you type against TYPING_TRAPS: the CKA bank is clean. Q49's params gain the plural stem and shortName the solution spells out (no new draws); Q103, ported from CKAD Q23, takes a solution of its own. Audited from a copy with renamed hosts: Q3-Q67 (52 tasks) all solved by their parts, after two waits were added to the audit scripts (Q43's DNS record, Q53's Pods); Q68-Q147 still running. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BNSZLLsGXTMBsw17kDunkQ
847 lines
59 KiB
Python
847 lines
59 KiB
Python
"""Hard-difficulty CKA tasks (ids 44-67): multi-part, no hints, traps, exact output,
|
|
compound failures, collateral-damage checks, exam-style contexts. Modelled on the kind of
|
|
difficulty people report from killer.sh, not on its content."""
|
|
from sim.bank import C, NS, ns_pick, name_pick
|
|
|
|
# Tasks written for the pre-2025 exam opened by switching kubectl context. Every task now has a host
|
|
# of its own, named above the task, so the line is empty; the ctx parameter is still drawn and unused.
|
|
CTX = ""
|
|
|
|
def ctx_pick(r):
|
|
return r.choice(["k8s-c1-H", "k8s-c2-AC", "k8s-c3-CCC"])
|
|
|
|
QUESTIONS = [
|
|
# ================================================================ ARCH ====
|
|
{
|
|
"id": 44, "domain": "ARCH", "difficulty": "hard", "title": "Exact RBAC with an existing over-grant",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "ci"), "sa": name_pick(r, "ci-bot")},
|
|
"task": CTX + """
|
|
ServiceAccount `%{sa}` in namespace `%{ns}` is used by a CI system. Its permissions must be
|
|
**exactly**:
|
|
- in `%{ns}`: `get`, `list`, `watch`, `patch` on `deployments` and `replicasets` (apps),
|
|
and `create` on `pods/exec`,
|
|
- cluster-wide: `list` on `nodes`.
|
|
|
|
Nothing more. Audit what it can do now and correct it. Use Role `%{sa}-role`, RoleBinding
|
|
`%{sa}-rb`, ClusterRole `%{sa}-nodes`, ClusterRoleBinding `%{sa}-crb`.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
$K -n %{ns} create sa %{sa} --dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
$K -n %{ns} delete role,rolebinding --all >/dev/null 2>&1
|
|
$K delete clusterrole %{sa}-nodes --ignore-not-found >/dev/null; $K delete clusterrolebinding %{sa}-crb %{sa}-legacy-admin --ignore-not-found >/dev/null
|
|
# trap: someone bound cluster-admin to this SA "temporarily"
|
|
$K create clusterrolebinding %{sa}-legacy-admin --clusterrole=cluster-admin --serviceaccount=%{ns}:%{sa} >/dev/null
|
|
""",
|
|
"checks": [
|
|
C("can get/list/watch/patch deployments and replicasets in %{ns}", 2, r"""! $K get clusterrolebinding %{sa}-legacy-admin >/dev/null 2>&1 || exit 1 # not via cluster-admin
|
|
for res in deployments.apps replicasets.apps; do for v in get list watch patch; do eq "$($K auth can-i $v $res -n %{ns} --as=system:serviceaccount:%{ns}:%{sa} 2>/dev/null)" yes || exit 1; done; done"""),
|
|
C("can create pods/exec in %{ns}; can list nodes", 2, r"""! $K get clusterrolebinding %{sa}-legacy-admin >/dev/null 2>&1 || exit 1
|
|
eq "$($K auth can-i create pods --subresource=exec -n %{ns} --as=system:serviceaccount:%{ns}:%{sa} 2>/dev/null)" yes && eq "$($K auth can-i list nodes --as=system:serviceaccount:%{ns}:%{sa} 2>/dev/null)" yes"""),
|
|
C("no over-grant: cannot delete deployments, cannot get secrets, cannot get nodes", 3, r"""for t in "delete deployments -n %{ns}" "get secrets -n %{ns}" "get nodes" "create pods -n default"; do eq "$($K auth can-i $t --as=system:serviceaccount:%{ns}:%{sa} 2>/dev/null || true)" no || exit 1; done"""),
|
|
C("the cluster-admin binding is gone and named objects exist", 2, r"""! $K get clusterrolebinding %{sa}-legacy-admin >/dev/null 2>&1 && $K -n %{ns} get role/%{sa}-role rolebinding/%{sa}-rb >/dev/null 2>&1 && $K get clusterrole/%{sa}-nodes clusterrolebinding/%{sa}-crb >/dev/null 2>&1"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 45, "domain": "ARCH", "breaks": "etcd", "difficulty": "hard", "title": "etcd restore",
|
|
# solo: restoring rolls EVERY API object back to the snapshot - other tasks' seeds and answers too
|
|
"solo": True,
|
|
"params": lambda r: {"ctx": "k8s-c1-H", "cm": name_pick(r, "restore-proof")},
|
|
"task": CTX + """
|
|
An etcd snapshot taken earlier is at `/opt/cka/backup.db` on the control-plane node.
|
|
A ConfigMap `%{cm}` in namespace `default` existed when the snapshot was taken and has since
|
|
been deleted.
|
|
|
|
**Restore the cluster's etcd from that snapshot** so the ConfigMap exists again. The cluster
|
|
must be healthy afterwards (API server up, all kube-system pods Running).
|
|
Do not recreate the ConfigMap by hand — its data must come from the snapshot.
|
|
|
|
A restore rolls every API object back to the moment of the snapshot — on a real exam cluster that
|
|
would include the work you did on other tasks, so a restore is the last thing you do.
|
|
""",
|
|
"setup": r"""
|
|
nx "$CP" 'mkdir -p /opt/cka; rm -f /opt/cka/backup.db'
|
|
ensure_etcd_tools || { echo "no etcdctl/etcdutl on $CP - rebuild the exam host image: ./ksim build"; exit 1; }
|
|
$K -n default delete cm %{cm} --ignore-not-found >/dev/null
|
|
$K -n default create cm %{cm} --from-literal=proof=$(date +%s)-%{cm} >/dev/null
|
|
out=$(nx "$CP" 'etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key snapshot save /opt/cka/backup.db' 2>&1) \
|
|
|| { echo "snapshot save on $CP failed: $out"; exit 1; }
|
|
sleep 2; $K -n default delete cm %{cm} >/dev/null
|
|
nx "$CP" 'awk "/hostPath:/{getline; p=\$2} /name: etcd-data/{if (p) print p}" /etc/kubernetes/manifests/etcd.yaml' </dev/null > "$WS/.q45-datadir"
|
|
date +%s > "$WS/.q45-t0"
|
|
""",
|
|
"checks": [
|
|
C("ConfigMap %{cm} exists again with its original data", 6, r"""j -n default cm %{cm} '{.data.proof}' | grep -q -- '-%{cm}$'"""),
|
|
C("etcd runs on a data directory restored from the snapshot", 2, r"""
|
|
was=$(cat "$WS/.q45-datadir" 2>/dev/null); t0=$(cat "$WS/.q45-t0" 2>/dev/null); [ -n "$was" ] && [ -n "$t0" ] || exit 1
|
|
now=$(nx "$CP" 'awk "/hostPath:/{getline; p=\$2} /name: etcd-data/{if (p) print p}" /etc/kubernetes/manifests/etcd.yaml' </dev/null)
|
|
[ -n "$now" ] || exit 1
|
|
[ "$now" != "$was" ] && exit 0
|
|
# the same directory name as before is fine if it was rebuilt from the snapshot after seeding
|
|
m=$(nx "$CP" "stat -c %Y $now/member 2>/dev/null" </dev/null); [ -n "$m" ] && [ "$m" -gt "$t0" ]"""),
|
|
C("control plane healthy: etcd, apiserver, scheduler, controller-manager Running", 2, r"""j -n default cm %{cm} '{.data.proof}' | grep -q -- '-%{cm}$' || exit 1 # only once restored
|
|
# right after a restore the API holds the snapshot's mirror pods and the kubelet replaces them, so a
|
|
# grade in the first minute can find one missing - retry instead of failing on that window
|
|
for i in $(seq 1 20); do
|
|
ok=1; for c in etcd kube-apiserver kube-scheduler kube-controller-manager; do eq "$(j -n kube-system pod -l component=$c '{.items[0].status.phase}')" Running || ok=0; done
|
|
[ $ok = 1 ] && exit 0; sleep 3
|
|
done; exit 1"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 46, "domain": "ARCH", "breaks": ["w3-kubelet", "w3-staticpath"], "difficulty": "hard", "title": "Static pod on a relocated manifest path",
|
|
"params": lambda r: {"ctx": "k8s-c1-H", "ns": ns_pick(r, "edge"), "pod": name_pick(r, "edge-proxy")},
|
|
"task": CTX + """
|
|
Exactly one worker node is labelled `tier=edge`. On that node create a **static pod**:
|
|
- name `%{pod}`, namespace `%{ns}` (create the namespace),
|
|
- image `nginx:1.27`, requests cpu 50m / memory 32Mi, label `role=edge`.
|
|
|
|
The mirror pod must be Running. Note that the kubelet on that node may not be configured the
|
|
way you expect.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
for n in "$W1" "$W2" "$W3"; do $K label node $n tier- >/dev/null 2>&1 || true; done
|
|
$K label node "$W3" tier=edge --overwrite >/dev/null
|
|
nx "$W3" '
|
|
rm -f /etc/kubernetes/manifests/%{pod}*.yaml /etc/kubernetes/static/%{pod}*.yaml 2>/dev/null; mkdir -p /etc/kubernetes/static
|
|
sed -i "s|^staticPodPath: .*|staticPodPath: /etc/kubernetes/static|" /var/lib/kubelet/config.yaml
|
|
systemctl restart kubelet'
|
|
""",
|
|
"checks": [
|
|
C("mirror pod %{pod}-<edge node> Running in %{ns}", 4, r"""eq "$(j -n %{ns} pod "%{pod}-$W3" '{.status.phase}')" Running"""),
|
|
C("image, requests and label correct", 2, r"""p="$K -n %{ns} get pod %{pod}-$W3"; eq "$($p -o jsonpath='{.spec.containers[0].image}' 2>/dev/null)" nginx:1.27 && eq "$($p -o jsonpath='{.spec.containers[0].resources.requests.cpu}')" 50m && eq "$($p -o jsonpath='{.metadata.labels.role}')" edge"""),
|
|
C("manifest lives in the kubelet's actual staticPodPath", 2, r"""d=$(nx "$W3" 'grep staticPodPath /var/lib/kubelet/config.yaml' | awk '{print $2}'); nx "$W3" "grep -ls %{pod} $d/*.yaml" >/dev/null 2>&1"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 47, "domain": "ARCH", "difficulty": "hard", "title": "User onboarding to a working kubeconfig",
|
|
"params": lambda r: {"ctx": "k8s-c1-H", "user": r.choice(["maya", "theo", "iris"]), "ns": ns_pick(r, "dev")},
|
|
"task": CTX + """
|
|
Onboard user `%{user}` (group `developers`) end to end:
|
|
1. Key + CSR with CN=%{user}, O=developers; Kubernetes CSR object `%{user}`, approved, certificate issued.
|
|
2. A **self-contained** kubeconfig at `{WS}/q47/%{user}.kubeconfig` (certs embedded, not file paths)
|
|
with a context `%{user}@lab` whose default namespace is `%{ns}`.
|
|
3. `%{user}` may do everything to pods, deployments and services in `%{ns}` (edit-level), nothing in other namespaces.
|
|
|
|
`kubectl --kubeconfig {WS}/q47/%{user}.kubeconfig get pods` must work with no extra flags.
|
|
""",
|
|
"setup": NS + r"""mkns %{ns}; rm -rf "$WS/q47"; mkdir -p "$WS/q47"; $K delete csr %{user} --ignore-not-found >/dev/null; $K -n %{ns} delete rolebinding --all >/dev/null 2>&1""",
|
|
"checks": [
|
|
C("CSR %{user} approved with certificate, O=developers", 2, r"""[ -n "$(j csr %{user} '{.status.certificate}')" ] && j csr %{user} '{.status.certificate}' | base64 -d | openssl x509 -noout -subject 2>/dev/null | grep -q 'O *= *developers'"""),
|
|
C("kubeconfig is self-contained and works without flags", 3, r"""
|
|
# tested on the host: kubectl writes the file readable by its owner only, and the server address
|
|
# in it is the one the host reaches the API server by
|
|
nx "$CP" 'f=/opt/course/q47/%{user}.kubeconfig; [ -f "$f" ] || exit 1
|
|
grep -q client-certificate-data "$f" && grep -q certificate-authority-data "$f" && ! grep -q "client-certificate:" "$f" \
|
|
&& kubectl --kubeconfig "$f" get pods >/dev/null 2>&1' </dev/null"""),
|
|
C("context %{user}@lab defaults to %{ns}", 1, r"""
|
|
eq "$(nx "$CP" "kubectl --kubeconfig /opt/course/q47/%{user}.kubeconfig config view -o jsonpath='{.contexts[?(@.name==\"%{user}@lab\")].context.namespace}'" </dev/null 2>/dev/null | tr -d '\r')" %{ns}"""),
|
|
C("edit rights in %{ns}, none elsewhere", 2, r"""eq "$($K auth can-i create deployments -n %{ns} --as=%{user} --as-group=developers 2>/dev/null)" yes && eq "$($K auth can-i delete pods -n %{ns} --as=%{user} --as-group=developers 2>/dev/null)" yes && eq "$($K auth can-i get pods -n default --as=%{user} --as-group=developers 2>/dev/null || true)" no && eq "$($K auth can-i get secrets -n kube-system --as=%{user} --as-group=developers 2>/dev/null || true)" no"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 48, "domain": "ARCH", "difficulty": "hard", "title": "Kustomize with generators and patches",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "kz"), "rep": r.choice([3, 4]), "lvl": r.choice(["debug", "trace", "warn"])},
|
|
"task": CTX + """
|
|
Base at `{WS}/q48/base` (Deployment `web`, Service `web`). Build overlay `{WS}/q48/overlays/prod` that:
|
|
- targets namespace `%{ns}` and adds label `tier=prod` to all resources (labels only, not selectors),
|
|
- sets replicas to %{rep} via a **strategic merge patch** file `replicas.yaml`,
|
|
- adds a `configMapGenerator` named `web-config` with literal `LOG_LEVEL=%{lvl}` (hash suffix enabled),
|
|
- injects that ConfigMap into the container via `envFrom` (JSON 6902 patch file `envfrom.yaml`).
|
|
|
|
Apply with `kubectl apply -k`. The Deployment must reference the hashed ConfigMap name.
|
|
""",
|
|
"setup": r"""
|
|
rm -rf "$WS/q48"; mkdir -p "$WS/q48/base"
|
|
cat > "$WS/q48/base/kustomization.yaml" <<'EOF'
|
|
resources: [deployment.yaml, service.yaml]
|
|
EOF
|
|
cat > "$WS/q48/base/deployment.yaml" <<'EOF'
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: web}
|
|
spec:
|
|
replicas: 1
|
|
selector: {matchLabels: {app: web}}
|
|
template:
|
|
metadata: {labels: {app: web}}
|
|
spec:
|
|
containers: [{name: web, image: nginx:1.25, ports: [{containerPort: 80}]}]
|
|
EOF
|
|
cat > "$WS/q48/base/service.yaml" <<'EOF'
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata: {name: web}
|
|
spec: {selector: {app: web}, ports: [{port: 80}]}
|
|
EOF
|
|
$K delete ns %{ns} --ignore-not-found --wait=false >/dev/null
|
|
""",
|
|
"checks": [
|
|
C("overlay has replicas.yaml (SMP) and envfrom.yaml (JSON6902)", 1, r"""[ -f "$WS/q48/overlays/prod/replicas.yaml" ] && [ -f "$WS/q48/overlays/prod/envfrom.yaml" ] && grep -q 'op:' "$WS/q48/overlays/prod/envfrom.yaml" """),
|
|
C("deployed to %{ns}: %{rep} replicas, tier=prod label, selector untouched", 2, r"""d="$K -n %{ns} get deploy web"; eq "$($d -o jsonpath='{.spec.replicas}' 2>/dev/null)" %{rep} && eq "$($d -o jsonpath='{.metadata.labels.tier}')" prod && [ -z "$($d -o jsonpath='{.spec.selector.matchLabels.tier}')" ]"""),
|
|
C("hashed ConfigMap with LOG_LEVEL=%{lvl}, referenced via envFrom", 3, r"""cm=$(j -n %{ns} deploy web '{.spec.template.spec.containers[0].envFrom[0].configMapRef.name}'); [ -n "$cm" ] && echo "$cm" | grep -Eq '^web-config-[a-z0-9]{6,}$' && eq "$(j -n %{ns} cm "$cm" '{.data.LOG_LEVEL}')" %{lvl}"""),
|
|
C("pods available and env visible", 1, r"""eq "$(j -n %{ns} deploy web '{.status.availableReplicas}')" %{rep} && $K -n %{ns} exec deploy/web -- env 2>/dev/null | grep -q '^LOG_LEVEL=%{lvl}$'"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 49, "domain": "ARCH", "difficulty": "hard", "title": "CRD with validation and defaults",
|
|
# low, short: the plural's stem and the shortName, which the solution spells out
|
|
"params": lambda r: (lambda p: dict(p, low=p["kind"].lower(), short=p["kind"][:2].lower()))(
|
|
{"ctx": ctx_pick(r), "kind": r.choice(["Rollout", "Canary", "Release"]), "env": r.choice(["prod", "stage"])}),
|
|
"task": CTX + """
|
|
Create CRD `%{kind}` (group `deploy.example.com`, v1, Namespaced, plural = the kind in lowercase plus
|
|
`s`, shortName = the first two letters of the kind in lowercase) with schema:
|
|
- `spec.replicas` integer, **required**, minimum 1, maximum 10,
|
|
- `spec.environment` string, enum `prod|stage|dev`, **default** `dev`,
|
|
- `spec.image` string, required.
|
|
Then:
|
|
1. Create `%{kind}` `good` in `default` with replicas 3, image `nginx:1.27`, environment `%{env}`.
|
|
2. Save the API server's rejection message for a `%{kind}` with `replicas: 0` to `{WS}/q49/rejected.txt`.
|
|
3. Create `%{kind}` `defaulted` in `default` with only replicas 2 and image set; its environment
|
|
must be `dev` after creation.
|
|
|
|
This context's default namespace is not necessarily `default` — say `-n default` explicitly.
|
|
""",
|
|
"setup": r"""rm -rf "$WS/q49"; mkdir -p "$WS/q49"; k=$(echo %{kind} | tr A-Z a-z); $K delete crd ${k}s.deploy.example.com --ignore-not-found >/dev/null""",
|
|
"checks": [
|
|
C("CRD exists, Namespaced, replicas has min/max and is required", 2, r"""s=""; k=$(echo %{kind} | tr A-Z a-z); c="$K get crd ${k}s.deploy.example.com"; eq "$($c -o jsonpath='{.spec.scope}' 2>/dev/null)" Namespaced && s=$($c -o jsonpath='{.spec.versions[0].schema.openAPIV3Schema.properties.spec}'); echo "$s" | grep -q '"minimum":1' && echo "$s" | grep -q '"maximum":10' && echo "$s" | grep -q '"required":\[' """),
|
|
C("good created with environment %{env}", 1, r"""k=$(echo %{kind} | tr A-Z a-z); eq "$(j -n default ${k}s.deploy.example.com good '{.spec.environment}')" %{env}"""),
|
|
C("replicas 0 is rejected and the message was saved", 2, r"""grep -qiE 'minimum|greater than|>= 1|invalid' "$WS/q49/rejected.txt" 2>/dev/null"""),
|
|
C("defaulting works: defaulted.spec.environment == dev", 2, r"""k=$(echo %{kind} | tr A-Z a-z); eq "$(j -n default ${k}s.deploy.example.com defaulted '{.spec.environment}')" dev"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 65, "domain": "ARCH", "breaks": "certs", "difficulty": "hard", "title": "Renew control-plane certificates",
|
|
"params": lambda r: {"ctx": "k8s-c1-H"},
|
|
"task": CTX + """
|
|
Security policy requires the control-plane certificates to be renewed now.
|
|
On the control-plane node, renew **all** kubeadm-managed certificates, make every control-plane
|
|
component pick up the new certificates, and update the admin kubeconfig on the node
|
|
(`/etc/kubernetes/admin.conf`) so it uses the renewed client certificate.
|
|
Write the new `apiserver` certificate's expiry (as printed by `kubeadm certs check-expiration`) to `{WS}/q65/expiry.txt`.
|
|
""",
|
|
"setup": r"""rm -rf "$WS/q65"; mkdir -p "$WS/q65"; nx "$CP" 'openssl x509 -in /etc/kubernetes/pki/apiserver.crt -noout -startdate' > "$WS/.q65-before" """,
|
|
"checks": [
|
|
C("apiserver certificate was renewed (newer notBefore)", 3, r"""b=$(sed 's/notBefore=//' "$WS/.q65-before"); a=$(nx "$CP" 'openssl x509 -in /etc/kubernetes/pki/apiserver.crt -noout -startdate' | sed 's/notBefore=//'); [ "$(date -d "$a" +%s)" -gt "$(date -d "$b" +%s)" ]"""),
|
|
C("all components restarted and Running with the new certs (apiserver serving cert matches file)", 3, r"""
|
|
for c in kube-apiserver kube-scheduler kube-controller-manager etcd; do eq "$(j -n kube-system pod -l component=$c '{.items[0].status.phase}')" Running || exit 1; done
|
|
srv=$(nx "$CP" 'echo | openssl s_client -connect 127.0.0.1:6443 2>/dev/null | openssl x509 -noout -serial -startdate'); f=$(nx "$CP" 'openssl x509 -in /etc/kubernetes/pki/apiserver.crt -noout -serial -startdate'); [ -n "$srv" ] && eq "$srv" "$f" || exit 1
|
|
# ...and it is the RENEWED certificate: an untouched cluster also serves the file it has on disk
|
|
b=$(sed 's/notBefore=//' "$WS/.q65-before"); a=$(echo "$srv" | sed -n 's/notBefore=//p'); [ "$(date -d "$a" +%s)" -gt "$(date -d "$b" +%s)" ]"""),
|
|
C("admin.conf uses the renewed client cert", 2, r"""nx "$CP" 'grep client-certificate-data /etc/kubernetes/admin.conf | awk "{print \$2}" | base64 -d | openssl x509 -noout -startdate' | sed 's/notBefore=//' > /tmp/q65a; b=$(sed 's/notBefore=//' "$WS/.q65-before"); [ "$(date -d "$(cat /tmp/q65a)" +%s)" -gt "$(date -d "$b" +%s)" ]"""),
|
|
C("expiry.txt matches kubeadm's report for apiserver", 1, r"""exp=$(nx "$CP" 'kubeadm certs check-expiration 2>/dev/null' | awk '$1=="apiserver"{print $2,$3,$4,$5,$6}'); [ -n "$exp" ] && grep -qF "$exp" "$WS/q65/expiry.txt" 2>/dev/null"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 67, "domain": "ARCH", "difficulty": "hard", "title": "Prepare a node join without shell access to the control plane",
|
|
"params": lambda r: {"ctx": "k8s-c1-H"},
|
|
"task": CTX + """
|
|
A new worker node is about to join this cluster, but you can only hand the networking team a
|
|
**file** — no shell access to the control plane.
|
|
|
|
1. On the control-plane node, create a **new** bootstrap token with TTL `2h`.
|
|
2. Compute the CA certificate's discovery hash **yourself** with openssl — the same
|
|
SHA-256-of-the-SubjectPublicKeyInfo value `kubeadm join --discovery-token-ca-cert-hash`
|
|
checks against. Do not just copy whatever `kubeadm token create --print-join-command` prints.
|
|
3. Determine the API server's real address as seen **from inside the cluster network** — not the
|
|
`127.0.0.1:<port>` your local kubeconfig uses to reach it from this machine, which a joining
|
|
node cannot use.
|
|
4. Write exactly these three lines to `{WS}/q67/node-join.env`:
|
|
```
|
|
JOIN_TOKEN=<token>
|
|
CA_HASH=sha256:<hash>
|
|
API_ENDPOINT=<ip>:<port>
|
|
```
|
|
Every value must be genuine: plugged into `kubeadm join $API_ENDPOINT --token $JOIN_TOKEN
|
|
--discovery-token-ca-cert-hash $CA_HASH` right now, it would work.
|
|
""",
|
|
"setup": r"""
|
|
rm -rf "$WS/q67"; mkdir -p "$WS/q67"
|
|
for t in $($K -n kube-system get secrets -o jsonpath='{.items[?(@.type=="bootstrap.kubernetes.io/token")].metadata.name}'); do $K -n kube-system delete secret $t >/dev/null; done
|
|
""",
|
|
"checks": [
|
|
C("a real, unexpired bootstrap token", 2, r"""
|
|
tok=$(grep -oE '^JOIN_TOKEN=[a-z0-9]{6}\.[a-z0-9]{16}' "$WS/q67/node-join.env" 2>/dev/null | cut -d= -f2); [ -n "$tok" ] || exit 1
|
|
id=${tok%%.*}; exp=$(j -n kube-system secret bootstrap-token-$id '{.data.expiration}' | base64 -d 2>/dev/null); [ -n "$exp" ] || exit 1
|
|
[ "$(date -d "$exp" +%s)" -gt "$(date +%s)" ]"""),
|
|
C("CA_HASH computed correctly (matches the real CA's SPKI SHA-256)", 3, r"""
|
|
want=$(nx "$CP" "openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'")
|
|
got=$(grep -oE '^CA_HASH=sha256:[0-9a-f]{64}' "$WS/q67/node-join.env" 2>/dev/null | cut -d= -f2 | cut -d: -f2)
|
|
[ -n "$want" ] && eq "$got" "$want" """),
|
|
C("API_ENDPOINT is the real in-cluster address, not the local kubeconfig's", 2, r"""
|
|
want=$($K get endpointslices -l kubernetes.io/service-name=kubernetes -o jsonpath='{.items[0].endpoints[0].addresses[0]}:{.items[0].ports[0].port}' 2>/dev/null)
|
|
got=$(grep -oE '^API_ENDPOINT=.*' "$WS/q67/node-join.env" 2>/dev/null | cut -d= -f2 | tr -d '[:space:]')
|
|
[ -n "$want" ] && eq "$got" "$want" """),
|
|
],
|
|
},
|
|
# ================================================================ WORK ====
|
|
{
|
|
"id": 50, "domain": "WORK", "breaks": "w1-taint", "difficulty": "hard", "title": "Spread, tolerate, prioritise — under a quota",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "spread"), "dep": name_pick(r, "gateway"), "key": r.choice(["pool", "lane"]), "val": r.choice(["ingress", "edge"])},
|
|
"task": CTX + """
|
|
Node `{W1}` is tainted `%{key}=%{val}:NoSchedule`. In namespace `%{ns}` create Deployment `%{dep}`
|
|
(nginx:1.27, **3 replicas**) such that:
|
|
- exactly one pod runs on each worker node (required pod anti-affinity on `kubernetes.io/hostname`),
|
|
- it tolerates the taint on `{W1}`,
|
|
- it uses PriorityClass `%{dep}-prio` (value 50000, create it),
|
|
- requests cpu 100m / memory 64Mi.
|
|
|
|
All 3 pods must be Running. The namespace has a ResourceQuota — respect it or adjust it, but the
|
|
quota must still cap pods in the namespace.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
$K taint node "$W1" %{key}- >/dev/null 2>&1 || true; $K taint node "$W1" %{key}=%{val}:NoSchedule >/dev/null
|
|
$K -n %{ns} delete deploy %{dep} --ignore-not-found >/dev/null; $K delete priorityclass %{dep}-prio --ignore-not-found >/dev/null
|
|
$K apply -f - >/dev/null <<EOF
|
|
apiVersion: v1
|
|
kind: ResourceQuota
|
|
metadata: {name: %{ns}-quota, namespace: %{ns}}
|
|
spec: {hard: {pods: "2", requests.cpu: "250m"}}
|
|
EOF
|
|
""",
|
|
"checks": [
|
|
C("3 pods available, one per worker", 3, r"""eq "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" 3 && [ "$(j -n %{ns} pod -l app=%{dep} '{.items[*].spec.nodeName}' | tr ' ' '\n' | sort -u | wc -l)" = 3 ]"""),
|
|
C("required anti-affinity on hostname + toleration for %{key}", 2, r"""t=$(j -n %{ns} deploy %{dep} '{.spec.template.spec}'); echo "$t" | grep -q 'requiredDuringSchedulingIgnoredDuringExecution' && echo "$t" | grep -q '"topologyKey":"kubernetes.io/hostname"' && echo "$t" | grep -q '"key":"%{key}"' """),
|
|
C("PriorityClass %{dep}-prio (50000) in use", 1, r"""eq "$(j priorityclass %{dep}-prio '{.value}')" 50000 && eq "$(j -n %{ns} pod -l app=%{dep} '{.items[0].spec.priority}')" 50000"""),
|
|
C("quota still exists and caps pods (>=3)", 2, r"""p=$(j -n %{ns} quota %{ns}-quota '{.spec.hard.pods}'); [ -n "$p" ] && [ "$p" -ge 3 ]"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 51, "domain": "WORK", "difficulty": "hard", "title": "Rollback to a specific revision, then pause",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "rel"), "dep": name_pick(r, "portal")},
|
|
"task": CTX + """
|
|
Deployment `%{dep}` in `%{ns}` has gone through several releases; the current one is broken.
|
|
1. Roll it back to the revision that used image `nginx:1.25` (not simply the previous revision).
|
|
2. Scale it to 3 replicas and wait until available.
|
|
3. **Pause** the rollout, then change the image to `nginx:1.27` and add env `RELEASE=next`.
|
|
Because it is paused, no new ReplicaSet may be created and the running pods must stay on `nginx:1.25`.
|
|
Leave it paused.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}; $K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null
|
|
$K -n %{ns} create deploy %{dep} --image=nginx:1.25 >/dev/null
|
|
$K -n %{ns} rollout status deploy %{dep} --timeout=90s >/dev/null 2>&1
|
|
$K -n %{ns} set image deploy %{dep} nginx=nginx:1.26 >/dev/null; $K -n %{ns} rollout status deploy %{dep} --timeout=90s >/dev/null 2>&1
|
|
$K -n %{ns} set image deploy %{dep} nginx=nginx:1.27 >/dev/null; $K -n %{ns} rollout status deploy %{dep} --timeout=90s >/dev/null 2>&1
|
|
$K -n %{ns} set image deploy %{dep} nginx=nginx:9.99-broken >/dev/null
|
|
""",
|
|
"checks": [
|
|
C("running pods use nginx:1.25 and 3 are available", 3, r"""
|
|
for t in $(seq 1 15); do # pods of the replaced ReplicaSets linger for a moment while they terminate
|
|
ok=1; eq "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" 3 || ok=0
|
|
for i in $(j -n %{ns} pod -l app=%{dep} '{.items[*].spec.containers[0].image}'); do eq "$i" nginx:1.25 || ok=0; done
|
|
[ $ok = 1 ] && exit 0; sleep 2
|
|
done; exit 1"""),
|
|
C("deployment is paused with template image nginx:1.27 and RELEASE=next", 2, r"""d="$K -n %{ns} get deploy %{dep}"; eq "$($d -o jsonpath='{.spec.paused}' 2>/dev/null)" true && eq "$($d -o jsonpath='{.spec.template.spec.containers[0].image}')" nginx:1.27 && eq "$($d -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="RELEASE")].value}')" next"""),
|
|
C("no ReplicaSet exists for the paused template (1.27 + RELEASE)", 2, r"""eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].image}')" nginx:1.27 || exit 1
|
|
! $K -n %{ns} get rs -o json 2>/dev/null | python3 -c '
|
|
import sys,json
|
|
for rs in json.load(sys.stdin)["items"]:
|
|
c=rs["spec"]["template"]["spec"]["containers"][0]
|
|
if c["image"]=="nginx:1.27" and any(e.get("name")=="RELEASE" for e in c.get("env",[])): sys.exit(0)
|
|
sys.exit(1)'"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 52, "domain": "WORK", "difficulty": "hard", "title": "Native sidecar with probes and an exact report",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "side"), "pod": name_pick(r, "collector")},
|
|
"task": CTX + """
|
|
Create pod `%{pod}` in `%{ns}`:
|
|
- a **native sidecar** (an init container with `restartPolicy: Always`) named `agent` (busybox:1.36)
|
|
that appends `$(date)` to `/var/log/agent/agent.log` every second,
|
|
- main container `app` (nginx:1.27) with a readiness probe (HTTP GET / on 80, period 5s)
|
|
and a liveness probe (TCP 80),
|
|
- shared `emptyDir` `logs` mounted at `/var/log/agent` in both (not at `/var/log`: nginx keeps its
|
|
own logs under `/var/log/nginx`, and an emptyDir mounted over `/var/log` hides that directory),
|
|
- `terminationGracePeriodSeconds: 5`.
|
|
When it is Running, write the names of **all** containers of the pod (init and regular), sorted,
|
|
one per line, to `{WS}/q52/containers.txt`, and the pod's `.status.podIP` to `{WS}/q52/ip.txt`.
|
|
""",
|
|
"setup": NS + r"""mkns %{ns}; rm -rf "$WS/q52"; mkdir -p "$WS/q52"; $K -n %{ns} delete pod %{pod} --ignore-not-found --force --grace-period=0 --wait=true >/dev/null 2>&1""",
|
|
"checks": [
|
|
C("native sidecar: initContainer agent with restartPolicy Always", 2, r"""eq "$(j -n %{ns} pod %{pod} '{.spec.initContainers[?(@.name=="agent")].restartPolicy}')" Always"""),
|
|
C("probes on app as specified", 2, r"""c='{.spec.containers[?(@.name=="app")]'; eq "$(j -n %{ns} pod %{pod} "$c.readinessProbe.httpGet.path}")" / && eq "$(j -n %{ns} pod %{pod} "$c.readinessProbe.periodSeconds}")" 5 && eq "$(j -n %{ns} pod %{pod} "$c.livenessProbe.tcpSocket.port}")" 80"""),
|
|
C("Running, shared log written by agent, grace period 5", 2, r"""eq "$(j -n %{ns} pod %{pod} '{.status.phase}')" Running && eq "$(j -n %{ns} pod %{pod} '{.spec.terminationGracePeriodSeconds}')" 5 && [ -n "$($K -n %{ns} exec %{pod} -c app -- sh -c 'tail -1 /var/log/agent/agent.log' 2>/dev/null)" ]"""),
|
|
C("report files exact", 2, r"""want=$( (j -n %{ns} pod %{pod} '{.spec.initContainers[*].name}'; echo; j -n %{ns} pod %{pod} '{.spec.containers[*].name}') | tr ' ' '\n' | grep . | sort); [ -n "$want" ] && [ "$(grep . "$WS/q52/containers.txt" 2>/dev/null | sort)" = "$want" ] && eq "$(tr -d '[:space:]' < "$WS/q52/ip.txt" 2>/dev/null)" "$(j -n %{ns} pod %{pod} '{.status.podIP}')" """),
|
|
],
|
|
},
|
|
{
|
|
# Scoped to its own namespace: the cluster-wide version compared a file written mid-exam with the
|
|
# whole cluster at submit time, so every pod another task (or a grader probe) created or deleted
|
|
# in between failed it. killer.sh-style exact-output tasks work on data the task owns.
|
|
"id": 53, "domain": "WORK", "difficulty": "hard", "title": "Namespace audit to exact files",
|
|
"params": lambda r: {"ctx": "k8s-c1-H", "ns": ns_pick(r, "audit")},
|
|
"task": CTX + """
|
|
Namespace `%{ns}` runs a mix of workloads. Produce three files with contents **exactly** as
|
|
specified — no headers, one entry per line, sorted with `sort`:
|
|
1. `{WS}/q53/no-readiness.txt` — the name of every Pod in `%{ns}` whose **first** container has
|
|
no readiness probe (whatever the Pod's phase).
|
|
2. `{WS}/q53/per-node.txt` — `<node> <count>`: how many Pods in `%{ns}` are **Running** on each
|
|
node, listing only nodes that have at least one.
|
|
3. `{WS}/q53/images.txt` — every distinct container image used by Pods in `kube-system`.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}; rm -rf "$WS/q53"; mkdir -p "$WS/q53"
|
|
$K -n %{ns} delete pod --all --force --grace-period=0 --wait=true >/dev/null 2>&1
|
|
$K apply -f - >/dev/null <<EOF
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: web-a, namespace: %{ns}}
|
|
spec:
|
|
nodeSelector: {kubernetes.io/hostname: $W1}
|
|
containers: [{name: web, image: nginx:1.27, readinessProbe: {httpGet: {path: /, port: 80}}}]
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: web-b, namespace: %{ns}}
|
|
spec:
|
|
nodeSelector: {kubernetes.io/hostname: $W2}
|
|
containers: [{name: web, image: nginx:1.27, readinessProbe: {tcpSocket: {port: 80}}}]
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: batch-a, namespace: %{ns}}
|
|
spec:
|
|
nodeSelector: {kubernetes.io/hostname: $W1}
|
|
containers: [{name: batch, image: busybox:1.36, command: [sleep, "86400"]}]
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: cache-a, namespace: %{ns}}
|
|
spec:
|
|
nodeSelector: {kubernetes.io/hostname: $W3}
|
|
containers:
|
|
- {name: cache, image: redis:7-alpine}
|
|
- {name: exporter, image: nginx:1.27, readinessProbe: {httpGet: {path: /, port: 80}}}
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: once-a, namespace: %{ns}}
|
|
spec:
|
|
restartPolicy: Never
|
|
nodeSelector: {kubernetes.io/hostname: $W3}
|
|
containers: [{name: job, image: busybox:1.36, command: ["true"]}]
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: idle-a, namespace: %{ns}}
|
|
spec:
|
|
nodeSelector: {kubernetes.io/hostname: $W2}
|
|
containers: [{name: idle, image: busybox:1.36, command: [sleep, "86400"]}]
|
|
EOF
|
|
for i in $(seq 1 60); do [ "$($K -n %{ns} get pod once-a -o jsonpath='{.status.phase}' 2>/dev/null)" = Succeeded ] && break; sleep 2; done
|
|
""",
|
|
"checks": [
|
|
C("no-readiness.txt exact (first container only, every phase)", 3, r"""want=$($K -n %{ns} get pods -o json 2>/dev/null | python3 -c '
|
|
import sys,json
|
|
for p in json.load(sys.stdin)["items"]:
|
|
if not p["spec"]["containers"][0].get("readinessProbe"): print(p["metadata"]["name"])' | sort); [ -n "$want" ] && [ "$(grep . "$WS/q53/no-readiness.txt" 2>/dev/null)" = "$want" ]"""),
|
|
C("per-node.txt exact (Running pods only)", 2, r"""want=$($K -n %{ns} get pods -o json 2>/dev/null | python3 -c '
|
|
import sys,json,collections
|
|
c=collections.Counter(p["spec"].get("nodeName") for p in json.load(sys.stdin)["items"] if p["status"]["phase"]=="Running" and p["spec"].get("nodeName"))
|
|
for n in c: print(n, c[n])' | sort); [ -n "$want" ] && [ "$(grep . "$WS/q53/per-node.txt" 2>/dev/null)" = "$want" ]"""),
|
|
C("images.txt exact", 2, r"""want=$(j -n kube-system pods '{.items[*].spec.containers[*].image}' | tr ' ' '\n' | sort -u); [ -n "$want" ] && [ "$(grep . "$WS/q53/images.txt" 2>/dev/null)" = "$want" ]"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 64, "domain": "WORK", "difficulty": "hard", "title": "The context's namespace",
|
|
# ctx_ns: this task reads the context's own default namespace instead of being told one, so
|
|
# k8s-c2-AC must be created pointing at *this* task's ns (see context_namespace in ./ksim).
|
|
"ctx_ns": True,
|
|
"params": lambda r: {"ctx": "k8s-c2-AC", "ns": ns_pick(r, "ac"), "dep": name_pick(r, "ledger")},
|
|
"task": CTX + """
|
|
In the **current namespace of that context** (do not pass `-n`), create Deployment `%{dep}`
|
|
(busybox:1.36, `sleep 3600`, 2 replicas, label `owner=ac`) and a ClusterIP Service `%{dep}`
|
|
on port 9000 targeting port 9000. Then write the namespace name you deployed into, as reported
|
|
by `kubectl config view --minify`, to `{WS}/q64/ns.txt`.
|
|
""",
|
|
"setup": NS + r"""mkns %{ns}; rm -rf "$WS/q64"; mkdir -p "$WS/q64"; $K -n %{ns} delete deploy,svc %{dep} --ignore-not-found >/dev/null; $K -n default delete deploy,svc %{dep} --ignore-not-found >/dev/null""",
|
|
"checks": [
|
|
C("Deployment %{dep} in %{ns} (the context's namespace), 2 available, owner=ac", 3, r"""eq "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" 2 && { eq "$(j -n %{ns} deploy %{dep} '{.metadata.labels.owner}')" ac || eq "$(j -n %{ns} deploy %{dep} '{.spec.template.metadata.labels.owner}')" ac; }"""),
|
|
C("Service %{dep} port 9000 in the same namespace", 1, r"""eq "$(j -n %{ns} svc %{dep} '{.spec.ports[0].port}')" 9000"""),
|
|
C("nothing was created in namespace default by mistake", 1, r"""$K -n %{ns} get deploy %{dep} >/dev/null 2>&1 && ! $K -n default get deploy %{dep} >/dev/null 2>&1"""),
|
|
C("ns.txt has the namespace", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q64/ns.txt" 2>/dev/null)" %{ns}"""),
|
|
],
|
|
},
|
|
# ================================================================= NET ====
|
|
{
|
|
"id": 54, "domain": "NET", "difficulty": "hard", "title": "Zero-trust namespace",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "zt"), "from": ns_pick(r, "front")},
|
|
"task": CTX + """
|
|
Namespace `%{ns}` runs `api` (label `app=api`, port 8080) and `db` (label `app=db`, port 5432).
|
|
Implement zero trust with NetworkPolicies:
|
|
1. `default-deny`: deny all ingress **and** egress for every pod in `%{ns}`.
|
|
2. `allow-dns`: every pod may egress to DNS (UDP/TCP 53) in `kube-system` only.
|
|
3. `api-ingress`: `api` accepts ingress on 8080 only from pods in namespaces labelled `team=%{from}`.
|
|
4. `api-to-db`: `api` may egress to `db` on 5432; `db` accepts ingress on 5432 from `api` only.
|
|
Remove anything that contradicts this. Label namespace `%{from}` appropriately.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}; mkns %{from}
|
|
$K -n %{ns} create deploy api --image=nginx:1.25 --port=8080 --dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
$K -n %{ns} create deploy db --image=nginx:1.25 --port=5432 --dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
$K -n %{ns} delete netpol --all >/dev/null 2>&1
|
|
# trap: an allow-all left behind
|
|
$K apply -f - >/dev/null <<EOF
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata: {name: temp-allow-all, namespace: %{ns}}
|
|
spec: {podSelector: {}, policyTypes: [Ingress, Egress], ingress: [{}], egress: [{}]}
|
|
EOF
|
|
$K label ns %{from} team- >/dev/null 2>&1 || true
|
|
""",
|
|
"checks": [
|
|
C("default-deny covers Ingress and Egress for all pods; allow-all removed", 2, r"""n="$K -n %{ns} get netpol default-deny"; eq "$($n -o jsonpath='{.spec.podSelector}' 2>/dev/null)" '{}' && $n -o jsonpath='{.spec.policyTypes}' | grep -q Ingress && $n -o jsonpath='{.spec.policyTypes}' | grep -q Egress && [ -z "$($n -o jsonpath='{.spec.ingress}')" ] && [ -z "$($n -o jsonpath='{.spec.egress}')" ] && ! $K -n %{ns} get netpol temp-allow-all >/dev/null 2>&1"""),
|
|
C("allow-dns: egress 53 UDP+TCP to kube-system only", 2, r"""$K -n %{ns} get netpol allow-dns -o json 2>/dev/null | python3 -c '
|
|
import sys,json; s=json.load(sys.stdin)["spec"]; e=s.get("egress",[])
|
|
ok=s.get("podSelector")=={} and any({p.get("protocol") for p in r.get("ports",[]) if str(p.get("port"))=="53"}>={"UDP","TCP"} and all("kube-system" in json.dumps(t.get("namespaceSelector",{})) for t in r.get("to",[])) and r.get("to") for r in e)
|
|
sys.exit(0 if ok else 1)'"""),
|
|
C("api-ingress: from namespaces team=%{from} on 8080 only; namespace labelled", 2, r"""eq "$(j ns %{from} '{.metadata.labels.team}')" %{from} && $K -n %{ns} get netpol api-ingress -o json 2>/dev/null | python3 -c '
|
|
import sys,json; s=json.load(sys.stdin)["spec"]
|
|
ok=s["podSelector"].get("matchLabels",{}).get("app")=="api" and len(s.get("ingress",[]))==1 and all(str(p.get("port"))=="8080" for p in s["ingress"][0].get("ports",[])) and s["ingress"][0].get("ports") and all(t.get("namespaceSelector",{}).get("matchLabels",{}).get("team")=="%{from}" for t in s["ingress"][0].get("from",[])) and s["ingress"][0].get("from")
|
|
sys.exit(0 if ok else 1)'"""),
|
|
C("api-to-db: api egress to db:5432 and db ingress from api:5432", 2, r"""$K -n %{ns} get netpol -o json 2>/dev/null | python3 -c '
|
|
import sys,json; items=json.load(sys.stdin)["items"]; eg=ing=False
|
|
for n in items:
|
|
s=n["spec"]; app=s["podSelector"].get("matchLabels",{}).get("app")
|
|
for r in s.get("egress",[]):
|
|
if app=="api" and any(t.get("podSelector",{}).get("matchLabels",{}).get("app")=="db" for t in r.get("to",[])) and any(str(p.get("port"))=="5432" for p in r.get("ports",[])): eg=True
|
|
for r in s.get("ingress",[]):
|
|
if app=="db" and any(t.get("podSelector",{}).get("matchLabels",{}).get("app")=="api" for t in r.get("from",[])) and any(str(p.get("port"))=="5432" for p in r.get("ports",[])): ing=True
|
|
sys.exit(0 if eg and ing else 1)'"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 55, "domain": "NET", "difficulty": "hard", "title": "TLS Ingress with a generated certificate",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "tls"), "host": r.choice(["secure.lab.local", "portal.lab.local", "admin.lab.local"])},
|
|
"task": CTX + """
|
|
Namespace `%{ns}` has Deployment+Service `web` (port 80).
|
|
1. Generate a self-signed certificate (RSA 2048, 365 days) with CN `%{host}` and store it as a
|
|
Secret `%{host}-tls` of type `kubernetes.io/tls`.
|
|
2. Create Ingress `web` (class `nginx`) serving host `%{host}` with that TLS secret, path `/` Prefix → `web:80`,
|
|
and the annotation `nginx.ingress.kubernetes.io/ssl-redirect: "true"`.
|
|
3. Write the certificate's SHA-256 fingerprint (`openssl x509 -fingerprint -sha256`, the value only) to `{WS}/q55/fp.txt`.
|
|
""",
|
|
"setup": NS + r"""mkns %{ns}; rm -rf "$WS/q55"; mkdir -p "$WS/q55"; $K -n %{ns} create deploy web --image=nginx:1.25 --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null; $K -n %{ns} expose deploy web --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null; $K -n %{ns} delete secret %{host}-tls --ignore-not-found >/dev/null; $K -n %{ns} delete ingress web --ignore-not-found >/dev/null""",
|
|
"checks": [
|
|
C("Secret %{host}-tls is kubernetes.io/tls with CN %{host}", 2, r"""eq "$(j -n %{ns} secret %{host}-tls '{.type}')" kubernetes.io/tls && j -n %{ns} secret %{host}-tls '{.data.tls\.crt}' | base64 -d | openssl x509 -noout -subject 2>/dev/null | grep -q 'CN *= *%{host}'"""),
|
|
C("Ingress web: class nginx, host, TLS secret, ssl-redirect annotation", 3, r"""i="$K -n %{ns} get ingress web"; eq "$($i -o jsonpath='{.spec.ingressClassName}' 2>/dev/null)" nginx && eq "$($i -o jsonpath='{.spec.tls[0].secretName}')" %{host}-tls && eq "$($i -o jsonpath='{.spec.tls[0].hosts[0]}')" %{host} && eq "$($i -o jsonpath='{.spec.rules[0].host}')" %{host} && eq "$($i -o jsonpath='{.spec.rules[0].http.paths[0].backend.service.name}')" web && eq "$($i -o jsonpath='{.metadata.annotations.nginx\.ingress\.kubernetes\.io/ssl-redirect}')" true"""),
|
|
C("fp.txt matches the secret's certificate fingerprint", 2, r"""fp=$(j -n %{ns} secret %{host}-tls '{.data.tls\.crt}' | base64 -d | openssl x509 -noout -fingerprint -sha256 2>/dev/null | sed 's/.*=//'); [ -n "$fp" ] && grep -qiF "$fp" "$WS/q55/fp.txt" 2>/dev/null"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 56, "domain": "NET", "difficulty": "hard", "title": "Gateway API canary with header match",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "canary"), "host": r.choice(["shop", "pay", "cart"]), "w": r.choice([10, 20, 25])},
|
|
"task": CTX + """
|
|
Gateway API CRDs are installed. Namespace `%{ns}` has Services `v1` and `v2` (port 80).
|
|
Create in `%{ns}`:
|
|
- Gateway `%{host}-gw` (class `%{host}-gc`, create the GatewayClass, controllerName `example.com/gw`)
|
|
with two HTTP listeners: `web` for `%{host}.example.com` and `api` for `api.%{host}.example.com`, both port 80.
|
|
- HTTPRoute `%{host}-route` attached to listener `web` only (`sectionName`), with:
|
|
- a rule matching header `X-Canary: always` → `v2` (100%),
|
|
- a default rule splitting `v1` %{w2}% / `v2` %{w}% by weight.
|
|
""",
|
|
"params_post": None,
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
ensure_gateway_crds || echo "warn: could not install the Gateway API CRDs (no network?) - the task needs them"
|
|
for v in v1 v2; do $K -n %{ns} create deploy $v --image=nginx:1.25 --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null; $K -n %{ns} expose deploy $v --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null; done
|
|
$K delete gatewayclass %{host}-gc --ignore-not-found >/dev/null 2>&1; $K -n %{ns} delete gateway %{host}-gw --ignore-not-found >/dev/null 2>&1; $K -n %{ns} delete httproute %{host}-route --ignore-not-found >/dev/null 2>&1
|
|
""",
|
|
"checks": [
|
|
C("Gateway with listeners web and api on the right hostnames", 2, r"""g="$K -n %{ns} get gateway %{host}-gw"; eq "$($g -o jsonpath='{.spec.gatewayClassName}' 2>/dev/null)" %{host}-gc && eq "$($g -o jsonpath='{.spec.listeners[?(@.name=="web")].hostname}')" %{host}.example.com && eq "$($g -o jsonpath='{.spec.listeners[?(@.name=="api")].hostname}')" api.%{host}.example.com"""),
|
|
C("route attaches to listener web only (sectionName)", 1, r"""eq "$(j -n %{ns} httproute %{host}-route '{.spec.parentRefs[0].sectionName}')" web && [ -z "$(j -n %{ns} httproute %{host}-route '{.spec.parentRefs[1]}')" ]"""),
|
|
C("header rule X-Canary: always -> v2", 2, r"""$K -n %{ns} get httproute %{host}-route -o json 2>/dev/null | python3 -c '
|
|
import sys,json; r=json.load(sys.stdin)["spec"]["rules"]
|
|
ok=any(any(any(h.get("name","").lower()=="x-canary" and h.get("value")=="always" for h in m.get("headers",[])) for m in x.get("matches",[])) and x["backendRefs"][0]["name"]=="v2" and len(x["backendRefs"])==1 for x in r)
|
|
sys.exit(0 if ok else 1)'"""),
|
|
C("default rule weights v1 %{w2} / v2 %{w}", 2, r"""$K -n %{ns} get httproute %{host}-route -o json 2>/dev/null | python3 -c '
|
|
import sys,json; r=json.load(sys.stdin)["spec"]["rules"]
|
|
ok=any((not x.get("matches") or all(not m.get("headers") for m in x["matches"])) and {b["name"]:b.get("weight") for b in x["backendRefs"]}=={"v1":%{w2},"v2":%{w}} for x in r if len(x.get("backendRefs",[]))==2)
|
|
sys.exit(0 if ok else 1)'"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 57, "domain": "NET", "breaks": "coredns", "difficulty": "hard", "title": "DNS down: three faults", "disruptive": True,
|
|
"params": lambda r: {"ctx": "k8s-c1-H"},
|
|
"task": CTX + """
|
|
Nothing in the cluster resolves DNS. Restore it. There is more than one cause.
|
|
When done, `nslookup kubernetes.default.svc.cluster.local` and `nslookup kubernetes.io` must
|
|
succeed from a pod in `default` (use the FQDN: busybox's nslookup does not apply the pod's
|
|
`search` domains), CoreDNS must have 2 available replicas, and no NetworkPolicy may remain in
|
|
`kube-system`.
|
|
""",
|
|
"setup": r"""
|
|
$K -n kube-system get cm coredns -o jsonpath='{.data.Corefile}' > /tmp/Corefile.q57
|
|
sed -i 's|forward \. /etc/resolv.conf|forward . 10.255.255.254|' /tmp/Corefile.q57
|
|
$K -n kube-system create cm coredns --from-file=Corefile=/tmp/Corefile.q57 --dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
$K -n kube-system scale deploy coredns --replicas=0 >/dev/null
|
|
$K apply -f - >/dev/null <<EOF
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata: {name: maintenance-lockdown, namespace: kube-system}
|
|
spec: {podSelector: {matchLabels: {k8s-app: kube-dns}}, policyTypes: [Ingress], ingress: []}
|
|
EOF
|
|
""",
|
|
"checks": [
|
|
C("Corefile forwards to /etc/resolv.conf", 2, r"""j -n kube-system cm coredns '{.data.Corefile}' | grep -q 'forward \. /etc/resolv.conf'"""),
|
|
C("CoreDNS has 2 available replicas", 2, r"""for i in $(seq 1 12); do [ "$(j -n kube-system deploy coredns '{.status.availableReplicas}')" = 2 ] && exit 0; sleep 5; done; exit 1"""),
|
|
C("no NetworkPolicy in kube-system", 1, r"""[ "$($K -n kube-system get netpol --no-headers 2>/dev/null | wc -l)" = 0 ]"""),
|
|
C("resolution works from a pod", 3, r"""
|
|
$K -n default delete pod dns-probe --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K -n default run dns-probe --image=busybox:1.36 --restart=Never -- sh -c 'for i in 1 2 3 4 5 6; do nslookup kubernetes.default.svc.cluster.local >/dev/null 2>&1 && nslookup kubernetes.io >/dev/null 2>&1 && exit 0; sleep 5; done; exit 1' >/dev/null 2>&1
|
|
for i in $(seq 1 20); do ph=$(j -n default pod dns-probe '{.status.phase}'); [ "$ph" = Succeeded ] && { $K -n default delete pod dns-probe --wait=false >/dev/null; exit 0; }; [ "$ph" = Failed ] && break; sleep 3; done
|
|
$K -n default delete pod dns-probe --wait=false >/dev/null; exit 1"""),
|
|
],
|
|
},
|
|
# ================================================================ STOR ====
|
|
{
|
|
"id": 58, "domain": "STOR", "difficulty": "hard", "title": "Bind the right local volume",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "lv"), "pvc": name_pick(r, "fast-data")},
|
|
"task": CTX + """
|
|
Two PersistentVolumes of class `local-fast` exist. Only one of them lives on the node labelled
|
|
`disk=nvme`. Create PVC `%{pvc}` in `%{ns}` (RWO, 1Gi, class `local-fast`) that binds to **that**
|
|
PV — use a label selector, do not edit the PVs — and pod `consumer` (busybox:1.36, `sleep 3600`)
|
|
mounting it at `/data`. The pod must run on the nvme node. Do not delete either PV.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
for n in "$W1" "$W2" "$W3"; do $K label node $n disk- >/dev/null 2>&1 || true; done
|
|
$K label node "$W2" disk=nvme --overwrite >/dev/null
|
|
$K -n %{ns} delete pod consumer --ignore-not-found --wait=false >/dev/null; $K -n %{ns} delete pvc %{pvc} --ignore-not-found >/dev/null
|
|
# The PV names are fixed but every session draws a new namespace, so a claim from an earlier run
|
|
# (another namespace, often with its pod still running) keeps lv-hdd/lv-nvme Bound. pv-protection
|
|
# then holds `kubectl delete pv` forever and the seed timed out. Release them from whoever has them.
|
|
for pv in lv-hdd lv-nvme; do
|
|
cns=$(j pv $pv '{.spec.claimRef.namespace}'); cn=$(j pv $pv '{.spec.claimRef.name}')
|
|
[ -n "$cns" ] && [ -n "$cn" ] || continue
|
|
for p in $($K -n "$cns" get pod -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.spec.volumes[*].persistentVolumeClaim.claimName}{"\n"}{end}' 2>/dev/null | awk -v c="$cn" '{for(i=2;i<=NF;i++) if($i==c) print $1}'); do
|
|
$K -n "$cns" delete pod "$p" --grace-period=0 --force >/dev/null 2>&1
|
|
done
|
|
$K -n "$cns" delete pvc "$cn" --ignore-not-found --wait=false >/dev/null 2>&1
|
|
done
|
|
$K delete pv lv-hdd lv-nvme --ignore-not-found --timeout=90s >/dev/null 2>&1 || true
|
|
for pair in "lv-hdd:$W1:hdd:2Gi" "lv-nvme:$W2:nvme:1Gi"; do IFS=: read name node kind size <<<"$pair"
|
|
nx "$node" "mkdir -p /mnt/$name"
|
|
$K apply -f - >/dev/null <<EOF
|
|
apiVersion: v1
|
|
kind: PersistentVolume
|
|
metadata: {name: $name, labels: {media: $kind}}
|
|
spec:
|
|
capacity: {storage: $size}
|
|
accessModes: [ReadWriteOnce]
|
|
persistentVolumeReclaimPolicy: Retain
|
|
storageClassName: local-fast
|
|
local: {path: /mnt/$name}
|
|
nodeAffinity: {required: {nodeSelectorTerms: [{matchExpressions: [{key: kubernetes.io/hostname, operator: In, values: [$node]}]}]}}
|
|
EOF
|
|
done
|
|
""",
|
|
"checks": [
|
|
C("PVC uses a selector and is Bound to lv-nvme", 3, r"""[ -n "$(j -n %{ns} pvc %{pvc} '{.spec.selector}')" ] && eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}')" lv-nvme && eq "$(j -n %{ns} pvc %{pvc} '{.status.phase}')" Bound"""),
|
|
C("consumer Running on the nvme node with /data mounted", 2, r"""eq "$(j -n %{ns} pod consumer '{.status.phase}')" Running && eq "$(j -n %{ns} pod consumer '{.spec.nodeName}')" "$W2" && j -n %{ns} pod consumer '{.spec.containers[0].volumeMounts[*].mountPath}' | grep -qw /data"""),
|
|
C("both PVs still exist, unmodified affinity", 1, r"""eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}')" lv-nvme && eq "$(j pv lv-hdd '{.spec.nodeAffinity.required.nodeSelectorTerms[0].matchExpressions[0].values[0]}')" "$W1" && eq "$(j pv lv-nvme '{.spec.nodeAffinity.required.nodeSelectorTerms[0].matchExpressions[0].values[0]}')" "$W2" """),
|
|
],
|
|
},
|
|
{
|
|
"id": 59, "domain": "STOR", "difficulty": "hard", "title": "Migrate data between claims",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "mig"), "n": r.choice([7, 11, 13])},
|
|
"task": CTX + """
|
|
In `%{ns}`, PVC `old-data` holds files a running pod `legacy` wrote (`/data`). Storage is being
|
|
replaced:
|
|
1. Create PVC `new-data` (default StorageClass, 1Gi, RWO).
|
|
2. Copy **everything** from `old-data` to `new-data` using a Job named `migrate` (busybox:1.36).
|
|
The Job must complete; use whatever mounts you need.
|
|
3. Re-point pod `legacy` to `new-data` (same name, same namespace, mount at `/data`) and get it Running.
|
|
4. Delete PVC `old-data` afterwards. Its PV must be released, not deleted (check its reclaim policy first).
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
$K -n %{ns} delete pod legacy --ignore-not-found --wait=true >/dev/null; $K -n %{ns} delete job migrate --ignore-not-found >/dev/null; $K -n %{ns} delete pvc old-data new-data --ignore-not-found >/dev/null
|
|
$K apply -f - >/dev/null <<EOF
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: old-data, namespace: %{ns}}
|
|
spec: {accessModes: [ReadWriteOnce], resources: {requests: {storage: 1Gi}}}
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: legacy, namespace: %{ns}}
|
|
spec:
|
|
containers: [{name: app, image: busybox:1.36, command: [sh, -c, "for i in \$(seq 1 %{n}); do echo record-\$i > /data/file-\$i.txt; done; sleep 3600"], volumeMounts: [{name: d, mountPath: /data}]}]
|
|
volumes: [{name: d, persistentVolumeClaim: {claimName: old-data}}]
|
|
EOF
|
|
for i in $(seq 1 30); do [ "$(j -n %{ns} pod legacy '{.status.phase}')" = Running ] && break; sleep 2; done
|
|
pv=$(j -n %{ns} pvc old-data '{.spec.volumeName}'); [ -n "$pv" ] && $K patch pv $pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' >/dev/null
|
|
echo "$pv" > "$WS/.q59-pv"
|
|
""",
|
|
"checks": [
|
|
C("Job migrate completed", 2, r"""eq "$(j -n %{ns} job migrate '{.status.succeeded}')" 1"""),
|
|
C("legacy Running on new-data with all %{n} files intact", 3, r"""eq "$(j -n %{ns} pod legacy '{.status.phase}')" Running && eq "$(j -n %{ns} pod legacy '{.spec.volumes[?(@.name!="kube-api-access")].persistentVolumeClaim.claimName}' | tr ' ' '\n' | grep -v '^$' | head -1)" new-data && eq "$($K -n %{ns} exec legacy -- sh -c 'ls /data/file-*.txt | wc -l' 2>/dev/null | tr -d ' ')" %{n} && eq "$($K -n %{ns} exec legacy -- cat /data/file-%{n}.txt 2>/dev/null)" record-%{n}"""),
|
|
C("old-data deleted; its PV Released (not deleted)", 2, r"""! $K -n %{ns} get pvc old-data >/dev/null 2>&1 && pv=$(cat "$WS/.q59-pv" 2>/dev/null) && eq "$(j pv $pv '{.status.phase}')" Released"""),
|
|
],
|
|
},
|
|
# ============================================================== TSHOOT ====
|
|
{
|
|
"id": 60, "domain": "TSHOOT", "breaks": "w2-kubelet", "difficulty": "hard", "title": "Worker node: three things wrong", "disruptive": True,
|
|
"params": lambda r: {"ctx": "k8s-c1-H"},
|
|
"task": CTX + """
|
|
Node `{W2}` is unusable. Make it fully usable again: `Ready`, schedulable, kubelet running and
|
|
enabled, and a new pod pinned to it must reach Running. Nothing else on the cluster may be changed.
|
|
""",
|
|
"setup": r"""
|
|
$K cordon "$W2" >/dev/null 2>&1 || true
|
|
nx "$W2" '
|
|
sed -i "s|clientCAFile: /etc/kubernetes/pki/ca.crt|clientCAFile: /etc/kubernetes/pki/ca-wrong.crt|" /var/lib/kubelet/config.yaml
|
|
systemctl disable kubelet >/dev/null 2>&1; systemctl stop kubelet'
|
|
""",
|
|
"checks": [
|
|
C("node Ready", 3, r"""
|
|
eq "$(nx "$W2" 'systemctl is-active kubelet' </dev/null)" active || exit 1
|
|
for i in $(seq 1 12); do [ "$(j node "$W2" '{.status.conditions[?(@.type=="Ready")].status}')" = True ] && exit 0; sleep 5; done; exit 1"""),
|
|
C("node uncordoned (collateral check)", 2, r"""[ "$(j node "$W2" '{.spec.unschedulable}')" != true ]"""),
|
|
C("kubelet active and enabled, config points at a real CA", 2, r"""eq "$(nx "$W2" 'systemctl is-active kubelet')" active && eq "$(nx "$W2" 'systemctl is-enabled kubelet')" enabled && f=$(nx "$W2" 'grep clientCAFile /var/lib/kubelet/config.yaml' | awk '{print $2}') && nx "$W2" "test -f $f" """),
|
|
C("a pod pinned to the node runs", 2, r"""
|
|
$K -n default delete pod node-probe --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K -n default run node-probe --image=busybox:1.36 --restart=Never --overrides="{\"spec\":{\"nodeName\":\"$W2\"}}" -- sleep 120 >/dev/null 2>&1
|
|
ok=1; for i in $(seq 1 20); do [ "$(j -n default pod node-probe '{.status.phase}')" = Running ] && { ok=0; break; }; sleep 3; done
|
|
$K -n default delete pod node-probe --wait=false >/dev/null 2>&1; exit $ok"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 61, "domain": "TSHOOT", "breaks": ["scheduler", "controller-manager"], "difficulty": "hard", "title": "Two control-plane components down", "disruptive": True,
|
|
"params": lambda r: {"ctx": "k8s-c1-H"},
|
|
"task": CTX + """
|
|
New Deployments create no pods, and pods that do get created stay `Pending`. Two different
|
|
control-plane components are failing for two different reasons. Repair both on the control-plane node.
|
|
Afterwards a new Deployment must reach Available within a minute.
|
|
""",
|
|
"setup": r"""
|
|
nx "$CP" '
|
|
f=/etc/kubernetes/manifests/kube-scheduler.yaml; grep -q "^ *- kube-scheduler$" $f && sed -i "s|^\( *\)- kube-scheduler$|\1- kube-schedulerr|" $f
|
|
g=/etc/kubernetes/manifests/kube-controller-manager.yaml; grep -q "image: registry.k8s.io/kube-controller-manager:v" $g && sed -i "s|\(image: registry.k8s.io/kube-controller-manager:v[0-9.]*\)|\1-broken|" $g
|
|
'
|
|
""",
|
|
"checks": [
|
|
C("kube-scheduler manifest repaired and Running", 3, r"""nx "$CP" 'grep -q "^ *- kube-scheduler$" /etc/kubernetes/manifests/kube-scheduler.yaml' && eq "$(j -n kube-system pod -l component=kube-scheduler '{.items[0].status.phase}')" Running"""),
|
|
C("kube-controller-manager image repaired and Running", 3, r"""! nx "$CP" 'grep -q "\-broken" /etc/kubernetes/manifests/kube-controller-manager.yaml' && eq "$(j -n kube-system pod -l component=kube-controller-manager '{.items[0].status.phase}')" Running"""),
|
|
C("a new Deployment becomes Available", 3, r"""
|
|
$K -n default delete deploy cp-probe --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K -n default create deploy cp-probe --image=busybox:1.36 -- sleep 300 >/dev/null 2>&1
|
|
ok=1; for i in $(seq 1 20); do [ "$(j -n default deploy cp-probe '{.status.availableReplicas}')" = 1 ] && { ok=0; break; }; sleep 3; done
|
|
$K -n default delete deploy cp-probe --wait=false >/dev/null 2>&1; exit $ok"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 62, "domain": "TSHOOT", "difficulty": "hard", "title": "Application down for four reasons",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "checkout"), "dep": name_pick(r, "checkout")},
|
|
"task": CTX + """
|
|
Deployment `%{dep}` in `%{ns}` should have 3 Available replicas and Service `%{dep}` should route
|
|
to them on port 80. Right now nothing works. Fix it **in place** — the Deployment must keep its
|
|
UID (do not delete and recreate it) and the Service must keep its ClusterIP.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
$K -n %{ns} delete deploy,svc,quota,secret --all >/dev/null 2>&1
|
|
$K apply -f - >/dev/null <<EOF
|
|
apiVersion: v1
|
|
kind: ResourceQuota
|
|
metadata: {name: tight, namespace: %{ns}}
|
|
spec: {hard: {pods: "1"}}
|
|
---
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata: {name: %{dep}-env, namespace: %{ns}}
|
|
stringData: {API_KEY: abc123}
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: %{dep}, namespace: %{ns}}
|
|
spec:
|
|
replicas: 3
|
|
selector: {matchLabels: {app: %{dep}}}
|
|
template:
|
|
metadata: {labels: {app: %{dep}}}
|
|
spec:
|
|
containers:
|
|
- name: web
|
|
image: nginx:1.27-nope
|
|
ports: [{containerPort: 80}]
|
|
env: [{name: DB_URL, valueFrom: {secretKeyRef: {name: %{dep}-env, key: DATABASE_URL}}}]
|
|
resources: {requests: {cpu: 10m, memory: 16Mi}}
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata: {name: %{dep}, namespace: %{ns}}
|
|
spec:
|
|
selector: {app: %{dep}-web}
|
|
ports: [{port: 80, targetPort: 8080}]
|
|
EOF
|
|
j -n %{ns} deploy %{dep} '{.metadata.uid}' > "$WS/.q62-uid"; j -n %{ns} svc %{dep} '{.spec.clusterIP}' > "$WS/.q62-ip"
|
|
""",
|
|
"checks": [
|
|
C("3 replicas Available", 3, r"""for i in $(seq 1 6); do [ "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" = 3 ] && exit 0; sleep 5; done; exit 1"""),
|
|
C("Service routes: endpoints ready, targetPort 80", 2, r"""eq "$(j -n %{ns} svc %{dep} '{.spec.ports[0].targetPort}')" 80 && [ -n "$(j -n %{ns} endpointslices -l kubernetes.io/service-name=%{dep} '{.items[*].endpoints[?(@.conditions.ready==true)].addresses[*]}')" ]"""),
|
|
C("Deployment UID and Service ClusterIP unchanged (fixed in place)", 2, r"""eq "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" 3 || exit 1 # only once it is actually fixed
|
|
eq "$(j -n %{ns} deploy %{dep} '{.metadata.uid}')" "$(cat "$WS/.q62-uid")" && eq "$(j -n %{ns} svc %{dep} '{.spec.clusterIP}')" "$(cat "$WS/.q62-ip")" """),
|
|
C("quota still present and permits the 3 pods", 1, r"""p=$(j -n %{ns} quota tight '{.spec.hard.pods}'); [ -n "$p" ] && [ "$p" -ge 3 ]"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 63, "domain": "TSHOOT", "difficulty": "hard", "workers": 3, "title": "Cluster facts, exactly",
|
|
"params": lambda r: {"ctx": "k8s-c1-H", "node": r.choice(["{W1}", "{W2}", "{W3}"])},
|
|
"task": CTX + """
|
|
Write `{WS}/q63/facts.txt` with these lines, values taken from the running cluster:
|
|
```
|
|
pod-cidr=<cluster pod CIDR as configured on kube-controller-manager>
|
|
service-cidr=<service CIDR as configured on kube-apiserver>
|
|
dns-ip=<ClusterIP of the kube-dns Service>
|
|
proxy-mode=<kube-proxy mode from its ConfigMap (empty means iptables — write iptables)>
|
|
%{node}-pod-cidr=<.spec.podCIDR of that node>
|
|
```
|
|
""",
|
|
"setup": r"""rm -rf "$WS/q63"; mkdir -p "$WS/q63" """,
|
|
"checks": [
|
|
C("pod-cidr and service-cidr", 2, r"""f="$WS/q63/facts.txt"; pc=$(j -n kube-system pod -l component=kube-controller-manager '{.items[0].spec.containers[0].command}' | tr ',' '\n' | grep -o 'cluster-cidr=[^"]*' | cut -d= -f2); sc=$(j -n kube-system pod -l component=kube-apiserver '{.items[0].spec.containers[0].command}' | tr ',' '\n' | grep -o 'service-cluster-ip-range=[^"]*' | cut -d= -f2); grep -qx "pod-cidr=$pc" "$f" 2>/dev/null && grep -qx "service-cidr=$sc" "$f" """),
|
|
C("dns-ip and proxy-mode", 2, r"""f="$WS/q63/facts.txt"; d=$(j -n kube-system svc kube-dns '{.spec.clusterIP}'); m=$(j -n kube-system cm kube-proxy '{.data.config\.conf}' | grep '^mode:' | awk '{print $2}' | tr -d '"'); [ -n "$m" ] || m=iptables; grep -qx "dns-ip=$d" "$f" 2>/dev/null && grep -qx "proxy-mode=$m" "$f" """),
|
|
C("%{node} pod CIDR", 1, r"""grep -qx "%{node}-pod-cidr=$(j node %{node} '{.spec.podCIDR}')" "$WS/q63/facts.txt" 2>/dev/null"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 66, "domain": "TSHOOT", "breaks": "apiserver", "difficulty": "hard", "title": "Nothing works: kube-apiserver itself is down", "disruptive": True,
|
|
"params": lambda r: {"ctx": "k8s-c1-H"},
|
|
"task": CTX + """
|
|
Every `kubectl` command from every context times out or refuses to connect. This is not a
|
|
network problem — a control-plane component itself is failing to start. `kubectl` is unusable
|
|
until you fix it, so investigate directly on the control-plane node's containers.
|
|
|
|
Repair the failing component on the control-plane node (`{SHELL_CP}`). Afterwards, `kubectl get
|
|
nodes` must work again, and every other control-plane component must still be healthy.
|
|
""",
|
|
"setup": r"""
|
|
nx "$CP" '
|
|
f=/etc/kubernetes/manifests/kube-apiserver.yaml
|
|
grep -q "^ *- kube-apiserver$" $f && sed -i "s|^\( *\)- kube-apiserver$|\1- kube-apiserverr|" $f
|
|
'
|
|
""",
|
|
"checks": [
|
|
C("kube-apiserver manifest repaired and Running", 4, r"""nx "$CP" 'grep -q "^ *- kube-apiserver$" /etc/kubernetes/manifests/kube-apiserver.yaml' && eq "$(j -n kube-system pod -l component=kube-apiserver '{.items[0].status.phase}')" Running"""),
|
|
C("cluster fully reachable again (get nodes, create a pod)", 4, r"""
|
|
$K get nodes >/dev/null 2>&1 || exit 1
|
|
$K -n default delete pod api-probe --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K -n default run api-probe --image=busybox:1.36 --restart=Never -- sleep 60 >/dev/null 2>&1
|
|
ok=1; for i in $(seq 1 20); do [ "$(j -n default pod api-probe '{.status.phase}')" = Running ] && { ok=0; break; }; sleep 3; done
|
|
$K -n default delete pod api-probe --wait=false >/dev/null 2>&1; exit $ok"""),
|
|
C("etcd, scheduler and controller-manager unaffected (no collateral damage)", 2, r"""for c in etcd kube-scheduler kube-controller-manager; do eq "$(j -n kube-system pod -l component=$c '{.items[0].status.phase}')" Running || exit 1; done"""),
|
|
],
|
|
},
|
|
]
|
|
|
|
# derived params: w2 = 100 - w for the canary split
|
|
for _q in QUESTIONS:
|
|
if _q["id"] == 56:
|
|
_orig = _q["params"]
|
|
_q["params"] = lambda r, _o=_orig: (lambda p: {**p, "w2": 100 - p["w"]})(_o(r))
|