f3d0e536b6
kubestronaut-sim runs each exam task on a host of its own - a kind (kubeadm) cluster in rootless podman, or a QEMU/KVM Ubuntu VM for the CKS tasks that need their own kernel (Falco, AppArmor) - and puts a browser desktop in front of them, the way killer.sh and sailor.sh do. - ksim, sim/: the engine (sessions, drills, study mode, grading, coverage, audit), host and VM lifecycle, the desktop - images/: exam host images (common, CKS variant with its tools, VM guest) - cka/ (131 tasks), ckad/ (105), cks/ (110): task banks, curricula with study notes, reference solutions and scripted solutions Every domain of every track has at least 17 pool tasks. `./ksim audit` proves a task three ways: it seeds cleanly on its own host, scores nothing before anyone has worked on it, and reaches full marks when its scripted solution is run on the host as `candidate`. All 346 tasks pass (CKA audited 2026-10-13, CKAD and CKS 2026-10-12). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01JZPd3GGWyxTgAkL6a4gAJz
688 lines
40 KiB
Python
688 lines
40 KiB
Python
"""Extra normal-difficulty CKA tasks (ids 21-43). Parameters (%{...}) are drawn per session."""
|
|
from sim.bank import C, NS, ns_pick, name_pick
|
|
|
|
QUESTIONS = [
|
|
# ---------------------------------------------------------------- ARCH ----
|
|
{
|
|
"id": 21, "domain": "ARCH", "title": "Helm release install and upgrade",
|
|
"params": lambda r: {"ns": ns_pick(r, "helm"), "rel": name_pick(r, "web"), "n": r.choice([3, 4, 5])},
|
|
"task": """
|
|
A Helm chart is at `{WS}/q21/chart` (values: `replicaCount`, `image.*`, `service.type`, `service.port`).
|
|
|
|
1. Install it as release `%{rel}` into namespace `%{ns}` (create the namespace) with
|
|
`replicaCount=%{n}` overridden.
|
|
2. Then **upgrade** the same release so the Service type becomes `NodePort`
|
|
(value `service.type`). The release must be at revision 2 or higher.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
rm -rf "$WS/q21"; mkdir -p "$WS/q21/chart/templates"
|
|
cat > "$WS/q21/chart/Chart.yaml" <<'EOF'
|
|
apiVersion: v2
|
|
name: chart
|
|
description: A small web chart for the exam simulator
|
|
type: application
|
|
version: 0.1.0
|
|
appVersion: "1.27"
|
|
EOF
|
|
cat > "$WS/q21/chart/values.yaml" <<'EOF'
|
|
replicaCount: 1
|
|
image:
|
|
repository: nginx
|
|
tag: "1.27"
|
|
service:
|
|
type: ClusterIP
|
|
port: 80
|
|
EOF
|
|
cat > "$WS/q21/chart/templates/_helpers.tpl" <<'EOF'
|
|
{{- define "chart.fullname" -}}
|
|
{{- printf "%s-%s" .Release.Name .Chart.Name | trunc 63 | trimSuffix "-" -}}
|
|
{{- end -}}
|
|
EOF
|
|
cat > "$WS/q21/chart/templates/deployment.yaml" <<'EOF'
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: {{ include "chart.fullname" . }}
|
|
labels: {app: {{ include "chart.fullname" . }}}
|
|
spec:
|
|
replicas: {{ .Values.replicaCount }}
|
|
selector:
|
|
matchLabels: {app: {{ include "chart.fullname" . }}}
|
|
template:
|
|
metadata:
|
|
labels: {app: {{ include "chart.fullname" . }}}
|
|
spec:
|
|
containers:
|
|
- name: web
|
|
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
|
ports: [{containerPort: 80}]
|
|
EOF
|
|
cat > "$WS/q21/chart/templates/service.yaml" <<'EOF'
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: {{ include "chart.fullname" . }}
|
|
spec:
|
|
type: {{ .Values.service.type }}
|
|
selector: {app: {{ include "chart.fullname" . }}}
|
|
ports: [{port: {{ .Values.service.port }}, targetPort: 80}]
|
|
EOF
|
|
command -v helm >/dev/null || echo "warn: helm is not installed on this machine - Q21 cannot be completed or graded without it"
|
|
helm -n %{ns} uninstall %{rel} >/dev/null 2>&1 || true
|
|
""",
|
|
"checks": [
|
|
C("release %{rel} deployed in %{ns}", 1, r"""helm -n %{ns} status %{rel} -o json 2>/dev/null | python3 -c 'import sys,json;sys.exit(0 if json.load(sys.stdin)["info"]["status"]=="deployed" else 1)'"""),
|
|
C("replicaCount=%{n} applied", 1, r"""eq "$(j -n %{ns} deploy %{rel}-chart '{.spec.replicas}')" %{n}"""),
|
|
C("upgraded: revision >= 2 and Service is NodePort", 2, r"""
|
|
rev=$(helm -n %{ns} history %{rel} -o json 2>/dev/null | python3 -c 'import sys,json;print(max(r["revision"] for r in json.load(sys.stdin)))' 2>/dev/null)
|
|
[ -n "$rev" ] && [ "$rev" -ge 2 ] && eq "$(j -n %{ns} svc %{rel}-chart '{.spec.type}')" NodePort"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 33, "domain": "ARCH", "title": "Aggregated ClusterRole",
|
|
"params": lambda r: {"cr": name_pick(r, "ops"), "user": r.choice(["dana", "lee", "sam"])},
|
|
"task": """
|
|
1. Create a ClusterRole `%{cr}` that has **no rules of its own** but aggregates every
|
|
ClusterRole labelled `rbac.example.com/aggregate-to-%{cr}=true`.
|
|
2. Create two ClusterRoles carrying that label: `%{cr}-pods` (get/list/watch pods) and
|
|
`%{cr}-services` (get/list/watch services).
|
|
3. Bind `%{cr}` cluster-wide to user `%{user}` with ClusterRoleBinding `%{cr}-binding`.
|
|
|
|
`kubectl auth can-i list pods --as=%{user} -A` must say yes; deleting pods must say no.
|
|
""",
|
|
"setup": r"""
|
|
for x in %{cr} %{cr}-pods %{cr}-services; do $K delete clusterrole $x --ignore-not-found >/dev/null; done
|
|
$K delete clusterrolebinding %{cr}-binding --ignore-not-found >/dev/null
|
|
""",
|
|
"checks": [
|
|
C("%{cr} has an aggregationRule with the right label", 1, r"""j clusterrole %{cr} '{.aggregationRule.clusterRoleSelectors[*].matchLabels}' | grep -q 'aggregate-to-%{cr}":"true'"""),
|
|
C("aggregation populated rules from the two labelled roles", 2, r"""j clusterrole %{cr} '{.rules[*].resources}' | grep -q pods && j clusterrole %{cr} '{.rules[*].resources}' | grep -q services"""),
|
|
C("%{user} can list pods and services cluster-wide", 1, r"""eq "$($K auth can-i list pods -A --as=%{user} 2>/dev/null)" yes && eq "$($K auth can-i list services -A --as=%{user} 2>/dev/null)" yes"""),
|
|
C("... but cannot delete pods", 1, r"""
|
|
eq "$($K auth can-i list pods -A --as=%{user} 2>/dev/null)" yes || exit 1
|
|
eq "$($K auth can-i delete pods -A --as=%{user} 2>/dev/null || true)" no"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 34, "domain": "ARCH", "title": "Discover a CRD field with kubectl explain",
|
|
"params": lambda r: {"kind": r.choice(["Widget", "Gadget", "Gizmo"]), "field": r.choice(["retries", "attempts", "tries"]), "n": r.randint(2, 7)},
|
|
"task": """
|
|
A CRD for kind `%{kind}` (group `demo.io`) is installed. Somewhere in its schema is an
|
|
integer field whose description is exactly **"Number of retry attempts"**.
|
|
|
|
1. Using `kubectl explain`, find the field's full path (e.g. `spec.foo.bar`) and write it
|
|
on one line to `{WS}/q34/path.txt`.
|
|
2. Create a `%{kind}` named `sample` in namespace `default` with that field set to `%{n}`.
|
|
""",
|
|
"setup": r"""
|
|
rm -rf "$WS/q34"; mkdir -p "$WS/q34"
|
|
K_LOWER=$(echo %{kind} | tr A-Z a-z)
|
|
$K delete crd ${K_LOWER}s.demo.io --ignore-not-found >/dev/null
|
|
$K apply -f - >/dev/null <<EOF
|
|
apiVersion: apiextensions.k8s.io/v1
|
|
kind: CustomResourceDefinition
|
|
metadata: {name: ${K_LOWER}s.demo.io}
|
|
spec:
|
|
group: demo.io
|
|
scope: Namespaced
|
|
names: {kind: %{kind}, plural: ${K_LOWER}s, singular: ${K_LOWER}}
|
|
versions:
|
|
- name: v1
|
|
served: true
|
|
storage: true
|
|
schema:
|
|
openAPIV3Schema:
|
|
type: object
|
|
properties:
|
|
spec:
|
|
type: object
|
|
properties:
|
|
mode: {type: string, description: "Operating mode"}
|
|
tuning:
|
|
type: object
|
|
description: "Performance tuning knobs"
|
|
properties:
|
|
%{field}: {type: integer, description: "Number of retry attempts"}
|
|
backoffSeconds: {type: integer, description: "Delay between attempts"}
|
|
EOF
|
|
""",
|
|
"checks": [
|
|
C("path.txt has the correct field path", 2, r"""[ "$(tr -d '[:space:]' < "$WS/q34/path.txt" 2>/dev/null)" = "spec.tuning.%{field}" ]"""),
|
|
C("%{kind}/sample created with the field = %{n}", 2, r"""K_LOWER=$(echo %{kind} | tr A-Z a-z); eq "$(j -n default ${K_LOWER}s.demo.io sample '{.spec.tuning.%{field}}')" %{n}"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 40, "domain": "ARCH", "title": "kubeadm join token",
|
|
"params": lambda r: {"ttl": r.choice(["2h", "6h", "24h"])},
|
|
"task": """
|
|
A new worker is going to be added to the cluster.
|
|
|
|
On the control-plane node (`{SHELL_CP}`), create a **new** bootstrap token with TTL `%{ttl}`
|
|
and save the complete `kubeadm join` command for it (with `--discovery-token-ca-cert-hash`)
|
|
to `{WS}/q40/join.sh`.
|
|
""",
|
|
"setup": r"""
|
|
rm -rf "$WS/q40"; mkdir -p "$WS/q40"
|
|
for t in $($K -n kube-system get secrets -o jsonpath='{.items[?(@.type=="bootstrap.kubernetes.io/token")].metadata.name}'); do $K -n kube-system delete secret $t >/dev/null; done
|
|
""",
|
|
"checks": [
|
|
C("join.sh contains a kubeadm join with token and CA hash", 2, r"""grep -q 'kubeadm join' "$WS/q40/join.sh" 2>/dev/null && grep -q -- '--token' "$WS/q40/join.sh" && grep -q -- '--discovery-token-ca-cert-hash sha256:' "$WS/q40/join.sh" """),
|
|
C("the token in the file exists in the cluster with TTL ~%{ttl}", 3, r"""
|
|
tok=$(grep -o -- '--token [a-z0-9]\{6\}\.[a-z0-9]\{16\}' "$WS/q40/join.sh" 2>/dev/null | awk '{print $2}'); [ -n "$tok" ] || exit 1
|
|
id=${tok%%.*}; exp=$(j -n kube-system secret bootstrap-token-$id '{.data.expiration}' | base64 -d 2>/dev/null); [ -n "$exp" ] || exit 1
|
|
ct=$(j -n kube-system secret bootstrap-token-$id '{.metadata.creationTimestamp}'); [ -n "$ct" ] || exit 1
|
|
secs=$(( $(date -d "$exp" +%s) - $(date -d "$ct" +%s) )); want=$(( $(echo %{ttl} | tr -d h) * 3600 ))
|
|
[ $secs -ge $((want - 120)) ] && [ $secs -le $((want + 120)) ] && [ "$(date -d "$exp" +%s)" -gt "$(date +%s)" ]"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 41, "domain": "ARCH", "workers": 3, "title": "Component versions",
|
|
"params": lambda r: {"node": r.choice(["{W1}", "{W2}", "{W3}"])},
|
|
"task": """
|
|
Write the following to `{WS}/q41/versions.txt`, one per line, in this exact format:
|
|
|
|
```
|
|
apiserver=<version>
|
|
etcd=<version>
|
|
kubelet=<version>
|
|
runtime=<containerRuntimeVersion of node %{node}>
|
|
```
|
|
|
|
Use the values the cluster itself reports (`kubectl`, not what you remember).
|
|
""",
|
|
"setup": r"""rm -rf "$WS/q41"; mkdir -p "$WS/q41" """,
|
|
"checks": [
|
|
C("apiserver and kubelet versions correct", 2, r"""
|
|
f="$WS/q41/versions.txt"; [ -f "$f" ] || exit 1
|
|
api=$($K version -o json 2>/dev/null | python3 -c 'import sys,json;print(json.load(sys.stdin)["serverVersion"]["gitVersion"])')
|
|
kub=$(j node %{node} '{.status.nodeInfo.kubeletVersion}')
|
|
grep -qx "apiserver=$api" "$f" && grep -qx "kubelet=$kub" "$f" """),
|
|
C("etcd and runtime versions correct", 2, r"""
|
|
f="$WS/q41/versions.txt"
|
|
etcd=$(j -n kube-system pod -l component=etcd '{.items[0].spec.containers[0].image}' | sed 's/.*://'); ev=${etcd%-*}
|
|
rt=$(j node %{node} '{.status.nodeInfo.containerRuntimeVersion}')
|
|
grep -qxE "etcd=v?($etcd|$ev)" "$f" && grep -qx "runtime=$rt" "$f" """),
|
|
],
|
|
},
|
|
# ---------------------------------------------------------------- WORK ----
|
|
{
|
|
"id": 23, "domain": "WORK", "title": "metrics-server repair and HPA", "breaks": "metrics-server",
|
|
"params": lambda r: {"ns": ns_pick(r, "scale"), "dep": name_pick(r, "api"), "max": r.choice([5, 6, 8]), "cpu": r.choice([60, 70, 75])},
|
|
"task": """
|
|
`metrics-server` was installed in `kube-system` but `kubectl top nodes` does not work.
|
|
|
|
1. Fix metrics-server so `kubectl top nodes` returns data (hint: in this cluster the kubelet
|
|
serving certificates are not signed by the cluster CA).
|
|
2. Create an HPA `%{dep}` for Deployment `%{dep}` in namespace `%{ns}`:
|
|
min 2, max %{max}, target average CPU utilisation %{cpu}%.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
$K -n %{ns} create deploy %{dep} --image=nginx:1.25 --dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
$K -n %{ns} set resources deploy %{dep} --requests=cpu=50m >/dev/null
|
|
$K -n %{ns} delete hpa %{dep} --ignore-not-found >/dev/null
|
|
if ! $K -n kube-system get deploy metrics-server >/dev/null 2>&1; then
|
|
curl -sfL --max-time 60 "$METRICS_SERVER_URL" | $K apply -f - >/dev/null || echo "warn: could not download metrics-server manifest"
|
|
fi
|
|
# make it broken (kind kubelets use self-signed serving certs)
|
|
$K -n kube-system get deploy metrics-server -o json 2>/dev/null | python3 -c '
|
|
import sys,json; d=json.load(sys.stdin); c=d["spec"]["template"]["spec"]["containers"][0]
|
|
c["args"]=[a for a in c["args"] if a!="--kubelet-insecure-tls"]; print(json.dumps(d))' | $K apply -f - >/dev/null 2>&1 || true
|
|
""",
|
|
"checks": [
|
|
C("kubectl top nodes works", 2, r"""for i in $(seq 1 12); do $K top nodes >/dev/null 2>&1 && exit 0; sleep 5; done; exit 1"""),
|
|
C("HPA %{dep}: min 2, max %{max}", 1, r"""eq "$(j -n %{ns} hpa %{dep} '{.spec.minReplicas}')" 2 && eq "$(j -n %{ns} hpa %{dep} '{.spec.maxReplicas}')" %{max}"""),
|
|
C("HPA targets CPU %{cpu}% on the deployment", 1, r"""eq "$(j -n %{ns} hpa %{dep} '{.spec.scaleTargetRef.name}')" %{dep} && j -n %{ns} hpa %{dep} '{.spec.metrics[*].resource}' | grep -q '"averageUtilization":%{cpu}'"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 24, "domain": "WORK", "title": "PriorityClass",
|
|
"params": lambda r: {"ns": ns_pick(r, "prio"), "pc": name_pick(r, "critical"), "val": r.choice([100000, 500000, 900000])},
|
|
"task": """
|
|
1. Create a PriorityClass `%{pc}` with value %{val}, not the global default, description "%{pc} workloads".
|
|
2. In namespace `%{ns}`, Deployment `payments` already exists. Make its pods use `%{pc}` and
|
|
ensure a rollout completes (pods must show priority %{val}).
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
$K delete priorityclass %{pc} --ignore-not-found >/dev/null
|
|
$K -n %{ns} delete deploy payments --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K -n %{ns} create deploy payments --image=nginx:1.25 --replicas=2 --dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
""",
|
|
"checks": [
|
|
C("PriorityClass %{pc} value %{val}, not default", 2, r"""eq "$(j priorityclass %{pc} '{.value}')" %{val} && [ "$(j priorityclass %{pc} '{.globalDefault}')" != true ]"""),
|
|
C("payments pods running with priority %{val}", 2, r"""
|
|
for t in $(seq 1 15); do # the old ReplicaSet's pods linger for a moment while they terminate
|
|
ok=1; eq "$(j -n %{ns} deploy payments '{.status.availableReplicas}')" 2 || ok=0
|
|
for p in $(j -n %{ns} pod -l app=payments '{.items[*].spec.priority}'); do eq "$p" %{val} || ok=0; done
|
|
[ $ok = 1 ] && exit 0; sleep 2
|
|
done; exit 1"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 25, "domain": "WORK", "breaks": "w1-taint", "title": "Taints and tolerations",
|
|
"params": lambda r: {"ns": ns_pick(r, "sched"), "key": r.choice(["dedicated", "workload", "team"]), "val": r.choice(["gpu", "batch", "finance"]), "dep": name_pick(r, "worker")},
|
|
"task": """
|
|
1. Taint node `{W1}` with `%{key}=%{val}:NoSchedule`.
|
|
2. Create Deployment `%{dep}` in namespace `%{ns}` (image `busybox:1.36`, command `sleep 3600`,
|
|
2 replicas) that tolerates that taint **and** is pinned to `{W1}` with a nodeSelector on
|
|
`kubernetes.io/hostname`.
|
|
|
|
Both pods must be Running on `{W1}`. Do not remove the taint.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
$K taint node "$W1" %{key}- >/dev/null 2>&1 || true
|
|
$K -n %{ns} delete deploy %{dep} --ignore-not-found >/dev/null
|
|
""",
|
|
"checks": [
|
|
C("{W1} carries %{key}=%{val}:NoSchedule", 1, r"""j node "$W1" '{.spec.taints}' | grep -q '"key":"%{key}"' && j node "$W1" '{.spec.taints}' | grep -q '"value":"%{val}"' """),
|
|
C("deployment tolerates the taint", 1, r"""j -n %{ns} deploy %{dep} '{.spec.template.spec.tolerations}' | grep -q '"key":"%{key}"' """),
|
|
C("both pods Running on {W1}", 2, r"""eq "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" 2 && for n in $(j -n %{ns} pod -l app=%{dep} '{.items[*].spec.nodeName}'); do eq "$n" "$W1" || exit 1; done"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 26, "domain": "WORK", "title": "DaemonSet on every node",
|
|
# needs every node schedulable: Q25/Q50 taint W1 and must keep the taint
|
|
"breaks": ["w1-taint"],
|
|
"params": lambda r: {"ns": ns_pick(r, "agents"), "ds": name_pick(r, "node-agent")},
|
|
"task": """
|
|
Create a DaemonSet `%{ds}` in namespace `%{ns}` (image `busybox:1.36`, command `sleep 3600`)
|
|
that runs on **every** node, including the control-plane.
|
|
Requests: cpu 10m, memory 16Mi.
|
|
""",
|
|
"setup": NS + r"""mkns %{ns}; $K -n %{ns} delete ds %{ds} --ignore-not-found >/dev/null""",
|
|
"checks": [
|
|
C("DaemonSet exists with requests set", 1, r"""eq "$(j -n %{ns} ds %{ds} '{.spec.template.spec.containers[0].resources.requests.cpu}')" 10m"""),
|
|
C("tolerates the control-plane taint", 1, r"""$K -n %{ns} get ds %{ds} -o json 2>/dev/null | python3 -c '
|
|
import sys,json; t=json.load(sys.stdin)["spec"]["template"]["spec"].get("tolerations") or []
|
|
sys.exit(0 if any("control-plane" in (x.get("key") or "") or (x.get("operator")=="Exists" and not x.get("key")) for x in t) else 1)'"""),
|
|
C("one ready pod per node (including control-plane)", 2, r"""n=$($K get nodes --no-headers 2>/dev/null | wc -l); eq "$(j -n %{ns} ds %{ds} '{.status.numberReady}')" "$n" && eq "$(j -n %{ns} ds %{ds} '{.status.desiredNumberScheduled}')" "$n" """),
|
|
],
|
|
},
|
|
{
|
|
"id": 27, "domain": "WORK", "title": "CronJob with a manual run",
|
|
"params": lambda r: {"ns": ns_pick(r, "jobs"), "cj": name_pick(r, "report"), "every": r.choice([5, 10, 15])},
|
|
"task": """
|
|
1. Create CronJob `%{cj}` in `%{ns}`: runs every %{every} minutes, image `busybox:1.36`,
|
|
command `date`, `concurrencyPolicy: Forbid`, keep 3 successful and 1 failed job,
|
|
each run limited to 30 seconds (`activeDeadlineSeconds`), restartPolicy Never.
|
|
2. Trigger it manually once as a Job named `%{cj}-manual` and let it complete.
|
|
""",
|
|
"setup": NS + r"""mkns %{ns}; $K -n %{ns} delete cj %{cj} --ignore-not-found >/dev/null; $K -n %{ns} delete job %{cj}-manual --ignore-not-found >/dev/null""",
|
|
"checks": [
|
|
C("schedule every %{every} minutes, Forbid, history limits", 2, r"""c="$K -n %{ns} get cj %{cj}"; eq "$($c -o jsonpath='{.spec.schedule}' 2>/dev/null)" "*/%{every} * * * *" && eq "$($c -o jsonpath='{.spec.concurrencyPolicy}')" Forbid && eq "$($c -o jsonpath='{.spec.successfulJobsHistoryLimit}')" 3 && eq "$($c -o jsonpath='{.spec.failedJobsHistoryLimit}')" 1"""),
|
|
C("activeDeadlineSeconds 30, restartPolicy Never", 1, r"""eq "$(j -n %{ns} cj %{cj} '{.spec.jobTemplate.spec.activeDeadlineSeconds}')" 30 && eq "$(j -n %{ns} cj %{cj} '{.spec.jobTemplate.spec.template.spec.restartPolicy}')" Never"""),
|
|
C("manual job %{cj}-manual completed", 1, r"""for i in $(seq 1 10); do [ "$(j -n %{ns} job %{cj}-manual '{.status.succeeded}')" = 1 ] && exit 0; sleep 3; done; exit 1"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 28, "domain": "WORK", "title": "Init container prepares content",
|
|
"params": lambda r: {"ns": ns_pick(r, "web"), "pod": name_pick(r, "site"), "text": r.choice(["hello from init", "prepared by init", "init was here"])},
|
|
"task": """
|
|
Create pod `%{pod}` in namespace `%{ns}`:
|
|
- init container `setup` (busybox:1.36) writes the text `%{text}` into `/work/index.html`,
|
|
- main container `web` (nginx:1.25) serves that file from `/usr/share/nginx/html`,
|
|
- both share an `emptyDir` volume named `content`.
|
|
|
|
`curl` from inside the cluster to the pod on port 80 must return the text.
|
|
""",
|
|
"setup": NS + r"""mkns %{ns}; $K -n %{ns} delete pod %{pod} --ignore-not-found --wait=false >/dev/null""",
|
|
"checks": [
|
|
C("pod Running, init container named setup completed", 1, r"""eq "$(j -n %{ns} pod %{pod} '{.status.phase}')" Running && eq "$(j -n %{ns} pod %{pod} '{.spec.initContainers[0].name}')" setup"""),
|
|
C("emptyDir 'content' shared by both containers", 1, r"""[ -n "$(j -n %{ns} pod %{pod} '{.spec.volumes[?(@.name=="content")].emptyDir}')" ] && j -n %{ns} pod %{pod} '{.spec.containers[0].volumeMounts[*].name}' | grep -qw content"""),
|
|
C("nginx serves the init-written content", 2, r"""$K -n %{ns} exec %{pod} -c web -- cat /usr/share/nginx/html/index.html 2>/dev/null | grep -q '%{text}'"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 29, "domain": "WORK", "title": "Immutable ConfigMap rotation",
|
|
"params": lambda r: {"ns": ns_pick(r, "cfg"), "dep": name_pick(r, "app"), "cm": name_pick(r, "settings")},
|
|
"task": """
|
|
Deployment `%{dep}` in `%{ns}` mounts ConfigMap `%{cm}` at `/etc/cfg`. The ConfigMap is
|
|
**immutable** and its key `mode` is `old`.
|
|
|
|
The application must now read `mode=new`. Without deleting `%{cm}` and without editing it
|
|
(it cannot be edited), make the running pods see `new` at `/etc/cfg/mode`.
|
|
Hint: version the ConfigMap.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
$K -n %{ns} delete deploy %{dep} --ignore-not-found >/dev/null
|
|
$K -n %{ns} delete cm -l q=29 --ignore-not-found >/dev/null; $K -n %{ns} delete cm %{cm} --ignore-not-found >/dev/null
|
|
$K apply -f - >/dev/null <<EOF
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata: {name: %{cm}, namespace: %{ns}, labels: {q: "29"}}
|
|
immutable: true
|
|
data: {mode: old}
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: %{dep}, namespace: %{ns}}
|
|
spec:
|
|
replicas: 2
|
|
selector: {matchLabels: {app: %{dep}}}
|
|
template:
|
|
metadata: {labels: {app: %{dep}}}
|
|
spec:
|
|
containers:
|
|
- name: app
|
|
image: busybox:1.36
|
|
command: [sleep, "3600"]
|
|
volumeMounts: [{name: cfg, mountPath: /etc/cfg}]
|
|
volumes: [{name: cfg, configMap: {name: %{cm}}}]
|
|
EOF
|
|
""",
|
|
"checks": [
|
|
C("original ConfigMap %{cm} still exists and is immutable, next to the new one", 1, r"""
|
|
n=$(j -n %{ns} deploy %{dep} '{.spec.template.spec.volumes[0].configMap.name}'); [ -n "$n" ] && [ "$n" != %{cm} ] || exit 1
|
|
eq "$(j -n %{ns} cm %{cm} '{.immutable}')" true && eq "$(j -n %{ns} cm %{cm} '{.data.mode}')" old"""),
|
|
C("deployment now references a different ConfigMap", 1, r"""n=$(j -n %{ns} deploy %{dep} '{.spec.template.spec.volumes[0].configMap.name}'); [ -n "$n" ] && [ "$n" != %{cm} ]"""),
|
|
C("all pods available and see mode=new", 2, r"""eq "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" 2 && for p in $(j -n %{ns} pod -l app=%{dep} '{.items[*].metadata.name}'); do eq "$($K -n %{ns} exec $p -- cat /etc/cfg/mode 2>/dev/null)" new || exit 1; done"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 31, "domain": "WORK", "title": "Parallel Job",
|
|
"params": lambda r: {"ns": ns_pick(r, "batch"), "job": name_pick(r, "crunch"), "c": r.choice([6, 8, 9]), "p": r.choice([2, 3])},
|
|
"task": """
|
|
Create Job `%{job}` in `%{ns}`: image `busybox:1.36`, command `sh -c "echo done; sleep 2"`,
|
|
%{c} completions, %{p} in parallel, `backoffLimit: 2`, restartPolicy Never. Let it finish.
|
|
""",
|
|
"setup": NS + r"""mkns %{ns}; $K -n %{ns} delete job %{job} --ignore-not-found >/dev/null""",
|
|
"checks": [
|
|
C("completions %{c}, parallelism %{p}, backoffLimit 2", 2, r"""eq "$(j -n %{ns} job %{job} '{.spec.completions}')" %{c} && eq "$(j -n %{ns} job %{job} '{.spec.parallelism}')" %{p} && eq "$(j -n %{ns} job %{job} '{.spec.backoffLimit}')" 2"""),
|
|
C("job succeeded %{c} times", 2, r"""for i in $(seq 1 20); do [ "$(j -n %{ns} job %{job} '{.status.succeeded}')" = %{c} ] && exit 0; sleep 3; done; exit 1"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 32, "domain": "WORK", "title": "ResourceQuota and LimitRange",
|
|
"params": lambda r: {"ns": ns_pick(r, "team"), "pods": r.choice([4, 5, 6]), "dcpu": r.choice(["100m", "150m", "200m"])},
|
|
"task": """
|
|
In namespace `%{ns}`:
|
|
1. ResourceQuota `%{ns}-quota`: at most %{pods} pods, total CPU requests 1, total memory limits 2Gi.
|
|
2. LimitRange `%{ns}-defaults`: for containers, default request cpu %{dcpu} / memory 64Mi and
|
|
default limit cpu 500m / memory 256Mi.
|
|
3. Create Deployment `worker` (nginx:1.25, 2 replicas) **without** specifying resources.
|
|
Its pods must be Running with the defaults applied.
|
|
""",
|
|
"setup": NS + r"""mkns %{ns}; $K -n %{ns} delete quota,limitrange --all >/dev/null 2>&1; $K -n %{ns} delete deploy worker --ignore-not-found >/dev/null""",
|
|
"checks": [
|
|
C("ResourceQuota limits pods/%{pods}, requests.cpu 1, limits.memory 2Gi", 2, r"""q="$K -n %{ns} get quota %{ns}-quota"; eq "$($q -o jsonpath='{.spec.hard.pods}' 2>/dev/null)" %{pods} && eq "$($q -o jsonpath='{.spec.hard.requests\.cpu}')" 1 && eq "$($q -o jsonpath='{.spec.hard.limits\.memory}')" 2Gi"""),
|
|
C("LimitRange defaults set", 1, r"""l="$K -n %{ns} get limitrange %{ns}-defaults"; eq "$($l -o jsonpath='{.spec.limits[0].defaultRequest.cpu}' 2>/dev/null)" %{dcpu} && eq "$($l -o jsonpath='{.spec.limits[0].default.memory}')" 256Mi"""),
|
|
C("worker pods Running with defaulted requests", 1, r"""eq "$(j -n %{ns} deploy worker '{.status.availableReplicas}')" 2 && eq "$(j -n %{ns} pod -l app=worker '{.items[0].spec.containers[0].resources.requests.cpu}')" %{dcpu}"""),
|
|
],
|
|
},
|
|
# ----------------------------------------------------------------- NET ----
|
|
{
|
|
"id": 22, "domain": "NET", "title": "Gateway API HTTPRoute",
|
|
"params": lambda r: {"ns": ns_pick(r, "gw"), "gc": name_pick(r, "lab-gc"), "gw": name_pick(r, "edge"), "host": r.choice(["shop", "api", "app"])},
|
|
"task": """
|
|
1. Install the Gateway API **standard** channel CRDs (v1.2.1 or newer) into the cluster.
|
|
2. Create a GatewayClass `%{gc}` (controllerName `example.com/gateway`).
|
|
3. In namespace `%{ns}` create Gateway `%{gw}` using that class, one HTTP listener on port 80
|
|
for hostname `%{host}.example.com`.
|
|
4. Create HTTPRoute `%{host}-route` attached to `%{gw}` that sends `/api` (PathPrefix) to
|
|
Service `api` port 8080 and everything else to Service `web` port 80.
|
|
|
|
(No controller runs here; you are graded on the objects.)
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
$K -n %{ns} create deploy web --image=nginx:1.25 --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
$K -n %{ns} create deploy api --image=nginx:1.25 --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
$K -n %{ns} expose deploy web --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
$K -n %{ns} expose deploy api --port=8080 --target-port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
# these fail harmlessly when the Gateway API CRDs are not installed yet - installing them is step 1
|
|
$K delete gatewayclass %{gc} --ignore-not-found >/dev/null 2>&1; $K -n %{ns} delete gateway %{gw} --ignore-not-found >/dev/null 2>&1; $K -n %{ns} delete httproute %{host}-route --ignore-not-found >/dev/null 2>&1 || true
|
|
""",
|
|
"checks": [
|
|
C("Gateway API CRDs installed (gateways, httproutes)", 1, r"""$K get crd gateways.gateway.networking.k8s.io httproutes.gateway.networking.k8s.io >/dev/null 2>&1"""),
|
|
C("GatewayClass %{gc} with the controllerName", 1, r"""eq "$(j gatewayclass %{gc} '{.spec.controllerName}')" example.com/gateway"""),
|
|
C("Gateway %{gw}: class, HTTP listener :80 for %{host}.example.com", 2, r"""g="$K -n %{ns} get gateway %{gw}"; eq "$($g -o jsonpath='{.spec.gatewayClassName}' 2>/dev/null)" %{gc} && eq "$($g -o jsonpath='{.spec.listeners[0].port}')" 80 && eq "$($g -o jsonpath='{.spec.listeners[0].hostname}')" %{host}.example.com"""),
|
|
C("HTTPRoute: parentRef %{gw}, /api -> api:8080, default -> web:80", 2, r"""
|
|
h="$K -n %{ns} get httproute %{host}-route"; $h >/dev/null 2>&1 || exit 1
|
|
eq "$($h -o jsonpath='{.spec.parentRefs[0].name}')" %{gw} || exit 1
|
|
$h -o json | python3 -c '
|
|
import sys,json; r=json.load(sys.stdin)["spec"]["rules"]
|
|
api=[x for x in r if any(m.get("path",{}).get("value")=="/api" for m in x.get("matches",[]))]
|
|
web=[x for x in r if x not in api and x.get("backendRefs",[{}])[0].get("name")=="web"]
|
|
ok=api and api[0]["backendRefs"][0]["name"]=="api" and api[0]["backendRefs"][0]["port"]==8080 and web and web[0]["backendRefs"][0]["port"]==80
|
|
sys.exit(0 if ok else 1)'"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 30, "domain": "NET", "title": "Named ports",
|
|
"params": lambda r: {"ns": ns_pick(r, "ports"), "dep": name_pick(r, "catalog"), "pname": r.choice(["http-web", "app-port", "svc-http"])},
|
|
"task": """
|
|
1. In `%{ns}`, Deployment `%{dep}` (nginx:1.25, 1 replica) must declare its container port 80
|
|
with the **name** `%{pname}`.
|
|
2. Create Service `%{dep}` (ClusterIP, port 8080) whose `targetPort` references the port
|
|
**by name**, not by number.
|
|
3. Add a readiness probe on the container that also references the port by name (HTTP GET /).
|
|
""",
|
|
"setup": NS + r"""mkns %{ns}; $K -n %{ns} delete deploy,svc %{dep} --ignore-not-found >/dev/null""",
|
|
"checks": [
|
|
C("container port 80 named %{pname}", 1, r"""eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].ports[?(@.containerPort==80)].name}')" %{pname}"""),
|
|
C("Service port 8080 -> targetPort %{pname} (by name)", 2, r"""eq "$(j -n %{ns} svc %{dep} '{.spec.ports[0].port}')" 8080 && eq "$(j -n %{ns} svc %{dep} '{.spec.ports[0].targetPort}')" %{pname}"""),
|
|
C("readiness probe uses the named port; endpoints ready", 1, r"""eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].readinessProbe.httpGet.port}')" %{pname} && [ -n "$(j -n %{ns} endpointslices -l kubernetes.io/service-name=%{dep} '{.items[*].endpoints[?(@.conditions.ready==true)].addresses[*]}')" ]"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 42, "domain": "NET", "title": "Egress-only NetworkPolicy",
|
|
"params": lambda r: {"ns": ns_pick(r, "egress"), "app": name_pick(r, "client"), "cidr": r.choice(["10.96.0.0/12", "172.16.0.0/16", "192.168.100.0/24"])},
|
|
"task": """
|
|
Namespace `%{ns}` has a Deployment `%{app}` (label `app=%{app}`).
|
|
|
|
Create NetworkPolicy `%{app}-egress` that applies only to those pods and allows **egress** to:
|
|
- DNS (UDP and TCP 53) to pods in `kube-system` labelled `k8s-app=kube-dns`,
|
|
- TCP 443 to the CIDR `%{cidr}`,
|
|
and denies all other egress. Do not restrict ingress.
|
|
""",
|
|
"setup": NS + r"""mkns %{ns}; $K -n %{ns} create deploy %{app} --image=busybox:1.36 --dry-run=client -o yaml -- sleep 3600 | $K apply -f - >/dev/null; $K -n %{ns} delete netpol %{app}-egress --ignore-not-found >/dev/null""",
|
|
"checks": [
|
|
C("applies to app=%{app}, Egress only", 1, r"""n="$K -n %{ns} get netpol %{app}-egress"; eq "$($n -o jsonpath='{.spec.podSelector.matchLabels.app}' 2>/dev/null)" %{app} && eq "$($n -o jsonpath='{.spec.policyTypes}')" '["Egress"]'"""),
|
|
C("DNS rule: kube-system namespace + k8s-app=kube-dns, ports 53 UDP and TCP", 2, r"""
|
|
$K -n %{ns} get netpol %{app}-egress -o json 2>/dev/null | python3 -c '
|
|
import sys,json; e=json.load(sys.stdin)["spec"].get("egress",[])
|
|
dns=[r for r in e if any((p.get("port")==53 or p.get("port")=="53") for p in r.get("ports",[]))]
|
|
ok=False
|
|
for r in dns:
|
|
protos={p.get("protocol","TCP") for p in r["ports"] if str(p.get("port"))=="53"}
|
|
to=r.get("to",[])
|
|
nsok=any("kube-system" in json.dumps(t.get("namespaceSelector",{})) for t in to)
|
|
podok=any(t.get("podSelector",{}).get("matchLabels",{}).get("k8s-app")=="kube-dns" for t in to)
|
|
if protos>={"UDP","TCP"} and nsok and podok: ok=True
|
|
sys.exit(0 if ok else 1)'"""),
|
|
C("TCP 443 to %{cidr} only", 1, r"""
|
|
$K -n %{ns} get netpol %{app}-egress -o json 2>/dev/null | python3 -c '
|
|
import sys,json; e=json.load(sys.stdin)["spec"].get("egress",[])
|
|
ok=any(any(t.get("ipBlock",{}).get("cidr")=="%{cidr}" for t in r.get("to",[])) and any(str(p.get("port"))=="443" for p in r.get("ports",[])) for r in e)
|
|
sys.exit(0 if ok else 1)'"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 43, "domain": "NET", "title": "Headless and ExternalName services",
|
|
"params": lambda r: {"ns": ns_pick(r, "svc2"), "sts": name_pick(r, "db"), "ext": r.choice(["api.example.com", "db.corp.example.org", "cache.internal.example.net"])},
|
|
"task": """
|
|
1. In `%{ns}`, StatefulSet `%{sts}` (2 replicas) exists but has no Service. Create the **headless**
|
|
Service `%{sts}` it needs (port 5432) so that `%{sts}-0.%{sts}.%{ns}.svc.cluster.local` resolves.
|
|
2. Create Service `legacy` of type ExternalName pointing at `%{ext}`.
|
|
3. From a pod in the namespace, resolve pod 0's stable name
|
|
`%{sts}-0.%{sts}.%{ns}.svc.cluster.local` and write the full nslookup output to
|
|
`{WS}/q43/pod0.txt`. (busybox's nslookup applies no search domains, so a short name such as
|
|
`%{sts}-0.%{sts}` fails even when DNS is healthy.)
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}; rm -rf "$WS/q43"; mkdir -p "$WS/q43"
|
|
$K -n %{ns} delete svc %{sts} legacy --ignore-not-found >/dev/null
|
|
$K apply -f - >/dev/null <<EOF
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata: {name: %{sts}, namespace: %{ns}}
|
|
spec:
|
|
serviceName: %{sts}
|
|
replicas: 2
|
|
selector: {matchLabels: {app: %{sts}}}
|
|
template:
|
|
metadata: {labels: {app: %{sts}}}
|
|
spec:
|
|
containers: [{name: db, image: busybox:1.36, command: [sleep, "3600"], ports: [{containerPort: 5432}]}]
|
|
EOF
|
|
""",
|
|
"checks": [
|
|
C("headless Service %{sts} (clusterIP None) selecting app=%{sts}, port 5432", 2, r"""eq "$(j -n %{ns} svc %{sts} '{.spec.clusterIP}')" None && eq "$(j -n %{ns} svc %{sts} '{.spec.selector.app}')" %{sts} && eq "$(j -n %{ns} svc %{sts} '{.spec.ports[0].port}')" 5432"""),
|
|
C("ExternalName Service legacy -> %{ext}", 1, r"""eq "$(j -n %{ns} svc legacy '{.spec.type}')" ExternalName && eq "$(j -n %{ns} svc legacy '{.spec.externalName}')" %{ext}"""),
|
|
C("pod0.txt shows %{sts}-0 resolving to its pod IP", 1, r"""ip=$(j -n %{ns} pod %{sts}-0 '{.status.podIP}'); [ -n "$ip" ] && grep -q "$ip" "$WS/q43/pod0.txt" 2>/dev/null"""),
|
|
],
|
|
},
|
|
# ---------------------------------------------------------------- STOR ----
|
|
{
|
|
"id": 35, "domain": "STOR", "title": "Re-bind a Released PersistentVolume",
|
|
"params": lambda r: {"ns": ns_pick(r, "store"), "pv": name_pick(r, "pv-archive"), "pvc": name_pick(r, "archive-claim")},
|
|
"task": """
|
|
PersistentVolume `%{pv}` (1Gi, hostPath, class `manual`) is in status **Released**: the claim that used
|
|
it was deleted, but its data must be kept and reused.
|
|
|
|
Make `%{pv}` bindable again without deleting it, then create PVC `%{pvc}` in `%{ns}`
|
|
(class `manual`, 500Mi, RWO) so it binds to **exactly** that PV. Do not create any other PV.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
$K -n %{ns} delete pvc %{pvc} --ignore-not-found >/dev/null; $K delete pv %{pv} --ignore-not-found >/dev/null
|
|
$K apply -f - >/dev/null <<EOF
|
|
apiVersion: v1
|
|
kind: PersistentVolume
|
|
metadata: {name: %{pv}}
|
|
spec:
|
|
capacity: {storage: 1Gi}
|
|
accessModes: [ReadWriteOnce]
|
|
persistentVolumeReclaimPolicy: Retain
|
|
storageClassName: manual
|
|
hostPath: {path: /mnt/%{pv}}
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: old-claim, namespace: %{ns}}
|
|
spec: {accessModes: [ReadWriteOnce], storageClassName: manual, resources: {requests: {storage: 500Mi}}}
|
|
EOF
|
|
for i in $(seq 1 20); do [ "$(j -n %{ns} pvc old-claim '{.status.phase}')" = Bound ] && break; sleep 1; done
|
|
$K -n %{ns} delete pvc old-claim >/dev/null
|
|
# every PV that exists now; the "no extra PV" check compares against this, so Retain leftovers from
|
|
# earlier sessions (this task's own, Q15's pv-data, Set C's) can no longer fail it
|
|
$K get pv -o jsonpath='{.items[*].metadata.name}' > "$WS/.q35-pvs"
|
|
$K get pv %{pv} -o jsonpath='{.metadata.uid}' > "$PRIV/q35.uid"
|
|
""",
|
|
"checks": [
|
|
C("PV %{pv} is bindable again and was not recreated", 2, r"""
|
|
u=$(j pv %{pv} '{.metadata.uid}'); [ -n "$u" ] && eq "$u" "$(cat "$PRIV/q35.uid")" || exit 1
|
|
eq "$(j pv %{pv} '{.spec.persistentVolumeReclaimPolicy}')" Retain || exit 1
|
|
# no longer Released: Available, or already Bound to the new claim
|
|
case "$(j pv %{pv} '{.status.phase}')" in Available) ;; Bound) eq "$(j pv %{pv} '{.spec.claimRef.name}')" %{pvc} ;; *) exit 1 ;; esac"""),
|
|
C("PVC %{pvc} Bound to %{pv}", 3, r"""eq "$(j -n %{ns} pvc %{pvc} '{.status.phase}')" Bound && eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}')" %{pv}"""),
|
|
C("no extra PVs were created", 1, r"""
|
|
eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}')" %{pv} || exit 1
|
|
before=" $(cat "$WS/.q35-pvs" 2>/dev/null) "; [ "$before" != " " ] || exit 1
|
|
for pv in $($K get pv -o jsonpath='{range .items[?(@.spec.storageClassName=="manual")]}{.metadata.name}{" "}{end}' 2>/dev/null); do
|
|
case "$before" in *" $pv "*) ;; *) exit 1 ;; esac
|
|
done"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 36, "domain": "STOR", "title": "Data survives the pod",
|
|
"params": lambda r: {"ns": ns_pick(r, "persist"), "pvc": name_pick(r, "notes"), "text": r.choice(["order-9981", "ticket-4471", "batch-2026"])},
|
|
"task": """
|
|
1. In `%{ns}` create PVC `%{pvc}` (200Mi, RWO) using the cluster's default StorageClass.
|
|
2. Run pod `writer` (busybox:1.36) that mounts it at `/data` and writes the text `%{text}`
|
|
into `/data/marker.txt`, then exits (restartPolicy Never).
|
|
3. After `writer` has completed, run pod `reader` (busybox:1.36, `sleep 3600`) mounting the
|
|
same claim at `/data`. `cat /data/marker.txt` inside `reader` must show the text.
|
|
""",
|
|
"setup": NS + r"""mkns %{ns}; $K -n %{ns} delete pod writer reader --ignore-not-found --wait=false >/dev/null; $K -n %{ns} delete pvc %{pvc} --ignore-not-found >/dev/null""",
|
|
"checks": [
|
|
C("PVC %{pvc} Bound with the default StorageClass", 1, r"""eq "$(j -n %{ns} pvc %{pvc} '{.status.phase}')" Bound && [ -n "$(j -n %{ns} pvc %{pvc} '{.spec.storageClassName}')" ]"""),
|
|
C("writer completed (Succeeded)", 1, r"""eq "$(j -n %{ns} pod writer '{.status.phase}')" Succeeded"""),
|
|
C("reader sees the marker written by writer", 2, r"""eq "$($K -n %{ns} exec reader -- cat /data/marker.txt 2>/dev/null | tr -d '[:space:]')" %{text}"""),
|
|
],
|
|
},
|
|
# -------------------------------------------------------------- TSHOOT ----
|
|
{
|
|
"id": 37, "domain": "TSHOOT", "breaks": "coredns", "title": "Cluster DNS is broken", "disruptive": True,
|
|
"params": lambda r: {"bad": r.choice(["forward . 10.255.255.1", "forward . 192.0.2.53"])},
|
|
"task": """
|
|
Pods across the cluster can no longer resolve names (in-cluster or external).
|
|
Find the cause in CoreDNS and fix it. `nslookup kubernetes.default.svc.cluster.local` and
|
|
`nslookup kubernetes.io` from any pod must work again, and CoreDNS must be healthy.
|
|
(Use the FQDN: busybox's nslookup does not apply the pod's `search` domains.)
|
|
""",
|
|
"setup": r"""
|
|
$K -n kube-system get cm coredns -o jsonpath='{.data.Corefile}' > "$PRIV/Corefile"
|
|
sed -i 's|forward \. /etc/resolv.conf|%{bad}|' "$PRIV/Corefile"
|
|
$K -n kube-system create cm coredns --from-file=Corefile="$PRIV/Corefile" --dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
$K -n kube-system rollout restart deploy coredns >/dev/null
|
|
$K -n kube-system rollout status deploy coredns --timeout=120s >/dev/null 2>&1 || true
|
|
""",
|
|
"checks": [
|
|
C("CoreDNS Corefile forwards to the node resolver again", 2, r"""j -n kube-system cm coredns '{.data.Corefile}' | grep -q 'forward \. /etc/resolv.conf'"""),
|
|
C("CoreDNS pods available, running the repaired configuration", 1, r"""
|
|
j -n kube-system cm coredns '{.data.Corefile}' | grep -q 'forward \. /etc/resolv.conf' || exit 1
|
|
[ "$(j -n kube-system deploy coredns '{.status.availableReplicas}')" -ge 1 ]"""),
|
|
C("in-cluster and external names resolve from a pod", 3, r"""
|
|
j -n kube-system cm coredns '{.data.Corefile}' | grep -q 'forward \. /etc/resolv.conf' || exit 1
|
|
$K -n default delete pod dns-probe --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K -n default run dns-probe --image=busybox:1.36 --restart=Never -- sh -c 'for i in 1 2 3 4 5 6; do nslookup kubernetes.default.svc.cluster.local >/dev/null 2>&1 && nslookup kubernetes.io >/dev/null 2>&1 && echo OK && exit 0; sleep 5; done; exit 1' >/dev/null 2>&1
|
|
for i in $(seq 1 20); do ph=$(j -n default pod dns-probe '{.status.phase}'); [ "$ph" = Succeeded ] && { $K -n default delete pod dns-probe --wait=false >/dev/null; exit 0; }; [ "$ph" = Failed ] && break; sleep 3; done
|
|
$K -n default delete pod dns-probe --wait=false >/dev/null; exit 1"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 38, "domain": "TSHOOT", "breaks": "controller-manager", "title": "Deployments never create pods", "disruptive": True,
|
|
"task": """
|
|
Creating a Deployment results in **no** ReplicaSet or pods, and nothing is reported in
|
|
`kubectl describe deploy`. Existing pods still run.
|
|
|
|
Find the failing control-plane component, repair it on the control-plane node
|
|
(`{SHELL_CP}`) and make sure new Deployments get their pods.
|
|
""",
|
|
"setup": r"""
|
|
nx "$CP" '
|
|
f=/etc/kubernetes/manifests/kube-controller-manager.yaml
|
|
grep -q "^ *- kube-controller-manager$" $f && sed -i "s|^\( *\)- kube-controller-manager$|\1- kube-controller-manager\n\1- --leader-elect-lease-duration=1s|" $f
|
|
'
|
|
""",
|
|
"checks": [
|
|
C("kube-controller-manager manifest repaired and pod Running", 3, r"""! nx "$CP" 'grep -q "leader-elect-lease-duration=1s" /etc/kubernetes/manifests/kube-controller-manager.yaml' && [ "$(j -n kube-system pod -l component=kube-controller-manager '{.items[0].status.phase}')" = Running ]"""),
|
|
C("a new Deployment gets a ReplicaSet and running pods", 4, r"""
|
|
$K -n default delete deploy cm-probe --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K -n default create deploy cm-probe --image=busybox:1.36 -- sleep 300 >/dev/null 2>&1
|
|
ok=1; for i in $(seq 1 24); do [ "$(j -n default deploy cm-probe '{.status.availableReplicas}')" = 1 ] && { ok=0; break; }; sleep 5; done
|
|
$K -n default delete deploy cm-probe --wait=false >/dev/null 2>&1; exit $ok"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 39, "domain": "TSHOOT", "breaks": "w1-kubelet", "title": "kubelet misconfiguration", "disruptive": True,
|
|
"task": """
|
|
Node `{W1}` is `NotReady`. Someone "tuned" the kubelet on it last night.
|
|
Bring the node back to `Ready`; the kubelet must be running and enabled.
|
|
Reach the node with `{SHELL_W1}`. `journalctl -u kubelet` is your friend.
|
|
""",
|
|
"setup": r"""
|
|
nx "$W1" '
|
|
cp -n /var/lib/kubelet/config.yaml /var/lib/kubelet/config.yaml.orig 2>/dev/null
|
|
sed -i "s|clientCAFile: /etc/kubernetes/pki/ca.crt|clientCAFile: /etc/kubernetes/pki/ca-bundle.crt|" /var/lib/kubelet/config.yaml
|
|
systemctl restart kubelet
|
|
'
|
|
""",
|
|
"checks": [
|
|
C("kubelet config points at a CA file that exists", 2, r"""f=$(nx "$W1" 'grep clientCAFile /var/lib/kubelet/config.yaml' | awk '{print $2}'); [ -n "$f" ] && nx "$W1" "test -f $f" """),
|
|
C("node Ready", 4, r"""
|
|
eq "$(nx "$W1" 'systemctl is-active kubelet' </dev/null)" active || exit 1
|
|
for i in $(seq 1 12); do [ "$(j node "$W1" '{.status.conditions[?(@.type=="Ready")].status}')" = True ] && exit 0; sleep 5; done; exit 1"""),
|
|
C("kubelet active and enabled, on the repaired configuration", 1, r"""
|
|
# a kubelet that keeps crashing on its configuration is "active" for a moment after every restart
|
|
f=$(nx "$W1" 'grep clientCAFile /var/lib/kubelet/config.yaml' </dev/null | awk '{print $2}'); [ -n "$f" ] && nx "$W1" "test -f $f" </dev/null || exit 1
|
|
eq "$(nx "$W1" 'systemctl is-active kubelet' </dev/null)" active && eq "$(nx "$W1" 'systemctl is-enabled kubelet' </dev/null)" enabled"""),
|
|
],
|
|
},
|
|
]
|