661 lines
49 KiB
Python
661 lines
49 KiB
Python
"""Hard-difficulty CKA tasks (ids 101-122): multi-part, no hints, traps, exact output,
|
|
compound failures, collateral-damage checks, exam-style contexts. Modelled on the kind of
|
|
difficulty people report from killer.sh, not on its content."""
|
|
from tracks.cka.common import C, NS, ns_pick, name_pick
|
|
|
|
CTX = "Use context: `kubectl config use-context %{ctx}`\n\n"
|
|
|
|
def ctx_pick(r):
|
|
return r.choice(["k8s-c1-H", "k8s-c2-AC", "k8s-c3-CCC"])
|
|
|
|
QUESTIONS = [
|
|
# ================================================================ ARCH ====
|
|
{
|
|
"id": 101, "domain": "ARCH", "difficulty": "hard", "title": "Exact RBAC with an existing over-grant",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "ci"), "sa": name_pick(r, "ci-bot")},
|
|
"task": CTX + """
|
|
ServiceAccount `%{sa}` in namespace `%{ns}` is used by a CI system. Its permissions must be
|
|
**exactly**:
|
|
- in `%{ns}`: `get`, `list`, `watch`, `patch` on `deployments` and `replicasets` (apps),
|
|
and `create` on `pods/exec`,
|
|
- cluster-wide: `list` on `nodes`.
|
|
|
|
Nothing more. Audit what it can do now and correct it. Use Role `%{sa}-role`, RoleBinding
|
|
`%{sa}-rb`, ClusterRole `%{sa}-nodes`, ClusterRoleBinding `%{sa}-crb`.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
$K -n %{ns} create sa %{sa} --dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
$K -n %{ns} delete role,rolebinding --all >/dev/null 2>&1
|
|
$K delete clusterrole %{sa}-nodes --ignore-not-found >/dev/null; $K delete clusterrolebinding %{sa}-crb %{sa}-legacy-admin --ignore-not-found >/dev/null
|
|
# trap: someone bound cluster-admin to this SA "temporarily"
|
|
$K create clusterrolebinding %{sa}-legacy-admin --clusterrole=cluster-admin --serviceaccount=%{ns}:%{sa} >/dev/null
|
|
""",
|
|
"checks": [
|
|
C("can get/list/watch/patch deployments and replicasets in %{ns}", 2, r"""for res in deployments.apps replicasets.apps; do for v in get list watch patch; do eq "$($K auth can-i $v $res -n %{ns} --as=system:serviceaccount:%{ns}:%{sa} 2>/dev/null)" yes || exit 1; done; done"""),
|
|
C("can create pods/exec in %{ns}; can list nodes", 2, r"""eq "$($K auth can-i create pods/exec -n %{ns} --as=system:serviceaccount:%{ns}:%{sa} 2>/dev/null)" yes && eq "$($K auth can-i list nodes --as=system:serviceaccount:%{ns}:%{sa} 2>/dev/null)" yes"""),
|
|
C("no over-grant: cannot delete deployments, cannot get secrets, cannot get nodes", 3, r"""for t in "delete deployments -n %{ns}" "get secrets -n %{ns}" "get nodes" "create pods -n default"; do eq "$($K auth can-i $t --as=system:serviceaccount:%{ns}:%{sa} 2>/dev/null || true)" no || exit 1; done"""),
|
|
C("the cluster-admin binding is gone and named objects exist", 2, r"""! $K get clusterrolebinding %{sa}-legacy-admin >/dev/null 2>&1 && $K -n %{ns} get role %{sa}-role rolebinding %{sa}-rb >/dev/null 2>&1 && $K get clusterrole %{sa}-nodes clusterrolebinding %{sa}-crb >/dev/null 2>&1"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 102, "domain": "ARCH", "breaks": "etcd", "difficulty": "hard", "title": "etcd restore",
|
|
"params": lambda r: {"ctx": "k8s-c1-H", "cm": name_pick(r, "restore-proof")},
|
|
"task": CTX + """
|
|
An etcd snapshot taken earlier is at `/opt/cka/backup.db` on the control-plane node.
|
|
A ConfigMap `%{cm}` in namespace `default` existed when the snapshot was taken and has since
|
|
been deleted.
|
|
|
|
**Restore the cluster's etcd from that snapshot** so the ConfigMap exists again. The cluster
|
|
must be healthy afterwards (API server up, all kube-system pods Running).
|
|
Do not recreate the ConfigMap by hand — its data must come from the snapshot.
|
|
""",
|
|
"setup": r"""
|
|
nx "$CP" 'mkdir -p /opt/cka; rm -f /opt/cka/backup.db'
|
|
$K -n default delete cm %{cm} --ignore-not-found >/dev/null
|
|
$K -n default create cm %{cm} --from-literal=proof=$(date +%s)-%{cm} >/dev/null
|
|
POD=$($K -n kube-system get pod -l component=etcd -o jsonpath='{.items[0].metadata.name}')
|
|
$K -n kube-system exec "$POD" -- sh -c 'ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key snapshot save /var/lib/etcd/q102-backup.db' >/dev/null 2>&1
|
|
nx "$CP" 'mv /var/lib/etcd/q102-backup.db /opt/cka/backup.db'
|
|
for bin in etcdctl etcdutl; do $K -n kube-system exec "$POD" -- cat /usr/local/bin/$bin > /tmp/cka-$bin 2>/dev/null && [ -s /tmp/cka-$bin ] && nx "$CP" "cat > /usr/local/bin/$bin && chmod +x /usr/local/bin/$bin" < /tmp/cka-$bin; rm -f /tmp/cka-$bin; done
|
|
sleep 2; $K -n default delete cm %{cm} >/dev/null
|
|
""",
|
|
"checks": [
|
|
C("ConfigMap %{cm} exists again with its original data", 6, r"""j -n default cm %{cm} '{.data.proof}' | grep -q -- '-%{cm}$'"""),
|
|
C("etcd uses a restored data directory (manifest changed)", 2, r"""! nx "$CP" 'grep -q "path: /var/lib/etcd$" /etc/kubernetes/manifests/etcd.yaml' """),
|
|
C("control plane healthy: etcd, apiserver, scheduler, controller-manager Running", 2, r"""for c in etcd kube-apiserver kube-scheduler kube-controller-manager; do eq "$(j -n kube-system pod -l component=$c '{.items[0].status.phase}')" Running || exit 1; done"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 103, "domain": "ARCH", "breaks": "w3-kubelet", "difficulty": "hard", "title": "Static pod on a relocated manifest path",
|
|
"params": lambda r: {"ctx": "k8s-c1-H", "ns": ns_pick(r, "edge"), "pod": name_pick(r, "edge-proxy")},
|
|
"task": CTX + """
|
|
Exactly one worker node is labelled `tier=edge`. On that node create a **static pod**:
|
|
- name `%{pod}`, namespace `%{ns}` (create the namespace),
|
|
- image `nginx:1.27`, requests cpu 50m / memory 32Mi, label `role=edge`.
|
|
|
|
The mirror pod must be Running. Note that the kubelet on that node may not be configured the
|
|
way you expect.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
for n in "$W1" "$W2" "$W3"; do $K label node $n tier- >/dev/null 2>&1 || true; done
|
|
$K label node "$W3" tier=edge --overwrite >/dev/null
|
|
nx "$W3" '
|
|
rm -f /etc/kubernetes/manifests/%{pod}*.yaml /etc/kubernetes/static/%{pod}*.yaml 2>/dev/null; mkdir -p /etc/kubernetes/static
|
|
sed -i "s|^staticPodPath: .*|staticPodPath: /etc/kubernetes/static|" /var/lib/kubelet/config.yaml
|
|
systemctl restart kubelet'
|
|
""",
|
|
"checks": [
|
|
C("mirror pod %{pod}-<edge node> Running in %{ns}", 4, r"""eq "$(j -n %{ns} pod "%{pod}-$W3" '{.status.phase}')" Running"""),
|
|
C("image, requests and label correct", 2, r"""p="$K -n %{ns} get pod %{pod}-$W3"; eq "$($p -o jsonpath='{.spec.containers[0].image}' 2>/dev/null)" nginx:1.27 && eq "$($p -o jsonpath='{.spec.containers[0].resources.requests.cpu}')" 50m && eq "$($p -o jsonpath='{.metadata.labels.role}')" edge"""),
|
|
C("manifest lives in the kubelet's actual staticPodPath", 2, r"""d=$(nx "$W3" 'grep staticPodPath /var/lib/kubelet/config.yaml' | awk '{print $2}'); nx "$W3" "grep -ls %{pod} $d/*.yaml" >/dev/null 2>&1"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 104, "domain": "ARCH", "difficulty": "hard", "title": "User onboarding to a working kubeconfig",
|
|
"params": lambda r: {"ctx": "k8s-c1-H", "user": r.choice(["maya", "theo", "iris"]), "ns": ns_pick(r, "dev")},
|
|
"task": CTX + """
|
|
Onboard user `%{user}` (group `developers`) end to end:
|
|
1. Key + CSR with CN=%{user}, O=developers; Kubernetes CSR object `%{user}`, approved, certificate issued.
|
|
2. A **self-contained** kubeconfig at `{WS}/q104/%{user}.kubeconfig` (certs embedded, not file paths)
|
|
with a context `%{user}@lab` whose default namespace is `%{ns}`.
|
|
3. `%{user}` may do everything to pods, deployments and services in `%{ns}` (edit-level), nothing in other namespaces.
|
|
|
|
`kubectl --kubeconfig {WS}/q104/%{user}.kubeconfig get pods` must work with no extra flags.
|
|
""",
|
|
"setup": NS + r"""mkns %{ns}; rm -rf "$WS/q104"; mkdir -p "$WS/q104"; $K delete csr %{user} --ignore-not-found >/dev/null; $K -n %{ns} delete rolebinding --all >/dev/null 2>&1""",
|
|
"checks": [
|
|
C("CSR %{user} approved with certificate, O=developers", 2, r"""[ -n "$(j csr %{user} '{.status.certificate}')" ] && j csr %{user} '{.status.certificate}' | base64 -d | openssl x509 -noout -subject 2>/dev/null | grep -q 'O *= *developers'"""),
|
|
C("kubeconfig is self-contained and works without flags", 3, r"""f="$WS/q104/%{user}.kubeconfig"; [ -f "$f" ] || exit 1; grep -q 'client-certificate-data' "$f" && grep -q 'certificate-authority-data' "$f" && ! grep -q 'client-certificate:' "$f" && kubectl --kubeconfig "$f" get pods >/dev/null 2>&1"""),
|
|
C("context %{user}@lab defaults to %{ns}", 1, r"""eq "$(kubectl --kubeconfig "$WS/q104/%{user}.kubeconfig" config view -o jsonpath='{.contexts[?(@.name=="%{user}@lab")].context.namespace}')" %{ns}"""),
|
|
C("edit rights in %{ns}, none elsewhere", 2, r"""eq "$($K auth can-i create deployments -n %{ns} --as=%{user} 2>/dev/null)" yes && eq "$($K auth can-i delete pods -n %{ns} --as=%{user} 2>/dev/null)" yes && eq "$($K auth can-i get pods -n default --as=%{user} 2>/dev/null || true)" no && eq "$($K auth can-i get secrets -n kube-system --as=%{user} 2>/dev/null || true)" no"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 105, "domain": "ARCH", "difficulty": "hard", "title": "Kustomize with generators and patches",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "kz"), "rep": r.choice([3, 4]), "lvl": r.choice(["debug", "trace", "warn"])},
|
|
"task": CTX + """
|
|
Base at `{WS}/q105/base` (Deployment `web`, Service `web`). Build overlay `{WS}/q105/overlays/prod` that:
|
|
- targets namespace `%{ns}` and adds label `tier=prod` to all resources (labels only, not selectors),
|
|
- sets replicas to %{rep} via a **strategic merge patch** file `replicas.yaml`,
|
|
- adds a `configMapGenerator` named `web-config` with literal `LOG_LEVEL=%{lvl}` (hash suffix enabled),
|
|
- injects that ConfigMap into the container via `envFrom` (JSON 6902 patch file `envfrom.yaml`).
|
|
|
|
Apply with `kubectl apply -k`. The Deployment must reference the hashed ConfigMap name.
|
|
""",
|
|
"setup": r"""
|
|
rm -rf "$WS/q105"; mkdir -p "$WS/q105/base"
|
|
cat > "$WS/q105/base/kustomization.yaml" <<'EOF'
|
|
resources: [deployment.yaml, service.yaml]
|
|
EOF
|
|
cat > "$WS/q105/base/deployment.yaml" <<'EOF'
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: web}
|
|
spec:
|
|
replicas: 1
|
|
selector: {matchLabels: {app: web}}
|
|
template:
|
|
metadata: {labels: {app: web}}
|
|
spec:
|
|
containers: [{name: web, image: nginx:1.25, ports: [{containerPort: 80}]}]
|
|
EOF
|
|
cat > "$WS/q105/base/service.yaml" <<'EOF'
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata: {name: web}
|
|
spec: {selector: {app: web}, ports: [{port: 80}]}
|
|
EOF
|
|
$K delete ns %{ns} --ignore-not-found --wait=false >/dev/null
|
|
""",
|
|
"checks": [
|
|
C("overlay has replicas.yaml (SMP) and envfrom.yaml (JSON6902)", 1, r"""[ -f "$WS/q105/overlays/prod/replicas.yaml" ] && [ -f "$WS/q105/overlays/prod/envfrom.yaml" ] && grep -q 'op:' "$WS/q105/overlays/prod/envfrom.yaml" """),
|
|
C("deployed to %{ns}: %{rep} replicas, tier=prod label, selector untouched", 2, r"""d="$K -n %{ns} get deploy web"; eq "$($d -o jsonpath='{.spec.replicas}' 2>/dev/null)" %{rep} && eq "$($d -o jsonpath='{.metadata.labels.tier}')" prod && [ -z "$($d -o jsonpath='{.spec.selector.matchLabels.tier}')" ]"""),
|
|
C("hashed ConfigMap with LOG_LEVEL=%{lvl}, referenced via envFrom", 3, r"""cm=$(j -n %{ns} deploy web '{.spec.template.spec.containers[0].envFrom[0].configMapRef.name}'); [ -n "$cm" ] && echo "$cm" | grep -Eq '^web-config-[a-z0-9]{6,}$' && eq "$(j -n %{ns} cm "$cm" '{.data.LOG_LEVEL}')" %{lvl}"""),
|
|
C("pods available and env visible", 1, r"""eq "$(j -n %{ns} deploy web '{.status.availableReplicas}')" %{rep} && $K -n %{ns} exec deploy/web -- env 2>/dev/null | grep -q '^LOG_LEVEL=%{lvl}$'"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 106, "domain": "ARCH", "difficulty": "hard", "title": "CRD with validation and defaults",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "kind": r.choice(["Rollout", "Canary", "Release"]), "env": r.choice(["prod", "stage"])},
|
|
"task": CTX + """
|
|
Create CRD `%{kind}` (group `deploy.example.com`, v1, Namespaced, plural lowercase+s, shortName `%{kind}`-initials) with schema:
|
|
- `spec.replicas` integer, **required**, minimum 1, maximum 10,
|
|
- `spec.environment` string, enum `prod|stage|dev`, **default** `dev`,
|
|
- `spec.image` string, required.
|
|
Then:
|
|
1. Create `%{kind}` `good` in `default` with replicas 3, image `nginx:1.27`, environment `%{env}`.
|
|
2. Save the API server's rejection message for a `%{kind}` with `replicas: 0` to `{WS}/q106/rejected.txt`.
|
|
3. Create `%{kind}` `defaulted` with only replicas 2 and image set; its environment must be `dev` after creation.
|
|
""",
|
|
"setup": r"""rm -rf "$WS/q106"; mkdir -p "$WS/q106"; k=$(echo %{kind} | tr A-Z a-z); $K delete crd ${k}s.deploy.example.com --ignore-not-found >/dev/null""",
|
|
"checks": [
|
|
C("CRD exists, Namespaced, replicas has min/max and is required", 2, r"""k=$(echo %{kind} | tr A-Z a-z); c="$K get crd ${k}s.deploy.example.com"; eq "$($c -o jsonpath='{.spec.scope}' 2>/dev/null)" Namespaced && s=$($c -o jsonpath='{.spec.versions[0].schema.openAPIV3Schema.properties.spec}'); echo "$s" | grep -q '"minimum":1' && echo "$s" | grep -q '"maximum":10' && echo "$s" | grep -q '"required":\[' """),
|
|
C("good created with environment %{env}", 1, r"""k=$(echo %{kind} | tr A-Z a-z); eq "$(j -n default ${k}s.deploy.example.com good '{.spec.environment}')" %{env}"""),
|
|
C("replicas 0 is rejected and the message was saved", 2, r"""grep -qiE 'minimum|greater than|>= 1|invalid' "$WS/q106/rejected.txt" 2>/dev/null"""),
|
|
C("defaulting works: defaulted.spec.environment == dev", 2, r"""k=$(echo %{kind} | tr A-Z a-z); eq "$(j -n default ${k}s.deploy.example.com defaulted '{.spec.environment}')" dev"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 122, "domain": "ARCH", "breaks": "certs", "difficulty": "hard", "title": "Renew control-plane certificates",
|
|
"params": lambda r: {"ctx": "k8s-c1-H"},
|
|
"task": CTX + """
|
|
Security policy requires the control-plane certificates to be renewed now.
|
|
On the control-plane node, renew **all** kubeadm-managed certificates, make every control-plane
|
|
component pick up the new certificates, and update the admin kubeconfig on the node
|
|
(`/etc/kubernetes/admin.conf`) so it uses the renewed client certificate.
|
|
Write the new `apiserver` certificate's expiry (as printed by `kubeadm certs check-expiration`) to `{WS}/q122/expiry.txt`.
|
|
""",
|
|
"setup": r"""rm -rf "$WS/q122"; mkdir -p "$WS/q122"; nx "$CP" 'openssl x509 -in /etc/kubernetes/pki/apiserver.crt -noout -startdate' > "$WS/.q122-before" """,
|
|
"checks": [
|
|
C("apiserver certificate was renewed (newer notBefore)", 3, r"""b=$(sed 's/notBefore=//' "$WS/.q122-before"); a=$(nx "$CP" 'openssl x509 -in /etc/kubernetes/pki/apiserver.crt -noout -startdate' | sed 's/notBefore=//'); [ "$(date -d "$a" +%s)" -gt "$(date -d "$b" +%s)" ]"""),
|
|
C("all components restarted and Running with the new certs (apiserver serving cert matches file)", 3, r"""
|
|
for c in kube-apiserver kube-scheduler kube-controller-manager etcd; do eq "$(j -n kube-system pod -l component=$c '{.items[0].status.phase}')" Running || exit 1; done
|
|
srv=$(nx "$CP" 'echo | openssl s_client -connect 127.0.0.1:6443 2>/dev/null | openssl x509 -noout -serial'); f=$(nx "$CP" 'openssl x509 -in /etc/kubernetes/pki/apiserver.crt -noout -serial'); [ -n "$srv" ] && eq "$srv" "$f" """),
|
|
C("admin.conf uses the renewed client cert", 2, r"""nx "$CP" 'grep client-certificate-data /etc/kubernetes/admin.conf | awk "{print \$2}" | base64 -d | openssl x509 -noout -startdate' | sed 's/notBefore=//' > /tmp/q122a; b=$(sed 's/notBefore=//' "$WS/.q122-before"); [ "$(date -d "$(cat /tmp/q122a)" +%s)" -gt "$(date -d "$b" +%s)" ]"""),
|
|
C("expiry.txt matches kubeadm's report for apiserver", 1, r"""exp=$(nx "$CP" 'kubeadm certs check-expiration 2>/dev/null' | awk '$1=="apiserver"{print $2,$3,$4,$5,$6}'); [ -n "$exp" ] && grep -qF "$exp" "$WS/q122/expiry.txt" 2>/dev/null"""),
|
|
],
|
|
},
|
|
# ================================================================ WORK ====
|
|
{
|
|
"id": 107, "domain": "WORK", "breaks": "w1-taint", "difficulty": "hard", "title": "Spread, tolerate, prioritise — under a quota",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "spread"), "dep": name_pick(r, "gateway"), "key": r.choice(["pool", "lane"]), "val": r.choice(["ingress", "edge"])},
|
|
"task": CTX + """
|
|
Node `{W1}` is tainted `%{key}=%{val}:NoSchedule`. In namespace `%{ns}` create Deployment `%{dep}`
|
|
(nginx:1.27, **3 replicas**) such that:
|
|
- exactly one pod runs on each worker node (required pod anti-affinity on `kubernetes.io/hostname`),
|
|
- it tolerates the taint on `{W1}`,
|
|
- it uses PriorityClass `%{dep}-prio` (value 50000, create it),
|
|
- requests cpu 100m / memory 64Mi.
|
|
|
|
All 3 pods must be Running. The namespace has a ResourceQuota — respect it or adjust it, but the
|
|
quota must still cap pods in the namespace.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
$K taint node "$W1" %{key}- >/dev/null 2>&1 || true; $K taint node "$W1" %{key}=%{val}:NoSchedule >/dev/null
|
|
$K -n %{ns} delete deploy %{dep} --ignore-not-found >/dev/null; $K delete priorityclass %{dep}-prio --ignore-not-found >/dev/null
|
|
$K apply -f - >/dev/null <<EOF
|
|
apiVersion: v1
|
|
kind: ResourceQuota
|
|
metadata: {name: %{ns}-quota, namespace: %{ns}}
|
|
spec: {hard: {pods: "2", requests.cpu: "250m"}}
|
|
EOF
|
|
""",
|
|
"checks": [
|
|
C("3 pods available, one per worker", 3, r"""eq "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" 3 && [ "$(j -n %{ns} pod -l app=%{dep} '{.items[*].spec.nodeName}' | tr ' ' '\n' | sort -u | wc -l)" = 3 ]"""),
|
|
C("required anti-affinity on hostname + toleration for %{key}", 2, r"""t=$(j -n %{ns} deploy %{dep} '{.spec.template.spec}'); echo "$t" | grep -q 'requiredDuringSchedulingIgnoredDuringExecution' && echo "$t" | grep -q '"topologyKey":"kubernetes.io/hostname"' && echo "$t" | grep -q '"key":"%{key}"' """),
|
|
C("PriorityClass %{dep}-prio (50000) in use", 1, r"""eq "$(j priorityclass %{dep}-prio '{.value}')" 50000 && eq "$(j -n %{ns} pod -l app=%{dep} '{.items[0].spec.priority}')" 50000"""),
|
|
C("quota still exists and caps pods (>=3)", 2, r"""p=$(j -n %{ns} quota %{ns}-quota '{.spec.hard.pods}'); [ -n "$p" ] && [ "$p" -ge 3 ]"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 108, "domain": "WORK", "difficulty": "hard", "title": "Rollback to a specific revision, then pause",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "rel"), "dep": name_pick(r, "portal")},
|
|
"task": CTX + """
|
|
Deployment `%{dep}` in `%{ns}` has gone through several releases; the current one is broken.
|
|
1. Roll it back to the revision that used image `nginx:1.25` (not simply the previous revision).
|
|
2. Scale it to 3 replicas and wait until available.
|
|
3. **Pause** the rollout, then change the image to `nginx:1.27` and add env `RELEASE=next`.
|
|
Because it is paused, no new ReplicaSet may be created and the running pods must stay on `nginx:1.25`.
|
|
Leave it paused.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}; $K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null
|
|
$K -n %{ns} create deploy %{dep} --image=nginx:1.25 >/dev/null
|
|
$K -n %{ns} rollout status deploy %{dep} --timeout=90s >/dev/null 2>&1
|
|
$K -n %{ns} set image deploy %{dep} nginx=nginx:1.26 >/dev/null; $K -n %{ns} rollout status deploy %{dep} --timeout=90s >/dev/null 2>&1
|
|
$K -n %{ns} set image deploy %{dep} nginx=nginx:1.27 >/dev/null; $K -n %{ns} rollout status deploy %{dep} --timeout=90s >/dev/null 2>&1
|
|
$K -n %{ns} set image deploy %{dep} nginx=nginx:9.99-broken >/dev/null
|
|
""",
|
|
"checks": [
|
|
C("running pods use nginx:1.25 and 3 are available", 3, r"""eq "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" 3 && for i in $(j -n %{ns} pod -l app=%{dep} '{.items[*].spec.containers[0].image}'); do eq "$i" nginx:1.25 || exit 1; done"""),
|
|
C("deployment is paused with template image nginx:1.27 and RELEASE=next", 2, r"""d="$K -n %{ns} get deploy %{dep}"; eq "$($d -o jsonpath='{.spec.paused}' 2>/dev/null)" true && eq "$($d -o jsonpath='{.spec.template.spec.containers[0].image}')" nginx:1.27 && eq "$($d -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="RELEASE")].value}')" next"""),
|
|
C("no ReplicaSet exists for the paused template (1.27 + RELEASE)", 2, r"""! $K -n %{ns} get rs -o json 2>/dev/null | python3 -c '
|
|
import sys,json
|
|
for rs in json.load(sys.stdin)["items"]:
|
|
c=rs["spec"]["template"]["spec"]["containers"][0]
|
|
if c["image"]=="nginx:1.27" and any(e.get("name")=="RELEASE" for e in c.get("env",[])): sys.exit(0)
|
|
sys.exit(1)'"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 109, "domain": "WORK", "difficulty": "hard", "title": "Native sidecar with probes and an exact report",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "side"), "pod": name_pick(r, "collector")},
|
|
"task": CTX + """
|
|
Create pod `%{pod}` in `%{ns}`:
|
|
- a **native sidecar** (an init container with `restartPolicy: Always`) named `agent` (busybox:1.36)
|
|
that appends `$(date)` to `/var/log/agent.log` every second,
|
|
- main container `app` (nginx:1.27) with a readiness probe (HTTP GET / on 80, period 5s)
|
|
and a liveness probe (TCP 80),
|
|
- shared `emptyDir` `logs` mounted at `/var/log` in both,
|
|
- `terminationGracePeriodSeconds: 5`.
|
|
When it is Running, write the names of **all** containers of the pod (init and regular), sorted,
|
|
one per line, to `{WS}/q109/containers.txt`, and the pod's `.status.podIP` to `{WS}/q109/ip.txt`.
|
|
""",
|
|
"setup": NS + r"""mkns %{ns}; rm -rf "$WS/q109"; mkdir -p "$WS/q109"; $K -n %{ns} delete pod %{pod} --ignore-not-found --wait=false >/dev/null""",
|
|
"checks": [
|
|
C("native sidecar: initContainer agent with restartPolicy Always", 2, r"""eq "$(j -n %{ns} pod %{pod} '{.spec.initContainers[?(@.name=="agent")].restartPolicy}')" Always"""),
|
|
C("probes on app as specified", 2, r"""c='{.spec.containers[?(@.name=="app")]'; eq "$(j -n %{ns} pod %{pod} "$c.readinessProbe.httpGet.path}")" / && eq "$(j -n %{ns} pod %{pod} "$c.readinessProbe.periodSeconds}")" 5 && eq "$(j -n %{ns} pod %{pod} "$c.livenessProbe.tcpSocket.port}")" 80"""),
|
|
C("Running, shared log written by agent, grace period 5", 2, r"""eq "$(j -n %{ns} pod %{pod} '{.status.phase}')" Running && eq "$(j -n %{ns} pod %{pod} '{.spec.terminationGracePeriodSeconds}')" 5 && [ -n "$($K -n %{ns} exec %{pod} -c app -- sh -c 'tail -1 /var/log/agent.log' 2>/dev/null)" ]"""),
|
|
C("report files exact", 2, r"""want=$( (j -n %{ns} pod %{pod} '{.spec.initContainers[*].name}'; echo; j -n %{ns} pod %{pod} '{.spec.containers[*].name}') | tr ' ' '\n' | grep . | sort); [ "$(grep . "$WS/q109/containers.txt" 2>/dev/null | sort)" = "$want" ] && eq "$(tr -d '[:space:]' < "$WS/q109/ip.txt" 2>/dev/null)" "$(j -n %{ns} pod %{pod} '{.status.podIP}')" """),
|
|
],
|
|
},
|
|
{
|
|
"id": 110, "domain": "WORK", "difficulty": "hard", "title": "Cluster audit to exact files",
|
|
"params": lambda r: {"ctx": "k8s-c1-H"},
|
|
"task": CTX + """
|
|
Produce three files, contents **exactly** as specified (no headers, sorted with `sort`, one per line):
|
|
1. `{WS}/q110/no-readiness.txt` — `<namespace>/<pod>` for every pod in the cluster whose first
|
|
container has **no** readiness probe. Exclude namespace `kube-system`.
|
|
2. `{WS}/q110/per-node.txt` — `<node> <count>` of Running pods per node, all namespaces.
|
|
3. `{WS}/q110/images.txt` — every distinct container image running in namespace `kube-system`.
|
|
""",
|
|
"setup": r"""rm -rf "$WS/q110"; mkdir -p "$WS/q110" """,
|
|
"checks": [
|
|
C("no-readiness.txt exact", 3, r"""want=$($K get pods -A -o json 2>/dev/null | python3 -c '
|
|
import sys,json
|
|
for p in json.load(sys.stdin)["items"]:
|
|
if p["metadata"]["namespace"]=="kube-system": continue
|
|
if not p["spec"]["containers"][0].get("readinessProbe"): print(p["metadata"]["namespace"]+"/"+p["metadata"]["name"])' | sort); [ "$(grep . "$WS/q110/no-readiness.txt" 2>/dev/null | sort)" = "$want" ]"""),
|
|
C("per-node.txt exact", 2, r"""want=$($K get pods -A -o json 2>/dev/null | python3 -c '
|
|
import sys,json,collections
|
|
c=collections.Counter(p["spec"].get("nodeName") for p in json.load(sys.stdin)["items"] if p["status"]["phase"]=="Running" and p["spec"].get("nodeName"))
|
|
for n in sorted(c): print(n, c[n])'); [ "$(grep . "$WS/q110/per-node.txt" 2>/dev/null | sort)" = "$(echo "$want" | sort)" ]"""),
|
|
C("images.txt exact", 2, r"""want=$(j -n kube-system pods '{.items[*].spec.containers[*].image}' | tr ' ' '\n' | sort -u); [ "$(grep . "$WS/q110/images.txt" 2>/dev/null | sort -u)" = "$want" ]"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 121, "domain": "WORK", "difficulty": "hard", "title": "The context's namespace",
|
|
"params": lambda r: {"ctx": "k8s-c2-AC", "ns": ns_pick(r, "ac"), "dep": name_pick(r, "ledger")},
|
|
"task": CTX + """
|
|
In the **current namespace of that context** (do not pass `-n`), create Deployment `%{dep}`
|
|
(busybox:1.36, `sleep 3600`, 2 replicas, label `owner=ac`) and a ClusterIP Service `%{dep}`
|
|
on port 9000 targeting port 9000. Then write the namespace name you deployed into, as reported
|
|
by `kubectl config view --minify`, to `{WS}/q121/ns.txt`.
|
|
""",
|
|
"setup": NS + r"""mkns %{ns}; rm -rf "$WS/q121"; mkdir -p "$WS/q121"; $K -n %{ns} delete deploy,svc %{dep} --ignore-not-found >/dev/null; $K -n default delete deploy,svc %{dep} --ignore-not-found >/dev/null""",
|
|
"checks": [
|
|
C("Deployment %{dep} in %{ns} (the context's namespace), 2 available, owner=ac", 3, r"""eq "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" 2 && eq "$(j -n %{ns} deploy %{dep} '{.spec.template.metadata.labels.owner}')" ac"""),
|
|
C("Service %{dep} port 9000 in the same namespace", 1, r"""eq "$(j -n %{ns} svc %{dep} '{.spec.ports[0].port}')" 9000"""),
|
|
C("nothing was created in namespace default by mistake", 1, r"""! $K -n default get deploy %{dep} >/dev/null 2>&1"""),
|
|
C("ns.txt has the namespace", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q121/ns.txt" 2>/dev/null)" %{ns}"""),
|
|
],
|
|
},
|
|
# ================================================================= NET ====
|
|
{
|
|
"id": 111, "domain": "NET", "difficulty": "hard", "title": "Zero-trust namespace",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "zt"), "from": ns_pick(r, "front")},
|
|
"task": CTX + """
|
|
Namespace `%{ns}` runs `api` (label `app=api`, port 8080) and `db` (label `app=db`, port 5432).
|
|
Implement zero trust with NetworkPolicies:
|
|
1. `default-deny`: deny all ingress **and** egress for every pod in `%{ns}`.
|
|
2. `allow-dns`: every pod may egress to DNS (UDP/TCP 53) in `kube-system` only.
|
|
3. `api-ingress`: `api` accepts ingress on 8080 only from pods in namespaces labelled `team=%{from}`.
|
|
4. `api-to-db`: `api` may egress to `db` on 5432; `db` accepts ingress on 5432 from `api` only.
|
|
Remove anything that contradicts this. Label namespace `%{from}` appropriately.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}; mkns %{from}
|
|
$K -n %{ns} create deploy api --image=nginx:1.25 --port=8080 --dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
$K -n %{ns} create deploy db --image=nginx:1.25 --port=5432 --dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
$K -n %{ns} delete netpol --all >/dev/null 2>&1
|
|
# trap: an allow-all left behind
|
|
$K apply -f - >/dev/null <<EOF
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata: {name: temp-allow-all, namespace: %{ns}}
|
|
spec: {podSelector: {}, policyTypes: [Ingress, Egress], ingress: [{}], egress: [{}]}
|
|
EOF
|
|
$K label ns %{from} team- >/dev/null 2>&1 || true
|
|
""",
|
|
"checks": [
|
|
C("default-deny covers Ingress and Egress for all pods; allow-all removed", 2, r"""n="$K -n %{ns} get netpol default-deny"; eq "$($n -o jsonpath='{.spec.podSelector}' 2>/dev/null)" '{}' && $n -o jsonpath='{.spec.policyTypes}' | grep -q Ingress && $n -o jsonpath='{.spec.policyTypes}' | grep -q Egress && [ -z "$($n -o jsonpath='{.spec.ingress}')" ] && [ -z "$($n -o jsonpath='{.spec.egress}')" ] && ! $K -n %{ns} get netpol temp-allow-all >/dev/null 2>&1"""),
|
|
C("allow-dns: egress 53 UDP+TCP to kube-system only", 2, r"""$K -n %{ns} get netpol allow-dns -o json 2>/dev/null | python3 -c '
|
|
import sys,json; s=json.load(sys.stdin)["spec"]; e=s.get("egress",[])
|
|
ok=s.get("podSelector")=={} and any({p.get("protocol") for p in r.get("ports",[]) if str(p.get("port"))=="53"}>={"UDP","TCP"} and all("kube-system" in json.dumps(t.get("namespaceSelector",{})) for t in r.get("to",[])) and r.get("to") for r in e)
|
|
sys.exit(0 if ok else 1)'"""),
|
|
C("api-ingress: from namespaces team=%{from} on 8080 only; namespace labelled", 2, r"""eq "$(j ns %{from} '{.metadata.labels.team}')" %{from} && $K -n %{ns} get netpol api-ingress -o json 2>/dev/null | python3 -c '
|
|
import sys,json; s=json.load(sys.stdin)["spec"]
|
|
ok=s["podSelector"].get("matchLabels",{}).get("app")=="api" and len(s.get("ingress",[]))==1 and all(str(p.get("port"))=="8080" for p in s["ingress"][0].get("ports",[])) and s["ingress"][0].get("ports") and all(t.get("namespaceSelector",{}).get("matchLabels",{}).get("team")=="%{from}" for t in s["ingress"][0].get("from",[])) and s["ingress"][0].get("from")
|
|
sys.exit(0 if ok else 1)'"""),
|
|
C("api-to-db: api egress to db:5432 and db ingress from api:5432", 2, r"""$K -n %{ns} get netpol -o json 2>/dev/null | python3 -c '
|
|
import sys,json; items=json.load(sys.stdin)["items"]; eg=ing=False
|
|
for n in items:
|
|
s=n["spec"]; app=s["podSelector"].get("matchLabels",{}).get("app")
|
|
for r in s.get("egress",[]):
|
|
if app=="api" and any(t.get("podSelector",{}).get("matchLabels",{}).get("app")=="db" for t in r.get("to",[])) and any(str(p.get("port"))=="5432" for p in r.get("ports",[])): eg=True
|
|
for r in s.get("ingress",[]):
|
|
if app=="db" and any(t.get("podSelector",{}).get("matchLabels",{}).get("app")=="api" for t in r.get("from",[])) and any(str(p.get("port"))=="5432" for p in r.get("ports",[])): ing=True
|
|
sys.exit(0 if eg and ing else 1)'"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 112, "domain": "NET", "difficulty": "hard", "title": "TLS Ingress with a generated certificate",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "tls"), "host": r.choice(["secure.lab.local", "portal.lab.local", "admin.lab.local"])},
|
|
"task": CTX + """
|
|
Namespace `%{ns}` has Deployment+Service `web` (port 80).
|
|
1. Generate a self-signed certificate (RSA 2048, 365 days) with CN `%{host}` and store it as a
|
|
Secret `%{host}-tls` of type `kubernetes.io/tls`.
|
|
2. Create Ingress `web` (class `nginx`) serving host `%{host}` with that TLS secret, path `/` Prefix → `web:80`,
|
|
and the annotation `nginx.ingress.kubernetes.io/ssl-redirect: "true"`.
|
|
3. Write the certificate's SHA-256 fingerprint (`openssl x509 -fingerprint -sha256`, the value only) to `{WS}/q112/fp.txt`.
|
|
""",
|
|
"setup": NS + r"""mkns %{ns}; rm -rf "$WS/q112"; mkdir -p "$WS/q112"; $K -n %{ns} create deploy web --image=nginx:1.25 --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null; $K -n %{ns} expose deploy web --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null; $K -n %{ns} delete secret %{host}-tls --ignore-not-found >/dev/null; $K -n %{ns} delete ingress web --ignore-not-found >/dev/null""",
|
|
"checks": [
|
|
C("Secret %{host}-tls is kubernetes.io/tls with CN %{host}", 2, r"""eq "$(j -n %{ns} secret %{host}-tls '{.type}')" kubernetes.io/tls && j -n %{ns} secret %{host}-tls '{.data.tls\.crt}' | base64 -d | openssl x509 -noout -subject 2>/dev/null | grep -q 'CN *= *%{host}'"""),
|
|
C("Ingress web: class nginx, host, TLS secret, ssl-redirect annotation", 3, r"""i="$K -n %{ns} get ingress web"; eq "$($i -o jsonpath='{.spec.ingressClassName}' 2>/dev/null)" nginx && eq "$($i -o jsonpath='{.spec.tls[0].secretName}')" %{host}-tls && eq "$($i -o jsonpath='{.spec.tls[0].hosts[0]}')" %{host} && eq "$($i -o jsonpath='{.spec.rules[0].host}')" %{host} && eq "$($i -o jsonpath='{.spec.rules[0].http.paths[0].backend.service.name}')" web && eq "$($i -o jsonpath='{.metadata.annotations.nginx\.ingress\.kubernetes\.io/ssl-redirect}')" true"""),
|
|
C("fp.txt matches the secret's certificate fingerprint", 2, r"""fp=$(j -n %{ns} secret %{host}-tls '{.data.tls\.crt}' | base64 -d | openssl x509 -noout -fingerprint -sha256 2>/dev/null | sed 's/.*=//'); [ -n "$fp" ] && grep -qiF "$fp" "$WS/q112/fp.txt" 2>/dev/null"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 113, "domain": "NET", "difficulty": "hard", "title": "Gateway API canary with header match",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "canary"), "host": r.choice(["shop", "pay", "cart"]), "w": r.choice([10, 20, 25])},
|
|
"task": CTX + """
|
|
Gateway API CRDs are installed. Namespace `%{ns}` has Services `v1` and `v2` (port 80).
|
|
Create in `%{ns}`:
|
|
- Gateway `%{host}-gw` (class `%{host}-gc`, create the GatewayClass, controllerName `example.com/gw`)
|
|
with two HTTP listeners: `web` for `%{host}.example.com` and `api` for `api.%{host}.example.com`, both port 80.
|
|
- HTTPRoute `%{host}-route` attached to listener `web` only (`sectionName`), with:
|
|
- a rule matching header `X-Canary: always` → `v2` (100%),
|
|
- a default rule splitting `v1` %{w2}% / `v2` %{w}% by weight.
|
|
""",
|
|
"params_post": None,
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
$K get crd gateways.gateway.networking.k8s.io >/dev/null 2>&1 || { curl -sfL --max-time 60 https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/standard-install.yaml -o /tmp/gwapi.yaml && $K apply -f /tmp/gwapi.yaml >/dev/null 2>&1; } || echo "warn: could not install Gateway API CRDs"
|
|
for v in v1 v2; do $K -n %{ns} create deploy $v --image=nginx:1.25 --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null; $K -n %{ns} expose deploy $v --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null; done
|
|
$K delete gatewayclass %{host}-gc --ignore-not-found >/dev/null 2>&1; $K -n %{ns} delete gateway %{host}-gw --ignore-not-found >/dev/null 2>&1; $K -n %{ns} delete httproute %{host}-route --ignore-not-found >/dev/null 2>&1
|
|
""",
|
|
"checks": [
|
|
C("Gateway with listeners web and api on the right hostnames", 2, r"""g="$K -n %{ns} get gateway %{host}-gw"; eq "$($g -o jsonpath='{.spec.gatewayClassName}' 2>/dev/null)" %{host}-gc && eq "$($g -o jsonpath='{.spec.listeners[?(@.name=="web")].hostname}')" %{host}.example.com && eq "$($g -o jsonpath='{.spec.listeners[?(@.name=="api")].hostname}')" api.%{host}.example.com"""),
|
|
C("route attaches to listener web only (sectionName)", 1, r"""eq "$(j -n %{ns} httproute %{host}-route '{.spec.parentRefs[0].sectionName}')" web && [ -z "$(j -n %{ns} httproute %{host}-route '{.spec.parentRefs[1]}')" ]"""),
|
|
C("header rule X-Canary: always -> v2", 2, r"""$K -n %{ns} get httproute %{host}-route -o json 2>/dev/null | python3 -c '
|
|
import sys,json; r=json.load(sys.stdin)["spec"]["rules"]
|
|
ok=any(any(any(h.get("name","").lower()=="x-canary" and h.get("value")=="always" for h in m.get("headers",[])) for m in x.get("matches",[])) and x["backendRefs"][0]["name"]=="v2" and len(x["backendRefs"])==1 for x in r)
|
|
sys.exit(0 if ok else 1)'"""),
|
|
C("default rule weights v1 %{w2} / v2 %{w}", 2, r"""$K -n %{ns} get httproute %{host}-route -o json 2>/dev/null | python3 -c '
|
|
import sys,json; r=json.load(sys.stdin)["spec"]["rules"]
|
|
ok=any(not x.get("matches") or all(not m.get("headers") for m in x["matches"]) and {b["name"]:b.get("weight") for b in x["backendRefs"]}=={"v1":%{w2},"v2":%{w}} for x in r if len(x.get("backendRefs",[]))==2)
|
|
sys.exit(0 if ok else 1)'"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 114, "domain": "NET", "breaks": "coredns", "difficulty": "hard", "title": "DNS down: three faults", "disruptive": True,
|
|
"params": lambda r: {"ctx": "k8s-c1-H"},
|
|
"task": CTX + """
|
|
Nothing in the cluster resolves DNS. Restore it. There is more than one cause.
|
|
When done, `nslookup kubernetes.default` and `nslookup kubernetes.io` must succeed from a pod
|
|
in `default`, CoreDNS must have 2 available replicas, and no NetworkPolicy may remain in `kube-system`.
|
|
""",
|
|
"setup": r"""
|
|
$K -n kube-system get cm coredns -o jsonpath='{.data.Corefile}' > /tmp/Corefile.q114
|
|
sed -i 's|forward \. /etc/resolv.conf|forward . 10.255.255.254|' /tmp/Corefile.q114
|
|
$K -n kube-system create cm coredns --from-file=Corefile=/tmp/Corefile.q114 --dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
$K -n kube-system scale deploy coredns --replicas=0 >/dev/null
|
|
$K apply -f - >/dev/null <<EOF
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata: {name: maintenance-lockdown, namespace: kube-system}
|
|
spec: {podSelector: {matchLabels: {k8s-app: kube-dns}}, policyTypes: [Ingress], ingress: []}
|
|
EOF
|
|
""",
|
|
"checks": [
|
|
C("Corefile forwards to /etc/resolv.conf", 2, r"""j -n kube-system cm coredns '{.data.Corefile}' | grep -q 'forward \. /etc/resolv.conf'"""),
|
|
C("CoreDNS has 2 available replicas", 2, r"""for i in $(seq 1 12); do [ "$(j -n kube-system deploy coredns '{.status.availableReplicas}')" = 2 ] && exit 0; sleep 5; done; exit 1"""),
|
|
C("no NetworkPolicy in kube-system", 1, r"""[ "$($K -n kube-system get netpol --no-headers 2>/dev/null | wc -l)" = 0 ]"""),
|
|
C("resolution works from a pod", 3, r"""
|
|
$K -n default delete pod dns-probe --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K -n default run dns-probe --image=busybox:1.36 --restart=Never -- sh -c 'for i in 1 2 3 4 5 6; do nslookup kubernetes.default >/dev/null 2>&1 && nslookup kubernetes.io >/dev/null 2>&1 && exit 0; sleep 5; done; exit 1' >/dev/null 2>&1
|
|
for i in $(seq 1 20); do ph=$(j -n default pod dns-probe '{.status.phase}'); [ "$ph" = Succeeded ] && { $K -n default delete pod dns-probe --wait=false >/dev/null; exit 0; }; [ "$ph" = Failed ] && break; sleep 3; done
|
|
$K -n default delete pod dns-probe --wait=false >/dev/null; exit 1"""),
|
|
],
|
|
},
|
|
# ================================================================ STOR ====
|
|
{
|
|
"id": 115, "domain": "STOR", "difficulty": "hard", "title": "Bind the right local volume",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "lv"), "pvc": name_pick(r, "fast-data")},
|
|
"task": CTX + """
|
|
Two PersistentVolumes of class `local-fast` exist. Only one of them lives on the node labelled
|
|
`disk=nvme`. Create PVC `%{pvc}` in `%{ns}` (RWO, 1Gi, class `local-fast`) that binds to **that**
|
|
PV — use a label selector, do not edit the PVs — and pod `consumer` (busybox:1.36, `sleep 3600`)
|
|
mounting it at `/data`. The pod must run on the nvme node. Do not delete either PV.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
for n in "$W1" "$W2" "$W3"; do $K label node $n disk- >/dev/null 2>&1 || true; done
|
|
$K label node "$W2" disk=nvme --overwrite >/dev/null
|
|
$K -n %{ns} delete pod consumer --ignore-not-found --wait=false >/dev/null; $K -n %{ns} delete pvc %{pvc} --ignore-not-found >/dev/null
|
|
$K delete pv lv-hdd lv-nvme --ignore-not-found >/dev/null
|
|
for pair in "lv-hdd:$W1:hdd:2Gi" "lv-nvme:$W2:nvme:1Gi"; do IFS=: read name node kind size <<<"$pair"
|
|
nx "$node" "mkdir -p /mnt/$name"
|
|
$K apply -f - >/dev/null <<EOF
|
|
apiVersion: v1
|
|
kind: PersistentVolume
|
|
metadata: {name: $name, labels: {media: $kind}}
|
|
spec:
|
|
capacity: {storage: $size}
|
|
accessModes: [ReadWriteOnce]
|
|
persistentVolumeReclaimPolicy: Retain
|
|
storageClassName: local-fast
|
|
local: {path: /mnt/$name}
|
|
nodeAffinity: {required: {nodeSelectorTerms: [{matchExpressions: [{key: kubernetes.io/hostname, operator: In, values: [$node]}]}]}}
|
|
EOF
|
|
done
|
|
""",
|
|
"checks": [
|
|
C("PVC uses a selector and is Bound to lv-nvme", 3, r"""[ -n "$(j -n %{ns} pvc %{pvc} '{.spec.selector}')" ] && eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}')" lv-nvme && eq "$(j -n %{ns} pvc %{pvc} '{.status.phase}')" Bound"""),
|
|
C("consumer Running on the nvme node with /data mounted", 2, r"""eq "$(j -n %{ns} pod consumer '{.status.phase}')" Running && eq "$(j -n %{ns} pod consumer '{.spec.nodeName}')" "$W2" && j -n %{ns} pod consumer '{.spec.containers[0].volumeMounts[*].mountPath}' | grep -qw /data"""),
|
|
C("both PVs still exist, unmodified affinity", 1, r"""eq "$(j pv lv-hdd '{.spec.nodeAffinity.required.nodeSelectorTerms[0].matchExpressions[0].values[0]}')" "$W1" && eq "$(j pv lv-nvme '{.spec.nodeAffinity.required.nodeSelectorTerms[0].matchExpressions[0].values[0]}')" "$W2" """),
|
|
],
|
|
},
|
|
{
|
|
"id": 116, "domain": "STOR", "difficulty": "hard", "title": "Migrate data between claims",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "mig"), "n": r.choice([7, 11, 13])},
|
|
"task": CTX + """
|
|
In `%{ns}`, PVC `old-data` holds files a running pod `legacy` wrote (`/data`). Storage is being
|
|
replaced:
|
|
1. Create PVC `new-data` (default StorageClass, 1Gi, RWO).
|
|
2. Copy **everything** from `old-data` to `new-data` using a Job named `migrate` (busybox:1.36).
|
|
The Job must complete; use whatever mounts you need.
|
|
3. Re-point pod `legacy` to `new-data` (same name, same namespace, mount at `/data`) and get it Running.
|
|
4. Delete PVC `old-data` afterwards. Its PV must be released, not deleted (check its reclaim policy first).
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
$K -n %{ns} delete pod legacy --ignore-not-found --wait=true >/dev/null; $K -n %{ns} delete job migrate --ignore-not-found >/dev/null; $K -n %{ns} delete pvc old-data new-data --ignore-not-found >/dev/null
|
|
$K apply -f - >/dev/null <<EOF
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: old-data, namespace: %{ns}}
|
|
spec: {accessModes: [ReadWriteOnce], resources: {requests: {storage: 1Gi}}}
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: legacy, namespace: %{ns}}
|
|
spec:
|
|
containers: [{name: app, image: busybox:1.36, command: [sh, -c, "for i in \$(seq 1 %{n}); do echo record-\$i > /data/file-\$i.txt; done; sleep 3600"], volumeMounts: [{name: d, mountPath: /data}]}]
|
|
volumes: [{name: d, persistentVolumeClaim: {claimName: old-data}}]
|
|
EOF
|
|
for i in $(seq 1 30); do [ "$(j -n %{ns} pod legacy '{.status.phase}')" = Running ] && break; sleep 2; done
|
|
pv=$(j -n %{ns} pvc old-data '{.spec.volumeName}'); [ -n "$pv" ] && $K patch pv $pv -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' >/dev/null
|
|
echo "$pv" > "$WS/.q116-pv"
|
|
""",
|
|
"checks": [
|
|
C("Job migrate completed", 2, r"""eq "$(j -n %{ns} job migrate '{.status.succeeded}')" 1"""),
|
|
C("legacy Running on new-data with all %{n} files intact", 3, r"""eq "$(j -n %{ns} pod legacy '{.status.phase}')" Running && eq "$(j -n %{ns} pod legacy '{.spec.volumes[?(@.name!="kube-api-access")].persistentVolumeClaim.claimName}' | tr ' ' '\n' | grep -v '^$' | head -1)" new-data && eq "$($K -n %{ns} exec legacy -- sh -c 'ls /data/file-*.txt | wc -l' 2>/dev/null | tr -d ' ')" %{n} && eq "$($K -n %{ns} exec legacy -- cat /data/file-%{n}.txt 2>/dev/null)" record-%{n}"""),
|
|
C("old-data deleted; its PV Released (not deleted)", 2, r"""! $K -n %{ns} get pvc old-data >/dev/null 2>&1 && pv=$(cat "$WS/.q116-pv" 2>/dev/null) && eq "$(j pv $pv '{.status.phase}')" Released"""),
|
|
],
|
|
},
|
|
# ============================================================== TSHOOT ====
|
|
{
|
|
"id": 117, "domain": "TSHOOT", "breaks": "w2-kubelet", "difficulty": "hard", "title": "Worker node: three things wrong", "disruptive": True,
|
|
"params": lambda r: {"ctx": "k8s-c1-H"},
|
|
"task": CTX + """
|
|
Node `{W2}` is unusable. Make it fully usable again: `Ready`, schedulable, kubelet running and
|
|
enabled, and a new pod pinned to it must reach Running. Nothing else on the cluster may be changed.
|
|
""",
|
|
"setup": r"""
|
|
$K cordon "$W2" >/dev/null 2>&1 || true
|
|
nx "$W2" '
|
|
sed -i "s|clientCAFile: /etc/kubernetes/pki/ca.crt|clientCAFile: /etc/kubernetes/pki/ca-wrong.crt|" /var/lib/kubelet/config.yaml
|
|
systemctl disable kubelet >/dev/null 2>&1; systemctl stop kubelet'
|
|
""",
|
|
"checks": [
|
|
C("node Ready", 3, r"""for i in $(seq 1 12); do [ "$(j node "$W2" '{.status.conditions[?(@.type=="Ready")].status}')" = True ] && exit 0; sleep 5; done; exit 1"""),
|
|
C("node uncordoned (collateral check)", 2, r"""[ "$(j node "$W2" '{.spec.unschedulable}')" != true ]"""),
|
|
C("kubelet active and enabled, config points at a real CA", 2, r"""eq "$(nx "$W2" 'systemctl is-active kubelet')" active && eq "$(nx "$W2" 'systemctl is-enabled kubelet')" enabled && f=$(nx "$W2" 'grep clientCAFile /var/lib/kubelet/config.yaml' | awk '{print $2}') && nx "$W2" "test -f $f" """),
|
|
C("a pod pinned to the node runs", 2, r"""
|
|
$K -n default delete pod node-probe --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K -n default run node-probe --image=busybox:1.36 --restart=Never --overrides="{\"spec\":{\"nodeName\":\"$W2\"}}" -- sleep 120 >/dev/null 2>&1
|
|
ok=1; for i in $(seq 1 20); do [ "$(j -n default pod node-probe '{.status.phase}')" = Running ] && { ok=0; break; }; sleep 3; done
|
|
$K -n default delete pod node-probe --wait=false >/dev/null 2>&1; exit $ok"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 118, "domain": "TSHOOT", "breaks": "scheduler", "difficulty": "hard", "title": "Two control-plane components down", "disruptive": True,
|
|
"params": lambda r: {"ctx": "k8s-c1-H"},
|
|
"task": CTX + """
|
|
New Deployments create no pods, and pods that do get created stay `Pending`. Two different
|
|
control-plane components are failing for two different reasons. Repair both on the control-plane node.
|
|
Afterwards a new Deployment must reach Available within a minute.
|
|
""",
|
|
"setup": r"""
|
|
nx "$CP" '
|
|
f=/etc/kubernetes/manifests/kube-scheduler.yaml; grep -q "^ *- kube-scheduler$" $f && sed -i "s|^\( *\)- kube-scheduler$|\1- kube-schedulerr|" $f
|
|
g=/etc/kubernetes/manifests/kube-controller-manager.yaml; grep -q "image: registry.k8s.io/kube-controller-manager:v" $g && sed -i "s|\(image: registry.k8s.io/kube-controller-manager:v[0-9.]*\)|\1-broken|" $g
|
|
'
|
|
""",
|
|
"checks": [
|
|
C("kube-scheduler manifest repaired and Running", 3, r"""nx "$CP" 'grep -q "^ *- kube-scheduler$" /etc/kubernetes/manifests/kube-scheduler.yaml' && eq "$(j -n kube-system pod -l component=kube-scheduler '{.items[0].status.phase}')" Running"""),
|
|
C("kube-controller-manager image repaired and Running", 3, r"""! nx "$CP" 'grep -q "\-broken" /etc/kubernetes/manifests/kube-controller-manager.yaml' && eq "$(j -n kube-system pod -l component=kube-controller-manager '{.items[0].status.phase}')" Running"""),
|
|
C("a new Deployment becomes Available", 3, r"""
|
|
$K -n default delete deploy cp-probe --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K -n default create deploy cp-probe --image=busybox:1.36 -- sleep 300 >/dev/null 2>&1
|
|
ok=1; for i in $(seq 1 20); do [ "$(j -n default deploy cp-probe '{.status.availableReplicas}')" = 1 ] && { ok=0; break; }; sleep 3; done
|
|
$K -n default delete deploy cp-probe --wait=false >/dev/null 2>&1; exit $ok"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 119, "domain": "TSHOOT", "difficulty": "hard", "title": "Application down for four reasons",
|
|
"params": lambda r: {"ctx": ctx_pick(r), "ns": ns_pick(r, "checkout"), "dep": name_pick(r, "checkout")},
|
|
"task": CTX + """
|
|
Deployment `%{dep}` in `%{ns}` should have 3 Available replicas and Service `%{dep}` should route
|
|
to them on port 80. Right now nothing works. Fix it **in place** — the Deployment must keep its
|
|
UID (do not delete and recreate it) and the Service must keep its ClusterIP.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
$K -n %{ns} delete deploy,svc,quota,secret --all >/dev/null 2>&1
|
|
$K apply -f - >/dev/null <<EOF
|
|
apiVersion: v1
|
|
kind: ResourceQuota
|
|
metadata: {name: tight, namespace: %{ns}}
|
|
spec: {hard: {pods: "1"}}
|
|
---
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata: {name: %{dep}-env, namespace: %{ns}}
|
|
stringData: {API_KEY: abc123}
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: %{dep}, namespace: %{ns}}
|
|
spec:
|
|
replicas: 3
|
|
selector: {matchLabels: {app: %{dep}}}
|
|
template:
|
|
metadata: {labels: {app: %{dep}}}
|
|
spec:
|
|
containers:
|
|
- name: web
|
|
image: nginx:1.27-nope
|
|
ports: [{containerPort: 80}]
|
|
env: [{name: DB_URL, valueFrom: {secretKeyRef: {name: %{dep}-env, key: DATABASE_URL}}}]
|
|
resources: {requests: {cpu: 10m, memory: 16Mi}}
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata: {name: %{dep}, namespace: %{ns}}
|
|
spec:
|
|
selector: {app: %{dep}-web}
|
|
ports: [{port: 80, targetPort: 8080}]
|
|
EOF
|
|
j -n %{ns} deploy %{dep} '{.metadata.uid}' > "$WS/.q119-uid"; j -n %{ns} svc %{dep} '{.spec.clusterIP}' > "$WS/.q119-ip"
|
|
""",
|
|
"checks": [
|
|
C("3 replicas Available", 3, r"""for i in $(seq 1 6); do [ "$(j -n %{ns} deploy %{dep} '{.status.availableReplicas}')" = 3 ] && exit 0; sleep 5; done; exit 1"""),
|
|
C("Service routes: endpoints ready, targetPort 80", 2, r"""eq "$(j -n %{ns} svc %{dep} '{.spec.ports[0].targetPort}')" 80 && [ -n "$(j -n %{ns} endpointslices -l kubernetes.io/service-name=%{dep} '{.items[*].endpoints[?(@.conditions.ready==true)].addresses[*]}')" ]"""),
|
|
C("Deployment UID and Service ClusterIP unchanged (fixed in place)", 2, r"""eq "$(j -n %{ns} deploy %{dep} '{.metadata.uid}')" "$(cat "$WS/.q119-uid")" && eq "$(j -n %{ns} svc %{dep} '{.spec.clusterIP}')" "$(cat "$WS/.q119-ip")" """),
|
|
C("quota still present and permits the 3 pods", 1, r"""p=$(j -n %{ns} quota tight '{.spec.hard.pods}'); [ -n "$p" ] && [ "$p" -ge 3 ]"""),
|
|
],
|
|
},
|
|
{
|
|
"id": 120, "domain": "TSHOOT", "difficulty": "hard", "title": "Cluster facts, exactly",
|
|
"params": lambda r: {"ctx": "k8s-c1-H", "node": r.choice(["{W1}", "{W2}", "{W3}"])},
|
|
"task": CTX + """
|
|
Write `{WS}/q120/facts.txt` with these lines, values taken from the running cluster:
|
|
```
|
|
pod-cidr=<cluster pod CIDR as configured on kube-controller-manager>
|
|
service-cidr=<service CIDR as configured on kube-apiserver>
|
|
dns-ip=<ClusterIP of the kube-dns Service>
|
|
proxy-mode=<kube-proxy mode from its ConfigMap (empty means iptables — write iptables)>
|
|
%{node}-pod-cidr=<.spec.podCIDR of that node>
|
|
```
|
|
""",
|
|
"setup": r"""rm -rf "$WS/q120"; mkdir -p "$WS/q120" """,
|
|
"checks": [
|
|
C("pod-cidr and service-cidr", 2, r"""f="$WS/q120/facts.txt"; pc=$(j -n kube-system pod -l component=kube-controller-manager '{.items[0].spec.containers[0].command}' | tr ',' '\n' | grep -o 'cluster-cidr=[^"]*' | cut -d= -f2); sc=$(j -n kube-system pod -l component=kube-apiserver '{.items[0].spec.containers[0].command}' | tr ',' '\n' | grep -o 'service-cluster-ip-range=[^"]*' | cut -d= -f2); grep -qx "pod-cidr=$pc" "$f" 2>/dev/null && grep -qx "service-cidr=$sc" "$f" """),
|
|
C("dns-ip and proxy-mode", 2, r"""f="$WS/q120/facts.txt"; d=$(j -n kube-system svc kube-dns '{.spec.clusterIP}'); m=$(j -n kube-system cm kube-proxy '{.data.config\.conf}' | grep '^mode:' | awk '{print $2}' | tr -d '"'); [ -n "$m" ] || m=iptables; grep -qx "dns-ip=$d" "$f" 2>/dev/null && grep -qx "proxy-mode=$m" "$f" """),
|
|
C("%{node} pod CIDR", 1, r"""grep -qx "%{node}-pod-cidr=$(j node %{node} '{.spec.podCIDR}')" "$WS/q120/facts.txt" 2>/dev/null"""),
|
|
],
|
|
},
|
|
]
|
|
|
|
# derived params: w2 = 100 - w for the canary split
|
|
for _q in QUESTIONS:
|
|
if _q["id"] == 113:
|
|
_orig = _q["params"]
|
|
_q["params"] = lambda r, _o=_orig: (lambda p: {**p, "w2": 100 - p["w"]})(_o(r))
|