f3d0e536b6
kubestronaut-sim runs each exam task on a host of its own - a kind (kubeadm) cluster in rootless podman, or a QEMU/KVM Ubuntu VM for the CKS tasks that need their own kernel (Falco, AppArmor) - and puts a browser desktop in front of them, the way killer.sh and sailor.sh do. - ksim, sim/: the engine (sessions, drills, study mode, grading, coverage, audit), host and VM lifecycle, the desktop - images/: exam host images (common, CKS variant with its tools, VM guest) - cka/ (131 tasks), ckad/ (105), cks/ (110): task banks, curricula with study notes, reference solutions and scripted solutions Every domain of every track has at least 17 pool tasks. `./ksim audit` proves a task three ways: it seeds cleanly on its own host, scores nothing before anyone has worked on it, and reaches full marks when its scripted solution is run on the host as `candidate`. All 346 tasks pass (CKA audited 2026-10-13, CKAD and CKS 2026-10-12). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01JZPd3GGWyxTgAkL6a4gAJz
990 lines
49 KiB
Python
990 lines
49 KiB
Python
"""CKA pool tasks 114-124: Storage, the domain the bank was thinnest in.
|
|
|
|
Written for the per-task hosts: each task has a cluster of its own (kind's `standard` StorageClass,
|
|
served by the local-path provisioner in Namespace local-path-storage), so a task may break the
|
|
provisioner or change the default class. Volumes are tested for real - a claim that binds, a file
|
|
that is still on the node, a Pod that reads it.
|
|
"""
|
|
from cka.common import C, NS, PY, STOR, ns_pick, name_pick
|
|
|
|
# what a setup records about the objects a task must leave alone
|
|
_UID = r"""
|
|
uid() { $K get "$@" -o jsonpath='{.metadata.uid}' 2>/dev/null; }
|
|
"""
|
|
|
|
QUESTIONS = [
|
|
# ------------------------------------------------------------------ 114 ----
|
|
{
|
|
"id": 114, "domain": "STOR", "title": "A StorageClass that keeps its volumes",
|
|
"params": lambda r: {"ns": ns_pick(r, "vault"), "sc": r.choice(["keep-local", "durable-local", "retain-local"]),
|
|
"pvc": name_pick(r, "ledger"), "pod": name_pick(r, "writer"), "text": f"ledger-{r.randint(1000, 9999)}"},
|
|
"task": """
|
|
1. Create StorageClass `%{sc}`: provisioner `rancher.io/local-path` (the one behind this cluster's
|
|
`standard` class), reclaim policy `Retain`, volume binding mode `WaitForFirstConsumer`. It must
|
|
not be the default class.
|
|
2. In Namespace `%{ns}` create PVC `%{pvc}` (300Mi, ReadWriteOnce, class `%{sc}`) and Pod `%{pod}`
|
|
(`busybox:1.36`) that mounts it at `/data`, writes the line `%{text}` to `/data/keep.txt` and
|
|
then sleeps.
|
|
3. Write the name of the PersistentVolume that was provisioned to `{WS}/q114/pv.txt`.
|
|
4. Delete the Pod and the claim. Write the phase the volume is in afterwards to
|
|
`{WS}/q114/phase.txt`.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}; rm -rf "$WS/q114"; mkdir -p "$WS/q114"
|
|
$K delete sc %{sc} --ignore-not-found >/dev/null 2>&1
|
|
""",
|
|
"checks": [
|
|
C("StorageClass %{sc}: local-path, Retain, WaitForFirstConsumer, not the default", 2, r"""
|
|
eq "$(j sc %{sc} '{.provisioner}/{.reclaimPolicy}/{.volumeBindingMode}')" rancher.io/local-path/Retain/WaitForFirstConsumer \
|
|
&& [ "$(j sc %{sc} '{.metadata.annotations.storageclass\.kubernetes\.io/is-default-class}')" != true ]"""),
|
|
C("pv.txt names the volume that was provisioned for %{pvc}, and it is kept (Retain)", 2, STOR + r"""
|
|
pv=$(one "$WS/q114/pv.txt"); [ -n "$pv" ] || exit 1
|
|
eq "$(j pv "$pv" '{.spec.storageClassName}/{.spec.claimRef.namespace}/{.spec.claimRef.name}/{.spec.persistentVolumeReclaimPolicy}')" %{sc}/%{ns}/%{pvc}/Retain"""),
|
|
C("Pod and claim are deleted, the volume is Released, and phase.txt says so", 2, STOR + r"""
|
|
pv=$(one "$WS/q114/pv.txt"); [ -n "$pv" ] || exit 1
|
|
gone -n %{ns} pod %{pod} && gone -n %{ns} pvc %{pvc} || exit 1
|
|
eq "$(j pv "$pv" '{.status.phase}')" Released && eq "$(one "$WS/q114/phase.txt")" Released"""),
|
|
C("keep.txt is still on the node, with the line %{text}", 3, STOR + r"""
|
|
pv=$(one "$WS/q114/pv.txt"); [ -n "$pv" ] || exit 1
|
|
gone -n %{ns} pvc %{pvc} || exit 1
|
|
node=$(pvnode "$pv"); path=$(pvpath "$pv"); [ -n "$node" ] && [ -n "$path" ] || exit 1
|
|
eq "$(nx "$node" "cat $path/keep.txt" </dev/null 2>/dev/null | tr -d '\r')" "%{text}" """),
|
|
],
|
|
"solution": """
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: storage.k8s.io/v1
|
|
kind: StorageClass
|
|
metadata: {name: %{sc}}
|
|
provisioner: rancher.io/local-path # k get sc standard -o yaml shows it
|
|
reclaimPolicy: Retain
|
|
volumeBindingMode: WaitForFirstConsumer
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: %{pvc}, namespace: %{ns}}
|
|
spec:
|
|
storageClassName: %{sc}
|
|
accessModes: [ReadWriteOnce]
|
|
resources: {requests: {storage: 300Mi}}
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: %{pod}, namespace: %{ns}}
|
|
spec:
|
|
volumes: [{name: data, persistentVolumeClaim: {claimName: %{pvc}}}]
|
|
containers:
|
|
- name: writer
|
|
image: busybox:1.36
|
|
command: [sh, -c, "echo %{text} > /data/keep.txt; sleep 86400"]
|
|
volumeMounts: [{name: data, mountPath: /data}]
|
|
EOF
|
|
k -n %{ns} get pvc %{pvc} # Pending until the Pod is scheduled: WaitForFirstConsumer
|
|
k -n %{ns} get pvc %{pvc} -o jsonpath='{.spec.volumeName}' > {WS}/q114/pv.txt
|
|
k -n %{ns} delete pod %{pod}; k -n %{ns} delete pvc %{pvc}
|
|
k get pv $(cat {WS}/q114/pv.txt) # STATUS Released, RECLAIM POLICY Retain
|
|
echo Released > {WS}/q114/phase.txt
|
|
# A class's reclaimPolicy is copied into every volume it provisions. Delete (the default) removes
|
|
# the volume and its data with the claim; Retain leaves a Released volume that no new claim can
|
|
# bind until an administrator clears spec.claimRef. A StorageClass cannot be edited afterwards,
|
|
# except for a few fields - to change the policy of one volume, patch the PV.
|
|
""",
|
|
},
|
|
# ------------------------------------------------------------------ 115 ----
|
|
{
|
|
"id": 115, "domain": "STOR", "title": "A volume that lives and dies with its Pod",
|
|
"params": lambda r: {"ns": ns_pick(r, "render"), "pod": name_pick(r, "encoder")},
|
|
"task": """
|
|
In Namespace `%{ns}` create Pod `%{pod}` (image `busybox:1.36`, command `sleep 86400`) with two
|
|
volumes:
|
|
|
|
- `scratch`, a **generic ephemeral volume**: 100Mi, ReadWriteOnce, StorageClass `standard`,
|
|
mounted at `/scratch`,
|
|
- `cache`, an `emptyDir` that is kept in memory and limited to 32Mi, mounted at `/cache`.
|
|
|
|
Then write
|
|
1. the name of the PersistentVolumeClaim that Kubernetes created for the Pod to
|
|
`{WS}/q115/claim.txt`,
|
|
2. the owner of that claim to `{WS}/q115/owner.txt`, as `<kind>/<name>`.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}; rm -rf "$WS/q115"; mkdir -p "$WS/q115"
|
|
$K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1
|
|
""",
|
|
"checks": [
|
|
C("Pod %{pod} runs with ephemeral volume scratch (100Mi, RWO, standard) at /scratch", 3, PY + r"""
|
|
py -n %{ns} pod %{pod} '
|
|
v = {x["name"]: x for x in o["spec"]["volumes"]}
|
|
t = (v.get("scratch", {}).get("ephemeral") or {}).get("volumeClaimTemplate", {}).get("spec", {})
|
|
m = {x["name"]: x["mountPath"] for x in o["spec"]["containers"][0].get("volumeMounts", [])}
|
|
o["status"].get("phase") == "Running" and t.get("storageClassName") == "standard" and t.get("accessModes") == ["ReadWriteOnce"] and t.get("resources", {}).get("requests", {}).get("storage") == "100Mi" and m.get("scratch") == "/scratch"'"""),
|
|
C("/scratch is a provisioned volume the Pod can write to", 2, r"""
|
|
eq "$(j -n %{ns} pvc %{pod}-scratch '{.status.phase}')" Bound || exit 1
|
|
pv=$(j -n %{ns} pvc %{pod}-scratch '{.spec.volumeName}'); eq "$(j pv "$pv" '{.spec.storageClassName}')" standard || exit 1
|
|
eq "$($K -n %{ns} exec %{pod} -- sh -c 'echo probe > /scratch/.ksim && cat /scratch/.ksim' 2>/dev/null)" probe"""),
|
|
C("cache is an emptyDir in memory, limited to 32Mi, at /cache", 2, PY + r"""
|
|
py -n %{ns} pod %{pod} '
|
|
v = {x["name"]: x for x in o["spec"]["volumes"]}
|
|
e = v.get("cache", {}).get("emptyDir") or {}
|
|
m = {x["name"]: x["mountPath"] for x in o["spec"]["containers"][0].get("volumeMounts", [])}
|
|
e.get("medium") == "Memory" and e.get("sizeLimit") == "32Mi" and m.get("cache") == "/cache"' || exit 1
|
|
$K -n %{ns} exec %{pod} -- sh -c 'mount | grep " /cache " | grep -q tmpfs' 2>/dev/null"""),
|
|
C("claim.txt and owner.txt: %{pod}-scratch, owned by the Pod", 2, STOR + r"""
|
|
eq "$(one "$WS/q115/claim.txt" | sed 's#.*/##')" %{pod}-scratch || exit 1
|
|
eq "$(j -n %{ns} pvc %{pod}-scratch '{.metadata.ownerReferences[0].kind}/{.metadata.ownerReferences[0].name}')" Pod/%{pod} || exit 1
|
|
eq "$(one "$WS/q115/owner.txt" | tr A-Z a-z)" pod/%{pod}"""),
|
|
],
|
|
"solution": """
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: %{pod}, namespace: %{ns}}
|
|
spec:
|
|
volumes:
|
|
- name: scratch
|
|
ephemeral:
|
|
volumeClaimTemplate:
|
|
spec:
|
|
storageClassName: standard
|
|
accessModes: [ReadWriteOnce]
|
|
resources: {requests: {storage: 100Mi}}
|
|
- name: cache
|
|
emptyDir: {medium: Memory, sizeLimit: 32Mi}
|
|
containers:
|
|
- name: encoder
|
|
image: busybox:1.36
|
|
command: [sleep, "86400"]
|
|
volumeMounts:
|
|
- {name: scratch, mountPath: /scratch}
|
|
- {name: cache, mountPath: /cache}
|
|
EOF
|
|
k -n %{ns} get pvc # %{pod}-scratch: <pod name>-<volume name>
|
|
echo %{pod}-scratch > {WS}/q115/claim.txt
|
|
k -n %{ns} get pvc %{pod}-scratch -o jsonpath='{.metadata.ownerReferences[0].kind}/{.metadata.ownerReferences[0].name}'
|
|
echo Pod/%{pod} > {WS}/q115/owner.txt
|
|
# A generic ephemeral volume is a claim written into the Pod: Kubernetes creates the PVC, the Pod
|
|
# owns it, and deleting the Pod deletes claim and volume. Unlike emptyDir it can come from any
|
|
# StorageClass - size, speed, snapshots - and unlike a normal claim it never outlives the Pod.
|
|
# An emptyDir with medium Memory is a tmpfs: what is written there counts as the container's memory.
|
|
""",
|
|
},
|
|
# ------------------------------------------------------------------ 116 ----
|
|
{
|
|
"id": 116, "domain": "STOR", "title": "Claims that go when the replica goes",
|
|
"params": lambda r: {"ns": ns_pick(r, "queue"), "sts": name_pick(r, "broker")},
|
|
"task": """
|
|
StatefulSet `%{sts}` in Namespace `%{ns}` runs 3 replicas, each with a claim of its own. It is
|
|
being shrunk for good, and the claims of replicas that no longer exist only take up storage.
|
|
|
|
1. Configure the StatefulSet so that the claims of replicas removed by **scaling down** are
|
|
deleted automatically, while deleting the StatefulSet itself would still keep its claims. Do
|
|
not recreate the StatefulSet.
|
|
2. Scale it down to 1 replica.
|
|
3. Write the names of the StatefulSet's claims that exist afterwards to `{WS}/q116/claims.txt`,
|
|
one per line.
|
|
""",
|
|
"setup": NS + _UID + r"""
|
|
mkns %{ns}; rm -rf "$WS/q116"; mkdir -p "$WS/q116"
|
|
$K -n %{ns} delete sts %{sts} --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K -n %{ns} delete pvc -l app=%{sts} --wait=true >/dev/null 2>&1
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata: {name: %{sts}, namespace: %{ns}}
|
|
spec: {clusterIP: None, selector: {app: %{sts}}, ports: [{port: 5672}]}
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata: {name: %{sts}, namespace: %{ns}}
|
|
spec:
|
|
serviceName: %{sts}
|
|
replicas: 3
|
|
podManagementPolicy: Parallel
|
|
selector: {matchLabels: {app: %{sts}}}
|
|
template:
|
|
metadata: {labels: {app: %{sts}}}
|
|
spec:
|
|
terminationGracePeriodSeconds: 2
|
|
containers:
|
|
- name: broker
|
|
image: busybox:1.36
|
|
command: [sh, -c, "hostname > /data/id; sleep 86400"]
|
|
volumeMounts: [{name: data, mountPath: /data}]
|
|
volumeClaimTemplates:
|
|
- metadata: {name: data, labels: {app: %{sts}}}
|
|
spec:
|
|
storageClassName: standard
|
|
accessModes: [ReadWriteOnce]
|
|
resources: {requests: {storage: 50Mi}}
|
|
EOF
|
|
$K -n %{ns} rollout status sts %{sts} --timeout=240s >/dev/null 2>&1 || { echo "StatefulSet %{sts} did not come up"; exit 1; }
|
|
uid -n %{ns} sts %{sts} > "$PRIV/q116.uid"
|
|
""",
|
|
"checks": [
|
|
C("retention policy: claims deleted when scaled down, kept when the StatefulSet is deleted", 2, r"""
|
|
eq "$(j -n %{ns} sts %{sts} '{.spec.persistentVolumeClaimRetentionPolicy.whenScaled}/{.spec.persistentVolumeClaimRetentionPolicy.whenDeleted}')" Delete/Retain"""),
|
|
C("the same StatefulSet, scaled to 1 ready replica", 2, r"""
|
|
eq "$(j -n %{ns} sts %{sts} '{.metadata.uid}')" "$(cat "$PRIV/q116.uid")" || exit 1
|
|
for i in $(seq 1 20); do eq "$(j -n %{ns} sts %{sts} '{.spec.replicas}/{.status.replicas}/{.status.readyReplicas}')" 1/1/1 && exit 0; sleep 3; done; exit 1"""),
|
|
C("the claims of replicas 1 and 2 are gone, data-%{sts}-0 is still Bound", 3, STOR + r"""
|
|
eq "$(j -n %{ns} sts %{sts} '{.spec.replicas}')" 1 || exit 1
|
|
for i in $(seq 1 20); do
|
|
gone -n %{ns} pvc data-%{sts}-1 && gone -n %{ns} pvc data-%{sts}-2 && break; sleep 3
|
|
done
|
|
gone -n %{ns} pvc data-%{sts}-1 && gone -n %{ns} pvc data-%{sts}-2 && eq "$(j -n %{ns} pvc data-%{sts}-0 '{.status.phase}')" Bound"""),
|
|
C("claims.txt lists what is left: data-%{sts}-0", 1, r"""
|
|
eq "$(tr -d ' \t\r' < "$WS/q116/claims.txt" 2>/dev/null | sed 's#.*/##' | grep . | sort | tr '\n' ' ')" "data-%{sts}-0 " """),
|
|
],
|
|
"solution": """
|
|
k -n %{ns} get sts %{sts} -o jsonpath='{.spec.persistentVolumeClaimRetentionPolicy}' # whenDeleted Retain, whenScaled Retain
|
|
k -n %{ns} patch sts %{sts} -p '{"spec":{"persistentVolumeClaimRetentionPolicy":{"whenScaled":"Delete","whenDeleted":"Retain"}}}'
|
|
k -n %{ns} scale sts %{sts} --replicas=1
|
|
k -n %{ns} get pvc -w # data-%{sts}-2 and data-%{sts}-1 disappear with their Pods
|
|
k -n %{ns} get pvc -o name | cut -d/ -f2 > {WS}/q116/claims.txt
|
|
# By default a StatefulSet never deletes a claim: scale down and up again and each replica finds
|
|
# its old data. persistentVolumeClaimRetentionPolicy changes that per event - whenScaled for the
|
|
# replicas a scale-down removes, whenDeleted for all of them when the StatefulSet goes. Set the
|
|
# policy BEFORE scaling: it is applied to the Pods being removed, not to claims orphaned earlier
|
|
# (those you delete by hand). It is one of the few StatefulSet fields that can be changed in place.
|
|
""",
|
|
},
|
|
# ------------------------------------------------------------------ 117 ----
|
|
{
|
|
"id": 117, "domain": "STOR", "difficulty": "hard", "title": "Reserve a volume for one claim",
|
|
"params": lambda r: {"ns": ns_pick(r, "atlas"), "oth": ns_pick(r, "pipeline"), "pv": name_pick(r, "pv-atlas"),
|
|
"pvc": name_pick(r, "maps")},
|
|
"task": """
|
|
Claim `any-volume` in Namespace `%{oth}` is Pending: it waits for any 1Gi volume of class `manual`,
|
|
and it has to keep waiting - its volume arrives next week.
|
|
|
|
Team `%{ns}` needs storage today. Create PersistentVolume `%{pv}` (1Gi, ReadWriteOnce, class
|
|
`manual`, `hostPath` `/mnt/%{pv}`, reclaim policy `Retain`) **reserved** for a claim `%{pvc}` in
|
|
Namespace `%{ns}`, then create that claim (1Gi, ReadWriteOnce, class `manual`). It must bind
|
|
`%{pv}`.
|
|
|
|
`any-volume` must not be changed, deleted or recreated, and it must not get the volume.
|
|
""",
|
|
"setup": NS + _UID + r"""
|
|
mkns %{ns}; mkns %{oth}
|
|
$K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K -n %{oth} delete pvc any-volume --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K delete pv %{pv} --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: any-volume, namespace: %{oth}}
|
|
spec:
|
|
storageClassName: manual
|
|
accessModes: [ReadWriteOnce]
|
|
resources: {requests: {storage: 1Gi}}
|
|
EOF
|
|
uid -n %{oth} pvc any-volume > "$PRIV/q117.uid"
|
|
""",
|
|
"checks": [
|
|
C("PersistentVolume %{pv}: 1Gi, RWO, manual, hostPath /mnt/%{pv}, Retain", 2, r"""
|
|
eq "$(j pv %{pv} '{.spec.capacity.storage}/{.spec.accessModes[*]}/{.spec.storageClassName}/{.spec.persistentVolumeReclaimPolicy}')" 1Gi/ReadWriteOnce/manual/Retain \
|
|
&& eq "$(j pv %{pv} '{.spec.hostPath.path}')" /mnt/%{pv}"""),
|
|
C("it is reserved: its claimRef names %{pvc} in %{ns}", 2, r"""
|
|
eq "$(j pv %{pv} '{.spec.claimRef.namespace}/{.spec.claimRef.name}')" %{ns}/%{pvc}"""),
|
|
C("claim %{pvc} (1Gi, RWO, manual) is Bound to %{pv}", 3, STOR + r"""
|
|
bound %{ns} %{pvc} 30 || exit 1
|
|
eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}/{.spec.storageClassName}/{.spec.accessModes[*]}/{.spec.resources.requests.storage}')" %{pv}/manual/ReadWriteOnce/1Gi"""),
|
|
C("any-volume is the same claim as before and still Pending", 2, r"""
|
|
[ -n "$(j pv %{pv} '{.metadata.name}')" ] || exit 1
|
|
eq "$(j -n %{oth} pvc any-volume '{.metadata.uid}')" "$(cat "$PRIV/q117.uid")" && eq "$(j -n %{oth} pvc any-volume '{.status.phase}')" Pending"""),
|
|
],
|
|
"solution": """
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: PersistentVolume
|
|
metadata: {name: %{pv}}
|
|
spec:
|
|
capacity: {storage: 1Gi}
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: manual
|
|
persistentVolumeReclaimPolicy: Retain
|
|
hostPath: {path: /mnt/%{pv}}
|
|
claimRef: {namespace: %{ns}, name: %{pvc}} # the reservation: no other claim can bind this volume
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: %{pvc}, namespace: %{ns}}
|
|
spec:
|
|
storageClassName: manual
|
|
accessModes: [ReadWriteOnce]
|
|
resources: {requests: {storage: 1Gi}}
|
|
EOF
|
|
k get pv %{pv}; k -n %{ns} get pvc %{pvc}; k -n %{oth} get pvc any-volume # Bound, Bound, Pending
|
|
# Without claimRef the volume would be Available, and the control plane hands an Available volume
|
|
# to whichever matching claim it looks at first - here the one that has been waiting. That cannot
|
|
# be undone cleanly: a claim whose volume is taken away ends up Lost. claimRef on the PV reserves
|
|
# it; volumeName on a claim is the other half (it asks for one volume but does not keep others
|
|
# away from it). Use both when it has to be this pair and no other.
|
|
""",
|
|
},
|
|
# ------------------------------------------------------------------ 118 ----
|
|
{
|
|
"id": 118, "domain": "STOR", "difficulty": "hard", "workers": 1, "title": "The claim took the wrong volume",
|
|
"params": lambda r: {"ns": ns_pick(r, "books"), "dep": name_pick(r, "ledger"), "pvc": name_pick(r, "ledger-data"),
|
|
"pv": name_pick(r, "pv-ledger"), "tok": f"{r.randint(100000, 999999)}"},
|
|
"task": """
|
|
Deployment `%{dep}` in Namespace `%{ns}` is meant to read its ledger from PersistentVolume `%{pv}`,
|
|
which an administrator prepared by hand - but its log says `no ledger found`. The manifest it was
|
|
created from is `{WS}/q118/app.yaml`.
|
|
|
|
1. Write the name of the volume that claim `%{pvc}` got instead to `{WS}/q118/wrong-pv.txt`.
|
|
2. Make claim `%{pvc}` - same name, same size - bind `%{pv}`, so that the Deployment's Pod prints
|
|
the ledger. The volume it got by mistake must be gone in the end; `%{pv}` itself and the files
|
|
on it must not be changed.
|
|
""",
|
|
"setup": NS + _UID + r"""
|
|
mkns %{ns}; rm -rf "$WS/q118"; mkdir -p "$WS/q118"
|
|
$K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K delete pv %{pv} --ignore-not-found --wait=true >/dev/null 2>&1
|
|
nx "$W1" "mkdir -p /mnt/%{pv} && echo 'ledger %{tok} balanced' > /mnt/%{pv}/ledger.db" </dev/null || exit 1
|
|
$K apply -f - >/dev/null <<EOF
|
|
apiVersion: v1
|
|
kind: PersistentVolume
|
|
metadata: {name: %{pv}}
|
|
spec:
|
|
capacity: {storage: 500Mi}
|
|
accessModes: [ReadWriteOnce]
|
|
persistentVolumeReclaimPolicy: Retain
|
|
storageClassName: ""
|
|
hostPath: {path: /mnt/%{pv}}
|
|
nodeAffinity:
|
|
required:
|
|
nodeSelectorTerms:
|
|
- matchExpressions: [{key: kubernetes.io/hostname, operator: In, values: ["$W1"]}]
|
|
EOF
|
|
cat > "$WS/q118/app.yaml" <<'EOF'
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: %{pvc}, namespace: %{ns}}
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
resources: {requests: {storage: 500Mi}}
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: %{dep}, namespace: %{ns}}
|
|
spec:
|
|
replicas: 1
|
|
strategy: {type: Recreate}
|
|
selector: {matchLabels: {app: %{dep}}}
|
|
template:
|
|
metadata: {labels: {app: %{dep}}}
|
|
spec:
|
|
terminationGracePeriodSeconds: 2
|
|
volumes: [{name: data, persistentVolumeClaim: {claimName: %{pvc}}}]
|
|
containers:
|
|
- name: app
|
|
image: busybox:1.36
|
|
command: [sh, -c, "cat /data/ledger.db 2>/dev/null || echo 'no ledger found'; sleep 86400"]
|
|
volumeMounts: [{name: data, mountPath: /data}]
|
|
EOF
|
|
$K apply -f "$WS/q118/app.yaml" >/dev/null
|
|
$K -n %{ns} rollout status deploy %{dep} --timeout=240s >/dev/null 2>&1 || { echo "Deployment %{dep} did not come up"; exit 1; }
|
|
$K -n %{ns} get pvc %{pvc} -o jsonpath='{.spec.volumeName}' > "$PRIV/q118.wrong"
|
|
[ -s "$PRIV/q118.wrong" ] || { echo "claim %{pvc} was not provisioned"; exit 1; }
|
|
uid pv %{pv} > "$PRIV/q118.uid"
|
|
""",
|
|
"checks": [
|
|
C("wrong-pv.txt names the volume the claim was given by mistake", 1, STOR + r"""
|
|
eq "$(one "$WS/q118/wrong-pv.txt" | sed 's#.*/##')" "$(cat "$PRIV/q118.wrong")" """),
|
|
C("claim %{pvc} (500Mi) is Bound to %{pv}, which is the volume it always was", 3, STOR + r"""
|
|
bound %{ns} %{pvc} 30 || exit 1
|
|
eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}/{.spec.resources.requests.storage}')" %{pv}/500Mi || exit 1
|
|
eq "$(j pv %{pv} '{.metadata.uid}/{.spec.hostPath.path}')" "$(cat "$PRIV/q118.uid")//mnt/%{pv}" """),
|
|
C("the Deployment's Pod prints the ledger", 3, r"""
|
|
eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}')" %{pv} || exit 1
|
|
for i in $(seq 1 20); do
|
|
$K -n %{ns} logs deploy/%{dep} 2>/dev/null | grep -q 'ledger %{tok} balanced' && exit 0; sleep 3
|
|
done; exit 1"""),
|
|
C("the volume provisioned by mistake is gone", 1, STOR + r"""
|
|
eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}')" %{pv} || exit 1
|
|
for i in $(seq 1 10); do gone pv "$(cat "$PRIV/q118.wrong")" && exit 0; sleep 3; done; exit 1"""),
|
|
],
|
|
"solution": """
|
|
k -n %{ns} get pvc %{pvc} # Bound to pvc-..., STORAGECLASS standard - not to %{pv}
|
|
k -n %{ns} get pvc %{pvc} -o jsonpath='{.spec.volumeName}' > {WS}/q118/wrong-pv.txt
|
|
k get pv %{pv} # Available, STORAGECLASS empty
|
|
# The manifest names no class, so the claim was given the cluster's default one (standard) and a
|
|
# brand-new, empty volume was provisioned for it. A claim's class and volume cannot be edited:
|
|
k -n %{ns} scale deploy %{dep} --replicas=0 # a claim in use is not deleted (pvc-protection)
|
|
k -n %{ns} delete pvc %{pvc} # its volume goes with it: reclaim policy Delete
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: %{pvc}, namespace: %{ns}}
|
|
spec:
|
|
storageClassName: "" # "no class": never the default, never provisioned
|
|
volumeName: %{pv}
|
|
accessModes: [ReadWriteOnce]
|
|
resources: {requests: {storage: 500Mi}}
|
|
EOF
|
|
k -n %{ns} scale deploy %{dep} --replicas=1
|
|
k -n %{ns} logs deploy/%{dep} # ledger %{tok} balanced
|
|
# storageClassName has three states: a name; "" = only volumes without a class, no provisioning;
|
|
# and absent = "give me the default class". volumeName alone would not have helped: the default
|
|
# class is still filled in, and it does not match the class-less volume.
|
|
""",
|
|
},
|
|
# ------------------------------------------------------------------ 119 ----
|
|
{
|
|
"id": 119, "domain": "STOR", "difficulty": "hard", "title": "Three claims, three reasons",
|
|
"params": lambda r: {"ns": ns_pick(r, "studio"), "p": name_pick(r, "pv")},
|
|
"task": """
|
|
Three claims in Namespace `%{ns}` are Pending: `reports`, `archive` and `media`. Each is meant to
|
|
bind the PersistentVolume that carries its `use` label - `%{p}-reports`, `%{p}-archive`,
|
|
`%{p}-media` - and each fails for a different reason.
|
|
|
|
1. Write the reason for each to `{WS}/q119/reasons.txt`, one line per claim as
|
|
`<claim> <reason>`, where the reason is one of `capacity`, `accessmode`, `class`.
|
|
2. Get all three Bound to their volumes. The PersistentVolumes must not be changed or recreated;
|
|
a claim may be recreated under its name, keeping its selector.
|
|
""",
|
|
"setup": NS + _UID + r"""
|
|
mkns %{ns}; rm -rf "$WS/q119"; mkdir -p "$WS/q119"
|
|
$K -n %{ns} delete pvc reports archive media --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K delete pv %{p}-reports %{p}-archive %{p}-media --ignore-not-found --wait=true >/dev/null 2>&1
|
|
pv() { # use size
|
|
cat <<EOF
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolume
|
|
metadata: {name: %{p}-$1, labels: {use: $1}}
|
|
spec:
|
|
capacity: {storage: $2}
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: manual
|
|
persistentVolumeReclaimPolicy: Retain
|
|
hostPath: {path: /mnt/%{p}-$1}
|
|
EOF
|
|
}
|
|
pvc() { # name size mode class
|
|
cat <<EOF
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: $1, namespace: %{ns}}
|
|
spec:
|
|
storageClassName: $4
|
|
accessModes: [$3]
|
|
selector: {matchLabels: {use: $1}}
|
|
resources: {requests: {storage: $2}}
|
|
EOF
|
|
}
|
|
{ pv reports 1Gi; pv archive 2Gi; pv media 1Gi
|
|
pvc reports 5Gi ReadWriteOnce manual; pvc archive 2Gi ReadWriteMany manual; pvc media 1Gi ReadWriteOnce manuel
|
|
} | $K apply -f - >/dev/null
|
|
for u in reports archive media; do uid pv %{p}-$u; echo; done > "$PRIV/q119.uids"
|
|
""",
|
|
"checks": [
|
|
C("reasons.txt: reports capacity, archive accessmode, media class", 2, r"""
|
|
eq "$(tr -s ' \t' ' ' < "$WS/q119/reasons.txt" 2>/dev/null | tr -d '\r' | sed 's/^ //; s/ $//' | tr A-Z a-z | grep . | sort | tr '\n' ';')" \
|
|
"archive accessmode;media class;reports capacity;" """),
|
|
] + [
|
|
C(f"claim {u} is Bound to %{{p}}-{u}, and the volume is as it was", 2, STOR + rf"""
|
|
bound %{{ns}} {u} 30 || exit 1
|
|
eq "$(j -n %{{ns}} pvc {u} '{{.spec.volumeName}}/{{.spec.selector.matchLabels.use}}')" %{{p}}-{u}/{u} || exit 1
|
|
eq "$(j pv %{{p}}-{u} '{{.metadata.uid}}')" "$(sed -n {n}p "$PRIV/q119.uids")" || exit 1
|
|
eq "$(j pv %{{p}}-{u} '{{.spec.capacity.storage}}/{{.spec.accessModes[*]}}/{{.spec.storageClassName}}/{{.metadata.labels.use}}')" {size}/ReadWriteOnce/manual/{u}""")
|
|
for n, (u, size) in enumerate([("reports", "1Gi"), ("archive", "2Gi"), ("media", "1Gi")], 1)
|
|
],
|
|
"solution": """
|
|
k -n %{ns} get pvc; k get pv --show-labels
|
|
k -n %{ns} describe pvc reports archive media | grep -E '^Name:|Warning|Normal'
|
|
# media: storageclass.storage.k8s.io "manuel" not found -> class (a typo for manual)
|
|
# reports, archive: no persistent volumes available for this claim -> compare claim and volume:
|
|
k -n %{ns} get pvc -o custom-columns=NAME:.metadata.name,SIZE:.spec.resources.requests.storage,MODES:.spec.accessModes,CLASS:.spec.storageClassName
|
|
k get pv -o custom-columns=NAME:.metadata.name,SIZE:.spec.capacity.storage,MODES:.spec.accessModes,CLASS:.spec.storageClassName
|
|
# reports asks for 5Gi, its volume has 1Gi -> capacity
|
|
# archive asks for ReadWriteMany, its volume offers ReadWriteOnce -> accessmode
|
|
printf 'reports capacity\\narchive accessmode\\nmedia class\\n' > {WS}/q119/reasons.txt
|
|
# size (downwards), access modes and class of a claim cannot be edited: recreate the three
|
|
k -n %{ns} delete pvc reports archive media
|
|
for c in "reports 1Gi" "archive 2Gi" "media 1Gi"; do set -- $c; cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: $1, namespace: %{ns}}
|
|
spec:
|
|
storageClassName: manual
|
|
accessModes: [ReadWriteOnce]
|
|
selector: {matchLabels: {use: $1}}
|
|
resources: {requests: {storage: $2}}
|
|
EOF
|
|
done
|
|
k -n %{ns} get pvc # all Bound, each to its own volume
|
|
# A claim binds a volume only if ALL of these hold: same class, the volume is at least as large,
|
|
# it offers every access mode the claim asks for, and it matches the claim's selector. The event
|
|
# only names the class problem - for the rest you compare the two objects yourself.
|
|
""",
|
|
},
|
|
# ------------------------------------------------------------------ 120 ----
|
|
{
|
|
"id": 120, "domain": "STOR", "difficulty": "hard", "workers": 2, "title": "Volume on one node, Pod sent to another",
|
|
"params": lambda r: {"ns": ns_pick(r, "shop"), "dep": name_pick(r, "orders"), "pv": name_pick(r, "pv-orders"),
|
|
"tok": f"{r.randint(100000, 999999)}"},
|
|
"task": """
|
|
The Pod of Deployment `%{dep}` in Namespace `%{ns}` is Pending.
|
|
|
|
1. Write the scheduler's message from the Pod's events to `{WS}/q120/event.txt`.
|
|
2. Get the Pod running so that it prints its data file: its log must show the `orders` line that
|
|
is stored on the volume. That data exists on one node only - do not change, recreate or copy
|
|
the PersistentVolume, its claim or the files.
|
|
""",
|
|
"setup": NS + _UID + r"""
|
|
mkns %{ns}; rm -rf "$WS/q120"; mkdir -p "$WS/q120"
|
|
$K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K -n %{ns} delete pvc %{dep}-data --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K delete pv %{pv} --ignore-not-found --wait=true >/dev/null 2>&1
|
|
nx "$W1" "mkdir -p /mnt/%{pv} && echo 'orders %{tok} open' > /mnt/%{pv}/orders.db" </dev/null || exit 1
|
|
$K label node "$W1" disk- >/dev/null 2>&1; $K label node "$W2" disk=ssd --overwrite >/dev/null
|
|
$K apply -f - >/dev/null <<EOF
|
|
apiVersion: v1
|
|
kind: PersistentVolume
|
|
metadata: {name: %{pv}}
|
|
spec:
|
|
capacity: {storage: 1Gi}
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: manual
|
|
persistentVolumeReclaimPolicy: Retain
|
|
local: {path: /mnt/%{pv}}
|
|
nodeAffinity:
|
|
required:
|
|
nodeSelectorTerms:
|
|
- matchExpressions: [{key: kubernetes.io/hostname, operator: In, values: ["$W1"]}]
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: %{dep}-data, namespace: %{ns}}
|
|
spec:
|
|
storageClassName: manual
|
|
volumeName: %{pv}
|
|
accessModes: [ReadWriteOnce]
|
|
resources: {requests: {storage: 1Gi}}
|
|
EOF
|
|
# the claim first, so that the Pod's only scheduling problem is the one the task is about
|
|
for i in $(seq 1 30); do [ "$(j -n %{ns} pvc %{dep}-data '{.status.phase}')" = Bound ] && break; sleep 2; done
|
|
[ "$(j -n %{ns} pvc %{dep}-data '{.status.phase}')" = Bound ] || { echo "claim %{dep}-data did not bind"; exit 1; }
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: %{dep}, namespace: %{ns}}
|
|
spec:
|
|
replicas: 1
|
|
strategy: {type: Recreate}
|
|
selector: {matchLabels: {app: %{dep}}}
|
|
template:
|
|
metadata: {labels: {app: %{dep}}}
|
|
spec:
|
|
terminationGracePeriodSeconds: 2
|
|
nodeSelector: {disk: ssd}
|
|
volumes: [{name: data, persistentVolumeClaim: {claimName: %{dep}-data}}]
|
|
containers:
|
|
- name: app
|
|
image: busybox:1.36
|
|
command: [sh, -c, "cat /data/orders.db; sleep 86400"]
|
|
volumeMounts: [{name: data, mountPath: /data}]
|
|
EOF
|
|
{ uid pv %{pv}; echo; uid -n %{ns} pvc %{dep}-data; echo; } > "$PRIV/q120.uids"
|
|
""",
|
|
"checks": [
|
|
C("event.txt holds the scheduler's message: no node matches the volume's node affinity", 2, r"""
|
|
grep -Eqi "match PersistentVolume's node affinity|volume node affinity conflict" "$WS/q120/event.txt" 2>/dev/null"""),
|
|
C("the Deployment's Pod runs on the node that holds the volume", 3, STOR + r"""
|
|
for i in $(seq 1 20); do p=$(newest %{ns} app=%{dep}); [ -n "$p" ] && break; sleep 3; done
|
|
[ -n "$p" ] && eq "$(j -n %{ns} pod "$p" '{.spec.nodeName}')" "$W1" """),
|
|
C("its log shows the orders line from the volume", 2, r"""
|
|
for i in $(seq 1 10); do
|
|
$K -n %{ns} logs deploy/%{dep} 2>/dev/null | grep -q 'orders %{tok} open' && exit 0; sleep 3
|
|
done; exit 1"""),
|
|
C("PersistentVolume and claim are the ones that were there", 1, STOR + r"""
|
|
[ -n "$(newest %{ns} app=%{dep})" ] || exit 1
|
|
eq "$(j pv %{pv} '{.metadata.uid}')" "$(sed -n 1p "$PRIV/q120.uids")" && eq "$(j -n %{ns} pvc %{dep}-data '{.metadata.uid}')" "$(sed -n 2p "$PRIV/q120.uids")" \
|
|
&& eq "$(pvnode %{pv})/$(pvpath %{pv})" "$W1//mnt/%{pv}" """),
|
|
],
|
|
"solution": """
|
|
k -n %{ns} describe pod -l app=%{dep} | tail -5
|
|
# 0/3 nodes are available: 1 node(s) didn't match PersistentVolume's node affinity, 1 node(s)
|
|
# didn't match Pod's node affinity/selector, 1 node(s) had untolerated taint(s). ...
|
|
k -n %{ns} get events --field-selector reason=FailedScheduling -o jsonpath='{.items[-1:].message}' > {WS}/q120/event.txt
|
|
k get pv %{pv} -o jsonpath='{.spec.nodeAffinity}'; echo # the volume is on {W1}
|
|
k -n %{ns} get deploy %{dep} -o jsonpath='{.spec.template.spec.nodeSelector}'; echo # disk=ssd
|
|
k get nodes -L disk # only {W2} carries disk=ssd
|
|
# The volume cannot move, so the Pod has to: drop the nodeSelector (the volume's own node
|
|
# affinity already sends the Pod to the right node) ...
|
|
k -n %{ns} patch deploy %{dep} --type=json -p '[{"op":"remove","path":"/spec/template/spec/nodeSelector"}]'
|
|
# ... or, if the selector has a reason to exist, give {W1} the label: k label node {W1} disk=ssd
|
|
k -n %{ns} rollout status deploy %{dep}; k -n %{ns} logs deploy/%{dep}
|
|
# The scheduler ANDs every constraint: taints, the Pod's nodeSelector/affinity, and the node
|
|
# affinity of each volume it mounts. "didn't match PersistentVolume's node affinity" (before v1.35:
|
|
# "had volume node affinity conflict") always means: the Pod is not allowed onto the one node its
|
|
# volume lives on.
|
|
""",
|
|
},
|
|
# ------------------------------------------------------------------ 121 ----
|
|
{
|
|
"id": 121, "domain": "STOR", "title": "A claim that will not go away",
|
|
"params": lambda r: {"ns": ns_pick(r, "gallery"), "dep": name_pick(r, "thumbs"), "pvc": name_pick(r, "uploads")},
|
|
"task": """
|
|
Claim `%{pvc}` in Namespace `%{ns}` was deleted yesterday, and it is still there, `Terminating`.
|
|
|
|
1. Write the finalizer that keeps it to `{WS}/q121/finalizer.txt`.
|
|
2. Write what still uses the claim to `{WS}/q121/used-by.txt`, as `<kind>/<name>` of the workload
|
|
(not of a Pod).
|
|
3. The data is not needed any more. Let the deletion finish - do not edit the claim - by making
|
|
that workload keep its files in an `emptyDir` at the same path instead. It must keep its name
|
|
and end up with 1 running replica.
|
|
""",
|
|
"setup": NS + _UID + r"""
|
|
mkns %{ns}; rm -rf "$WS/q121"; mkdir -p "$WS/q121"
|
|
$K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: %{pvc}, namespace: %{ns}}
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
resources: {requests: {storage: 100Mi}}
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: %{dep}, namespace: %{ns}}
|
|
spec:
|
|
replicas: 1
|
|
strategy: {type: Recreate}
|
|
selector: {matchLabels: {app: %{dep}}}
|
|
template:
|
|
metadata: {labels: {app: %{dep}}}
|
|
spec:
|
|
terminationGracePeriodSeconds: 2
|
|
volumes: [{name: files, persistentVolumeClaim: {claimName: %{pvc}}}]
|
|
containers:
|
|
- name: worker
|
|
image: busybox:1.36
|
|
command: [sh, -c, "date > /srv/files/started; sleep 86400"]
|
|
volumeMounts: [{name: files, mountPath: /srv/files}]
|
|
EOF
|
|
$K -n %{ns} rollout status deploy %{dep} --timeout=240s >/dev/null 2>&1 || { echo "Deployment %{dep} did not come up"; exit 1; }
|
|
$K -n %{ns} get pvc %{pvc} -o jsonpath='{.spec.volumeName}' > "$PRIV/q121.pv"
|
|
uid -n %{ns} deploy %{dep} > "$PRIV/q121.uid"
|
|
$K -n %{ns} delete pvc %{pvc} --wait=false >/dev/null
|
|
""",
|
|
"checks": [
|
|
C("finalizer.txt holds kubernetes.io/pvc-protection", 1, STOR + r"""eq "$(one "$WS/q121/finalizer.txt")" kubernetes.io/pvc-protection"""),
|
|
C("used-by.txt names Deployment %{dep}", 2, STOR + r"""
|
|
case "$(one "$WS/q121/used-by.txt" | tr A-Z a-z)" in
|
|
deployment/%{dep}|deployments/%{dep}|deploy/%{dep}|deployment.apps/%{dep}|deployments.apps/%{dep}) ;; *) exit 1 ;; esac"""),
|
|
C("the claim is gone, and its volume with it", 3, STOR + r"""
|
|
for i in $(seq 1 20); do gone -n %{ns} pvc %{pvc} && gone pv "$(cat "$PRIV/q121.pv")" && exit 0; sleep 3; done; exit 1"""),
|
|
C("the same Deployment runs 1 replica with an emptyDir at /srv/files", 2, PY + r"""
|
|
eq "$(j -n %{ns} deploy %{dep} '{.metadata.uid}')" "$(cat "$PRIV/q121.uid")" || exit 1
|
|
py -n %{ns} deploy %{dep} '
|
|
s = o["spec"]["template"]["spec"]
|
|
v = {x["name"]: x for x in s.get("volumes", [])}
|
|
m = [x["name"] for x in s["containers"][0].get("volumeMounts", []) if x["mountPath"] == "/srv/files"]
|
|
len(m) == 1 and "emptyDir" in v.get(m[0], {}) and not any("persistentVolumeClaim" in x for x in v.values())' || exit 1
|
|
for i in $(seq 1 20); do eq "$(j -n %{ns} deploy %{dep} '{.status.updatedReplicas}/{.status.availableReplicas}/{.status.replicas}')" 1/1/1 && exit 0; sleep 3; done; exit 1"""),
|
|
],
|
|
"solution": """
|
|
k -n %{ns} get pvc %{pvc} # STATUS Terminating
|
|
k -n %{ns} get pvc %{pvc} -o jsonpath='{.metadata.finalizers}'; echo # ["kubernetes.io/pvc-protection"]
|
|
echo kubernetes.io/pvc-protection > {WS}/q121/finalizer.txt
|
|
k -n %{ns} describe pvc %{pvc} | grep 'Used By' # Used By: %{dep}-... (a Pod; find its owner)
|
|
k -n %{ns} get deploy -o custom-columns=NAME:.metadata.name,CLAIMS:.spec.template.spec.volumes[*].persistentVolumeClaim.claimName
|
|
echo deployment/%{dep} > {WS}/q121/used-by.txt
|
|
k -n %{ns} edit deploy %{dep}
|
|
# volumes:
|
|
# - name: files
|
|
# emptyDir: {} <- instead of persistentVolumeClaim: {claimName: %{pvc}}
|
|
k -n %{ns} rollout status deploy %{dep}
|
|
k -n %{ns} get pvc; k get pv # the claim is gone, and so is its volume (reclaim policy Delete)
|
|
# "Storage object in use protection": a claim that a Pod still mounts is only marked for deletion;
|
|
# the pvc-protection finalizer is removed by the control plane when the last such Pod is gone.
|
|
# Removing the finalizer by hand would pull the claim out from under a running Pod - the fix is
|
|
# always to stop or change whatever uses it.
|
|
""",
|
|
},
|
|
# ------------------------------------------------------------------ 122 ----
|
|
{
|
|
"id": 122, "domain": "STOR", "difficulty": "hard", "title": "Dynamic provisioning has stopped",
|
|
"params": lambda r: {"ns": ns_pick(r, "lab"), "sc": r.choice(["node-local", "scratch-local", "team-local"]),
|
|
"pvc": name_pick(r, "results"), "pod": name_pick(r, "runner")},
|
|
"task": """
|
|
Pod `%{pod}` in Namespace `%{ns}` is Pending because its claim `%{pvc}` never gets a volume. Until
|
|
last week, volumes of StorageClass `%{sc}` were provisioned within seconds. More than one thing is
|
|
wrong.
|
|
|
|
1. Fix it: the claim must be Bound to a dynamically provisioned volume of class `%{sc}`, and the
|
|
Pod must run. The class keeps its name, its reclaim policy and its volume binding mode. Claim
|
|
and Pod may be recreated from `{WS}/q122/app.yaml` if you need to.
|
|
2. Write the provisioner that class `%{sc}` uses in the end to `{WS}/q122/provisioner.txt`.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}; rm -rf "$WS/q122"; mkdir -p "$WS/q122"
|
|
$K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1
|
|
$K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K delete sc %{sc} --ignore-not-found >/dev/null 2>&1
|
|
$K -n local-path-storage get deploy local-path-provisioner >/dev/null 2>&1 || { echo "this cluster has no local-path provisioner"; exit 1; }
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: storage.k8s.io/v1
|
|
kind: StorageClass
|
|
metadata: {name: %{sc}}
|
|
provisioner: rancher.io/localpath
|
|
reclaimPolicy: Delete
|
|
volumeBindingMode: WaitForFirstConsumer
|
|
EOF
|
|
$K -n local-path-storage scale deploy local-path-provisioner --replicas=0 >/dev/null
|
|
cat > "$WS/q122/app.yaml" <<'EOF'
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: %{pvc}, namespace: %{ns}}
|
|
spec:
|
|
storageClassName: %{sc}
|
|
accessModes: [ReadWriteOnce]
|
|
resources: {requests: {storage: 200Mi}}
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: %{pod}, namespace: %{ns}}
|
|
spec:
|
|
terminationGracePeriodSeconds: 2
|
|
volumes: [{name: out, persistentVolumeClaim: {claimName: %{pvc}}}]
|
|
containers:
|
|
- name: runner
|
|
image: busybox:1.36
|
|
command: [sh, -c, "date > /out/started; sleep 86400"]
|
|
volumeMounts: [{name: out, mountPath: /out}]
|
|
EOF
|
|
$K apply -f "$WS/q122/app.yaml" >/dev/null
|
|
""",
|
|
"checks": [
|
|
C("the provisioner is running again", 2, r"""
|
|
[ "$(j -n local-path-storage deploy local-path-provisioner '{.status.availableReplicas}')" -ge 1 ] 2>/dev/null"""),
|
|
C("StorageClass %{sc}: provisioner rancher.io/local-path, still Delete and WaitForFirstConsumer", 2, r"""
|
|
eq "$(j sc %{sc} '{.provisioner}/{.reclaimPolicy}/{.volumeBindingMode}')" rancher.io/local-path/Delete/WaitForFirstConsumer"""),
|
|
C("claim %{pvc} is Bound to a volume of class %{sc}, and Pod %{pod} runs", 3, STOR + r"""
|
|
bound %{ns} %{pvc} 60 && running %{ns} %{pod} 60 || exit 1
|
|
pv=$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}')
|
|
eq "$(j pv "$pv" '{.spec.storageClassName}')" %{sc} && eq "$(j -n %{ns} pod %{pod} '{.spec.volumes[0].persistentVolumeClaim.claimName}')" %{pvc}"""),
|
|
C("a new claim of class %{sc} gets a volume within a minute", 2, STOR + r"""
|
|
$K -n %{ns} delete pod ksim-probe --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1
|
|
$K -n %{ns} delete pvc ksim-probe --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K apply -f - >/dev/null 2>&1 <<'EOF'
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: ksim-probe, namespace: %{ns}}
|
|
spec:
|
|
storageClassName: %{sc}
|
|
accessModes: [ReadWriteOnce]
|
|
resources: {requests: {storage: 10Mi}}
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: ksim-probe, namespace: %{ns}}
|
|
spec:
|
|
terminationGracePeriodSeconds: 1
|
|
volumes: [{name: v, persistentVolumeClaim: {claimName: ksim-probe}}]
|
|
containers: [{name: c, image: "busybox:1.36", command: [sleep, "600"], volumeMounts: [{name: v, mountPath: /v}]}]
|
|
EOF
|
|
rc=1; bound %{ns} ksim-probe 60 && rc=0
|
|
$K -n %{ns} delete pod ksim-probe --wait=false --grace-period=1 >/dev/null 2>&1
|
|
$K -n %{ns} delete pvc ksim-probe --wait=false >/dev/null 2>&1
|
|
exit $rc"""),
|
|
C("provisioner.txt holds rancher.io/local-path", 1, STOR + r"""
|
|
eq "$(j sc %{sc} '{.provisioner}')" rancher.io/local-path && eq "$(one "$WS/q122/provisioner.txt")" rancher.io/local-path"""),
|
|
],
|
|
"solution": """
|
|
k -n %{ns} describe pvc %{pvc} | tail -3
|
|
# waiting for a volume to be created either by the external provisioner 'rancher.io/localpath'
|
|
# or manually by the system administrator ... verify that the provisioner is running and correctly registered
|
|
k get sc # %{sc}: rancher.io/localpath standard: rancher.io/local-path <- a typo
|
|
k get pods -A | grep -i provision # nothing: who serves rancher.io/local-path?
|
|
k -n local-path-storage get deploy # local-path-provisioner 0/0 - scaled to zero
|
|
k -n local-path-storage scale deploy local-path-provisioner --replicas=1
|
|
|
|
# a StorageClass's provisioner cannot be edited: write it out, correct it, replace it
|
|
k get sc %{sc} -o yaml > sc.yaml
|
|
vi sc.yaml # provisioner: rancher.io/local-path (drop uid, resourceVersion, creationTimestamp)
|
|
k delete sc %{sc}; k apply -f sc.yaml
|
|
k -n %{ns} get pvc,pod -w # the waiting claim is picked up by itself within half a minute
|
|
# (if it is not: k -n %{ns} delete pod %{pod}; k -n %{ns} delete pvc %{pvc}; k apply -f {WS}/q122/app.yaml)
|
|
echo rancher.io/local-path > {WS}/q122/provisioner.txt
|
|
# Dynamic provisioning is a chain: the claim names a class, the class names a provisioner, and a
|
|
# controller that answers to exactly that name has to be running. The claim's event tells you
|
|
# which name it is waiting for - then check the class for a typo and the cluster for the controller.
|
|
""",
|
|
},
|
|
# ------------------------------------------------------------------ 123 ----
|
|
{
|
|
"id": 123, "domain": "STOR", "title": "A claim that waits for a default class",
|
|
"params": lambda r: {"ns": ns_pick(r, "intake"), "sc": name_pick(r, "general"), "pvc": name_pick(r, "inbox"),
|
|
"pod": name_pick(r, "scanner")},
|
|
"task": """
|
|
Claim `%{pvc}` in Namespace `%{ns}` is Pending, and so is Pod `%{pod}` that mounts it: the claim
|
|
names no StorageClass, and this cluster has no default one at the moment.
|
|
|
|
1. Without changing, deleting or recreating the claim or the Pod, get the claim Bound to a volume
|
|
of class `%{sc}` and the Pod running.
|
|
2. Exactly one StorageClass must be the default in the end.
|
|
3. Write the StorageClass the claim ended up with to `{WS}/q123/class.txt`.
|
|
""",
|
|
"setup": NS + _UID + r"""
|
|
mkns %{ns}; rm -rf "$WS/q123"; mkdir -p "$WS/q123"
|
|
$K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1
|
|
$K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K delete sc %{sc} --ignore-not-found >/dev/null 2>&1
|
|
$K annotate sc standard storageclass.kubernetes.io/is-default-class- >/dev/null 2>&1
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: storage.k8s.io/v1
|
|
kind: StorageClass
|
|
metadata: {name: %{sc}}
|
|
provisioner: rancher.io/local-path
|
|
reclaimPolicy: Delete
|
|
volumeBindingMode: WaitForFirstConsumer
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: %{pvc}, namespace: %{ns}}
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
resources: {requests: {storage: 200Mi}}
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: %{pod}, namespace: %{ns}}
|
|
spec:
|
|
terminationGracePeriodSeconds: 2
|
|
volumes: [{name: inbox, persistentVolumeClaim: {claimName: %{pvc}}}]
|
|
containers:
|
|
- name: scanner
|
|
image: busybox:1.36
|
|
command: [sleep, "86400"]
|
|
volumeMounts: [{name: inbox, mountPath: /inbox}]
|
|
EOF
|
|
[ -z "$(j -n %{ns} pvc %{pvc} '{.spec.storageClassName}')" ] || { echo "claim %{pvc} was given a class although no default exists"; exit 1; }
|
|
{ uid -n %{ns} pvc %{pvc}; echo; uid -n %{ns} pod %{pod}; echo; } > "$PRIV/q123.uids"
|
|
""",
|
|
"checks": [
|
|
C("%{sc} is the default StorageClass, and the only one", 2, PY + r"""
|
|
py sc '
|
|
d = sorted(x["metadata"]["name"] for x in o["items"] if (x["metadata"].get("annotations") or {}).get("storageclass.kubernetes.io/is-default-class") == "true")
|
|
d == ["%{sc}"]'"""),
|
|
C("the same claim now has class %{sc} and is Bound", 3, STOR + r"""
|
|
eq "$(j -n %{ns} pvc %{pvc} '{.metadata.uid}')" "$(sed -n 1p "$PRIV/q123.uids")" || exit 1
|
|
bound %{ns} %{pvc} 60 && eq "$(j -n %{ns} pvc %{pvc} '{.spec.storageClassName}')" %{sc}"""),
|
|
C("the same Pod runs with the claim mounted", 2, STOR + r"""
|
|
eq "$(j -n %{ns} pod %{pod} '{.metadata.uid}')" "$(sed -n 2p "$PRIV/q123.uids")" || exit 1
|
|
running %{ns} %{pod} 60"""),
|
|
C("class.txt holds %{sc}", 1, STOR + r"""
|
|
eq "$(j -n %{ns} pvc %{pvc} '{.spec.storageClassName}')" %{sc} && eq "$(one "$WS/q123/class.txt")" %{sc}"""),
|
|
],
|
|
"solution": """
|
|
k get sc # no "(default)" behind any name
|
|
k -n %{ns} describe pvc %{pvc} | tail -2 # no persistent volumes available for this claim and no storage class is set
|
|
k patch sc %{sc} -p '{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
|
|
k get sc # %{sc} (default)
|
|
k -n %{ns} get pvc %{pvc} -w # within seconds: STORAGECLASS %{sc}, then Bound once the Pod is scheduled
|
|
k -n %{ns} get pvc %{pvc} -o jsonpath='{.spec.storageClassName}' > {WS}/q123/class.txt
|
|
# A claim without storageClassName gets the default class when it is created. If there is none
|
|
# it stays Pending with the field unset - and is given the default as soon as one appears: the
|
|
# claim does not have to be recreated. Marking `standard` instead would have bound it as well,
|
|
# to the wrong class. With two defaults, new claims get the most recently created one - never
|
|
# leave two.
|
|
""",
|
|
},
|
|
# ------------------------------------------------------------------ 124 ----
|
|
{
|
|
"id": 124, "domain": "STOR", "workers": 2, "title": "A local volume with a class of its own",
|
|
"params": lambda r: {"ns": ns_pick(r, "metrics"), "sc": name_pick(r, "local-disks"), "pv": name_pick(r, "pv-disk"),
|
|
"pvc": name_pick(r, "tsdb"), "pod": name_pick(r, "prom"), "tok": f"{r.randint(100000, 999999)}"},
|
|
"task": """
|
|
A disk has been mounted on node `{W2}` at `/mnt/disks/%{pv}`; it holds a file `disk.id`. Make it
|
|
available to Pods as a **local** volume:
|
|
|
|
1. StorageClass `%{sc}` for volumes that are created by hand: provisioner
|
|
`kubernetes.io/no-provisioner`, volume binding mode `WaitForFirstConsumer`.
|
|
2. PersistentVolume `%{pv}`: 1Gi, ReadWriteOnce, class `%{sc}`, reclaim policy `Retain`, volume
|
|
type `local` with that path, tied to node `{W2}`.
|
|
3. In Namespace `%{ns}`: claim `%{pvc}` (1Gi, ReadWriteOnce, class `%{sc}`) and Pod `%{pod}`
|
|
(`busybox:1.36`, command `sleep 86400`) that mounts it at `/disk`. Do not tell the Pod where to
|
|
run - no nodeSelector, no affinity.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
$K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1
|
|
$K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K delete pv %{pv} --ignore-not-found --wait=true >/dev/null 2>&1; $K delete sc %{sc} --ignore-not-found >/dev/null 2>&1
|
|
nx "$W2" "mkdir -p /mnt/disks/%{pv} && echo 'disk %{tok}' > /mnt/disks/%{pv}/disk.id" </dev/null || exit 1
|
|
""",
|
|
"checks": [
|
|
C("StorageClass %{sc}: no-provisioner, WaitForFirstConsumer", 2, r"""
|
|
eq "$(j sc %{sc} '{.provisioner}/{.volumeBindingMode}')" kubernetes.io/no-provisioner/WaitForFirstConsumer"""),
|
|
C("PersistentVolume %{pv}: a local volume at /mnt/disks/%{pv}, tied to {W2}; 1Gi, RWO, Retain", 3, PY + r"""
|
|
export W2; py pv %{pv} '
|
|
s = o["spec"]
|
|
terms = s.get("nodeAffinity", {}).get("required", {}).get("nodeSelectorTerms", [])
|
|
nodes = [e.get("values") for t in terms for e in t.get("matchExpressions", []) if e.get("key") == "kubernetes.io/hostname" and e.get("operator") == "In"]
|
|
s.get("local", {}).get("path") == "/mnt/disks/%{pv}" and nodes == [[os.environ["W2"]]] and s["capacity"]["storage"] == "1Gi" and s["accessModes"] == ["ReadWriteOnce"] and s.get("persistentVolumeReclaimPolicy") == "Retain" and s.get("storageClassName") == "%{sc}"'"""),
|
|
C("claim %{pvc} (1Gi, RWO, %{sc}) is Bound to %{pv}", 2, STOR + r"""
|
|
bound %{ns} %{pvc} 60 && eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}/{.spec.storageClassName}/{.spec.resources.requests.storage}')" %{pv}/%{sc}/1Gi"""),
|
|
C("Pod %{pod} was sent to {W2} by its volume alone, and reads disk.id at /disk", 3, STOR + r"""
|
|
running %{ns} %{pod} 60 || exit 1
|
|
eq "$(j -n %{ns} pod %{pod} '{.spec.nodeName}')" "$W2" || exit 1
|
|
[ -z "$(j -n %{ns} pod %{pod} '{.spec.nodeSelector}{.spec.affinity}')" ] || exit 1
|
|
eq "$($K -n %{ns} exec %{pod} -- cat /disk/disk.id 2>/dev/null | tr -d '\r')" "disk %{tok}" """),
|
|
],
|
|
"solution": """
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: storage.k8s.io/v1
|
|
kind: StorageClass
|
|
metadata: {name: %{sc}}
|
|
provisioner: kubernetes.io/no-provisioner
|
|
volumeBindingMode: WaitForFirstConsumer
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolume
|
|
metadata: {name: %{pv}}
|
|
spec:
|
|
capacity: {storage: 1Gi}
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: %{sc}
|
|
persistentVolumeReclaimPolicy: Retain
|
|
local: {path: /mnt/disks/%{pv}}
|
|
nodeAffinity: # required for a local volume
|
|
required:
|
|
nodeSelectorTerms:
|
|
- matchExpressions: [{key: kubernetes.io/hostname, operator: In, values: [{W2}]}]
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: %{pvc}, namespace: %{ns}}
|
|
spec:
|
|
storageClassName: %{sc}
|
|
accessModes: [ReadWriteOnce]
|
|
resources: {requests: {storage: 1Gi}}
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: %{pod}, namespace: %{ns}}
|
|
spec:
|
|
volumes: [{name: disk, persistentVolumeClaim: {claimName: %{pvc}}}]
|
|
containers:
|
|
- name: prom
|
|
image: busybox:1.36
|
|
command: [sleep, "86400"]
|
|
volumeMounts: [{name: disk, mountPath: /disk}]
|
|
EOF
|
|
k -n %{ns} get pod %{pod} -o wide # on {W2}
|
|
k -n %{ns} exec %{pod} -- cat /disk/disk.id
|
|
# hostPath is a directory on whatever node the Pod happens to land on; a `local` volume knows
|
|
# which node it is on, and the scheduler follows it. WaitForFirstConsumer delays binding until a
|
|
# Pod needs the claim, so that the Pod's other constraints are weighed together with the volume's.
|
|
# The page to search for: "Storage Classes" -> Local.
|
|
""",
|
|
},
|
|
]
|