"""CKA pool tasks 114-124: Storage, the domain the bank was thinnest in. Written for the per-task hosts: each task has a cluster of its own (kind's `standard` StorageClass, served by the local-path provisioner in Namespace local-path-storage), so a task may break the provisioner or change the default class. Volumes are tested for real - a claim that binds, a file that is still on the node, a Pod that reads it. """ from cka.common import C, NS, PY, STOR, ns_pick, name_pick # what a setup records about the objects a task must leave alone _UID = r""" uid() { $K get "$@" -o jsonpath='{.metadata.uid}' 2>/dev/null; } """ QUESTIONS = [ # ------------------------------------------------------------------ 114 ---- { "id": 114, "domain": "STOR", "title": "A StorageClass that keeps its volumes", "params": lambda r: {"ns": ns_pick(r, "vault"), "sc": r.choice(["keep-local", "durable-local", "retain-local"]), "pvc": name_pick(r, "ledger"), "pod": name_pick(r, "writer"), "text": f"ledger-{r.randint(1000, 9999)}"}, "task": """ 1. Create StorageClass `%{sc}`: provisioner `rancher.io/local-path` (the one behind this cluster's `standard` class), reclaim policy `Retain`, volume binding mode `WaitForFirstConsumer`. It must not be the default class. 2. In Namespace `%{ns}` create PVC `%{pvc}` (300Mi, ReadWriteOnce, class `%{sc}`) and Pod `%{pod}` (`busybox:1.36`) that mounts it at `/data`, writes the line `%{text}` to `/data/keep.txt` and then sleeps. 3. Write the name of the PersistentVolume that was provisioned to `{WS}/q114/pv.txt`. 4. Delete the Pod and the claim. Write the phase the volume is in afterwards to `{WS}/q114/phase.txt`. """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q114"; mkdir -p "$WS/q114" $K delete sc %{sc} --ignore-not-found >/dev/null 2>&1 """, "checks": [ C("StorageClass %{sc}: local-path, Retain, WaitForFirstConsumer, not the default", 2, r""" eq "$(j sc %{sc} '{.provisioner}/{.reclaimPolicy}/{.volumeBindingMode}')" rancher.io/local-path/Retain/WaitForFirstConsumer \ && [ "$(j sc %{sc} '{.metadata.annotations.storageclass\.kubernetes\.io/is-default-class}')" != true ]"""), C("pv.txt names the volume that was provisioned for %{pvc}, and it is kept (Retain)", 2, STOR + r""" pv=$(one "$WS/q114/pv.txt"); [ -n "$pv" ] || exit 1 eq "$(j pv "$pv" '{.spec.storageClassName}/{.spec.claimRef.namespace}/{.spec.claimRef.name}/{.spec.persistentVolumeReclaimPolicy}')" %{sc}/%{ns}/%{pvc}/Retain"""), C("Pod and claim are deleted, the volume is Released, and phase.txt says so", 2, STOR + r""" pv=$(one "$WS/q114/pv.txt"); [ -n "$pv" ] || exit 1 gone -n %{ns} pod %{pod} && gone -n %{ns} pvc %{pvc} || exit 1 eq "$(j pv "$pv" '{.status.phase}')" Released && eq "$(one "$WS/q114/phase.txt")" Released"""), C("keep.txt is still on the node, with the line %{text}", 3, STOR + r""" pv=$(one "$WS/q114/pv.txt"); [ -n "$pv" ] || exit 1 gone -n %{ns} pvc %{pvc} || exit 1 node=$(pvnode "$pv"); path=$(pvpath "$pv"); [ -n "$node" ] && [ -n "$path" ] || exit 1 eq "$(nx "$node" "cat $path/keep.txt" /dev/null | tr -d '\r')" "%{text}" """), ], "solution": """ cat < /data/keep.txt; sleep 86400"] volumeMounts: [{name: data, mountPath: /data}] EOF k -n %{ns} get pvc %{pvc} # Pending until the Pod is scheduled: WaitForFirstConsumer k -n %{ns} get pvc %{pvc} -o jsonpath='{.spec.volumeName}' > {WS}/q114/pv.txt k -n %{ns} delete pod %{pod}; k -n %{ns} delete pvc %{pvc} k get pv $(cat {WS}/q114/pv.txt) # STATUS Released, RECLAIM POLICY Retain echo Released > {WS}/q114/phase.txt # A class's reclaimPolicy is copied into every volume it provisions. Delete (the default) removes # the volume and its data with the claim; Retain leaves a Released volume that no new claim can # bind until an administrator clears spec.claimRef. A StorageClass cannot be edited afterwards, # except for a few fields - to change the policy of one volume, patch the PV. """, }, # ------------------------------------------------------------------ 115 ---- { "id": 115, "domain": "STOR", "title": "A volume that lives and dies with its Pod", "params": lambda r: {"ns": ns_pick(r, "render"), "pod": name_pick(r, "encoder")}, "task": """ In Namespace `%{ns}` create Pod `%{pod}` (image `busybox:1.36`, command `sleep 86400`) with two volumes: - `scratch`, a **generic ephemeral volume**: 100Mi, ReadWriteOnce, StorageClass `standard`, mounted at `/scratch`, - `cache`, an `emptyDir` that is kept in memory and limited to 32Mi, mounted at `/cache`. Then write 1. the name of the PersistentVolumeClaim that Kubernetes created for the Pod to `{WS}/q115/claim.txt`, 2. the owner of that claim to `{WS}/q115/owner.txt`, as `/`. """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q115"; mkdir -p "$WS/q115" $K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1 """, "checks": [ C("Pod %{pod} runs with ephemeral volume scratch (100Mi, RWO, standard) at /scratch", 3, PY + r""" py -n %{ns} pod %{pod} ' v = {x["name"]: x for x in o["spec"]["volumes"]} t = (v.get("scratch", {}).get("ephemeral") or {}).get("volumeClaimTemplate", {}).get("spec", {}) m = {x["name"]: x["mountPath"] for x in o["spec"]["containers"][0].get("volumeMounts", [])} o["status"].get("phase") == "Running" and t.get("storageClassName") == "standard" and t.get("accessModes") == ["ReadWriteOnce"] and t.get("resources", {}).get("requests", {}).get("storage") == "100Mi" and m.get("scratch") == "/scratch"'"""), C("/scratch is a provisioned volume the Pod can write to", 2, r""" eq "$(j -n %{ns} pvc %{pod}-scratch '{.status.phase}')" Bound || exit 1 pv=$(j -n %{ns} pvc %{pod}-scratch '{.spec.volumeName}'); eq "$(j pv "$pv" '{.spec.storageClassName}')" standard || exit 1 eq "$($K -n %{ns} exec %{pod} -- sh -c 'echo probe > /scratch/.ksim && cat /scratch/.ksim' 2>/dev/null)" probe"""), C("cache is an emptyDir in memory, limited to 32Mi, at /cache", 2, PY + r""" py -n %{ns} pod %{pod} ' v = {x["name"]: x for x in o["spec"]["volumes"]} e = v.get("cache", {}).get("emptyDir") or {} m = {x["name"]: x["mountPath"] for x in o["spec"]["containers"][0].get("volumeMounts", [])} e.get("medium") == "Memory" and e.get("sizeLimit") == "32Mi" and m.get("cache") == "/cache"' || exit 1 $K -n %{ns} exec %{pod} -- sh -c 'mount | grep " /cache " | grep -q tmpfs' 2>/dev/null"""), C("claim.txt and owner.txt: %{pod}-scratch, owned by the Pod", 2, STOR + r""" eq "$(one "$WS/q115/claim.txt" | sed 's#.*/##')" %{pod}-scratch || exit 1 eq "$(j -n %{ns} pvc %{pod}-scratch '{.metadata.ownerReferences[0].kind}/{.metadata.ownerReferences[0].name}')" Pod/%{pod} || exit 1 eq "$(one "$WS/q115/owner.txt" | tr A-Z a-z)" pod/%{pod}"""), ], "solution": """ cat <- echo %{pod}-scratch > {WS}/q115/claim.txt k -n %{ns} get pvc %{pod}-scratch -o jsonpath='{.metadata.ownerReferences[0].kind}/{.metadata.ownerReferences[0].name}' echo Pod/%{pod} > {WS}/q115/owner.txt # A generic ephemeral volume is a claim written into the Pod: Kubernetes creates the PVC, the Pod # owns it, and deleting the Pod deletes claim and volume. Unlike emptyDir it can come from any # StorageClass - size, speed, snapshots - and unlike a normal claim it never outlives the Pod. # An emptyDir with medium Memory is a tmpfs: what is written there counts as the container's memory. """, }, # ------------------------------------------------------------------ 116 ---- { "id": 116, "domain": "STOR", "title": "Claims that go when the replica goes", "params": lambda r: {"ns": ns_pick(r, "queue"), "sts": name_pick(r, "broker")}, "task": """ StatefulSet `%{sts}` in Namespace `%{ns}` runs 3 replicas, each with a claim of its own. It is being shrunk for good, and the claims of replicas that no longer exist only take up storage. 1. Configure the StatefulSet so that the claims of replicas removed by **scaling down** are deleted automatically, while deleting the StatefulSet itself would still keep its claims. Do not recreate the StatefulSet. 2. Scale it down to 1 replica. 3. Write the names of the StatefulSet's claims that exist afterwards to `{WS}/q116/claims.txt`, one per line. """, "setup": NS + _UID + r""" mkns %{ns}; rm -rf "$WS/q116"; mkdir -p "$WS/q116" $K -n %{ns} delete sts %{sts} --ignore-not-found --wait=true >/dev/null 2>&1 $K -n %{ns} delete pvc -l app=%{sts} --wait=true >/dev/null 2>&1 $K apply -f - >/dev/null <<'EOF' apiVersion: v1 kind: Service metadata: {name: %{sts}, namespace: %{ns}} spec: {clusterIP: None, selector: {app: %{sts}}, ports: [{port: 5672}]} --- apiVersion: apps/v1 kind: StatefulSet metadata: {name: %{sts}, namespace: %{ns}} spec: serviceName: %{sts} replicas: 3 podManagementPolicy: Parallel selector: {matchLabels: {app: %{sts}}} template: metadata: {labels: {app: %{sts}}} spec: terminationGracePeriodSeconds: 2 containers: - name: broker image: busybox:1.36 command: [sh, -c, "hostname > /data/id; sleep 86400"] volumeMounts: [{name: data, mountPath: /data}] volumeClaimTemplates: - metadata: {name: data, labels: {app: %{sts}}} spec: storageClassName: standard accessModes: [ReadWriteOnce] resources: {requests: {storage: 50Mi}} EOF $K -n %{ns} rollout status sts %{sts} --timeout=240s >/dev/null 2>&1 || { echo "StatefulSet %{sts} did not come up"; exit 1; } uid -n %{ns} sts %{sts} > "$PRIV/q116.uid" """, "checks": [ C("retention policy: claims deleted when scaled down, kept when the StatefulSet is deleted", 2, r""" eq "$(j -n %{ns} sts %{sts} '{.spec.persistentVolumeClaimRetentionPolicy.whenScaled}/{.spec.persistentVolumeClaimRetentionPolicy.whenDeleted}')" Delete/Retain"""), C("the same StatefulSet, scaled to 1 ready replica", 2, r""" eq "$(j -n %{ns} sts %{sts} '{.metadata.uid}')" "$(cat "$PRIV/q116.uid")" || exit 1 for i in $(seq 1 20); do eq "$(j -n %{ns} sts %{sts} '{.spec.replicas}/{.status.replicas}/{.status.readyReplicas}')" 1/1/1 && exit 0; sleep 3; done; exit 1"""), C("the claims of replicas 1 and 2 are gone, data-%{sts}-0 is still Bound", 3, STOR + r""" eq "$(j -n %{ns} sts %{sts} '{.spec.replicas}')" 1 || exit 1 for i in $(seq 1 20); do gone -n %{ns} pvc data-%{sts}-1 && gone -n %{ns} pvc data-%{sts}-2 && break; sleep 3 done gone -n %{ns} pvc data-%{sts}-1 && gone -n %{ns} pvc data-%{sts}-2 && eq "$(j -n %{ns} pvc data-%{sts}-0 '{.status.phase}')" Bound"""), C("claims.txt lists what is left: data-%{sts}-0", 1, r""" eq "$(tr -d ' \t\r' < "$WS/q116/claims.txt" 2>/dev/null | sed 's#.*/##' | grep . | sort | tr '\n' ' ')" "data-%{sts}-0 " """), ], "solution": """ k -n %{ns} get sts %{sts} -o jsonpath='{.spec.persistentVolumeClaimRetentionPolicy}' # whenDeleted Retain, whenScaled Retain k -n %{ns} patch sts %{sts} -p '{"spec":{"persistentVolumeClaimRetentionPolicy":{"whenScaled":"Delete","whenDeleted":"Retain"}}}' k -n %{ns} scale sts %{sts} --replicas=1 k -n %{ns} get pvc -w # data-%{sts}-2 and data-%{sts}-1 disappear with their Pods k -n %{ns} get pvc -o name | cut -d/ -f2 > {WS}/q116/claims.txt # By default a StatefulSet never deletes a claim: scale down and up again and each replica finds # its old data. persistentVolumeClaimRetentionPolicy changes that per event - whenScaled for the # replicas a scale-down removes, whenDeleted for all of them when the StatefulSet goes. Set the # policy BEFORE scaling: it is applied to the Pods being removed, not to claims orphaned earlier # (those you delete by hand). It is one of the few StatefulSet fields that can be changed in place. """, }, # ------------------------------------------------------------------ 117 ---- { "id": 117, "domain": "STOR", "difficulty": "hard", "title": "Reserve a volume for one claim", "params": lambda r: {"ns": ns_pick(r, "atlas"), "oth": ns_pick(r, "pipeline"), "pv": name_pick(r, "pv-atlas"), "pvc": name_pick(r, "maps")}, "task": """ Claim `any-volume` in Namespace `%{oth}` is Pending: it waits for any 1Gi volume of class `manual`, and it has to keep waiting - its volume arrives next week. Team `%{ns}` needs storage today. Create PersistentVolume `%{pv}` (1Gi, ReadWriteOnce, class `manual`, `hostPath` `/mnt/%{pv}`, reclaim policy `Retain`) **reserved** for a claim `%{pvc}` in Namespace `%{ns}`, then create that claim (1Gi, ReadWriteOnce, class `manual`). It must bind `%{pv}`. `any-volume` must not be changed, deleted or recreated, and it must not get the volume. """, "setup": NS + _UID + r""" mkns %{ns}; mkns %{oth} $K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null 2>&1 $K -n %{oth} delete pvc any-volume --ignore-not-found --wait=true >/dev/null 2>&1 $K delete pv %{pv} --ignore-not-found --wait=true >/dev/null 2>&1 $K apply -f - >/dev/null <<'EOF' apiVersion: v1 kind: PersistentVolumeClaim metadata: {name: any-volume, namespace: %{oth}} spec: storageClassName: manual accessModes: [ReadWriteOnce] resources: {requests: {storage: 1Gi}} EOF uid -n %{oth} pvc any-volume > "$PRIV/q117.uid" """, "checks": [ C("PersistentVolume %{pv}: 1Gi, RWO, manual, hostPath /mnt/%{pv}, Retain", 2, r""" eq "$(j pv %{pv} '{.spec.capacity.storage}/{.spec.accessModes[*]}/{.spec.storageClassName}/{.spec.persistentVolumeReclaimPolicy}')" 1Gi/ReadWriteOnce/manual/Retain \ && eq "$(j pv %{pv} '{.spec.hostPath.path}')" /mnt/%{pv}"""), C("it is reserved: its claimRef names %{pvc} in %{ns}", 2, r""" eq "$(j pv %{pv} '{.spec.claimRef.namespace}/{.spec.claimRef.name}')" %{ns}/%{pvc}"""), C("claim %{pvc} (1Gi, RWO, manual) is Bound to %{pv}", 3, STOR + r""" bound %{ns} %{pvc} 30 || exit 1 eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}/{.spec.storageClassName}/{.spec.accessModes[*]}/{.spec.resources.requests.storage}')" %{pv}/manual/ReadWriteOnce/1Gi"""), C("any-volume is the same claim as before and still Pending", 2, r""" [ -n "$(j pv %{pv} '{.metadata.name}')" ] || exit 1 eq "$(j -n %{oth} pvc any-volume '{.metadata.uid}')" "$(cat "$PRIV/q117.uid")" && eq "$(j -n %{oth} pvc any-volume '{.status.phase}')" Pending"""), ], "solution": """ cat </dev/null 2>&1 $K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null 2>&1 $K delete pv %{pv} --ignore-not-found --wait=true >/dev/null 2>&1 nx "$W1" "mkdir -p /mnt/%{pv} && echo 'ledger %{tok} balanced' > /mnt/%{pv}/ledger.db" /dev/null < "$WS/q118/app.yaml" <<'EOF' apiVersion: v1 kind: PersistentVolumeClaim metadata: {name: %{pvc}, namespace: %{ns}} spec: accessModes: [ReadWriteOnce] resources: {requests: {storage: 500Mi}} --- apiVersion: apps/v1 kind: Deployment metadata: {name: %{dep}, namespace: %{ns}} spec: replicas: 1 strategy: {type: Recreate} selector: {matchLabels: {app: %{dep}}} template: metadata: {labels: {app: %{dep}}} spec: terminationGracePeriodSeconds: 2 volumes: [{name: data, persistentVolumeClaim: {claimName: %{pvc}}}] containers: - name: app image: busybox:1.36 command: [sh, -c, "cat /data/ledger.db 2>/dev/null || echo 'no ledger found'; sleep 86400"] volumeMounts: [{name: data, mountPath: /data}] EOF $K apply -f "$WS/q118/app.yaml" >/dev/null $K -n %{ns} rollout status deploy %{dep} --timeout=240s >/dev/null 2>&1 || { echo "Deployment %{dep} did not come up"; exit 1; } $K -n %{ns} get pvc %{pvc} -o jsonpath='{.spec.volumeName}' > "$PRIV/q118.wrong" [ -s "$PRIV/q118.wrong" ] || { echo "claim %{pvc} was not provisioned"; exit 1; } uid pv %{pv} > "$PRIV/q118.uid" """, "checks": [ C("wrong-pv.txt names the volume the claim was given by mistake", 1, STOR + r""" eq "$(one "$WS/q118/wrong-pv.txt" | sed 's#.*/##')" "$(cat "$PRIV/q118.wrong")" """), C("claim %{pvc} (500Mi) is Bound to %{pv}, which is the volume it always was", 3, STOR + r""" bound %{ns} %{pvc} 30 || exit 1 eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}/{.spec.resources.requests.storage}')" %{pv}/500Mi || exit 1 eq "$(j pv %{pv} '{.metadata.uid}/{.spec.hostPath.path}')" "$(cat "$PRIV/q118.uid")//mnt/%{pv}" """), C("the Deployment's Pod prints the ledger", 3, r""" eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}')" %{pv} || exit 1 for i in $(seq 1 20); do $K -n %{ns} logs deploy/%{dep} 2>/dev/null | grep -q 'ledger %{tok} balanced' && exit 0; sleep 3 done; exit 1"""), C("the volume provisioned by mistake is gone", 1, STOR + r""" eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}')" %{pv} || exit 1 for i in $(seq 1 10); do gone pv "$(cat "$PRIV/q118.wrong")" && exit 0; sleep 3; done; exit 1"""), ], "solution": """ k -n %{ns} get pvc %{pvc} # Bound to pvc-..., STORAGECLASS standard - not to %{pv} k -n %{ns} get pvc %{pvc} -o jsonpath='{.spec.volumeName}' > {WS}/q118/wrong-pv.txt k get pv %{pv} # Available, STORAGECLASS empty # The manifest names no class, so the claim was given the cluster's default one (standard) and a # brand-new, empty volume was provisioned for it. A claim's class and volume cannot be edited: k -n %{ns} scale deploy %{dep} --replicas=0 # a claim in use is not deleted (pvc-protection) k -n %{ns} delete pvc %{pvc} # its volume goes with it: reclaim policy Delete cat < `, where the reason is one of `capacity`, `accessmode`, `class`. 2. Get all three Bound to their volumes. The PersistentVolumes must not be changed or recreated; a claim may be recreated under its name, keeping its selector. """, "setup": NS + _UID + r""" mkns %{ns}; rm -rf "$WS/q119"; mkdir -p "$WS/q119" $K -n %{ns} delete pvc reports archive media --ignore-not-found --wait=true >/dev/null 2>&1 $K delete pv %{p}-reports %{p}-archive %{p}-media --ignore-not-found --wait=true >/dev/null 2>&1 pv() { # use size cat </dev/null for u in reports archive media; do uid pv %{p}-$u; echo; done > "$PRIV/q119.uids" """, "checks": [ C("reasons.txt: reports capacity, archive accessmode, media class", 2, r""" eq "$(tr -s ' \t' ' ' < "$WS/q119/reasons.txt" 2>/dev/null | tr -d '\r' | sed 's/^ //; s/ $//' | tr A-Z a-z | grep . | sort | tr '\n' ';')" \ "archive accessmode;media class;reports capacity;" """), ] + [ C(f"claim {u} is Bound to %{{p}}-{u}, and the volume is as it was", 2, STOR + rf""" bound %{{ns}} {u} 30 || exit 1 eq "$(j -n %{{ns}} pvc {u} '{{.spec.volumeName}}/{{.spec.selector.matchLabels.use}}')" %{{p}}-{u}/{u} || exit 1 eq "$(j pv %{{p}}-{u} '{{.metadata.uid}}')" "$(sed -n {n}p "$PRIV/q119.uids")" || exit 1 eq "$(j pv %{{p}}-{u} '{{.spec.capacity.storage}}/{{.spec.accessModes[*]}}/{{.spec.storageClassName}}/{{.metadata.labels.use}}')" {size}/ReadWriteOnce/manual/{u}""") for n, (u, size) in enumerate([("reports", "1Gi"), ("archive", "2Gi"), ("media", "1Gi")], 1) ], "solution": """ k -n %{ns} get pvc; k get pv --show-labels k -n %{ns} describe pvc reports archive media | grep -E '^Name:|Warning|Normal' # media: storageclass.storage.k8s.io "manuel" not found -> class (a typo for manual) # reports, archive: no persistent volumes available for this claim -> compare claim and volume: k -n %{ns} get pvc -o custom-columns=NAME:.metadata.name,SIZE:.spec.resources.requests.storage,MODES:.spec.accessModes,CLASS:.spec.storageClassName k get pv -o custom-columns=NAME:.metadata.name,SIZE:.spec.capacity.storage,MODES:.spec.accessModes,CLASS:.spec.storageClassName # reports asks for 5Gi, its volume has 1Gi -> capacity # archive asks for ReadWriteMany, its volume offers ReadWriteOnce -> accessmode printf 'reports capacity\\narchive accessmode\\nmedia class\\n' > {WS}/q119/reasons.txt # size (downwards), access modes and class of a claim cannot be edited: recreate the three k -n %{ns} delete pvc reports archive media for c in "reports 1Gi" "archive 2Gi" "media 1Gi"; do set -- $c; cat </dev/null 2>&1 $K -n %{ns} delete pvc %{dep}-data --ignore-not-found --wait=true >/dev/null 2>&1 $K delete pv %{pv} --ignore-not-found --wait=true >/dev/null 2>&1 nx "$W1" "mkdir -p /mnt/%{pv} && echo 'orders %{tok} open' > /mnt/%{pv}/orders.db" /dev/null 2>&1; $K label node "$W2" disk=ssd --overwrite >/dev/null $K apply -f - >/dev/null </dev/null <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: {name: %{dep}, namespace: %{ns}} spec: replicas: 1 strategy: {type: Recreate} selector: {matchLabels: {app: %{dep}}} template: metadata: {labels: {app: %{dep}}} spec: terminationGracePeriodSeconds: 2 nodeSelector: {disk: ssd} volumes: [{name: data, persistentVolumeClaim: {claimName: %{dep}-data}}] containers: - name: app image: busybox:1.36 command: [sh, -c, "cat /data/orders.db; sleep 86400"] volumeMounts: [{name: data, mountPath: /data}] EOF { uid pv %{pv}; echo; uid -n %{ns} pvc %{dep}-data; echo; } > "$PRIV/q120.uids" """, "checks": [ C("event.txt holds the scheduler's message: no node matches the volume's node affinity", 2, r""" grep -Eqi "match PersistentVolume's node affinity|volume node affinity conflict" "$WS/q120/event.txt" 2>/dev/null"""), C("the Deployment's Pod runs on the node that holds the volume", 3, STOR + r""" for i in $(seq 1 20); do p=$(newest %{ns} app=%{dep}); [ -n "$p" ] && break; sleep 3; done [ -n "$p" ] && eq "$(j -n %{ns} pod "$p" '{.spec.nodeName}')" "$W1" """), C("its log shows the orders line from the volume", 2, r""" for i in $(seq 1 10); do $K -n %{ns} logs deploy/%{dep} 2>/dev/null | grep -q 'orders %{tok} open' && exit 0; sleep 3 done; exit 1"""), C("PersistentVolume and claim are the ones that were there", 1, STOR + r""" [ -n "$(newest %{ns} app=%{dep})" ] || exit 1 eq "$(j pv %{pv} '{.metadata.uid}')" "$(sed -n 1p "$PRIV/q120.uids")" && eq "$(j -n %{ns} pvc %{dep}-data '{.metadata.uid}')" "$(sed -n 2p "$PRIV/q120.uids")" \ && eq "$(pvnode %{pv})/$(pvpath %{pv})" "$W1//mnt/%{pv}" """), ], "solution": """ k -n %{ns} describe pod -l app=%{dep} | tail -5 # 0/3 nodes are available: 1 node(s) didn't match PersistentVolume's node affinity, 1 node(s) # didn't match Pod's node affinity/selector, 1 node(s) had untolerated taint(s). ... k -n %{ns} get events --field-selector reason=FailedScheduling -o jsonpath='{.items[-1:].message}' > {WS}/q120/event.txt k get pv %{pv} -o jsonpath='{.spec.nodeAffinity}'; echo # the volume is on {W1} k -n %{ns} get deploy %{dep} -o jsonpath='{.spec.template.spec.nodeSelector}'; echo # disk=ssd k get nodes -L disk # only {W2} carries disk=ssd # The volume cannot move, so the Pod has to: drop the nodeSelector (the volume's own node # affinity already sends the Pod to the right node) ... k -n %{ns} patch deploy %{dep} --type=json -p '[{"op":"remove","path":"/spec/template/spec/nodeSelector"}]' # ... or, if the selector has a reason to exist, give {W1} the label: k label node {W1} disk=ssd k -n %{ns} rollout status deploy %{dep}; k -n %{ns} logs deploy/%{dep} # The scheduler ANDs every constraint: taints, the Pod's nodeSelector/affinity, and the node # affinity of each volume it mounts. "didn't match PersistentVolume's node affinity" (before v1.35: # "had volume node affinity conflict") always means: the Pod is not allowed onto the one node its # volume lives on. """, }, # ------------------------------------------------------------------ 121 ---- { "id": 121, "domain": "STOR", "title": "A claim that will not go away", "params": lambda r: {"ns": ns_pick(r, "gallery"), "dep": name_pick(r, "thumbs"), "pvc": name_pick(r, "uploads")}, "task": """ Claim `%{pvc}` in Namespace `%{ns}` was deleted yesterday, and it is still there, `Terminating`. 1. Write the finalizer that keeps it to `{WS}/q121/finalizer.txt`. 2. Write what still uses the claim to `{WS}/q121/used-by.txt`, as `/` of the workload (not of a Pod). 3. The data is not needed any more. Let the deletion finish - do not edit the claim - by making that workload keep its files in an `emptyDir` at the same path instead. It must keep its name and end up with 1 running replica. """, "setup": NS + _UID + r""" mkns %{ns}; rm -rf "$WS/q121"; mkdir -p "$WS/q121" $K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null 2>&1 $K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null 2>&1 $K apply -f - >/dev/null <<'EOF' apiVersion: v1 kind: PersistentVolumeClaim metadata: {name: %{pvc}, namespace: %{ns}} spec: accessModes: [ReadWriteOnce] resources: {requests: {storage: 100Mi}} --- apiVersion: apps/v1 kind: Deployment metadata: {name: %{dep}, namespace: %{ns}} spec: replicas: 1 strategy: {type: Recreate} selector: {matchLabels: {app: %{dep}}} template: metadata: {labels: {app: %{dep}}} spec: terminationGracePeriodSeconds: 2 volumes: [{name: files, persistentVolumeClaim: {claimName: %{pvc}}}] containers: - name: worker image: busybox:1.36 command: [sh, -c, "date > /srv/files/started; sleep 86400"] volumeMounts: [{name: files, mountPath: /srv/files}] EOF $K -n %{ns} rollout status deploy %{dep} --timeout=240s >/dev/null 2>&1 || { echo "Deployment %{dep} did not come up"; exit 1; } $K -n %{ns} get pvc %{pvc} -o jsonpath='{.spec.volumeName}' > "$PRIV/q121.pv" uid -n %{ns} deploy %{dep} > "$PRIV/q121.uid" $K -n %{ns} delete pvc %{pvc} --wait=false >/dev/null """, "checks": [ C("finalizer.txt holds kubernetes.io/pvc-protection", 1, STOR + r"""eq "$(one "$WS/q121/finalizer.txt")" kubernetes.io/pvc-protection"""), C("used-by.txt names Deployment %{dep}", 2, STOR + r""" case "$(one "$WS/q121/used-by.txt" | tr A-Z a-z)" in deployment/%{dep}|deployments/%{dep}|deploy/%{dep}|deployment.apps/%{dep}|deployments.apps/%{dep}) ;; *) exit 1 ;; esac"""), C("the claim is gone, and its volume with it", 3, STOR + r""" for i in $(seq 1 20); do gone -n %{ns} pvc %{pvc} && gone pv "$(cat "$PRIV/q121.pv")" && exit 0; sleep 3; done; exit 1"""), C("the same Deployment runs 1 replica with an emptyDir at /srv/files", 2, PY + r""" eq "$(j -n %{ns} deploy %{dep} '{.metadata.uid}')" "$(cat "$PRIV/q121.uid")" || exit 1 py -n %{ns} deploy %{dep} ' s = o["spec"]["template"]["spec"] v = {x["name"]: x for x in s.get("volumes", [])} m = [x["name"] for x in s["containers"][0].get("volumeMounts", []) if x["mountPath"] == "/srv/files"] len(m) == 1 and "emptyDir" in v.get(m[0], {}) and not any("persistentVolumeClaim" in x for x in v.values())' || exit 1 for i in $(seq 1 20); do eq "$(j -n %{ns} deploy %{dep} '{.status.updatedReplicas}/{.status.availableReplicas}/{.status.replicas}')" 1/1/1 && exit 0; sleep 3; done; exit 1"""), ], "solution": """ k -n %{ns} get pvc %{pvc} # STATUS Terminating k -n %{ns} get pvc %{pvc} -o jsonpath='{.metadata.finalizers}'; echo # ["kubernetes.io/pvc-protection"] echo kubernetes.io/pvc-protection > {WS}/q121/finalizer.txt k -n %{ns} describe pvc %{pvc} | grep 'Used By' # Used By: %{dep}-... (a Pod; find its owner) k -n %{ns} get deploy -o custom-columns=NAME:.metadata.name,CLAIMS:.spec.template.spec.volumes[*].persistentVolumeClaim.claimName echo deployment/%{dep} > {WS}/q121/used-by.txt k -n %{ns} edit deploy %{dep} # volumes: # - name: files # emptyDir: {} <- instead of persistentVolumeClaim: {claimName: %{pvc}} k -n %{ns} rollout status deploy %{dep} k -n %{ns} get pvc; k get pv # the claim is gone, and so is its volume (reclaim policy Delete) # "Storage object in use protection": a claim that a Pod still mounts is only marked for deletion; # the pvc-protection finalizer is removed by the control plane when the last such Pod is gone. # Removing the finalizer by hand would pull the claim out from under a running Pod - the fix is # always to stop or change whatever uses it. """, }, # ------------------------------------------------------------------ 122 ---- { "id": 122, "domain": "STOR", "difficulty": "hard", "title": "Dynamic provisioning has stopped", "params": lambda r: {"ns": ns_pick(r, "lab"), "sc": r.choice(["node-local", "scratch-local", "team-local"]), "pvc": name_pick(r, "results"), "pod": name_pick(r, "runner")}, "task": """ Pod `%{pod}` in Namespace `%{ns}` is Pending because its claim `%{pvc}` never gets a volume. Until last week, volumes of StorageClass `%{sc}` were provisioned within seconds. More than one thing is wrong. 1. Fix it: the claim must be Bound to a dynamically provisioned volume of class `%{sc}`, and the Pod must run. The class keeps its name, its reclaim policy and its volume binding mode. Claim and Pod may be recreated from `{WS}/q122/app.yaml` if you need to. 2. Write the provisioner that class `%{sc}` uses in the end to `{WS}/q122/provisioner.txt`. """, "setup": NS + r""" mkns %{ns}; rm -rf "$WS/q122"; mkdir -p "$WS/q122" $K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1 $K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null 2>&1 $K delete sc %{sc} --ignore-not-found >/dev/null 2>&1 $K -n local-path-storage get deploy local-path-provisioner >/dev/null 2>&1 || { echo "this cluster has no local-path provisioner"; exit 1; } $K apply -f - >/dev/null <<'EOF' apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: {name: %{sc}} provisioner: rancher.io/localpath reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer EOF $K -n local-path-storage scale deploy local-path-provisioner --replicas=0 >/dev/null cat > "$WS/q122/app.yaml" <<'EOF' apiVersion: v1 kind: PersistentVolumeClaim metadata: {name: %{pvc}, namespace: %{ns}} spec: storageClassName: %{sc} accessModes: [ReadWriteOnce] resources: {requests: {storage: 200Mi}} --- apiVersion: v1 kind: Pod metadata: {name: %{pod}, namespace: %{ns}} spec: terminationGracePeriodSeconds: 2 volumes: [{name: out, persistentVolumeClaim: {claimName: %{pvc}}}] containers: - name: runner image: busybox:1.36 command: [sh, -c, "date > /out/started; sleep 86400"] volumeMounts: [{name: out, mountPath: /out}] EOF $K apply -f "$WS/q122/app.yaml" >/dev/null """, "checks": [ C("the provisioner is running again", 2, r""" [ "$(j -n local-path-storage deploy local-path-provisioner '{.status.availableReplicas}')" -ge 1 ] 2>/dev/null"""), C("StorageClass %{sc}: provisioner rancher.io/local-path, still Delete and WaitForFirstConsumer", 2, r""" eq "$(j sc %{sc} '{.provisioner}/{.reclaimPolicy}/{.volumeBindingMode}')" rancher.io/local-path/Delete/WaitForFirstConsumer"""), C("claim %{pvc} is Bound to a volume of class %{sc}, and Pod %{pod} runs", 3, STOR + r""" bound %{ns} %{pvc} 60 && running %{ns} %{pod} 60 || exit 1 pv=$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}') eq "$(j pv "$pv" '{.spec.storageClassName}')" %{sc} && eq "$(j -n %{ns} pod %{pod} '{.spec.volumes[0].persistentVolumeClaim.claimName}')" %{pvc}"""), C("a new claim of class %{sc} gets a volume within a minute", 2, STOR + r""" $K -n %{ns} delete pod ksim-probe --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1 $K -n %{ns} delete pvc ksim-probe --ignore-not-found --wait=true >/dev/null 2>&1 $K apply -f - >/dev/null 2>&1 <<'EOF' apiVersion: v1 kind: PersistentVolumeClaim metadata: {name: ksim-probe, namespace: %{ns}} spec: storageClassName: %{sc} accessModes: [ReadWriteOnce] resources: {requests: {storage: 10Mi}} --- apiVersion: v1 kind: Pod metadata: {name: ksim-probe, namespace: %{ns}} spec: terminationGracePeriodSeconds: 1 volumes: [{name: v, persistentVolumeClaim: {claimName: ksim-probe}}] containers: [{name: c, image: "busybox:1.36", command: [sleep, "600"], volumeMounts: [{name: v, mountPath: /v}]}] EOF rc=1; bound %{ns} ksim-probe 60 && rc=0 $K -n %{ns} delete pod ksim-probe --wait=false --grace-period=1 >/dev/null 2>&1 $K -n %{ns} delete pvc ksim-probe --wait=false >/dev/null 2>&1 exit $rc"""), C("provisioner.txt holds rancher.io/local-path", 1, STOR + r""" eq "$(j sc %{sc} '{.provisioner}')" rancher.io/local-path && eq "$(one "$WS/q122/provisioner.txt")" rancher.io/local-path"""), ], "solution": """ k -n %{ns} describe pvc %{pvc} | tail -3 # waiting for a volume to be created either by the external provisioner 'rancher.io/localpath' # or manually by the system administrator ... verify that the provisioner is running and correctly registered k get sc # %{sc}: rancher.io/localpath standard: rancher.io/local-path <- a typo k get pods -A | grep -i provision # nothing: who serves rancher.io/local-path? k -n local-path-storage get deploy # local-path-provisioner 0/0 - scaled to zero k -n local-path-storage scale deploy local-path-provisioner --replicas=1 # a StorageClass's provisioner cannot be edited: write it out, correct it, replace it k get sc %{sc} -o yaml > sc.yaml vi sc.yaml # provisioner: rancher.io/local-path (drop uid, resourceVersion, creationTimestamp) k delete sc %{sc}; k apply -f sc.yaml k -n %{ns} get pvc,pod -w # the waiting claim is picked up by itself within half a minute # (if it is not: k -n %{ns} delete pod %{pod}; k -n %{ns} delete pvc %{pvc}; k apply -f {WS}/q122/app.yaml) echo rancher.io/local-path > {WS}/q122/provisioner.txt # Dynamic provisioning is a chain: the claim names a class, the class names a provisioner, and a # controller that answers to exactly that name has to be running. The claim's event tells you # which name it is waiting for - then check the class for a typo and the cluster for the controller. """, }, # ------------------------------------------------------------------ 123 ---- { "id": 123, "domain": "STOR", "title": "A claim that waits for a default class", "params": lambda r: {"ns": ns_pick(r, "intake"), "sc": name_pick(r, "general"), "pvc": name_pick(r, "inbox"), "pod": name_pick(r, "scanner")}, "task": """ Claim `%{pvc}` in Namespace `%{ns}` is Pending, and so is Pod `%{pod}` that mounts it: the claim names no StorageClass, and this cluster has no default one at the moment. 1. Without changing, deleting or recreating the claim or the Pod, get the claim Bound to a volume of class `%{sc}` and the Pod running. 2. Exactly one StorageClass must be the default in the end. 3. Write the StorageClass the claim ended up with to `{WS}/q123/class.txt`. """, "setup": NS + _UID + r""" mkns %{ns}; rm -rf "$WS/q123"; mkdir -p "$WS/q123" $K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1 $K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null 2>&1 $K delete sc %{sc} --ignore-not-found >/dev/null 2>&1 $K annotate sc standard storageclass.kubernetes.io/is-default-class- >/dev/null 2>&1 $K apply -f - >/dev/null <<'EOF' apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: {name: %{sc}} provisioner: rancher.io/local-path reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer --- apiVersion: v1 kind: PersistentVolumeClaim metadata: {name: %{pvc}, namespace: %{ns}} spec: accessModes: [ReadWriteOnce] resources: {requests: {storage: 200Mi}} --- apiVersion: v1 kind: Pod metadata: {name: %{pod}, namespace: %{ns}} spec: terminationGracePeriodSeconds: 2 volumes: [{name: inbox, persistentVolumeClaim: {claimName: %{pvc}}}] containers: - name: scanner image: busybox:1.36 command: [sleep, "86400"] volumeMounts: [{name: inbox, mountPath: /inbox}] EOF [ -z "$(j -n %{ns} pvc %{pvc} '{.spec.storageClassName}')" ] || { echo "claim %{pvc} was given a class although no default exists"; exit 1; } { uid -n %{ns} pvc %{pvc}; echo; uid -n %{ns} pod %{pod}; echo; } > "$PRIV/q123.uids" """, "checks": [ C("%{sc} is the default StorageClass, and the only one", 2, PY + r""" py sc ' d = sorted(x["metadata"]["name"] for x in o["items"] if (x["metadata"].get("annotations") or {}).get("storageclass.kubernetes.io/is-default-class") == "true") d == ["%{sc}"]'"""), C("the same claim now has class %{sc} and is Bound", 3, STOR + r""" eq "$(j -n %{ns} pvc %{pvc} '{.metadata.uid}')" "$(sed -n 1p "$PRIV/q123.uids")" || exit 1 bound %{ns} %{pvc} 60 && eq "$(j -n %{ns} pvc %{pvc} '{.spec.storageClassName}')" %{sc}"""), C("the same Pod runs with the claim mounted", 2, STOR + r""" eq "$(j -n %{ns} pod %{pod} '{.metadata.uid}')" "$(sed -n 2p "$PRIV/q123.uids")" || exit 1 running %{ns} %{pod} 60"""), C("class.txt holds %{sc}", 1, STOR + r""" eq "$(j -n %{ns} pvc %{pvc} '{.spec.storageClassName}')" %{sc} && eq "$(one "$WS/q123/class.txt")" %{sc}"""), ], "solution": """ k get sc # no "(default)" behind any name k -n %{ns} describe pvc %{pvc} | tail -2 # no persistent volumes available for this claim and no storage class is set k patch sc %{sc} -p '{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}' k get sc # %{sc} (default) k -n %{ns} get pvc %{pvc} -w # within seconds: STORAGECLASS %{sc}, then Bound once the Pod is scheduled k -n %{ns} get pvc %{pvc} -o jsonpath='{.spec.storageClassName}' > {WS}/q123/class.txt # A claim without storageClassName gets the default class when it is created. If there is none # it stays Pending with the field unset - and is given the default as soon as one appears: the # claim does not have to be recreated. Marking `standard` instead would have bound it as well, # to the wrong class. With two defaults, new claims get the most recently created one - never # leave two. """, }, # ------------------------------------------------------------------ 124 ---- { "id": 124, "domain": "STOR", "workers": 2, "title": "A local volume with a class of its own", "params": lambda r: {"ns": ns_pick(r, "metrics"), "sc": name_pick(r, "local-disks"), "pv": name_pick(r, "pv-disk"), "pvc": name_pick(r, "tsdb"), "pod": name_pick(r, "prom"), "tok": f"{r.randint(100000, 999999)}"}, "task": """ A disk has been mounted on node `{W2}` at `/mnt/disks/%{pv}`; it holds a file `disk.id`. Make it available to Pods as a **local** volume: 1. StorageClass `%{sc}` for volumes that are created by hand: provisioner `kubernetes.io/no-provisioner`, volume binding mode `WaitForFirstConsumer`. 2. PersistentVolume `%{pv}`: 1Gi, ReadWriteOnce, class `%{sc}`, reclaim policy `Retain`, volume type `local` with that path, tied to node `{W2}`. 3. In Namespace `%{ns}`: claim `%{pvc}` (1Gi, ReadWriteOnce, class `%{sc}`) and Pod `%{pod}` (`busybox:1.36`, command `sleep 86400`) that mounts it at `/disk`. Do not tell the Pod where to run - no nodeSelector, no affinity. """, "setup": NS + r""" mkns %{ns} $K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1 $K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null 2>&1 $K delete pv %{pv} --ignore-not-found --wait=true >/dev/null 2>&1; $K delete sc %{sc} --ignore-not-found >/dev/null 2>&1 nx "$W2" "mkdir -p /mnt/disks/%{pv} && echo 'disk %{tok}' > /mnt/disks/%{pv}/disk.id" /dev/null | tr -d '\r')" "disk %{tok}" """), ], "solution": """ cat < Local. """, }, ]