"""CKA pool tasks 114-124: Storage, the domain the bank was thinnest in.

Written for the per-task hosts: each task has a cluster of its own (kind's `standard` StorageClass,
served by the local-path provisioner in Namespace local-path-storage), so a task may break the
provisioner or change the default class. Volumes are tested for real - a claim that binds, a file
that is still on the node, a Pod that reads it.
"""

from sim.bank import cmd, hand
from cka.common import C, NS, PY, STOR, ns_pick, name_pick

# what a setup records about the objects a task must leave alone
_UID = r"""
uid() { $K get "$@" -o jsonpath='{.metadata.uid}' 2>/dev/null; }
"""

QUESTIONS = [
# ------------------------------------------------------------------ 114 ----
{
 "id": 114, "domain": "STOR", "title": "A StorageClass that keeps its volumes",
 "params": lambda r: {"ns": ns_pick(r, "vault"), "sc": r.choice(["keep-local", "durable-local", "retain-local"]),
                      "pvc": name_pick(r, "ledger"), "pod": name_pick(r, "writer"), "text": f"ledger-{r.randint(1000, 9999)}"},
 "task": """
1. Create StorageClass `%{sc}`: provisioner `rancher.io/local-path` (the one behind this cluster's
   `standard` class), reclaim policy `Retain`, volume binding mode `WaitForFirstConsumer`. It must
   not be the default class.
2. In Namespace `%{ns}` create PVC `%{pvc}` (300Mi, ReadWriteOnce, class `%{sc}`) and Pod `%{pod}`
   (`busybox:1.36`) that mounts it at `/data`, writes the line `%{text}` to `/data/keep.txt` and
   then sleeps.
3. Write the name of the PersistentVolume that was provisioned to `{WS}/q114/pv.txt`.
4. Delete the Pod and the claim. Write the phase the volume is in afterwards to
   `{WS}/q114/phase.txt`.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q114"; mkdir -p "$WS/q114"
$K delete sc %{sc} --ignore-not-found >/dev/null 2>&1
""",
 "checks": [
  C("StorageClass %{sc}: local-path, Retain, WaitForFirstConsumer, not the default", 2, r"""
eq "$(j sc %{sc} '{.provisioner}/{.reclaimPolicy}/{.volumeBindingMode}')" rancher.io/local-path/Retain/WaitForFirstConsumer \
 && [ "$(j sc %{sc} '{.metadata.annotations.storageclass\.kubernetes\.io/is-default-class}')" != true ]"""),
  C("pv.txt names the volume that was provisioned for %{pvc}, and it is kept (Retain)", 2, STOR + r"""
pv=$(one "$WS/q114/pv.txt"); [ -n "$pv" ] || exit 1
eq "$(j pv "$pv" '{.spec.storageClassName}/{.spec.claimRef.namespace}/{.spec.claimRef.name}/{.spec.persistentVolumeReclaimPolicy}')" %{sc}/%{ns}/%{pvc}/Retain"""),
  C("Pod and claim are deleted, the volume is Released, and phase.txt says so", 2, STOR + r"""
pv=$(one "$WS/q114/pv.txt"); [ -n "$pv" ] || exit 1
gone -n %{ns} pod %{pod} && gone -n %{ns} pvc %{pvc} || exit 1
eq "$(j pv "$pv" '{.status.phase}')" Released && eq "$(one "$WS/q114/phase.txt")" Released"""),
  C("keep.txt is still on the node, with the line %{text}", 3, STOR + r"""
pv=$(one "$WS/q114/pv.txt"); [ -n "$pv" ] || exit 1
gone -n %{ns} pvc %{pvc} || exit 1
node=$(pvnode "$pv"); path=$(pvpath "$pv"); [ -n "$node" ] && [ -n "$path" ] || exit 1
eq "$(nx "$node" "cat $path/keep.txt" </dev/null 2>/dev/null | tr -d '\r')" "%{text}" """),
 ],
 "solution": [cmd(r"""k get sc standard -o yaml | grep provisioner""", r"""`rancher.io/local-path`."""),
              hand(r"""vim sc.yaml""",
                   r"""The class, from the docs page *Storage Classes* - no `is-default-class` annotation, so not the default - and the claim:
```
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: %{sc}
provisioner: rancher.io/local-path
reclaimPolicy: Retain
volumeBindingMode: WaitForFirstConsumer
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: %{pvc}
  namespace: %{ns}
spec:
  storageClassName: %{sc}
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 300Mi
```""",
                   run=r"""cat > sc.yaml <<'EOF'
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: %{sc}
provisioner: rancher.io/local-path
reclaimPolicy: Retain
volumeBindingMode: WaitForFirstConsumer
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: %{pvc}
  namespace: %{ns}
spec:
  storageClassName: %{sc}
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 300Mi
EOF"""),
              cmd(r"""k apply -f sc.yaml
k -n %{ns} run %{pod} --image=busybox:1.36 --dry-run=client -o yaml -- sh -c "echo %{text} > /data/keep.txt; sleep 86400" > pod.yaml""", r"""The claim stays `Pending` until a Pod uses it: `WaitForFirstConsumer`."""),
              hand(r"""vim pod.yaml""",
                   r"""Add to the container, below `name: %{pod}`:
```
    volumeMounts:
    - name: data
      mountPath: /data
```
and to the Pod's `spec:`, right below `spec:`:
```
  volumes:
  - name: data
    persistentVolumeClaim:
      claimName: %{pvc}
```""",
                   run=r"""yq -i '.spec.containers[0].volumeMounts = [{"name": "data", "mountPath": "/data"}]
  | .spec.volumes = [{"name": "data", "persistentVolumeClaim": {"claimName": "%{pvc}"}}]' pod.yaml"""),
              cmd(r"""k apply -f pod.yaml"""),
              hand(r"""k -n %{ns} get pod,pvc""",
                   r"""Again until the Pod is `Running` and the claim `Bound`; note its `VOLUME`.""",
                   run=r"""kubectl -n %{ns} wait --for=condition=Ready pod/%{pod} --timeout=240s"""),
              hand(r"""echo pvc-0b1c2d3e-... > {WS}/q114/pv.txt""",
                   r"""With the volume's name.""",
                   run=r"""kubectl -n %{ns} get pvc %{pvc} -o jsonpath='{.spec.volumeName}' > {WS}/q114/pv.txt"""),
              cmd(r"""k -n %{ns} delete pod %{pod} --now
k -n %{ns} delete pvc %{pvc}"""),
              hand(r"""k get pv
echo Released > {WS}/q114/phase.txt""",
                   r"""`STATUS Released`, `RECLAIM POLICY Retain`: the class's policy is copied into every volume it provisions. A Released volume binds no new claim until its `claimRef` is cleared.""",
                   run=r"""sleep 4
kubectl get pv $(cat {WS}/q114/pv.txt) -o jsonpath='{.status.phase}' > {WS}/q114/phase.txt""")],
},
# ------------------------------------------------------------------ 115 ----
{
 "id": 115, "domain": "STOR", "title": "A volume that lives and dies with its Pod",
 "params": lambda r: {"ns": ns_pick(r, "render"), "pod": name_pick(r, "encoder")},
 "task": """
In Namespace `%{ns}` create Pod `%{pod}` (image `busybox:1.36`, command `sleep 86400`) with two
volumes:

- `scratch`, a **generic ephemeral volume**: 100Mi, ReadWriteOnce, StorageClass `standard`,
  mounted at `/scratch`,
- `cache`, an `emptyDir` that is kept in memory and limited to 32Mi, mounted at `/cache`.

Then write
1. the name of the PersistentVolumeClaim that Kubernetes created for the Pod to
   `{WS}/q115/claim.txt`,
2. the owner of that claim to `{WS}/q115/owner.txt`, as `<kind>/<name>`.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q115"; mkdir -p "$WS/q115"
$K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1
""",
 "checks": [
  C("Pod %{pod} runs with ephemeral volume scratch (100Mi, RWO, standard) at /scratch", 3, PY + r"""
py -n %{ns} pod %{pod} '
v = {x["name"]: x for x in o["spec"]["volumes"]}
t = (v.get("scratch", {}).get("ephemeral") or {}).get("volumeClaimTemplate", {}).get("spec", {})
m = {x["name"]: x["mountPath"] for x in o["spec"]["containers"][0].get("volumeMounts", [])}
o["status"].get("phase") == "Running" and t.get("storageClassName") == "standard" and t.get("accessModes") == ["ReadWriteOnce"] and t.get("resources", {}).get("requests", {}).get("storage") == "100Mi" and m.get("scratch") == "/scratch"'"""),
  C("/scratch is a provisioned volume the Pod can write to", 2, r"""
eq "$(j -n %{ns} pvc %{pod}-scratch '{.status.phase}')" Bound || exit 1
pv=$(j -n %{ns} pvc %{pod}-scratch '{.spec.volumeName}'); eq "$(j pv "$pv" '{.spec.storageClassName}')" standard || exit 1
eq "$($K -n %{ns} exec %{pod} -- sh -c 'echo probe > /scratch/.ksim && cat /scratch/.ksim' 2>/dev/null)" probe"""),
  C("cache is an emptyDir in memory, limited to 32Mi, at /cache", 2, PY + r"""
py -n %{ns} pod %{pod} '
v = {x["name"]: x for x in o["spec"]["volumes"]}
e = v.get("cache", {}).get("emptyDir") or {}
m = {x["name"]: x["mountPath"] for x in o["spec"]["containers"][0].get("volumeMounts", [])}
e.get("medium") == "Memory" and e.get("sizeLimit") == "32Mi" and m.get("cache") == "/cache"' || exit 1
$K -n %{ns} exec %{pod} -- sh -c 'mount | grep " /cache " | grep -q tmpfs' 2>/dev/null"""),
  C("claim.txt and owner.txt: %{pod}-scratch, owned by the Pod", 2, STOR + r"""
eq "$(one "$WS/q115/claim.txt" | sed 's#.*/##')" %{pod}-scratch || exit 1
eq "$(j -n %{ns} pvc %{pod}-scratch '{.metadata.ownerReferences[0].kind}/{.metadata.ownerReferences[0].name}')" Pod/%{pod} || exit 1
eq "$(one "$WS/q115/owner.txt" | tr A-Z a-z)" pod/%{pod}"""),
 ],
 "solution": [cmd(r"""k -n %{ns} run %{pod} --image=busybox:1.36 --dry-run=client -o yaml -- sleep 86400 > pod.yaml"""),
              hand(r"""vim pod.yaml""",
                   r"""Add to the container, below `name: %{pod}`:
```
    volumeMounts:
    - name: scratch
      mountPath: /scratch
    - name: cache
      mountPath: /cache
```
and to the Pod's `spec:`, right below `spec:` (the docs page *Ephemeral Volumes* has the generic ephemeral volume):
```
  volumes:
  - name: scratch
    ephemeral:
      volumeClaimTemplate:
        spec:
          storageClassName: standard
          accessModes:
          - ReadWriteOnce
          resources:
            requests:
              storage: 100Mi
  - name: cache
    emptyDir:
      medium: Memory
      sizeLimit: 32Mi
```""",
                   run=r"""yq -i '.spec.containers[0].volumeMounts = [{"name": "scratch", "mountPath": "/scratch"}, {"name": "cache", "mountPath": "/cache"}]
  | .spec.volumes = [{"name": "scratch", "ephemeral": {"volumeClaimTemplate": {"spec": {"storageClassName": "standard", "accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "100Mi"}}}}}}, {"name": "cache", "emptyDir": {"medium": "Memory", "sizeLimit": "32Mi"}}]' pod.yaml"""),
              cmd(r"""k apply -f pod.yaml"""),
              hand(r"""k -n %{ns} get pod,pvc""",
                   r"""Again until the Pod is `Running`. The claim is `<pod>-<volume>`: `%{pod}-scratch`.""",
                   run=r"""kubectl -n %{ns} wait --for=condition=Ready pod/%{pod} --timeout=240s"""),
              cmd(r"""echo %{pod}-scratch > {WS}/q115/claim.txt
k -n %{ns} get pvc %{pod}-scratch -o yaml | grep -A5 ownerReferences
echo Pod/%{pod} > {WS}/q115/owner.txt""", r"""The Pod owns it: delete the Pod, and the claim and its volume go too. An emptyDir in `Memory` is a tmpfs, counted as the container's memory.""")],
},
# ------------------------------------------------------------------ 116 ----
{
 "id": 116, "domain": "STOR", "title": "Claims that go when the replica goes",
 "params": lambda r: {"ns": ns_pick(r, "queue"), "sts": name_pick(r, "broker")},
 "task": """
StatefulSet `%{sts}` in Namespace `%{ns}` runs 3 replicas, each with a claim of its own. It is
being shrunk for good, and the claims of replicas that no longer exist only take up storage.

1. Configure the StatefulSet so that the claims of replicas removed by **scaling down** are
   deleted automatically, while deleting the StatefulSet itself would still keep its claims. Do
   not recreate the StatefulSet.
2. Scale it down to 1 replica.
3. Write the names of the StatefulSet's claims that exist afterwards to `{WS}/q116/claims.txt`,
   one per line.
""",
 "setup": NS + _UID + r"""
mkns %{ns}; rm -rf "$WS/q116"; mkdir -p "$WS/q116"
$K -n %{ns} delete sts %{sts} --ignore-not-found --wait=true >/dev/null 2>&1
$K -n %{ns} delete pvc -l app=%{sts} --wait=true >/dev/null 2>&1
$K apply -f - >/dev/null <<'EOF'
apiVersion: v1
kind: Service
metadata: {name: %{sts}, namespace: %{ns}}
spec: {clusterIP: None, selector: {app: %{sts}}, ports: [{port: 5672}]}
---
apiVersion: apps/v1
kind: StatefulSet
metadata: {name: %{sts}, namespace: %{ns}}
spec:
  serviceName: %{sts}
  replicas: 3
  podManagementPolicy: Parallel
  selector: {matchLabels: {app: %{sts}}}
  template:
    metadata: {labels: {app: %{sts}}}
    spec:
      terminationGracePeriodSeconds: 2
      containers:
        - name: broker
          image: busybox:1.36
          command: [sh, -c, "hostname > /data/id; sleep 86400"]
          volumeMounts: [{name: data, mountPath: /data}]
  volumeClaimTemplates:
    - metadata: {name: data, labels: {app: %{sts}}}
      spec:
        storageClassName: standard
        accessModes: [ReadWriteOnce]
        resources: {requests: {storage: 50Mi}}
EOF
$K -n %{ns} rollout status sts %{sts} --timeout=240s >/dev/null 2>&1 || { echo "StatefulSet %{sts} did not come up"; exit 1; }
uid -n %{ns} sts %{sts} > "$PRIV/q116.uid"
""",
 "checks": [
  C("retention policy: claims deleted when scaled down, kept when the StatefulSet is deleted", 2, r"""
eq "$(j -n %{ns} sts %{sts} '{.spec.persistentVolumeClaimRetentionPolicy.whenScaled}/{.spec.persistentVolumeClaimRetentionPolicy.whenDeleted}')" Delete/Retain"""),
  C("the same StatefulSet, scaled to 1 ready replica", 2, r"""
eq "$(j -n %{ns} sts %{sts} '{.metadata.uid}')" "$(cat "$PRIV/q116.uid")" || exit 1
for i in $(seq 1 20); do eq "$(j -n %{ns} sts %{sts} '{.spec.replicas}/{.status.replicas}/{.status.readyReplicas}')" 1/1/1 && exit 0; sleep 3; done; exit 1"""),
  C("the claims of replicas 1 and 2 are gone, data-%{sts}-0 is still Bound", 3, STOR + r"""
eq "$(j -n %{ns} sts %{sts} '{.spec.replicas}')" 1 || exit 1
for i in $(seq 1 20); do
  gone -n %{ns} pvc data-%{sts}-1 && gone -n %{ns} pvc data-%{sts}-2 && break; sleep 3
done
gone -n %{ns} pvc data-%{sts}-1 && gone -n %{ns} pvc data-%{sts}-2 && eq "$(j -n %{ns} pvc data-%{sts}-0 '{.status.phase}')" Bound"""),
  C("claims.txt lists what is left: data-%{sts}-0", 1, r"""
eq "$(tr -d ' \t\r' < "$WS/q116/claims.txt" 2>/dev/null | sed 's#.*/##' | grep . | sort | tr '\n' ' ')" "data-%{sts}-0 " """),
 ],
 "solution": [cmd(r"""k -n %{ns} get sts %{sts} -o yaml | grep -A2 persistentVolumeClaimRetentionPolicy""", r"""`Retain` for both: by default a StatefulSet never deletes a claim."""),
              hand(r"""k -n %{ns} edit sts %{sts}""",
                   r"""Set it before scaling - it applies to the Pods being removed - then `:wq`:
```
  persistentVolumeClaimRetentionPolicy:
    whenDeleted: Retain
    whenScaled: Delete
```""",
                   run=r"""kubectl -n %{ns} patch sts %{sts} -p '{"spec":{"persistentVolumeClaimRetentionPolicy":{"whenScaled":"Delete","whenDeleted":"Retain"}}}'"""),
              cmd(r"""k -n %{ns} scale sts %{sts} --replicas=1"""),
              hand(r"""k -n %{ns} get pvc""",
                   r"""Again until `data-%{sts}-2` and `data-%{sts}-1` have gone with their Pods.""",
                   run=r"""for i in $(seq 1 40); do [ "$(kubectl -n %{ns} get pvc --no-headers | wc -l)" = 1 ] && break; sleep 3; done"""),
              cmd(r"""echo data-%{sts}-0 > {WS}/q116/claims.txt""")],
},
# ------------------------------------------------------------------ 117 ----
{
 "id": 117, "domain": "STOR", "difficulty": "hard", "title": "Reserve a volume for one claim",
 "params": lambda r: {"ns": ns_pick(r, "atlas"), "oth": ns_pick(r, "pipeline"), "pv": name_pick(r, "pv-atlas"),
                      "pvc": name_pick(r, "maps")},
 "task": """
Claim `any-volume` in Namespace `%{oth}` is Pending: it waits for any 1Gi volume of class `manual`,
and it has to keep waiting - its volume arrives next week.

Team `%{ns}` needs storage today. Create PersistentVolume `%{pv}` (1Gi, ReadWriteOnce, class
`manual`, `hostPath` `/mnt/%{pv}`, reclaim policy `Retain`) **reserved** for a claim `%{pvc}` in
Namespace `%{ns}`, then create that claim (1Gi, ReadWriteOnce, class `manual`). It must bind
`%{pv}`.

`any-volume` must not be changed, deleted or recreated, and it must not get the volume.
""",
 "setup": NS + _UID + r"""
mkns %{ns}; mkns %{oth}
$K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null 2>&1
$K -n %{oth} delete pvc any-volume --ignore-not-found --wait=true >/dev/null 2>&1
$K delete pv %{pv} --ignore-not-found --wait=true >/dev/null 2>&1
$K apply -f - >/dev/null <<'EOF'
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: any-volume, namespace: %{oth}}
spec:
  storageClassName: manual
  accessModes: [ReadWriteOnce]
  resources: {requests: {storage: 1Gi}}
EOF
uid -n %{oth} pvc any-volume > "$PRIV/q117.uid"
""",
 "checks": [
  C("PersistentVolume %{pv}: 1Gi, RWO, manual, hostPath /mnt/%{pv}, Retain", 2, r"""
eq "$(j pv %{pv} '{.spec.capacity.storage}/{.spec.accessModes[*]}/{.spec.storageClassName}/{.spec.persistentVolumeReclaimPolicy}')" 1Gi/ReadWriteOnce/manual/Retain \
 && eq "$(j pv %{pv} '{.spec.hostPath.path}')" /mnt/%{pv}"""),
  C("it is reserved: its claimRef names %{pvc} in %{ns}", 2, r"""
eq "$(j pv %{pv} '{.spec.claimRef.namespace}/{.spec.claimRef.name}')" %{ns}/%{pvc}"""),
  C("claim %{pvc} (1Gi, RWO, manual) is Bound to %{pv}", 3, STOR + r"""
bound %{ns} %{pvc} 30 || exit 1
eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}/{.spec.storageClassName}/{.spec.accessModes[*]}/{.spec.resources.requests.storage}')" %{pv}/manual/ReadWriteOnce/1Gi"""),
  C("any-volume is the same claim as before and still Pending", 2, r"""
[ -n "$(j pv %{pv} '{.metadata.name}')" ] || exit 1
eq "$(j -n %{oth} pvc any-volume '{.metadata.uid}')" "$(cat "$PRIV/q117.uid")" && eq "$(j -n %{oth} pvc any-volume '{.status.phase}')" Pending"""),
 ],
 "solution": [cmd(r"""k -n %{oth} get pvc any-volume""", r"""`Pending`, for any 1Gi `manual` volume: an Available one would go to it."""),
              hand(r"""vim pv.yaml""",
                   r"""A `claimRef` on the PV reserves it for one claim, so the waiting one cannot take it:
```
apiVersion: v1
kind: PersistentVolume
metadata:
  name: %{pv}
spec:
  storageClassName: manual
  capacity:
    storage: 1Gi
  accessModes:
  - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  hostPath:
    path: /mnt/%{pv}
  claimRef:
    namespace: %{ns}
    name: %{pvc}
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: %{pvc}
  namespace: %{ns}
spec:
  storageClassName: manual
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
```
`volumeName` on a claim is the other half: it asks for one volume, but keeps no other claim away from it.""",
                   run=r"""cat > pv.yaml <<'EOF'
apiVersion: v1
kind: PersistentVolume
metadata:
  name: %{pv}
spec:
  storageClassName: manual
  capacity:
    storage: 1Gi
  accessModes:
  - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  hostPath:
    path: /mnt/%{pv}
  claimRef:
    namespace: %{ns}
    name: %{pvc}
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: %{pvc}
  namespace: %{ns}
spec:
  storageClassName: manual
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
EOF"""),
              cmd(r"""k apply -f pv.yaml
k get pv %{pv}
k -n %{ns} get pvc %{pvc}
k -n %{oth} get pvc any-volume""", r"""`Bound`, `Bound`, still `Pending`.""")],
},
# ------------------------------------------------------------------ 118 ----
{
 "id": 118, "domain": "STOR", "difficulty": "hard", "workers": 1, "title": "The claim took the wrong volume",
 "params": lambda r: {"ns": ns_pick(r, "books"), "dep": name_pick(r, "ledger"), "pvc": name_pick(r, "ledger-data"),
                      "pv": name_pick(r, "pv-ledger"), "tok": f"{r.randint(100000, 999999)}"},
 "task": """
Deployment `%{dep}` in Namespace `%{ns}` is meant to read its ledger from PersistentVolume `%{pv}`,
which an administrator prepared by hand - but its log says `no ledger found`. The manifest it was
created from is `{WS}/q118/app.yaml`.

1. Write the name of the volume that claim `%{pvc}` got instead to `{WS}/q118/wrong-pv.txt`.
2. Make claim `%{pvc}` - same name, same size - bind `%{pv}`, so that the Deployment's Pod prints
   the ledger. The volume it got by mistake must be gone in the end; `%{pv}` itself and the files
   on it must not be changed.
""",
 "setup": NS + _UID + r"""
mkns %{ns}; rm -rf "$WS/q118"; mkdir -p "$WS/q118"
$K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null 2>&1
$K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null 2>&1
$K delete pv %{pv} --ignore-not-found --wait=true >/dev/null 2>&1
nx "$W1" "mkdir -p /mnt/%{pv} && echo 'ledger %{tok} balanced' > /mnt/%{pv}/ledger.db" </dev/null || exit 1
$K apply -f - >/dev/null <<EOF
apiVersion: v1
kind: PersistentVolume
metadata: {name: %{pv}}
spec:
  capacity: {storage: 500Mi}
  accessModes: [ReadWriteOnce]
  persistentVolumeReclaimPolicy: Retain
  storageClassName: ""
  hostPath: {path: /mnt/%{pv}}
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions: [{key: kubernetes.io/hostname, operator: In, values: ["$W1"]}]
EOF
cat > "$WS/q118/app.yaml" <<'EOF'
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: %{pvc}, namespace: %{ns}}
spec:
  accessModes: [ReadWriteOnce]
  resources: {requests: {storage: 500Mi}}
---
apiVersion: apps/v1
kind: Deployment
metadata: {name: %{dep}, namespace: %{ns}}
spec:
  replicas: 1
  strategy: {type: Recreate}
  selector: {matchLabels: {app: %{dep}}}
  template:
    metadata: {labels: {app: %{dep}}}
    spec:
      terminationGracePeriodSeconds: 2
      volumes: [{name: data, persistentVolumeClaim: {claimName: %{pvc}}}]
      containers:
        - name: app
          image: busybox:1.36
          command: [sh, -c, "cat /data/ledger.db 2>/dev/null || echo 'no ledger found'; sleep 86400"]
          volumeMounts: [{name: data, mountPath: /data}]
EOF
$K apply -f "$WS/q118/app.yaml" >/dev/null
$K -n %{ns} rollout status deploy %{dep} --timeout=240s >/dev/null 2>&1 || { echo "Deployment %{dep} did not come up"; exit 1; }
$K -n %{ns} get pvc %{pvc} -o jsonpath='{.spec.volumeName}' > "$PRIV/q118.wrong"
[ -s "$PRIV/q118.wrong" ] || { echo "claim %{pvc} was not provisioned"; exit 1; }
uid pv %{pv} > "$PRIV/q118.uid"
""",
 "checks": [
  C("wrong-pv.txt names the volume the claim was given by mistake", 1, STOR + r"""
eq "$(one "$WS/q118/wrong-pv.txt" | sed 's#.*/##')" "$(cat "$PRIV/q118.wrong")" """),
  C("claim %{pvc} (500Mi) is Bound to %{pv}, which is the volume it always was", 3, STOR + r"""
bound %{ns} %{pvc} 30 || exit 1
eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}/{.spec.resources.requests.storage}')" %{pv}/500Mi || exit 1
eq "$(j pv %{pv} '{.metadata.uid}/{.spec.hostPath.path}')" "$(cat "$PRIV/q118.uid")//mnt/%{pv}" """),
  C("the Deployment's Pod prints the ledger", 3, r"""
eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}')" %{pv} || exit 1
for i in $(seq 1 20); do
  $K -n %{ns} logs deploy/%{dep} 2>/dev/null | grep -q 'ledger %{tok} balanced' && exit 0; sleep 3
done; exit 1"""),
  C("the volume provisioned by mistake is gone", 1, STOR + r"""
eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}')" %{pv} || exit 1
for i in $(seq 1 10); do gone pv "$(cat "$PRIV/q118.wrong")" && exit 0; sleep 3; done; exit 1"""),
 ],
 "solution": [cmd(r"""k -n %{ns} get pvc %{pvc}
k get pv %{pv}""", r"""Bound to a `pvc-...` volume of class `standard`, while `%{pv}` is `Available` with no class: the manifest names no class, so the claim got the default one, and a new empty volume."""),
              hand(r"""echo pvc-0b1c2d3e-... > {WS}/q118/wrong-pv.txt""",
                   r"""With the claim's `VOLUME`.""",
                   run=r"""kubectl -n %{ns} get pvc %{pvc} -o jsonpath='{.spec.volumeName}' > {WS}/q118/wrong-pv.txt"""),
              cmd(r"""k -n %{ns} scale deploy %{dep} --replicas=0""", r"""A claim's class and volume cannot be edited, and a claim in use is not deleted (pvc-protection): stop the Pod first."""),
              hand(r"""k -n %{ns} get pods""",
                   r"""Again until the Pod has gone.""",
                   run=r"""kubectl -n %{ns} wait --for=delete pod -l app=%{dep} --timeout=120s"""),
              cmd(r"""k -n %{ns} delete pvc %{pvc}""", r"""Its volume goes with it: reclaim policy `Delete`."""),
              hand(r"""vim pvc.yaml""",
                   r"""`storageClassName: ""` - no class: never the default, never provisioned - and the volume by name:
```
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: %{pvc}
  namespace: %{ns}
spec:
  storageClassName: ""
  volumeName: %{pv}
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 500Mi
```
`volumeName` alone would not do: the default class is still filled in, and it does not match the class-less volume.""",
                   run=r"""cat > pvc.yaml <<'EOF'
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: %{pvc}
  namespace: %{ns}
spec:
  storageClassName: ""
  volumeName: %{pv}
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 500Mi
EOF"""),
              cmd(r"""k apply -f pvc.yaml
k -n %{ns} scale deploy %{dep} --replicas=1
k -n %{ns} rollout status deploy %{dep}
k -n %{ns} logs deploy/%{dep}""", r"""`ledger %{tok} balanced`.""")],
},
# ------------------------------------------------------------------ 119 ----
{
 "id": 119, "domain": "STOR", "difficulty": "hard", "title": "Three claims, three reasons",
 "params": lambda r: {"ns": ns_pick(r, "studio"), "p": name_pick(r, "pv")},
 "task": """
Three claims in Namespace `%{ns}` are Pending: `reports`, `archive` and `media`. Each is meant to
bind the PersistentVolume that carries its `use` label - `%{p}-reports`, `%{p}-archive`,
`%{p}-media` - and each fails for a different reason.

1. Write the reason for each to `{WS}/q119/reasons.txt`, one line per claim as
   `<claim> <reason>`, where the reason is one of `capacity`, `accessmode`, `class`.
2. Get all three Bound to their volumes. The PersistentVolumes must not be changed or recreated;
   a claim may be recreated under its name, keeping its selector.
""",
 "setup": NS + _UID + r"""
mkns %{ns}; rm -rf "$WS/q119"; mkdir -p "$WS/q119"
$K -n %{ns} delete pvc reports archive media --ignore-not-found --wait=true >/dev/null 2>&1
$K delete pv %{p}-reports %{p}-archive %{p}-media --ignore-not-found --wait=true >/dev/null 2>&1
pv() {  # use size
  cat <<EOF
---
apiVersion: v1
kind: PersistentVolume
metadata: {name: %{p}-$1, labels: {use: $1}}
spec:
  capacity: {storage: $2}
  accessModes: [ReadWriteOnce]
  storageClassName: manual
  persistentVolumeReclaimPolicy: Retain
  hostPath: {path: /mnt/%{p}-$1}
EOF
}
pvc() {  # name size mode class
  cat <<EOF
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: $1, namespace: %{ns}}
spec:
  storageClassName: $4
  accessModes: [$3]
  selector: {matchLabels: {use: $1}}
  resources: {requests: {storage: $2}}
EOF
}
{ pv reports 1Gi; pv archive 2Gi; pv media 1Gi
  pvc reports 5Gi ReadWriteOnce manual; pvc archive 2Gi ReadWriteMany manual; pvc media 1Gi ReadWriteOnce manuel
} | $K apply -f - >/dev/null
for u in reports archive media; do uid pv %{p}-$u; echo; done > "$PRIV/q119.uids"
""",
 "checks": [
  C("reasons.txt: reports capacity, archive accessmode, media class", 2, r"""
eq "$(tr -s ' \t' '  ' < "$WS/q119/reasons.txt" 2>/dev/null | tr -d '\r' | sed 's/^ //; s/ $//' | tr A-Z a-z | grep . | sort | tr '\n' ';')" \
   "archive accessmode;media class;reports capacity;" """),
 ] + [
  C(f"claim {u} is Bound to %{{p}}-{u}, and the volume is as it was", 2, STOR + rf"""
bound %{{ns}} {u} 30 || exit 1
eq "$(j -n %{{ns}} pvc {u} '{{.spec.volumeName}}/{{.spec.selector.matchLabels.use}}')" %{{p}}-{u}/{u} || exit 1
eq "$(j pv %{{p}}-{u} '{{.metadata.uid}}')" "$(sed -n {n}p "$PRIV/q119.uids")" || exit 1
eq "$(j pv %{{p}}-{u} '{{.spec.capacity.storage}}/{{.spec.accessModes[*]}}/{{.spec.storageClassName}}/{{.metadata.labels.use}}')" {size}/ReadWriteOnce/manual/{u}""")
  for n, (u, size) in enumerate([("reports", "1Gi"), ("archive", "2Gi"), ("media", "1Gi")], 1)
 ],
 "solution": [cmd(r"""k -n %{ns} get pvc
k -n %{ns} describe pvc reports archive media | grep -e ^Name: -e Warning -e Normal""", r"""`media`: `storageclass.storage.k8s.io "manuel" not found` - the class, a typo. The other two only say no volume fits: compare claim and volume yourself."""),
              cmd(r"""k -n %{ns} get pvc -o custom-columns=NAME:.metadata.name,SIZE:.spec.resources.requests.storage,MODES:.spec.accessModes,CLASS:.spec.storageClassName
k get pv -o custom-columns=NAME:.metadata.name,SIZE:.spec.capacity.storage,MODES:.spec.accessModes,CLASS:.spec.storageClassName""", r"""`reports` asks for 5Gi of a 1Gi volume; `archive` for `ReadWriteMany`, its volume offers `ReadWriteOnce`. A claim binds only when the class matches, the volume is large enough, it has every access mode asked for, and the selector matches."""),
              hand(r"""vim {WS}/q119/reasons.txt""",
                   r"""```
reports capacity
archive accessmode
media class
```""",
                   run=r"""printf 'reports capacity\narchive accessmode\nmedia class\n' > {WS}/q119/reasons.txt"""),
              cmd(r"""k -n %{ns} delete pvc reports archive media""", r"""None of the three fields can be edited on a claim: recreate them."""),
              hand(r"""vim pvcs.yaml""",
                   r"""The three claims again, each keeping its selector:
```
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: reports
  namespace: %{ns}
spec:
  storageClassName: manual
  accessModes:
  - ReadWriteOnce
  selector:
    matchLabels:
      use: reports
  resources:
    requests:
      storage: 1Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: archive
  namespace: %{ns}
spec:
  storageClassName: manual
  accessModes:
  - ReadWriteOnce
  selector:
    matchLabels:
      use: archive
  resources:
    requests:
      storage: 2Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: media
  namespace: %{ns}
spec:
  storageClassName: manual
  accessModes:
  - ReadWriteOnce
  selector:
    matchLabels:
      use: media
  resources:
    requests:
      storage: 1Gi
```""",
                   run=r"""cat > pvcs.yaml <<'EOF'
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: reports
  namespace: %{ns}
spec:
  storageClassName: manual
  accessModes:
  - ReadWriteOnce
  selector:
    matchLabels:
      use: reports
  resources:
    requests:
      storage: 1Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: archive
  namespace: %{ns}
spec:
  storageClassName: manual
  accessModes:
  - ReadWriteOnce
  selector:
    matchLabels:
      use: archive
  resources:
    requests:
      storage: 2Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: media
  namespace: %{ns}
spec:
  storageClassName: manual
  accessModes:
  - ReadWriteOnce
  selector:
    matchLabels:
      use: media
  resources:
    requests:
      storage: 1Gi
EOF"""),
              cmd(r"""k apply -f pvcs.yaml
k -n %{ns} get pvc""", r"""All three `Bound`, each to its own volume.""")],
},
# ------------------------------------------------------------------ 120 ----
{
 "id": 120, "domain": "STOR", "difficulty": "hard", "workers": 2, "title": "Volume on one node, Pod sent to another",
 "params": lambda r: {"ns": ns_pick(r, "shop"), "dep": name_pick(r, "orders"), "pv": name_pick(r, "pv-orders"),
                      "tok": f"{r.randint(100000, 999999)}"},
 "task": """
The Pod of Deployment `%{dep}` in Namespace `%{ns}` is Pending.

1. Write the scheduler's message from the Pod's events to `{WS}/q120/event.txt`.
2. Get the Pod running so that it prints its data file: its log must show the `orders` line that
   is stored on the volume. That data exists on one node only - do not change, recreate or copy
   the PersistentVolume, its claim or the files.
""",
 "setup": NS + _UID + r"""
mkns %{ns}; rm -rf "$WS/q120"; mkdir -p "$WS/q120"
$K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null 2>&1
$K -n %{ns} delete pvc %{dep}-data --ignore-not-found --wait=true >/dev/null 2>&1
$K delete pv %{pv} --ignore-not-found --wait=true >/dev/null 2>&1
nx "$W1" "mkdir -p /mnt/%{pv} && echo 'orders %{tok} open' > /mnt/%{pv}/orders.db" </dev/null || exit 1
$K label node "$W1" disk- >/dev/null 2>&1; $K label node "$W2" disk=ssd --overwrite >/dev/null
$K apply -f - >/dev/null <<EOF
apiVersion: v1
kind: PersistentVolume
metadata: {name: %{pv}}
spec:
  capacity: {storage: 1Gi}
  accessModes: [ReadWriteOnce]
  storageClassName: manual
  persistentVolumeReclaimPolicy: Retain
  local: {path: /mnt/%{pv}}
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions: [{key: kubernetes.io/hostname, operator: In, values: ["$W1"]}]
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: %{dep}-data, namespace: %{ns}}
spec:
  storageClassName: manual
  volumeName: %{pv}
  accessModes: [ReadWriteOnce]
  resources: {requests: {storage: 1Gi}}
EOF
# the claim first, so that the Pod's only scheduling problem is the one the task is about
for i in $(seq 1 30); do [ "$(j -n %{ns} pvc %{dep}-data '{.status.phase}')" = Bound ] && break; sleep 2; done
[ "$(j -n %{ns} pvc %{dep}-data '{.status.phase}')" = Bound ] || { echo "claim %{dep}-data did not bind"; exit 1; }
$K apply -f - >/dev/null <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata: {name: %{dep}, namespace: %{ns}}
spec:
  replicas: 1
  strategy: {type: Recreate}
  selector: {matchLabels: {app: %{dep}}}
  template:
    metadata: {labels: {app: %{dep}}}
    spec:
      terminationGracePeriodSeconds: 2
      nodeSelector: {disk: ssd}
      volumes: [{name: data, persistentVolumeClaim: {claimName: %{dep}-data}}]
      containers:
        - name: app
          image: busybox:1.36
          command: [sh, -c, "cat /data/orders.db; sleep 86400"]
          volumeMounts: [{name: data, mountPath: /data}]
EOF
{ uid pv %{pv}; echo; uid -n %{ns} pvc %{dep}-data; echo; } > "$PRIV/q120.uids"
""",
 "checks": [
  C("event.txt holds the scheduler's message: no node matches the volume's node affinity", 2, r"""
grep -Eqi "match PersistentVolume's node affinity|volume node affinity conflict" "$WS/q120/event.txt" 2>/dev/null"""),
  C("the Deployment's Pod runs on the node that holds the volume", 3, STOR + r"""
for i in $(seq 1 20); do p=$(newest %{ns} app=%{dep}); [ -n "$p" ] && break; sleep 3; done
[ -n "$p" ] && eq "$(j -n %{ns} pod "$p" '{.spec.nodeName}')" "$W1" """),
  C("its log shows the orders line from the volume", 2, r"""
for i in $(seq 1 10); do
  $K -n %{ns} logs deploy/%{dep} 2>/dev/null | grep -q 'orders %{tok} open' && exit 0; sleep 3
done; exit 1"""),
  C("PersistentVolume and claim are the ones that were there", 1, STOR + r"""
[ -n "$(newest %{ns} app=%{dep})" ] || exit 1
eq "$(j pv %{pv} '{.metadata.uid}')" "$(sed -n 1p "$PRIV/q120.uids")" && eq "$(j -n %{ns} pvc %{dep}-data '{.metadata.uid}')" "$(sed -n 2p "$PRIV/q120.uids")" \
 && eq "$(pvnode %{pv})/$(pvpath %{pv})" "$W1//mnt/%{pv}" """),
 ],
 "solution": [hand(r"""k -n %{ns} describe pod -l app=%{dep} | grep FailedScheduling > {WS}/q120/event.txt
cat {WS}/q120/event.txt""",
                   r"""`1 node(s) didn't match PersistentVolume's node affinity, 1 node(s) didn't match Pod's node affinity/selector`: the scheduler ANDs every constraint - taints, the Pod's selector, and the node affinity of each volume it mounts.""",
                   run=r"""for i in $(seq 1 40); do
  kubectl -n %{ns} describe pod -l app=%{dep} | grep FailedScheduling > {WS}/q120/event.txt && break; sleep 3
done"""),
              cmd(r"""k describe pv %{pv} | grep -A3 'Node Affinity'
k -n %{ns} get deploy %{dep} -o yaml | grep -A1 nodeSelector
k get nodes -L disk""", r"""The volume is on `{W1}`; the Pod must go to `disk=ssd`, which only `{W2}` has. The volume cannot move, so the Pod has to."""),
              hand(r"""k -n %{ns} edit deploy %{dep}""",
                   r"""Delete the two lines of the `nodeSelector` (`nodeSelector:` and `disk: ssd`), then `:wq` - the volume's node affinity already sends the Pod to the right node. (If the selector has a reason to exist, `k label node {W1} disk=ssd` instead.)""",
                   run=r"""kubectl -n %{ns} patch deploy %{dep} --type=json -p '[{"op":"remove","path":"/spec/template/spec/nodeSelector"}]'"""),
              cmd(r"""k -n %{ns} rollout status deploy %{dep}
k -n %{ns} logs deploy/%{dep}""", r"""`orders %{tok} open`.""")],
},
# ------------------------------------------------------------------ 121 ----
{
 "id": 121, "domain": "STOR", "title": "A claim that will not go away",
 "params": lambda r: {"ns": ns_pick(r, "gallery"), "dep": name_pick(r, "thumbs"), "pvc": name_pick(r, "uploads")},
 "task": """
Claim `%{pvc}` in Namespace `%{ns}` was deleted yesterday, and it is still there, `Terminating`.

1. Write the finalizer that keeps it to `{WS}/q121/finalizer.txt`.
2. Write what still uses the claim to `{WS}/q121/used-by.txt`, as `<kind>/<name>` of the workload
   (not of a Pod).
3. The data is not needed any more. Let the deletion finish - do not edit the claim - by making
   that workload keep its files in an `emptyDir` at the same path instead. It must keep its name
   and end up with 1 running replica.
""",
 "setup": NS + _UID + r"""
mkns %{ns}; rm -rf "$WS/q121"; mkdir -p "$WS/q121"
$K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null 2>&1
$K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null 2>&1
$K apply -f - >/dev/null <<'EOF'
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: %{pvc}, namespace: %{ns}}
spec:
  accessModes: [ReadWriteOnce]
  resources: {requests: {storage: 100Mi}}
---
apiVersion: apps/v1
kind: Deployment
metadata: {name: %{dep}, namespace: %{ns}}
spec:
  replicas: 1
  strategy: {type: Recreate}
  selector: {matchLabels: {app: %{dep}}}
  template:
    metadata: {labels: {app: %{dep}}}
    spec:
      terminationGracePeriodSeconds: 2
      volumes: [{name: files, persistentVolumeClaim: {claimName: %{pvc}}}]
      containers:
        - name: worker
          image: busybox:1.36
          command: [sh, -c, "date > /srv/files/started; sleep 86400"]
          volumeMounts: [{name: files, mountPath: /srv/files}]
EOF
$K -n %{ns} rollout status deploy %{dep} --timeout=240s >/dev/null 2>&1 || { echo "Deployment %{dep} did not come up"; exit 1; }
$K -n %{ns} get pvc %{pvc} -o jsonpath='{.spec.volumeName}' > "$PRIV/q121.pv"
uid -n %{ns} deploy %{dep} > "$PRIV/q121.uid"
$K -n %{ns} delete pvc %{pvc} --wait=false >/dev/null
""",
 "checks": [
  C("finalizer.txt holds kubernetes.io/pvc-protection", 1, STOR + r"""eq "$(one "$WS/q121/finalizer.txt")" kubernetes.io/pvc-protection"""),
  C("used-by.txt names Deployment %{dep}", 2, STOR + r"""
case "$(one "$WS/q121/used-by.txt" | tr A-Z a-z)" in
  deployment/%{dep}|deployments/%{dep}|deploy/%{dep}|deployment.apps/%{dep}|deployments.apps/%{dep}) ;; *) exit 1 ;; esac"""),
  C("the claim is gone, and its volume with it", 3, STOR + r"""
for i in $(seq 1 20); do gone -n %{ns} pvc %{pvc} && gone pv "$(cat "$PRIV/q121.pv")" && exit 0; sleep 3; done; exit 1"""),
  C("the same Deployment runs 1 replica with an emptyDir at /srv/files", 2, PY + r"""
eq "$(j -n %{ns} deploy %{dep} '{.metadata.uid}')" "$(cat "$PRIV/q121.uid")" || exit 1
py -n %{ns} deploy %{dep} '
s = o["spec"]["template"]["spec"]
v = {x["name"]: x for x in s.get("volumes", [])}
m = [x["name"] for x in s["containers"][0].get("volumeMounts", []) if x["mountPath"] == "/srv/files"]
len(m) == 1 and "emptyDir" in v.get(m[0], {}) and not any("persistentVolumeClaim" in x for x in v.values())' || exit 1
for i in $(seq 1 20); do eq "$(j -n %{ns} deploy %{dep} '{.status.updatedReplicas}/{.status.availableReplicas}/{.status.replicas}')" 1/1/1 && exit 0; sleep 3; done; exit 1"""),
 ],
 "solution": """
k -n %{ns} get pvc %{pvc}                              # STATUS Terminating
k -n %{ns} get pvc %{pvc} -o jsonpath='{.metadata.finalizers}'; echo      # ["kubernetes.io/pvc-protection"]
echo kubernetes.io/pvc-protection > {WS}/q121/finalizer.txt
k -n %{ns} describe pvc %{pvc} | grep 'Used By'        # Used By: %{dep}-...   (a Pod; find its owner)
k -n %{ns} get deploy -o custom-columns=NAME:.metadata.name,CLAIMS:.spec.template.spec.volumes[*].persistentVolumeClaim.claimName
echo deployment/%{dep} > {WS}/q121/used-by.txt
k -n %{ns} edit deploy %{dep}
#   volumes:
#     - name: files
#       emptyDir: {}              <- instead of persistentVolumeClaim: {claimName: %{pvc}}
k -n %{ns} rollout status deploy %{dep}
k -n %{ns} get pvc; k get pv                           # the claim is gone, and so is its volume (reclaim policy Delete)
# "Storage object in use protection": a claim that a Pod still mounts is only marked for deletion;
# the pvc-protection finalizer is removed by the control plane when the last such Pod is gone.
# Removing the finalizer by hand would pull the claim out from under a running Pod - the fix is
# always to stop or change whatever uses it.
""",
},
# ------------------------------------------------------------------ 122 ----
{
 "id": 122, "domain": "STOR", "difficulty": "hard", "title": "Dynamic provisioning has stopped",
 "params": lambda r: {"ns": ns_pick(r, "lab"), "sc": r.choice(["node-local", "scratch-local", "team-local"]),
                      "pvc": name_pick(r, "results"), "pod": name_pick(r, "runner")},
 "task": """
Pod `%{pod}` in Namespace `%{ns}` is Pending because its claim `%{pvc}` never gets a volume. Until
last week, volumes of StorageClass `%{sc}` were provisioned within seconds. More than one thing is
wrong.

1. Fix it: the claim must be Bound to a dynamically provisioned volume of class `%{sc}`, and the
   Pod must run. The class keeps its name, its reclaim policy and its volume binding mode. Claim
   and Pod may be recreated from `{WS}/q122/app.yaml` if you need to.
2. Write the provisioner that class `%{sc}` uses in the end to `{WS}/q122/provisioner.txt`.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q122"; mkdir -p "$WS/q122"
$K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1
$K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null 2>&1
$K delete sc %{sc} --ignore-not-found >/dev/null 2>&1
$K -n local-path-storage get deploy local-path-provisioner >/dev/null 2>&1 || { echo "this cluster has no local-path provisioner"; exit 1; }
$K apply -f - >/dev/null <<'EOF'
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata: {name: %{sc}}
provisioner: rancher.io/localpath
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer
EOF
$K -n local-path-storage scale deploy local-path-provisioner --replicas=0 >/dev/null
cat > "$WS/q122/app.yaml" <<'EOF'
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: %{pvc}, namespace: %{ns}}
spec:
  storageClassName: %{sc}
  accessModes: [ReadWriteOnce]
  resources: {requests: {storage: 200Mi}}
---
apiVersion: v1
kind: Pod
metadata: {name: %{pod}, namespace: %{ns}}
spec:
  terminationGracePeriodSeconds: 2
  volumes: [{name: out, persistentVolumeClaim: {claimName: %{pvc}}}]
  containers:
    - name: runner
      image: busybox:1.36
      command: [sh, -c, "date > /out/started; sleep 86400"]
      volumeMounts: [{name: out, mountPath: /out}]
EOF
$K apply -f "$WS/q122/app.yaml" >/dev/null
""",
 "checks": [
  C("the provisioner is running again", 2, r"""
[ "$(j -n local-path-storage deploy local-path-provisioner '{.status.availableReplicas}')" -ge 1 ] 2>/dev/null"""),
  C("StorageClass %{sc}: provisioner rancher.io/local-path, still Delete and WaitForFirstConsumer", 2, r"""
eq "$(j sc %{sc} '{.provisioner}/{.reclaimPolicy}/{.volumeBindingMode}')" rancher.io/local-path/Delete/WaitForFirstConsumer"""),
  C("claim %{pvc} is Bound to a volume of class %{sc}, and Pod %{pod} runs", 3, STOR + r"""
bound %{ns} %{pvc} 60 && running %{ns} %{pod} 60 || exit 1
pv=$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}')
eq "$(j pv "$pv" '{.spec.storageClassName}')" %{sc} && eq "$(j -n %{ns} pod %{pod} '{.spec.volumes[0].persistentVolumeClaim.claimName}')" %{pvc}"""),
  C("a new claim of class %{sc} gets a volume within a minute", 2, STOR + r"""
$K -n %{ns} delete pod ksim-probe --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1
$K -n %{ns} delete pvc ksim-probe --ignore-not-found --wait=true >/dev/null 2>&1
$K apply -f - >/dev/null 2>&1 <<'EOF'
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: ksim-probe, namespace: %{ns}}
spec:
  storageClassName: %{sc}
  accessModes: [ReadWriteOnce]
  resources: {requests: {storage: 10Mi}}
---
apiVersion: v1
kind: Pod
metadata: {name: ksim-probe, namespace: %{ns}}
spec:
  terminationGracePeriodSeconds: 1
  volumes: [{name: v, persistentVolumeClaim: {claimName: ksim-probe}}]
  containers: [{name: c, image: "busybox:1.36", command: [sleep, "600"], volumeMounts: [{name: v, mountPath: /v}]}]
EOF
rc=1; bound %{ns} ksim-probe 60 && rc=0
$K -n %{ns} delete pod ksim-probe --wait=false --grace-period=1 >/dev/null 2>&1
$K -n %{ns} delete pvc ksim-probe --wait=false >/dev/null 2>&1
exit $rc"""),
  C("provisioner.txt holds rancher.io/local-path", 1, STOR + r"""
eq "$(j sc %{sc} '{.provisioner}')" rancher.io/local-path && eq "$(one "$WS/q122/provisioner.txt")" rancher.io/local-path"""),
 ],
 "solution": """
k -n %{ns} describe pvc %{pvc} | tail -3
#   waiting for a volume to be created either by the external provisioner 'rancher.io/localpath'
#   or manually by the system administrator ... verify that the provisioner is running and correctly registered
k get sc                                   # %{sc}: rancher.io/localpath   standard: rancher.io/local-path  <- a typo
k get pods -A | grep -i provision          # nothing: who serves rancher.io/local-path?
k -n local-path-storage get deploy         # local-path-provisioner 0/0 - scaled to zero
k -n local-path-storage scale deploy local-path-provisioner --replicas=1

# a StorageClass's provisioner cannot be edited: write it out, correct it, replace it
k get sc %{sc} -o yaml > sc.yaml
vi sc.yaml                                 # provisioner: rancher.io/local-path   (drop uid, resourceVersion, creationTimestamp)
k delete sc %{sc}; k apply -f sc.yaml
k -n %{ns} get pvc,pod -w                  # the waiting claim is picked up by itself within half a minute
#   (if it is not:  k -n %{ns} delete pod %{pod}; k -n %{ns} delete pvc %{pvc}; k apply -f {WS}/q122/app.yaml)
echo rancher.io/local-path > {WS}/q122/provisioner.txt
# Dynamic provisioning is a chain: the claim names a class, the class names a provisioner, and a
# controller that answers to exactly that name has to be running. The claim's event tells you
# which name it is waiting for - then check the class for a typo and the cluster for the controller.
""",
},
# ------------------------------------------------------------------ 123 ----
{
 "id": 123, "domain": "STOR", "title": "A claim that waits for a default class",
 "params": lambda r: {"ns": ns_pick(r, "intake"), "sc": name_pick(r, "general"), "pvc": name_pick(r, "inbox"),
                      "pod": name_pick(r, "scanner")},
 "task": """
Claim `%{pvc}` in Namespace `%{ns}` is Pending, and so is Pod `%{pod}` that mounts it: the claim
names no StorageClass, and this cluster has no default one at the moment.

1. Without changing, deleting or recreating the claim or the Pod, get the claim Bound to a volume
   of class `%{sc}` and the Pod running.
2. Exactly one StorageClass must be the default in the end.
3. Write the StorageClass the claim ended up with to `{WS}/q123/class.txt`.
""",
 "setup": NS + _UID + r"""
mkns %{ns}; rm -rf "$WS/q123"; mkdir -p "$WS/q123"
$K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1
$K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null 2>&1
$K delete sc %{sc} --ignore-not-found >/dev/null 2>&1
$K annotate sc standard storageclass.kubernetes.io/is-default-class- >/dev/null 2>&1
$K apply -f - >/dev/null <<'EOF'
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata: {name: %{sc}}
provisioner: rancher.io/local-path
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: %{pvc}, namespace: %{ns}}
spec:
  accessModes: [ReadWriteOnce]
  resources: {requests: {storage: 200Mi}}
---
apiVersion: v1
kind: Pod
metadata: {name: %{pod}, namespace: %{ns}}
spec:
  terminationGracePeriodSeconds: 2
  volumes: [{name: inbox, persistentVolumeClaim: {claimName: %{pvc}}}]
  containers:
    - name: scanner
      image: busybox:1.36
      command: [sleep, "86400"]
      volumeMounts: [{name: inbox, mountPath: /inbox}]
EOF
[ -z "$(j -n %{ns} pvc %{pvc} '{.spec.storageClassName}')" ] || { echo "claim %{pvc} was given a class although no default exists"; exit 1; }
{ uid -n %{ns} pvc %{pvc}; echo; uid -n %{ns} pod %{pod}; echo; } > "$PRIV/q123.uids"
""",
 "checks": [
  C("%{sc} is the default StorageClass, and the only one", 2, PY + r"""
py sc '
d = sorted(x["metadata"]["name"] for x in o["items"] if (x["metadata"].get("annotations") or {}).get("storageclass.kubernetes.io/is-default-class") == "true")
d == ["%{sc}"]'"""),
  C("the same claim now has class %{sc} and is Bound", 3, STOR + r"""
eq "$(j -n %{ns} pvc %{pvc} '{.metadata.uid}')" "$(sed -n 1p "$PRIV/q123.uids")" || exit 1
bound %{ns} %{pvc} 60 && eq "$(j -n %{ns} pvc %{pvc} '{.spec.storageClassName}')" %{sc}"""),
  C("the same Pod runs with the claim mounted", 2, STOR + r"""
eq "$(j -n %{ns} pod %{pod} '{.metadata.uid}')" "$(sed -n 2p "$PRIV/q123.uids")" || exit 1
running %{ns} %{pod} 60"""),
  C("class.txt holds %{sc}", 1, STOR + r"""
eq "$(j -n %{ns} pvc %{pvc} '{.spec.storageClassName}')" %{sc} && eq "$(one "$WS/q123/class.txt")" %{sc}"""),
 ],
 "solution": [cmd(r"""k get sc
k -n %{ns} describe pvc %{pvc} | tail -2""", r"""No `(default)` anywhere, and the claim waits: `no storage class is set`."""),
              cmd(r"""k annotate sc %{sc} storageclass.kubernetes.io/is-default-class=true
k get sc""", r"""`%{sc} (default)`. A claim without a class gets the default when it is created - or, if there is none, as soon as one appears: no need to recreate it."""),
              hand(r"""k -n %{ns} get pvc,pod""",
                   r"""Again until the claim shows `STORAGECLASS %{sc}` and `Bound`, and the Pod `Running`.""",
                   run=r"""for i in $(seq 1 40); do [ "$(kubectl -n %{ns} get pvc %{pvc} -o jsonpath='{.status.phase}')" = Bound ] && break; sleep 3; done
kubectl -n %{ns} wait --for=condition=Ready pod/%{pod} --timeout=240s"""),
              cmd(r"""echo %{sc} > {WS}/q123/class.txt""", r"""Marking `standard` would have bound it too - to the wrong class. With two defaults, new claims take the newest; never leave two.""")],
},
# ------------------------------------------------------------------ 124 ----
{
 "id": 124, "domain": "STOR", "workers": 2, "title": "A local volume with a class of its own",
 "params": lambda r: {"ns": ns_pick(r, "metrics"), "sc": name_pick(r, "local-disks"), "pv": name_pick(r, "pv-disk"),
                      "pvc": name_pick(r, "tsdb"), "pod": name_pick(r, "prom"), "tok": f"{r.randint(100000, 999999)}"},
 "task": """
A disk has been mounted on node `{W2}` at `/mnt/disks/%{pv}`; it holds a file `disk.id`. Make it
available to Pods as a **local** volume:

1. StorageClass `%{sc}` for volumes that are created by hand: provisioner
   `kubernetes.io/no-provisioner`, volume binding mode `WaitForFirstConsumer`.
2. PersistentVolume `%{pv}`: 1Gi, ReadWriteOnce, class `%{sc}`, reclaim policy `Retain`, volume
   type `local` with that path, tied to node `{W2}`.
3. In Namespace `%{ns}`: claim `%{pvc}` (1Gi, ReadWriteOnce, class `%{sc}`) and Pod `%{pod}`
   (`busybox:1.36`, command `sleep 86400`) that mounts it at `/disk`. Do not tell the Pod where to
   run - no nodeSelector, no affinity.
""",
 "setup": NS + r"""
mkns %{ns}
$K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1
$K -n %{ns} delete pvc %{pvc} --ignore-not-found --wait=true >/dev/null 2>&1
$K delete pv %{pv} --ignore-not-found --wait=true >/dev/null 2>&1; $K delete sc %{sc} --ignore-not-found >/dev/null 2>&1
nx "$W2" "mkdir -p /mnt/disks/%{pv} && echo 'disk %{tok}' > /mnt/disks/%{pv}/disk.id" </dev/null || exit 1
""",
 "checks": [
  C("StorageClass %{sc}: no-provisioner, WaitForFirstConsumer", 2, r"""
eq "$(j sc %{sc} '{.provisioner}/{.volumeBindingMode}')" kubernetes.io/no-provisioner/WaitForFirstConsumer"""),
  C("PersistentVolume %{pv}: a local volume at /mnt/disks/%{pv}, tied to {W2}; 1Gi, RWO, Retain", 3, PY + r"""
export W2; py pv %{pv} '
s = o["spec"]
terms = s.get("nodeAffinity", {}).get("required", {}).get("nodeSelectorTerms", [])
nodes = [e.get("values") for t in terms for e in t.get("matchExpressions", []) if e.get("key") == "kubernetes.io/hostname" and e.get("operator") == "In"]
s.get("local", {}).get("path") == "/mnt/disks/%{pv}" and nodes == [[os.environ["W2"]]] and s["capacity"]["storage"] == "1Gi" and s["accessModes"] == ["ReadWriteOnce"] and s.get("persistentVolumeReclaimPolicy") == "Retain" and s.get("storageClassName") == "%{sc}"'"""),
  C("claim %{pvc} (1Gi, RWO, %{sc}) is Bound to %{pv}", 2, STOR + r"""
bound %{ns} %{pvc} 60 && eq "$(j -n %{ns} pvc %{pvc} '{.spec.volumeName}/{.spec.storageClassName}/{.spec.resources.requests.storage}')" %{pv}/%{sc}/1Gi"""),
  C("Pod %{pod} was sent to {W2} by its volume alone, and reads disk.id at /disk", 3, STOR + r"""
running %{ns} %{pod} 60 || exit 1
eq "$(j -n %{ns} pod %{pod} '{.spec.nodeName}')" "$W2" || exit 1
[ -z "$(j -n %{ns} pod %{pod} '{.spec.nodeSelector}{.spec.affinity}')" ] || exit 1
eq "$($K -n %{ns} exec %{pod} -- cat /disk/disk.id 2>/dev/null | tr -d '\r')" "disk %{tok}" """),
 ],
 "solution": [hand(r"""vim local.yaml""",
                   r"""The docs page *Storage Classes* (under *Local*) has the class, *Persistent Volumes* a `local` volume with its required node affinity:
```
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: %{sc}
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsumer
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: %{pv}
spec:
  capacity:
    storage: 1Gi
  accessModes:
  - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: %{sc}
  local:
    path: /mnt/disks/%{pv}
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - {W2}
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: %{pvc}
  namespace: %{ns}
spec:
  storageClassName: %{sc}
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
```""",
                   run=r"""cat > local.yaml <<'EOF'
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: %{sc}
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsumer
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: %{pv}
spec:
  capacity:
    storage: 1Gi
  accessModes:
  - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: %{sc}
  local:
    path: /mnt/disks/%{pv}
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - {W2}
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: %{pvc}
  namespace: %{ns}
spec:
  storageClassName: %{sc}
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
EOF"""),
              cmd(r"""k apply -f local.yaml
k -n %{ns} run %{pod} --image=busybox:1.36 --dry-run=client -o yaml -- sleep 86400 > pod.yaml"""),
              hand(r"""vim pod.yaml""",
                   r"""Add to the container, below `name: %{pod}`:
```
    volumeMounts:
    - name: disk
      mountPath: /disk
```
and to the Pod's `spec:`, right below `spec:`:
```
  volumes:
  - name: disk
    persistentVolumeClaim:
      claimName: %{pvc}
```""",
                   run=r"""yq -i '.spec.containers[0].volumeMounts = [{"name": "disk", "mountPath": "/disk"}]
  | .spec.volumes = [{"name": "disk", "persistentVolumeClaim": {"claimName": "%{pvc}"}}]' pod.yaml"""),
              cmd(r"""k apply -f pod.yaml"""),
              hand(r"""k -n %{ns} get pod %{pod} -o wide""",
                   r"""Again until `Running` - on `{W2}`, sent there by the volume alone. A `hostPath` is a directory on whatever node the Pod lands on; a `local` volume knows its node, and the scheduler follows it.""",
                   run=r"""kubectl -n %{ns} wait --for=condition=Ready pod/%{pod} --timeout=240s"""),
              cmd(r"""k -n %{ns} exec %{pod} -- cat /disk/disk.id""")],
},
]