kubestronaut-sim/cka/lessons_work.py

"""CKA lessons of the Workloads and Scheduling domain (3.1-3.5). See cka/lessons.py for what a lesson is."""
from sim.bank import NS, step

# ---- 3.1 rolling updates and rollbacks --------------------------------------------------------
L31 = {
    "id": 9301, "objective": "3.1", "workers": 1,
    "title": "Rollouts: strategy, a release that fails, undo, and pause",
    "intro": r"""Every change to a Deployment's Pod **template** is a new **revision**: a new ReplicaSet, filled while the old one is emptied. Changes elsewhere - `replicas`, the strategy - are not revisions.

```
strategy: RollingUpdate          maxSurge: extra Pods allowed     maxUnavailable: missing Pods allowed
strategy: Recreate               all old Pods go first, then the new ones come - a gap, on purpose
kubectl rollout status | history [--revision=N] | undo [--to-revision=N] | pause | resume | restart
```

A failed release does not roll itself back: the rollout stops, and with `maxUnavailable: 0` the old Pods keep serving until you act.

Namespace `rollout` runs Deployment `api`: `nginx:1.26`, 4 replicas.""",
    "setup": NS + r"""
mkns rollout
mkdir -p "$WS/lesson-3.1"
$K -n rollout create deploy api --image=nginx:1.26 --replicas=4 >/dev/null
$K -n rollout rollout status deploy api --timeout=180s >/dev/null
""",
    "steps": [
        step("Never fewer than four",
             r"""Change the update strategy of `api` so that an update first adds **one** Pod and never takes one away before its replacement is ready: `maxSurge: 1`, `maxUnavailable: 0`.""",
             check=r"""eq "$(j -n rollout deploy api '{.spec.strategy.type} {.spec.strategy.rollingUpdate.maxSurge} {.spec.strategy.rollingUpdate.maxUnavailable}')" "RollingUpdate 1 0" """,
             hint=r"""`kubectl -n rollout edit deploy api`, or `kubectl patch` with `{"spec":{"strategy":{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0}}}}`.""",
             show=r"""kubectl -n rollout patch deploy api -p '{"spec":{"strategy":{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0}}}}'
kubectl -n rollout get deploy api -o jsonpath='{.spec.strategy}'; echo"""),
        step("A release with a reason",
             r"""Update `api` to `nginx:1.27`, and record why: the annotation `kubernetes.io/change-cause` on the Deployment, set to `upgrade to 1.27`. `kubectl rollout history` shows it beside the revision.""",
             check=r"""
eq "$(j -n rollout deploy api '{.spec.template.spec.containers[0].image}')" nginx:1.27 || exit 1
eq "$(j -n rollout deploy api '{.metadata.annotations.kubernetes\.io/change-cause}')" "upgrade to 1.27" || exit 1
eq "$(j -n rollout deploy api '{.status.updatedReplicas} {.status.availableReplicas}')" "4 4" """,
             hint=r"""`kubectl set image deploy/api nginx=nginx:1.27` (the container is called `nginx`), then `kubectl annotate deploy api kubernetes.io/change-cause='...'`. The `--record` flag is gone.""",
             show=r"""kubectl -n rollout set image deploy/api nginx=nginx:1.27
kubectl -n rollout annotate deploy api kubernetes.io/change-cause='upgrade to 1.27' --overwrite
kubectl -n rollout rollout status deploy api --timeout=180s
kubectl -n rollout rollout history deploy api"""),
        step("A release that cannot start",
             r"""Now a mistake: set the image of `api` to `nginx:1.27-broken`, a tag that does not exist.

Watch what happens: one new Pod (the surge) fails to pull, and the rollout stops there. Because `maxUnavailable` is `0`, all four `nginx:1.27` Pods keep serving. `kubectl rollout status` would wait for ever (here, until `progressDeadlineSeconds` marks it failed).""",
             check=r"""
eq "$(j -n rollout deploy api '{.spec.template.spec.containers[0].image}')" nginx:1.27-broken || exit 1
[ "$(j -n rollout deploy api '{.status.availableReplicas}')" = 4 ] || exit 1
$K -n rollout get pods -l app=api --no-headers 2>/dev/null | grep -Eq 'ErrImagePull|ImagePullBackOff' """,
             hint=r"""`kubectl set image` again. `kubectl -n rollout get pods` shows the Pod that cannot pull; `kubectl -n rollout get rs` shows the two ReplicaSets side by side.""",
             show=r"""kubectl -n rollout set image deploy/api nginx=nginx:1.27-broken
sleep 10
kubectl -n rollout get pods
kubectl -n rollout get rs"""),
        step("Undo",
             r"""Roll `api` back to the revision before - `nginx:1.27` - and let the rollout finish.""",
             check=r"""
eq "$(j -n rollout deploy api '{.spec.template.spec.containers[0].image}')" nginx:1.27 || exit 1
eq "$(j -n rollout deploy api '{.status.updatedReplicas} {.status.availableReplicas} {.status.replicas}')" "4 4 4" """,
             hint=r"""`kubectl rollout undo deploy/api`. It does not create a ReplicaSet: it scales the old one back up, and that revision gets a new number.""",
             show=r"""kubectl -n rollout rollout undo deploy/api
kubectl -n rollout rollout status deploy api --timeout=180s
kubectl -n rollout rollout history deploy api"""),
        step("Back to a revision by number",
             r"""Look at the history, then go back to the revision that ran **`nginx:1.26`**, by its number. `kubectl rollout history deploy api --revision=N` shows what a revision's template was.""",
             check=r"""
eq "$(j -n rollout deploy api '{.spec.template.spec.containers[0].image}')" nginx:1.26 || exit 1
eq "$(j -n rollout deploy api '{.status.updatedReplicas} {.status.availableReplicas} {.status.replicas}')" "4 4 4" """,
             hint=r"""Revision 1 is where the Deployment started. `kubectl rollout undo deploy/api --to-revision=1`.""",
             show=r"""kubectl -n rollout rollout history deploy api
kubectl -n rollout rollout history deploy api --revision=1 | grep Image
kubectl -n rollout rollout undo deploy/api --to-revision=1
kubectl -n rollout rollout status deploy api --timeout=180s"""),
        step("Two changes, one rollout",
             r"""Each `set` command starts a rollout of its own. To make two changes at once, **pause** the Deployment, make them - image `nginx:1.27`, and an environment variable `MODE=prod` in the container - then **resume**: one new revision, one rollout.""",
             check=r"""
[ "$(j -n rollout deploy api '{.spec.paused}')" != true ] || exit 1
eq "$(j -n rollout deploy api '{.spec.template.spec.containers[0].image}')" nginx:1.27 || exit 1
eq "$(j -n rollout deploy api '{.spec.template.spec.containers[0].env[?(@.name=="MODE")].value}')" prod || exit 1
eq "$(j -n rollout deploy api '{.status.updatedReplicas} {.status.availableReplicas} {.status.replicas}')" "4 4 4" """,
             hint=r"""`kubectl rollout pause deploy/api`, `kubectl set image ...`, `kubectl set env deploy/api MODE=prod`, `kubectl rollout resume deploy/api`.""",
             show=r"""kubectl -n rollout rollout pause deploy/api
kubectl -n rollout set image deploy/api nginx=nginx:1.27
kubectl -n rollout set env deploy/api MODE=prod
kubectl -n rollout rollout resume deploy/api
kubectl -n rollout rollout status deploy api --timeout=180s
kubectl -n rollout rollout history deploy api"""),
    ],
    "outro": r"""You tuned the strategy, released with a recorded cause, watched a broken release stall safely, undid it, went back to a numbered revision, and batched two changes into one rollout.

**For the exam**

- `kubectl set image deploy/NAME CONTAINER=IMAGE` - the container's name, not the image's.
- `rollout undo` goes to the previous revision; `--to-revision=N` to any in `rollout history`. The history keeps `revisionHistoryLimit` old ReplicaSets (10 by default).
- `maxSurge` / `maxUnavailable` take numbers or percentages; `maxUnavailable: 0` keeps full capacity during an update. `Recreate` has no such fields.
- `rollout restart` makes new Pods with an unchanged spec (it stamps an annotation into the template).
- A stuck rollout: `kubectl rollout status` waits, `describe deploy` shows `Progressing=False` after `progressDeadlineSeconds`.

Next, prove it against the clock: **drill 3.1**.""",
}

# ---- 3.2 ConfigMaps and Secrets ---------------------------------------------------------------
_ENC_CHECK = r"""
nx "$CP" 'ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key get /registry/secrets/conf/db --print-value-only 2>/dev/null | head -c 64' </dev/null | grep -aq '^k8s:enc:aescbc:v1:'"""

L32 = {
    "id": 9302, "objective": "3.2", "workers": 1,
    "title": "ConfigMaps and Secrets: create, consume, update, and encrypt",
    "intro": r"""A ConfigMap holds configuration, a Secret holds the confidential part of it - and both reach a container the same three ways:

```
env:      - name: X  valueFrom: {configMapKeyRef | secretKeyRef}     one key, renamed if you like
envFrom:  - configMapRef | secretRef                                  every key, as it is called
volumes:  - configMap | secret                                        every key as a file
```

Only a mounted volume follows later changes (after a short delay, and never with `subPath`); environment variables are read once, when the container starts.

A Secret is **base64-encoded, not encrypted**: anyone who may read it, or read etcd, has it. Encryption at rest is the API server's job, set up with an `EncryptionConfiguration`.

Files for this lesson are in `{WS}/lesson-3.2/`; Namespace `conf` is empty.""",
    "setup": NS + r"""
mkns conf
mkdir -p "$WS/lesson-3.2"
printf 'log.level=info\ncache.size=128\n' > "$WS/lesson-3.2/app.properties"
printf 'DB_USER=ledger\nDB_PASS=t0ps3cret\n' > "$WS/lesson-3.2/db.env"
""",
    "steps": [
        step("A ConfigMap from a file and a literal",
             r"""Create ConfigMap `app-config` in `conf` with two keys: `app.properties`, holding the file `{WS}/lesson-3.2/app.properties`, and `MODE` with the value `prod`.""",
             check=r"""
eq "$(j -n conf cm app-config '{.data.MODE}')" prod && $K -n conf get cm app-config -o jsonpath='{.data.app\.properties}' | grep -q '^cache.size=128$'""",
             hint=r"""`kubectl create configmap NAME --from-file=PATH --from-literal=KEY=VALUE`. With `--from-file`, the key is the file's name.""",
             show=r"""kubectl -n conf create configmap app-config --from-file={WS}/lesson-3.2/app.properties --from-literal=MODE=prod
kubectl -n conf describe cm app-config"""),
        step("A Secret from an env file",
             r"""Create Secret `db` (type `Opaque`) in `conf` from `{WS}/lesson-3.2/db.env`, one key per line of the file: `DB_USER` and `DB_PASS`. Then read `DB_PASS` back, decoded.""",
             check=r"""
eq "$(j -n conf secret db '{.type}')" Opaque && eq "$(j -n conf secret db '{.data.DB_USER}' | base64 -d)" ledger && eq "$(j -n conf secret db '{.data.DB_PASS}' | base64 -d)" t0ps3cret""",
             hint=r"""`kubectl create secret generic NAME --from-env-file=PATH`. Reading one key: `-o jsonpath='{.data.DB_PASS}' | base64 -d`.""",
             show=r"""kubectl -n conf create secret generic db --from-env-file={WS}/lesson-3.2/db.env
kubectl -n conf get secret db -o jsonpath='{.data.DB_PASS}' | base64 -d; echo"""),
        step("Consume both",
             r"""Run Pod `app` in `conf` (`busybox:1.36`, `sleep 86400`) that gets:

- every key of Secret `db` as an environment variable (`envFrom`),
- the variable `APP_MODE` from key `MODE` of ConfigMap `app-config`,
- ConfigMap `app-config` as files under `/etc/app`.""",
             check=r"""
x() { $K -n conf exec app -- "$@" 2>/dev/null; }
eq "$(x printenv DB_PASS)" t0ps3cret && eq "$(x printenv APP_MODE)" prod && x cat /etc/app/app.properties | grep -q '^log.level=info$'""",
             hint=r"""`envFrom: [{secretRef: {name: db}}]`, `env: [{name: APP_MODE, valueFrom: {configMapKeyRef: {name: app-config, key: MODE}}}]`, and a `configMap` volume.""",
             show=r"""kubectl -n conf apply -f - <<'EOF'
apiVersion: v1
kind: Pod
metadata:
  name: app
spec:
  containers:
  - name: app
    image: busybox:1.36
    command: [sleep, '86400']
    envFrom:
    - secretRef: {name: db}
    env:
    - name: APP_MODE
      valueFrom:
        configMapKeyRef: {name: app-config, key: MODE}
    volumeMounts:
    - {name: config, mountPath: /etc/app}
  volumes:
  - name: config
    configMap: {name: app-config}
EOF
kubectl -n conf wait --for=condition=Ready pod/app --timeout=120s
kubectl -n conf exec app -- sh -c 'printenv DB_USER APP_MODE; cat /etc/app/app.properties'"""),
        step("Change the configuration, live",
             r"""Change `cache.size` to `256` in ConfigMap `app-config` - edit the file and re-create the ConfigMap's key from it, or `kubectl edit`.

Then watch `/etc/app/app.properties` inside Pod `app`: within a minute the kubelet replaces the file. `APP_MODE` would not change, even if you edited `MODE`: the environment is fixed at start.""",
             check=r"""$K -n conf exec app -- cat /etc/app/app.properties 2>/dev/null | grep -q '^cache.size=256$'""",
             hint=r"""Regenerate and apply: `kubectl create configmap app-config --from-file=... --from-literal=MODE=prod --dry-run=client -o yaml | kubectl apply -f -` - the usual way to update a ConfigMap from files.""",
             show=r"""sed -i 's/^cache.size=.*/cache.size=256/' {WS}/lesson-3.2/app.properties
kubectl -n conf create configmap app-config --from-file={WS}/lesson-3.2/app.properties --from-literal=MODE=prod --dry-run=client -o yaml | kubectl apply -f -
sleep 30
kubectl -n conf exec app -- cat /etc/app/app.properties"""),
        step("A TLS Secret",
             r"""Make a self-signed certificate for `shop.example.com` (`openssl` is on this host), and store it as Secret `shop-tls` in `conf`, of type `kubernetes.io/tls` - the type an Ingress or Gateway expects. Its keys are `tls.crt` and `tls.key`.""",
             check=r"""
eq "$(j -n conf secret shop-tls '{.type}')" kubernetes.io/tls || exit 1
j -n conf secret shop-tls '{.data.tls\.crt}' | base64 -d | openssl x509 -noout -subject 2>/dev/null | grep -q 'shop.example.com'""",
             hint=r"""`openssl req -x509 -newkey rsa:2048 -nodes -keyout tls.key -out tls.crt -days 30 -subj /CN=shop.example.com`, then `kubectl create secret tls NAME --cert=tls.crt --key=tls.key`.""",
             show=r"""cd {WS}/lesson-3.2
openssl req -x509 -newkey rsa:2048 -nodes -keyout tls.key -out tls.crt -days 30 -subj /CN=shop.example.com 2>/dev/null
kubectl -n conf create secret tls shop-tls --cert=tls.crt --key=tls.key
kubectl -n conf get secret shop-tls
cd"""),
        step("Encrypt Secrets at rest",
             r"""Read Secret `db` straight out of etcd on `{CP}`, and you see `t0ps3cret` in plain text. Make the API server encrypt Secrets before it stores them:

1. Write an `EncryptionConfiguration` to `/etc/kubernetes/enc/enc.yaml`: for resource `secrets`, provider `aescbc` with one key `key1` (32 random bytes, base64), then `identity: {}` so that what is still unencrypted can be read.
2. Start the API server with `--encryption-provider-config=/etc/kubernetes/enc/enc.yaml`, and mount `/etc/kubernetes/enc` into its Pod (a `hostPath` volume): it runs in a container and sees only what is mounted.
3. Once it is back, rewrite Secret `db`, so that it is stored again - encrypted this time.

Edit a copy of the manifest outside `/etc/kubernetes/manifests` and copy it back: the kubelet runs every file in that directory, backups too.""",
             check=_ENC_CHECK + r""" && eq "$(j -n conf secret db '{.data.DB_PASS}' | base64 -d)" t0ps3cret""",
             hint=r"""The docs page "Encrypting Confidential Data at Rest" has the file. Stored values of encrypted Secrets start with `k8s:enc:aescbc:v1:key1:`: `sudo ETCDCTL_API=3 etcdctl ... get /registry/secrets/conf/db | od -c | head` (or `hexdump -C`). Rewriting: `kubectl get secret db -o json | kubectl replace -f -`. If the API server does not come back, `sudo crictl ps -a` and the logs under `/var/log/pods/kube-system_kube-apiserver-*` say why.""",
             show=r"""sudo mkdir -p /etc/kubernetes/enc
sudo tee /etc/kubernetes/enc/enc.yaml >/dev/null <<EOF
apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
- resources: [secrets]
  providers:
  - aescbc:
      keys:
      - name: key1
        secret: $(head -c 32 /dev/urandom | base64)
  - identity: {}
EOF
sudo cp /etc/kubernetes/manifests/kube-apiserver.yaml /root/kube-apiserver.yaml.bak
sudo cp /etc/kubernetes/manifests/kube-apiserver.yaml /tmp/kube-apiserver.yaml
sudo yq -i '.spec.containers[0].command += ["--encryption-provider-config=/etc/kubernetes/enc/enc.yaml"]
  | .spec.containers[0].volumeMounts += [{"name": "enc", "mountPath": "/etc/kubernetes/enc", "readOnly": true}]
  | .spec.volumes += [{"name": "enc", "hostPath": {"path": "/etc/kubernetes/enc", "type": "DirectoryOrCreate"}}]' /tmp/kube-apiserver.yaml
sudo cp /tmp/kube-apiserver.yaml /etc/kubernetes/manifests/kube-apiserver.yaml
sleep 20
for i in $(seq 1 40); do kubectl get --raw /readyz >/dev/null 2>&1 && break; sleep 3; done
kubectl -n conf get secret db -o json | kubectl replace -f -
sudo ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key \
  get /registry/secrets/conf/db | od -c | head -4"""),
    ],
    "outro": r"""You built a ConfigMap and Secrets from files, consumed them three ways, updated configuration in a running Pod, and turned on encryption at rest for Secrets.

**For the exam**

- `kubectl create configmap|secret generic NAME --from-literal --from-file --from-env-file`; `kubectl create secret tls|docker-registry`. Generate, then apply: `--dry-run=client -o yaml | kubectl apply -f -`.
- `immutable: true` on a ConfigMap or Secret: no more changes, only delete and re-create - and Pods must restart to see the new one.
- Mounted keys follow updates; `subPath` mounts and environment variables do not.
- Encryption at rest: `--encryption-provider-config` and a mount for the file. The **first** provider encrypts; the others are tried when reading. `identity` first means "not encrypted". Existing Secrets stay as they are until rewritten (`kubectl get secrets -A -o json | kubectl replace -f -`).

Next, prove it against the clock: **drill 3.2**.""",
}

# ---- 3.3 autoscaling --------------------------------------------------------------------------
L33 = {
    "id": 9303, "objective": "3.3", "workers": 1,
    "title": "Autoscaling: the HorizontalPodAutoscaler, and resizing a Pod in place",
    "intro": r"""Three kinds of autoscaling exist, and only one is built in:

```
HorizontalPodAutoscaler   more or fewer Pods, from metrics          built in (needs metrics-server)
VerticalPodAutoscaler     bigger or smaller requests per Pod        an add-on, with its own CRDs
Cluster Autoscaler        more or fewer nodes                       an add-on, per cloud
```

An HPA compares each Pod's usage with its **request**: 50% CPU means half of what the Pod requested. A Pod without a request gives it nothing to compare against. The HPA checks every 15 seconds and waits before scaling down (`behavior.scaleDown.stabilizationWindowSeconds`, five minutes unless you say otherwise).

On v1.37 a running Pod's CPU and memory can also be changed **in place**, through its `resize` subresource, without a new Pod.

metrics-server runs here. In Namespace `scale`, Deployment `api` (nginx) requests 100m CPU and idles; Deployment `cruncher` requests 50m and burns CPU.""",
    "setup": NS + r"""
ensure_metrics_server || { echo "metrics-server did not become ready" >&2; exit 1; }
mkns scale
$K -n scale apply -f - >/dev/null <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata: {name: api}
spec:
  replicas: 1
  selector: {matchLabels: {app: api}}
  template:
    metadata: {labels: {app: api}}
    spec:
      containers:
      - name: nginx
        image: nginx:1.27
        resources: {requests: {cpu: 100m, memory: 64Mi}}
---
apiVersion: apps/v1
kind: Deployment
metadata: {name: cruncher}
spec:
  replicas: 1
  selector: {matchLabels: {app: cruncher}}
  template:
    metadata: {labels: {app: cruncher}}
    spec:
      containers:
      - name: cruncher
        image: busybox:1.36
        command: [sh, -c, 'while true; do :; done']
        resources: {requests: {cpu: 50m, memory: 16Mi}, limits: {cpu: 150m, memory: 32Mi}}
EOF
$K -n scale rollout status deploy api --timeout=180s >/dev/null && $K -n scale rollout status deploy cruncher --timeout=180s >/dev/null
""",
    "steps": [
        step("An HPA in one command",
             r"""Make `api` scale between **2** and **6** replicas, aiming at **50%** CPU. Within a minute the HPA shows a current value - `kubectl -n scale get hpa` - and `api` runs at least 2 replicas.""",
             check=r"""
eq "$(j -n scale hpa api '{.spec.minReplicas} {.spec.maxReplicas} {.spec.metrics[0].resource.name} {.spec.metrics[0].resource.target.averageUtilization}')" "2 6 cpu 50" || exit 1
[ -n "$(j -n scale hpa api '{.status.currentMetrics[0].resource.current.averageUtilization}')" ] && [ "$(j -n scale deploy api '{.status.readyReplicas}')" -ge 2 ] 2>/dev/null""",
             hint=r"""`kubectl autoscale deploy api --min=2 --max=6 --cpu=50%` (kubectl before 1.34: `--cpu-percent=50`). `<unknown>` in the TARGETS column means no metrics yet - or a Pod without a CPU request.""",
             show=r"""kubectl -n scale autoscale deploy api --min=2 --max=6 --cpu=50%
sleep 45
kubectl -n scale get hpa api"""),
        step("Scale down calmly",
             r"""Give the HPA of `api` a `behavior`: when scaling **down**, wait **120** seconds of stable recommendations (`stabilizationWindowSeconds`), and remove at most **1** Pod per **60** seconds.

`kubectl autoscale` cannot do that; edit the HPA (`autoscaling/v2`).""",
             check=r"""
eq "$(j -n scale hpa api '{.spec.behavior.scaleDown.stabilizationWindowSeconds}')" 120 || exit 1
eq "$(j -n scale hpa api '{.spec.behavior.scaleDown.policies[0].type} {.spec.behavior.scaleDown.policies[0].value} {.spec.behavior.scaleDown.policies[0].periodSeconds}')" "Pods 1 60" """,
             hint=r"""Under `spec`: `behavior: {scaleDown: {stabilizationWindowSeconds: 120, policies: [{type: Pods, value: 1, periodSeconds: 60}]}}`. `kubectl explain hpa.spec.behavior.scaleDown`.""",
             show=r"""kubectl -n scale patch hpa api --type merge -p '{"spec":{"behavior":{"scaleDown":{"stabilizationWindowSeconds":120,"policies":[{"type":"Pods","value":1,"periodSeconds":60}]}}}}'
kubectl -n scale describe hpa api | grep -A4 Behavior"""),
        step("Watch it scale up",
             r"""Write an HPA for `cruncher` as a manifest (`autoscaling/v2`): 1 to **4** replicas at **60%** average CPU. `cruncher` uses about three times its request, so the HPA soon adds Pods: wait until it wants more than one.""",
             check=r"""
eq "$(j -n scale hpa cruncher '{.spec.maxReplicas} {.spec.metrics[0].resource.target.averageUtilization}')" "4 60" || exit 1
[ "$(j -n scale hpa cruncher '{.status.desiredReplicas}')" -gt 1 ] 2>/dev/null""",
             hint=r"""`kubectl autoscale ... --dry-run=client -o yaml` writes the manifest for you. `kubectl -n scale get hpa cruncher -w` shows it react; `describe hpa` lists its decisions as events.""",
             show=r"""kubectl -n scale apply -f - <<'EOF'
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: cruncher
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: cruncher
  minReplicas: 1
  maxReplicas: 4
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 60
EOF
sleep 60
kubectl -n scale get hpa cruncher"""),
        step("Resize a running Pod",
             r"""Run Pod `sizer` in `scale` (`busybox:1.36`, `sleep 86400`) with CPU request `100m` and limit `200m`, memory request and limit `64Mi`.

Then give it more CPU **without replacing it**: request `150m`, limit `300m`, through the Pod's `resize` subresource. The container keeps running - its restart count stays 0 - and the Pod's status shows the new values once the kubelet has applied them.""",
             check=r"""
eq "$(j -n scale pod sizer '{.spec.containers[0].resources.requests.cpu} {.spec.containers[0].resources.limits.cpu}')" "150m 300m" || exit 1
eq "$(j -n scale pod sizer '{.status.containerStatuses[0].resources.limits.cpu} {.status.containerStatuses[0].restartCount}')" "300m 0" """,
             hint=r"""`kubectl patch pod sizer --subresource resize --patch '{"spec":{"containers":[{"name":"sizer","resources":{...}}]}}'`. A plain `kubectl edit` of a Pod's resources is refused; the subresource is the way in. A container's `resizePolicy` decides whether a change needs a restart.""",
             show=r"""kubectl -n scale run sizer --image=busybox:1.36 --overrides='{"spec":{"containers":[{"name":"sizer","image":"busybox:1.36","command":["sleep","86400"],"resources":{"requests":{"cpu":"100m","memory":"64Mi"},"limits":{"cpu":"200m","memory":"64Mi"}}}]}}'
kubectl -n scale wait --for=condition=Ready pod/sizer --timeout=120s
kubectl -n scale patch pod sizer --subresource resize --patch '{"spec":{"containers":[{"name":"sizer","resources":{"requests":{"cpu":"150m"},"limits":{"cpu":"300m"}}}]}}'
sleep 5
kubectl -n scale get pod sizer -o jsonpath='{.status.containerStatuses[0].resources}{"\n"}{.status.containerStatuses[0].restartCount}{"\n"}'"""),
    ],
    "outro": r"""You made HPAs both ways, slowed down their scale-down, watched one scale up, and resized a running Pod without replacing it.

**For the exam**

- `kubectl autoscale deploy NAME --min --max --cpu=N%` for the simple case; `autoscaling/v2` YAML for memory, several metrics, or `behavior`.
- The HPA needs metrics-server **and** requests on the target's containers. `kubectl describe hpa` explains `<unknown>`.
- Utilization is relative to the request; `averageValue` is absolute (`target: {type: AverageValue, averageValue: 200Mi}`).
- In-place resize: `kubectl patch pod NAME --subresource resize ...`; `resizePolicy` (`NotRequired` / `RestartContainer`) per resource. Deployments still replace Pods when their template changes.
- VPA and the Cluster Autoscaler are add-ons: know what they do, and that the VPA should not fight an HPA over the same metric.

Next, prove it against the clock: **drill 3.3**.""",
}

# ---- 3.4 robust, self-healing deployments ------------------------------------------------------
L34 = {
    "id": 9304, "objective": "3.4", "workers": 1,
    "title": "Self-healing: ReplicaSets, probes, disruption budgets, Jobs",
    "intro": r"""Kubernetes heals by comparing what is with what should be, over and over:

```
ReplicaSet        too few Pods? make more                          (a Deployment manages them)
livenessProbe     container stuck? restart it
readinessProbe    not ready? take it out of the Service's endpoints
startupProbe      slow to start? hold the other probes until it is up
PodDisruptionBudget   how many Pods a voluntary disruption (drain, eviction) may take
Job / CronJob     run to completion, retry on failure, on a schedule
```

Namespace `heal` runs Deployment `web` (nginx, 3 replicas) and Deployment `api` (nginx, 2 replicas), whose Pods never become ready.""",
    "setup": NS + r"""
mkns heal
mkdir -p "$WS/lesson-3.4"
$K -n heal create deploy web --image=nginx:1.27 --replicas=3 >/dev/null
$K -n heal apply -f - >/dev/null <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata: {name: api}
spec:
  replicas: 2
  selector: {matchLabels: {app: api}}
  template:
    metadata: {labels: {app: api}}
    spec:
      containers:
      - name: nginx
        image: nginx:1.27
        readinessProbe:
          httpGet: {path: /healthz, port: 80}
          periodSeconds: 5
EOF
$K -n heal rollout status deploy web --timeout=180s >/dev/null
$K -n heal get pods -l app=web -o jsonpath='{.items[*].metadata.uid}' > "$PRIV/lesson-3.4.uids"
""",
    "steps": [
        step("Delete a Pod, get a Pod",
             r"""Delete one Pod of Deployment `web`. Its ReplicaSet notices that it has 2 of 3, and makes a new one at once.

Write the name of that ReplicaSet to `{WS}/lesson-3.4/rs.txt`. `kubectl get pod POD -o yaml` names it under `ownerReferences`.""",
             check=r"""
rs=$(tr -d ' \r\n' < "$WS/lesson-3.4/rs.txt" 2>/dev/null); [ -n "$rs" ] || exit 1
eq "$(j -n heal rs "$rs" '{.metadata.ownerReferences[0].name} {.status.readyReplicas}')" "web 3" || exit 1
now=$(j -n heal pods -l app=web '{.items[*].metadata.uid}'); for u in $now; do grep -q "$u" "$PRIV/lesson-3.4.uids" || exit 0; done; exit 1""",
             hint=r"""`kubectl -n heal delete pod $(kubectl -n heal get pod -l app=web -o name | head -1)`. Pods of a Deployment are named DEPLOYMENT-RSHASH-RANDOM; the middle part is the ReplicaSet's suffix.""",
             show=r"""kubectl -n heal delete $(kubectl -n heal get pod -l app=web -o name | head -1)
kubectl -n heal get pods -l app=web
kubectl -n heal get pod -l app=web -o jsonpath='{.items[0].metadata.ownerReferences[0].name}' > {WS}/lesson-3.4/rs.txt
kubectl -n heal get rs"""),
        step("A readiness probe that asks the wrong thing",
             r"""The Pods of `api` run, but none is `Ready`: `kubectl -n heal describe pod -l app=api` shows the readiness probe failing with `404`. nginx has no `/healthz`. Make the probe ask for `/` instead, and both Pods become ready.""",
             check=r"""eq "$(j -n heal deploy api '{.spec.template.spec.containers[0].readinessProbe.httpGet.path}')" / && eq "$(j -n heal deploy api '{.status.readyReplicas}')" 2""",
             hint=r"""A JSON patch on `/spec/template/spec/containers/0/readinessProbe/httpGet/path`, or `kubectl edit deploy api`. The fix is a change of template: a rollout.""",
             show=r"""kubectl -n heal describe pod -l app=api | grep -m2 Readiness
kubectl -n heal patch deploy api --type json -p '[{"op":"replace","path":"/spec/template/spec/containers/0/readinessProbe/httpGet/path","value":"/"}]'
kubectl -n heal rollout status deploy api --timeout=120s"""),
        step("A liveness probe that restarts",
             r"""Run Pod `fragile` in `heal` (`busybox:1.36`) with the command `sh -c 'touch /tmp/healthy; sleep 20; rm /tmp/healthy; sleep 86400'` and a **liveness** probe that runs `cat /tmp/healthy` every 5 seconds, starting after 5.

After 20 seconds the file is gone, the probe fails three times, and the kubelet restarts the container - which makes the file again. Wait for the first restart.""",
             check=r"""
eq "$(j -n heal pod fragile '{.spec.containers[0].livenessProbe.exec.command[1]}')" /tmp/healthy || exit 1
[ "$(j -n heal pod fragile '{.status.containerStatuses[0].restartCount}')" -ge 1 ] 2>/dev/null""",
             hint=r"""`livenessProbe: {exec: {command: [cat, /tmp/healthy]}, initialDelaySeconds: 5, periodSeconds: 5}`. `kubectl -n heal get pod fragile -w` shows RESTARTS go up; `describe` shows `Liveness probe failed`.""",
             show=r"""kubectl -n heal apply -f - <<'EOF'
apiVersion: v1
kind: Pod
metadata:
  name: fragile
spec:
  containers:
  - name: fragile
    image: busybox:1.36
    command: [sh, -c, 'touch /tmp/healthy; sleep 20; rm /tmp/healthy; sleep 86400']
    livenessProbe:
      exec:
        command: [cat, /tmp/healthy]
      initialDelaySeconds: 5
      periodSeconds: 5
EOF
sleep 50
kubectl -n heal get pod fragile"""),
        step("A budget for disruptions",
             r"""Create PodDisruptionBudget `web-pdb` in `heal`: at least **2** Pods of `app=web` must stay available.

Delete Pod `fragile` first: it has done its job, and `drain` refuses a Pod that no controller would re-create (unless `--force`, which deletes it for good).

Then try to drain `{W1}`, where all three `web` Pods run: `kubectl drain {W1} --ignore-daemonsets --timeout=30s`. It evicts what it may and refuses the rest - "would violate the pod's disruption budget". Save its output in `{WS}/lesson-3.4/drain.txt`, then make `{W1}` schedulable again.""",
             check=r"""
eq "$(j -n heal pdb web-pdb '{.spec.minAvailable} {.spec.selector.matchLabels.app}')" "2 web" || exit 1
grep -qi 'disruption budget' "$WS/lesson-3.4/drain.txt" 2>/dev/null && [ "$(j node "$W1" '{.spec.unschedulable}')" != true ]""",
             hint=r"""`kubectl create pdb web-pdb --selector=app=web --min-available=2`. Drain prints the refusals on stderr: `2>&1 | tee FILE`. `kubectl uncordon {W1}` afterwards - drain cordons first.""",
             show=r"""kubectl -n heal create pdb web-pdb --selector=app=web --min-available=2
kubectl -n heal get pdb web-pdb
kubectl -n heal delete pod fragile --grace-period=1
kubectl drain {W1} --ignore-daemonsets --delete-emptydir-data --timeout=30s 2>&1 | tee {WS}/lesson-3.4/drain.txt | tail -3
kubectl uncordon {W1}"""),
        step("A Job that runs to completion",
             r"""Create Job `batch` in `heal`: `busybox:1.36` running `sh -c 'echo processed; sleep 2'`, **3** completions, **2** at a time, giving up after **2** retries (`backoffLimit`). Wait until it is complete.""",
             check=r"""eq "$(j -n heal job batch '{.spec.completions} {.spec.parallelism} {.spec.backoffLimit} {.status.succeeded}')" "3 2 2 3" """,
             hint=r"""`kubectl create job batch --image=busybox:1.36 --dry-run=client -o yaml -- sh -c '...'`, then add `completions`, `parallelism` and `backoffLimit` under `spec`. `kubectl wait --for=condition=complete job/batch`.""",
             show=r"""kubectl -n heal create job batch --image=busybox:1.36 --dry-run=client -o yaml -- sh -c 'echo processed; sleep 2' > {WS}/lesson-3.4/batch.yaml
yq -i '.spec.completions = 3 | .spec.parallelism = 2 | .spec.backoffLimit = 2' {WS}/lesson-3.4/batch.yaml
kubectl -n heal apply -f {WS}/lesson-3.4/batch.yaml
kubectl -n heal wait --for=condition=complete job/batch --timeout=120s
kubectl -n heal get pods -l job-name=batch"""),
        step("A CronJob, and a run on demand",
             r"""Create CronJob `report` in `heal`: every 30 minutes (`*/30 * * * *`), `busybox:1.36` running `date`, never two runs at once (`concurrencyPolicy: Forbid`), keeping the last **2** successful Jobs.

Do not wait half an hour: start a Job `report-now` from the CronJob yourself, and let it finish.""",
             check=r"""
eq "$(j -n heal cronjob report '{.spec.schedule} {.spec.concurrencyPolicy} {.spec.successfulJobsHistoryLimit}')" "*/30 * * * * Forbid 2" || exit 1
eq "$(j -n heal job report-now '{.metadata.annotations.cronjob\.kubernetes\.io/instantiate} {.status.succeeded}')" "manual 1" """,
             hint=r"""`kubectl create cronjob report --image=busybox:1.36 --schedule='*/30 * * * *' --dry-run=client -o yaml -- date`, then the two fields. `kubectl create job report-now --from=cronjob/report`.""",
             show=r"""kubectl -n heal create cronjob report --image=busybox:1.36 --schedule='*/30 * * * *' --dry-run=client -o yaml -- date > {WS}/lesson-3.4/report.yaml
yq -i '.spec.concurrencyPolicy = "Forbid" | .spec.successfulJobsHistoryLimit = 2' {WS}/lesson-3.4/report.yaml
kubectl -n heal apply -f {WS}/lesson-3.4/report.yaml
kubectl -n heal create job report-now --from=cronjob/report
kubectl -n heal wait --for=condition=complete job/report-now --timeout=120s"""),
    ],
    "outro": r"""You watched a ReplicaSet replace a Pod, fixed a readiness probe, saw a liveness probe restart a container, stopped a drain with a disruption budget, and ran a Job and a CronJob.

**For the exam**

- Readiness decides **traffic** (endpoints); liveness decides **restarts**; a startup probe holds both off until the app has started. Probes: `httpGet`, `tcpSocket`, `exec`, `grpc`.
- A PDB has `minAvailable` **or** `maxUnavailable`. It stops evictions - a drain, or anything else that uses the Eviction API - never a `kubectl delete pod`, and never a node that just dies.
- Jobs: `completions`, `parallelism`, `backoffLimit`, `activeDeadlineSeconds`, `ttlSecondsAfterFinished`; a Job's Pods use `restartPolicy: Never` or `OnFailure`.
- CronJobs: `schedule` in cron syntax (UTC unless `timeZone`), `concurrencyPolicy`, `suspend`, history limits; `kubectl create job NAME --from=cronjob/CJ` to test one.

Next, prove it against the clock: **drill 3.4**.""",
}

# ---- 3.5 Pod admission and scheduling ---------------------------------------------------------
L35 = {
    "id": 9305, "objective": "3.5", "workers": 2,
    "title": "Scheduling: QoS, affinity, spreading, taints, defaults, and nodeName",
    "intro": r"""The scheduler puts a Pod on a node that passes every **filter** and scores best:

```
requests             does it fit? (requests, not limits, and not usage)
nodeSelector / nodeAffinity      may it go there? (node labels)
podAffinity / podAntiAffinity    near or away from other Pods (their labels, per topologyKey)
taints and tolerations           is the node closed to it?  cordon = the unschedulable taint
```

`nodeName` skips the scheduler entirely. Requests and limits also give each Pod a **QoS class**, which decides who is evicted first when a node runs short: `BestEffort`, then `Burstable`, then `Guaranteed`. A **LimitRange** fills in requests and limits for Pods that bring none.

This cluster has two workers, `{W1}` and `{W2}`. Namespace `sched` is empty.""",
    "setup": NS + r"""
mkns sched
""",
    "steps": [
        step("Three QoS classes",
             r"""Create three Pods in `sched`, each `busybox:1.36` running `sleep 86400`:

- `gold`: requests **equal** to limits, for CPU (`100m`) and memory (`64Mi`) → `Guaranteed`
- `silver`: only a memory request of `32Mi` → `Burstable`
- `bronze`: no resources at all → `BestEffort`""",
             check=r"""
for p in gold:Guaranteed silver:Burstable bronze:BestEffort; do eq "$(j -n sched pod "${p%%:*}" '{.status.qosClass}')" "${p##*:}" || exit 1; done""",
             hint=r"""`kubectl run NAME --image=busybox:1.36 --dry-run=client -o yaml -- sleep 86400`, then `resources:` in the container. `.status.qosClass` shows the result.""",
             show=r"""kubectl -n sched run bronze --image=busybox:1.36 -- sleep 86400
kubectl -n sched run silver --image=busybox:1.36 --overrides='{"spec":{"containers":[{"name":"silver","image":"busybox:1.36","command":["sleep","86400"],"resources":{"requests":{"memory":"32Mi"}}}]}}'
kubectl -n sched run gold --image=busybox:1.36 --overrides='{"spec":{"containers":[{"name":"gold","image":"busybox:1.36","command":["sleep","86400"],"resources":{"requests":{"cpu":"100m","memory":"64Mi"},"limits":{"cpu":"100m","memory":"64Mi"}}}]}}'
kubectl -n sched get pods -o custom-columns=NAME:.metadata.name,QOS:.status.qosClass"""),
        step("Only on SSD nodes",
             r"""Label `{W2}` with `disk=ssd`. Then run Pod `fast` in `sched` (`busybox:1.36`, `sleep 86400`) with a **required** node affinity for `disk In (ssd)`. It lands on `{W2}` - and if no node had the label, it would stay `Pending`.""",
             check=r"""
eq "$(j node "$W2" '{.metadata.labels.disk}')" ssd && eq "$(j -n sched pod fast '{.spec.nodeName}')" "$W2" \
  && eq "$(j -n sched pod fast '{.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key}')" disk""",
             hint=r"""`kubectl label node {W2} disk=ssd`. The affinity block: `affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[].matchExpressions[]` with `key`, `operator: In`, `values`. The docs page "Assign Pods to Nodes using Node Affinity" has it to copy.""",
             show=r"""kubectl label node {W2} disk=ssd
kubectl -n sched apply -f - <<'EOF'
apiVersion: v1
kind: Pod
metadata:
  name: fast
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchExpressions:
          - {key: disk, operator: In, values: [ssd]}
  containers:
  - name: fast
    image: busybox:1.36
    command: [sleep, '86400']
EOF
kubectl -n sched get pod fast -o wide"""),
        step("One replica per node",
             r"""Create Deployment `spread` in `sched` (2 replicas, `busybox:1.36`, `sleep 86400`, label `app=spread`) whose Pods **refuse** to share a node with each other: a required pod anti-affinity on `app=spread`, with `topologyKey: kubernetes.io/hostname`. One Pod lands on each worker; a third replica would stay `Pending`.""",
             check=r"""
eq "$(j -n sched deploy spread '{.status.readyReplicas}')" 2 || exit 1
eq "$(j -n sched deploy spread '{.spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey}')" kubernetes.io/hostname || exit 1
[ "$($K -n sched get pods -l app=spread -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' | sort -u | grep -c .)" = 2 ]""",
             hint=r"""`podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution: [{labelSelector: {matchLabels: {app: spread}}, topologyKey: kubernetes.io/hostname}]` in the Pod template. `topologySpreadConstraints` is the softer, counting way to spread.""",
             show=r"""kubectl -n sched apply -f - <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata:
  name: spread
spec:
  replicas: 2
  selector:
    matchLabels: {app: spread}
  template:
    metadata:
      labels: {app: spread}
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchLabels: {app: spread}
            topologyKey: kubernetes.io/hostname
      containers:
      - name: spread
        image: busybox:1.36
        command: [sleep, '86400']
EOF
kubectl -n sched rollout status deploy spread --timeout=120s
kubectl -n sched get pods -l app=spread -o wide"""),
        step("A node for batch work only",
             r"""Taint `{W1}` with `dedicated=batch:NoSchedule`: from now on only Pods that tolerate it are scheduled there (Pods already running stay - `NoExecute` would evict them).

Then run Pod `nightly` in `sched` (`busybox:1.36`, `sleep 86400`) that tolerates the taint **and** is placed on `{W1}` with a `nodeSelector` on `kubernetes.io/hostname`. A toleration only opens the door; it does not lead the Pod there.""",
             check=r"""
[ -n "$($K get node "$W1" -o jsonpath='{range .spec.taints[*]}{.key}={.value}:{.effect}{"\n"}{end}' | grep -x 'dedicated=batch:NoSchedule')" ] || exit 1
eq "$(j -n sched pod nightly '{.spec.nodeName} {.status.phase}')" "$W1 Running" && eq "$(j -n sched pod nightly '{.spec.tolerations[?(@.key=="dedicated")].value}')" batch""",
             hint=r"""`kubectl taint node {W1} dedicated=batch:NoSchedule`. The toleration: `{key: dedicated, operator: Equal, value: batch, effect: NoSchedule}`.""",
             show=r"""kubectl taint node {W1} dedicated=batch:NoSchedule
kubectl -n sched apply -f - <<'EOF'
apiVersion: v1
kind: Pod
metadata:
  name: nightly
spec:
  nodeSelector:
    kubernetes.io/hostname: {W1}
  tolerations:
  - {key: dedicated, operator: Equal, value: batch, effect: NoSchedule}
  containers:
  - name: nightly
    image: busybox:1.36
    command: [sleep, '86400']
EOF
kubectl -n sched get pod nightly -o wide"""),
        step("Defaults for Pods that bring none",
             r"""Create LimitRange `defaults` in `sched`: containers that say nothing get a **request** of `50m` CPU and `32Mi` memory and a **limit** of `200m` and `128Mi`.

Then run Pod `plain` (`busybox:1.36`, `sleep 86400`, no resources) and look at what it got. A LimitRange acts when a Pod is created: Pods that already exist keep what they had.""",
             check=r"""
eq "$(j -n sched limitrange defaults '{.spec.limits[0].type}')" Container || exit 1
eq "$(j -n sched pod plain '{.spec.containers[0].resources.requests.cpu} {.spec.containers[0].resources.requests.memory} {.spec.containers[0].resources.limits.cpu} {.spec.containers[0].resources.limits.memory}')" "50m 32Mi 200m 128Mi" """,
             hint=r"""`spec.limits: [{type: Container, defaultRequest: {cpu: 50m, memory: 32Mi}, default: {cpu: 200m, memory: 128Mi}}]` - `default` is the limit.""",
             show=r"""kubectl -n sched apply -f - <<'EOF'
apiVersion: v1
kind: LimitRange
metadata:
  name: defaults
spec:
  limits:
  - type: Container
    defaultRequest: {cpu: 50m, memory: 32Mi}
    default: {cpu: 200m, memory: 128Mi}
EOF
kubectl -n sched run plain --image=busybox:1.36 -- sleep 86400
kubectl -n sched get pod plain -o jsonpath='{.spec.containers[0].resources}'; echo"""),
        step("Past the scheduler",
             r"""Cordon `{W2}`. Then create Pod `manual` in `sched` (`busybox:1.36`, `sleep 86400`) with **`nodeName: {W2}`**. It runs there anyway: `nodeName` hands the Pod straight to that node's kubelet, and cordoning only tells the *scheduler* to stay away.

Leave `{W2}` cordoned.""",
             check=r"""eq "$(j node "$W2" '{.spec.unschedulable}')" true && eq "$(j -n sched pod manual '{.spec.nodeName} {.status.phase}')" "$W2 Running" """,
             hint=r"""`kubectl cordon {W2}`, then a Pod manifest with `spec.nodeName`. Without a running scheduler, this is also how you place a Pod by hand.""",
             show=r"""kubectl cordon {W2}
kubectl -n sched run manual --image=busybox:1.36 --overrides='{"spec":{"nodeName":"{W2}"}}' -- sleep 86400
kubectl -n sched get pod manual -o wide
kubectl get nodes"""),
    ],
    "outro": r"""You produced all three QoS classes, steered Pods with node affinity, kept replicas apart, reserved a node with a taint, set defaults with a LimitRange, and bypassed the scheduler.

**For the exam**

- `Pending` with `FailedScheduling`: the event lists why each node was rejected - insufficient cpu/memory, a taint not tolerated, affinity not matched, unschedulable.
- Requests decide placement; limits are enforced at run time (CPU is throttled, memory over the limit is `OOMKilled`).
- `required...` affinity filters, `preferred...` only scores. Pod (anti-)affinity needs a `topologyKey`.
- Taint effects: `NoSchedule`, `PreferNoSchedule`, `NoExecute` (evicts running Pods without a toleration). Remove one with a trailing `-`: `kubectl taint node N dedicated=batch:NoSchedule-`.
- ResourceQuota caps a Namespace's total and then **requires** requests/limits on new Pods; a LimitRange supplies them. PriorityClasses decide who is preempted.

Next, prove it against the clock: **drill 3.5**.""",
}

LESSONS = [L31, L32, L33, L34, L35]