cka-sim/tracks/cka/bank_sim_a.py

"""Simulator Set A - 17 tasks, fixed set, killer.sh-grade difficulty.

Original scenarios (not copies of any commercial simulator) exercising the same techniques:
investigate-and-write-to-file, node-level work via crictl/openssl/systemd, Kustomize overlays,
Helm with CRDs, QoS reasoning, Gateway API conversion, NetworkPolicy match semantics, CoreDNS.

Run with:  ./exam sim a
"""
from tracks.cka.common import C, NS

Q = []


def task(**kw):
    kw.setdefault("difficulty", "hard")
    kw["set"] = "a"
    Q.append(kw)


# ------------------------------------------------------------------ 201 ----
task(
    id=201, domain="ARCH", title="kubeconfig forensics",
    task="""
The file `{WS}/q201/kubeconfig` was handed over by another team.

1. Write every context name, one per line, to `{WS}/q201/contexts`.
2. Write the name of the current context to `{WS}/q201/current-context`.
3. The user `svc-reporting` has an embedded client certificate. Write it **base64-decoded**
   (the PEM block) to `{WS}/q201/cert`.
4. Write the Common Name (CN) of that certificate to `{WS}/q201/cn`.

Do not modify the kubeconfig.
""",
    setup=r"""
rm -rf "$WS/q201"; mkdir -p "$WS/q201"; cd "$WS/q201"
openssl req -x509 -newkey rsa:2048 -nodes -days 30 -subj "/CN=svc-reporting/O=reporting" -keyout r.key -out r.crt >/dev/null 2>&1
openssl req -x509 -newkey rsa:2048 -nodes -days 30 -subj "/CN=svc-billing/O=billing"     -keyout b.key -out b.crt >/dev/null 2>&1
CA=$(base64 -w0 r.crt); RC=$(base64 -w0 r.crt); RK=$(base64 -w0 r.key); BC=$(base64 -w0 b.crt); BK=$(base64 -w0 b.key)
cat > kubeconfig <<EOF
apiVersion: v1
kind: Config
clusters:
  - name: prod
    cluster: {server: https://10.20.30.40:6443, certificate-authority-data: $CA}
  - name: staging
    cluster: {server: https://10.20.30.41:6443, certificate-authority-data: $CA}
contexts:
  - name: prod-billing
    context: {cluster: prod, user: svc-billing}
  - name: prod-reporting
    context: {cluster: prod, user: svc-reporting}
  - name: staging-reporting
    context: {cluster: staging, user: svc-reporting, namespace: reports}
current-context: prod-billing
users:
  - name: svc-billing
    user: {client-certificate-data: $BC, client-key-data: $BK}
  - name: svc-reporting
    user: {client-certificate-data: $RC, client-key-data: $RK}
EOF
rm -f r.key b.key b.crt; mv r.crt .expected.crt
""",
    checks=[
        C("contexts file lists exactly the three context names", 2, r"""
[ "$(grep . "$WS/q201/contexts" 2>/dev/null | sort | tr '\n' ' ')" = "prod-billing prod-reporting staging-reporting " ]"""),
        C("current-context file is correct", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q201/current-context" 2>/dev/null)" prod-billing"""),
        C("cert file is the decoded PEM of svc-reporting", 3, r"""
grep -q 'BEGIN CERTIFICATE' "$WS/q201/cert" 2>/dev/null || exit 1
a=$(openssl x509 -in "$WS/q201/cert" -noout -fingerprint -sha256 2>/dev/null); b=$(openssl x509 -in "$WS/q201/.expected.crt" -noout -fingerprint -sha256)
eq "$a" "$b" """),
        C("cn file contains svc-reporting", 1, r"""grep -q 'svc-reporting' "$WS/q201/cn" 2>/dev/null"""),
    ],
    solution="""
export KC={WS}/q201/kubeconfig
k --kubeconfig $KC config get-contexts -o name > {WS}/q201/contexts
k --kubeconfig $KC config current-context > {WS}/q201/current-context
k --kubeconfig $KC config view --raw \\
  -o jsonpath='{.users[?(@.name=="svc-reporting")].user.client-certificate-data}' | base64 -d > {WS}/q201/cert
openssl x509 -in {WS}/q201/cert -noout -subject | sed 's/.*CN *= *//' > {WS}/q201/cn
# --raw is what reveals the data fields; without it kubectl prints REDACTED
""",
)

# ------------------------------------------------------------------ 202 ----
task(
    id=202, domain="ARCH", title="Helm chart with CRDs, then a custom resource",
    task="""
A local Helm chart is at `{WS}/q202/chart`. It ships a CRD (`Ledger`, group `finance.example.com`).

1. Create Namespace `finance`.
2. Install the chart into `finance` as release `ledger-operator`, with value `watchAllNamespaces=true`.
3. The file `{WS}/q202/ledger.yaml` creates a `Ledger`. Add `spec.retentionDays: 30` to it
   (the CRD's schema accepts it — check with `kubectl explain`), then apply it.

The release must report `deployed` and `kubectl get ledger -A` must list it.
""",
    setup=r"""
rm -rf "$WS/q202"; mkdir -p "$WS/q202/chart/crds" "$WS/q202/chart/templates"
cat > "$WS/q202/chart/Chart.yaml" <<'EOF'
apiVersion: v2
name: ledger-operator
version: 0.2.0
appVersion: "0.2.0"
description: Toy operator shipping a Ledger CRD
EOF
cat > "$WS/q202/chart/values.yaml" <<'EOF'
watchAllNamespaces: false
replicaCount: 1
EOF
cat > "$WS/q202/chart/crds/ledger-crd.yaml" <<'EOF'
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata: {name: ledgers.finance.example.com}
spec:
  group: finance.example.com
  scope: Namespaced
  names: {kind: Ledger, plural: ledgers, singular: ledger, shortNames: [ldg]}
  versions:
    - name: v1
      served: true
      storage: true
      schema:
        openAPIV3Schema:
          type: object
          properties:
            spec:
              type: object
              properties:
                currency: {type: string, description: "ISO currency code"}
                retentionDays: {type: integer, description: "How many days entries are kept"}
EOF
cat > "$WS/q202/chart/templates/deployment.yaml" <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata: {name: {{ .Release.Name }}}
spec:
  replicas: {{ .Values.replicaCount }}
  selector: {matchLabels: {app: {{ .Release.Name }}}}
  template:
    metadata: {labels: {app: {{ .Release.Name }}}}
    spec:
      containers:
        - name: operator
          image: busybox:1.36
          command: [sleep, "86400"]
          env:
            - {name: WATCH_ALL_NAMESPACES, value: "{{ .Values.watchAllNamespaces }}"}
EOF
cat > "$WS/q202/ledger.yaml" <<'EOF'
apiVersion: finance.example.com/v1
kind: Ledger
metadata:
  name: main-ledger
  namespace: finance
spec:
  currency: USD
EOF
helm -n finance uninstall ledger-operator >/dev/null 2>&1 || true
$K delete crd ledgers.finance.example.com --ignore-not-found >/dev/null
$K delete ns finance --ignore-not-found --wait=false >/dev/null
command -v helm >/dev/null || echo "warn: helm not installed - this task needs it"
""",
    checks=[
        C("release ledger-operator deployed in finance", 2, r"""helm -n finance status ledger-operator -o json 2>/dev/null | python3 -c 'import sys,json;sys.exit(0 if json.load(sys.stdin)["info"]["status"]=="deployed" else 1)'"""),
        C("watchAllNamespaces=true reached the pod env", 2, r"""eq "$(j -n finance deploy ledger-operator '{.spec.template.spec.containers[0].env[?(@.name=="WATCH_ALL_NAMESPACES")].value}')" true"""),
        C("CRD installed and shortname works", 1, r"""$K get crd ledgers.finance.example.com >/dev/null 2>&1 && $K -n finance get ldg >/dev/null 2>&1"""),
        C("Ledger main-ledger has currency USD and retentionDays 30", 3, r"""eq "$(j -n finance ledgers main-ledger '{.spec.currency}')" USD && eq "$(j -n finance ledgers main-ledger '{.spec.retentionDays}')" 30"""),
    ],
    solution="""
k create ns finance
helm -n finance install ledger-operator {WS}/q202/chart --set watchAllNamespaces=true
helm -n finance ls; k get crd | grep ledgers
k explain ledger.spec                      # confirms retentionDays exists
vi {WS}/q202/ledger.yaml                   # add:  retentionDays: 30   under spec
k apply -f {WS}/q202/ledger.yaml
k -n finance get ldg
# CRDs in a chart's crds/ directory are installed by helm before the templates,
# which is why the Ledger can be created straight after install.
""",
)

# ------------------------------------------------------------------ 203 ----
task(
    id=203, domain="WORK", title="Find the controller, then scale it",
    task="""
Namespace `project-atlas` runs Pods named `atlas-cache-*`. Management wants them down to **one** replica.

1. Find out which resource manages them and write its kind and name, in the form `kind/name`
   (lowercase kind, as `kubectl get` prints it), to `{WS}/q203/owner.txt`.
2. Scale it to 1 replica without deleting anything.
3. Write the number of Pods still running in the Namespace after scaling to `{WS}/q203/count.txt`.
""",
    setup=NS + r"""
mkns project-atlas
rm -rf "$WS/q203"; mkdir -p "$WS/q203"
$K -n project-atlas delete sts atlas-cache --ignore-not-found >/dev/null
$K apply -f - >/dev/null <<'EOF'
apiVersion: v1
kind: Service
metadata: {name: atlas-cache, namespace: project-atlas}
spec: {clusterIP: None, selector: {app: atlas-cache}, ports: [{port: 6379}]}
---
apiVersion: apps/v1
kind: StatefulSet
metadata: {name: atlas-cache, namespace: project-atlas}
spec:
  serviceName: atlas-cache
  replicas: 3
  selector: {matchLabels: {app: atlas-cache}}
  template:
    metadata: {labels: {app: atlas-cache}}
    spec:
      containers: [{name: cache, image: busybox:1.36, command: [sleep, "86400"]}]
EOF
""",
    checks=[
        C("owner.txt identifies statefulset/atlas-cache", 2, r"""tr -d '[:space:]' < "$WS/q203/owner.txt" 2>/dev/null | grep -Eq '^(statefulset|statefulset.apps)/atlas-cache$'"""),
        C("StatefulSet scaled to 1 and still exists", 3, r"""eq "$(j -n project-atlas sts atlas-cache '{.spec.replicas}')" 1 && eq "$(j -n project-atlas sts atlas-cache '{.status.readyReplicas}')" 1"""),
        C("count.txt matches the running pod count", 1, r"""n=$($K -n project-atlas get pod --no-headers 2>/dev/null | grep -c Running); eq "$(tr -dc '0-9' < "$WS/q203/count.txt" 2>/dev/null)" "$n" """),
    ],
    solution="""
k -n project-atlas get pod --show-labels
k -n project-atlas get deploy,sts,ds,rs           # -> statefulset.apps/atlas-cache
# or from the pod itself:
k -n project-atlas get pod atlas-cache-0 -o jsonpath='{.metadata.ownerReferences[0].kind}{"/"}{.metadata.ownerReferences[0].name}'
echo statefulset/atlas-cache > {WS}/q203/owner.txt
k -n project-atlas scale sts atlas-cache --replicas 1
k -n project-atlas get pod | grep -c Running > {WS}/q203/count.txt
""",
)

# ------------------------------------------------------------------ 204 ----
task(
    id=204, domain="WORK", title="Which Pods get evicted first",
    task="""
Namespace `project-orion` is close to filling its nodes.

1. Write the names of all Pods in `project-orion` that the kubelet would evict **first** under
   node memory pressure — one name per line, sorted — to `{WS}/q204/evicted-first.txt`.
2. Write the Quality of Service class of Pod `orion-api-0` to `{WS}/q204/api-qos.txt`.
3. One Deployment's Pods are `Guaranteed`. Write its name to `{WS}/q204/guaranteed.txt`.
""",
    setup=NS + r"""
mkns project-orion
rm -rf "$WS/q204"; mkdir -p "$WS/q204"
# the pod is re-created below under the same name: a delete that does not wait lets the apply land on
# the terminating old pod, which then vanishes and leaves the task without orion-api-0
$K -n project-orion delete deploy --all >/dev/null 2>&1; $K -n project-orion delete pod orion-api-0 --ignore-not-found --force --grace-period=0 --wait=true >/dev/null 2>&1
$K apply -f - >/dev/null <<EOF
apiVersion: v1
kind: Pod
metadata: {name: orion-api-0, namespace: project-orion, labels: {app: orion-api}}
spec:
  affinity: {nodeAffinity: {requiredDuringSchedulingIgnoredDuringExecution: {nodeSelectorTerms: [{matchExpressions: [{key: kubernetes.io/hostname, operator: NotIn, values: [$W2]}]}]}}}
  containers:
    - name: api
      image: busybox:1.36
      command: [sleep, "86400"]
      resources: {requests: {cpu: 20m, memory: 24Mi}, limits: {cpu: 100m, memory: 64Mi}}
---
apiVersion: apps/v1
kind: Deployment
metadata: {name: orion-batch, namespace: project-orion}
spec:
  replicas: 2
  selector: {matchLabels: {app: orion-batch}}
  template:
    metadata: {labels: {app: orion-batch}}
    spec:
      affinity: {nodeAffinity: {requiredDuringSchedulingIgnoredDuringExecution: {nodeSelectorTerms: [{matchExpressions: [{key: kubernetes.io/hostname, operator: NotIn, values: [$W2]}]}]}}}
      containers: [{name: batch, image: busybox:1.36, command: [sleep, "86400"]}]
---
apiVersion: apps/v1
kind: Deployment
metadata: {name: orion-ledger, namespace: project-orion}
spec:
  replicas: 2
  selector: {matchLabels: {app: orion-ledger}}
  template:
    metadata: {labels: {app: orion-ledger}}
    spec:
      affinity: {nodeAffinity: {requiredDuringSchedulingIgnoredDuringExecution: {nodeSelectorTerms: [{matchExpressions: [{key: kubernetes.io/hostname, operator: NotIn, values: [$W2]}]}]}}}
      containers:
        - name: ledger
          image: busybox:1.36
          command: [sleep, "86400"]
          resources: {requests: {cpu: 25m, memory: 32Mi}, limits: {cpu: 25m, memory: 32Mi}}
EOF
""",
    checks=[
        # Every BestEffort pod this task seeded must be listed and no Burstable/Guaranteed one may be.
        # BestEffort pods other tasks add to the namespace later (212's spread-check has no
        # resources at all) are tolerated either way - they did not exist when the file was written.
        C("evicted-first.txt lists the BestEffort pods and nothing else", 4, r"""
f="$WS/q204/evicted-first.txt"; [ -s "$f" ] || exit 1
$K -n project-orion get pod -o json 2>/dev/null | python3 -c '
import sys,json
items=json.load(sys.stdin)["items"]; got={l.strip() for l in open(sys.argv[1]) if l.strip()}
qos={p["metadata"]["name"]:p["status"].get("qosClass") for p in items}
seeded={p["metadata"]["name"] for p in items if p["metadata"].get("labels",{}).get("app")=="orion-batch"}
ok=seeded and seeded<=got and not any(qos.get(n) in ("Burstable","Guaranteed") for n in got)
sys.exit(0 if ok else 1)' "$f" """),
        C("api-qos.txt says Burstable", 2, r"""grep -qi burstable "$WS/q204/api-qos.txt" 2>/dev/null"""),
        C("guaranteed.txt names orion-ledger", 2, r"""grep -q 'orion-ledger' "$WS/q204/guaranteed.txt" 2>/dev/null"""),
    ],
    solution="""
k -n project-orion get pod -o jsonpath='{range .items[*]}{.metadata.name} {.status.qosClass}{"\\n"}{end}'
# BestEffort (no requests and no limits at all) is evicted first, then Burstable
#   exceeding its requests, Guaranteed (requests == limits on every container) last.
k -n project-orion get pod -o jsonpath='{range .items[*]}{.metadata.name} {.status.qosClass}{"\\n"}{end}' \\
  | awk '$2=="BestEffort"{print $1}' | sort > {WS}/q204/evicted-first.txt
k -n project-orion get pod orion-api-0 -o jsonpath='{.status.qosClass}' > {WS}/q204/api-qos.txt
echo orion-ledger > {WS}/q204/guaranteed.txt
""",
)

# ------------------------------------------------------------------ 205 ----
task(
    id=205, domain="ARCH", title="Kustomize: drop a ConfigMap, add an HPA, override in prod",
    task="""
The app `checkout` is deployed from `{WS}/q205/checkout` into Namespaces `checkout-staging`
and `checkout-prod`:

```
kubectl kustomize {WS}/q205/checkout/staging | kubectl apply -f -
kubectl kustomize {WS}/q205/checkout/prod    | kubectl apply -f -
```

Change the Kustomize config so that:
1. the ConfigMap `legacy-scaler-config` is gone from the built output of **both** overlays,
2. a HorizontalPodAutoscaler named `checkout` targets the `checkout` Deployment with
   min 2 / max 5 replicas at 60% average CPU utilisation,
3. in **prod** the same HPA has max 8 replicas,
4. both overlays are applied, and the ConfigMap no longer exists in either Namespace.
""",
    setup=r"""
rm -rf "$WS/q205"; mkdir -p "$WS/q205/checkout/base" "$WS/q205/checkout/staging" "$WS/q205/checkout/prod"
B="$WS/q205/checkout"
cat > "$B/base/kustomization.yaml" <<'EOF'
resources: [checkout.yaml]
EOF
cat > "$B/base/checkout.yaml" <<'EOF'
apiVersion: v1
kind: ConfigMap
metadata: {name: legacy-scaler-config}
data: {threshold: "70"}
---
apiVersion: apps/v1
kind: Deployment
metadata: {name: checkout}
spec:
  replicas: 1
  selector: {matchLabels: {id: checkout}}
  template:
    metadata: {labels: {id: checkout}}
    spec:
      containers:
        - name: web
          image: httpd:2-alpine
          resources: {requests: {cpu: 10m, memory: 20Mi}}
EOF
for env in staging prod; do
cat > "$B/$env/kustomization.yaml" <<EOF
resources: [../base]
namespace: checkout-$env
patches: [{path: checkout.yaml}]
EOF
cat > "$B/$env/checkout.yaml" <<EOF
apiVersion: v1
kind: ConfigMap
metadata: {name: legacy-scaler-config}
data: {threshold: "$( [ $env = prod ] && echo 80 || echo 60 )"}
---
apiVersion: apps/v1
kind: Deployment
metadata: {name: checkout, labels: {env: $env}}
EOF
done
for ns in checkout-staging checkout-prod; do $K create ns $ns --dry-run=client -o yaml | $K apply -f - >/dev/null; done
$K kustomize "$B/staging" 2>/dev/null | $K apply -f - >/dev/null 2>&1
$K kustomize "$B/prod" 2>/dev/null | $K apply -f - >/dev/null 2>&1
""",
    checks=[
        C("built output of both overlays contains no ConfigMap", 3, r"""
for e in staging prod; do $K kustomize "$WS/q205/checkout/$e" 2>/dev/null | grep -q 'kind: ConfigMap' && exit 1; done
$K kustomize "$WS/q205/checkout/staging" >/dev/null 2>&1"""),
        C("HPA in staging: min 2, max 5, 60% CPU, targets the Deployment", 3, r"""
h="$K -n checkout-staging get hpa checkout"
eq "$($h -o jsonpath='{.spec.minReplicas}' 2>/dev/null)" 2 && eq "$($h -o jsonpath='{.spec.maxReplicas}')" 5 \
 && eq "$($h -o jsonpath='{.spec.scaleTargetRef.name}')" checkout \
 && $h -o jsonpath='{.spec.metrics[*].resource}' | grep -q '"averageUtilization":60'"""),
        C("HPA in prod has max 8 (min and target unchanged)", 2, r"""
h="$K -n checkout-prod get hpa checkout"
eq "$($h -o jsonpath='{.spec.maxReplicas}' 2>/dev/null)" 8 && eq "$($h -o jsonpath='{.spec.minReplicas}')" 2"""),
        C("legacy-scaler-config removed from both Namespaces", 2, r"""
! $K -n checkout-staging get cm legacy-scaler-config >/dev/null 2>&1 && ! $K -n checkout-prod get cm legacy-scaler-config >/dev/null 2>&1"""),
    ],
    solution="""
cd {WS}/q205/checkout
# 1. the ConfigMap is patched by BOTH overlays, so removing it only from base breaks the build.
#    Delete it from base/checkout.yaml AND from staging/checkout.yaml AND prod/checkout.yaml.
# 2. add the HPA to base/checkout.yaml (no namespace - the overlays set it):
#    apiVersion: autoscaling/v2
#    kind: HorizontalPodAutoscaler
#    metadata: {name: checkout}
#    spec:
#      scaleTargetRef: {apiVersion: apps/v1, kind: Deployment, name: checkout}
#      minReplicas: 2
#      maxReplicas: 5
#      metrics: [{type: Resource, resource: {name: cpu, target: {type: Utilization, averageUtilization: 60}}}]
# 3. in prod/checkout.yaml add a patch with only the changed field:
#    apiVersion: autoscaling/v2
#    kind: HorizontalPodAutoscaler
#    metadata: {name: checkout}
#    spec: {maxReplicas: 8}
k kustomize staging | k apply -f -
k kustomize prod    | k apply -f -
# 4. Kustomize keeps no state: it never deletes resources that only exist in the cluster.
k -n checkout-staging delete cm legacy-scaler-config
k -n checkout-prod    delete cm legacy-scaler-config
""",
)

# ------------------------------------------------------------------ 206 ----
task(
    id=206, domain="STOR", title="PV, PVC and a Deployment that mounts it",
    task="""
1. Create a PersistentVolume `orbit-pv`: 3Gi, accessMode `ReadWriteOnce`, `hostPath` `/Volumes/orbit`,
   and **no** storageClassName.
2. In Namespace `project-orbit` create PVC `orbit-pvc` requesting 3Gi, `ReadWriteOnce`, also with
   **no** storageClassName, bound to that PV.
3. Create Deployment `orbit` in the same Namespace, image `httpd:2-alpine`, that mounts the claim
   at `/usr/local/apache2/htdocs/data`.

The claim must be `Bound` and the Pod `Running`.
""",
    setup=NS + r"""
mkns project-orbit
$K -n project-orbit delete deploy orbit --ignore-not-found >/dev/null
$K -n project-orbit delete pvc orbit-pvc --ignore-not-found >/dev/null
$K delete pv orbit-pv --ignore-not-found >/dev/null
""",
    checks=[
        C("PV orbit-pv: 3Gi, RWO, hostPath /Volumes/orbit, no storageClassName", 3, r"""
p="$K get pv orbit-pv"
eq "$($p -o jsonpath='{.spec.capacity.storage}' 2>/dev/null)" 3Gi && eq "$($p -o jsonpath='{.spec.hostPath.path}')" /Volumes/orbit \
 && $p -o jsonpath='{.spec.accessModes}' | grep -q ReadWriteOnce && [ -z "$($p -o jsonpath='{.spec.storageClassName}')" ]"""),
        C("PVC orbit-pvc Bound to orbit-pv with no storageClassName", 3, r"""
eq "$(j -n project-orbit pvc orbit-pvc '{.status.phase}')" Bound && eq "$(j -n project-orbit pvc orbit-pvc '{.spec.volumeName}')" orbit-pv \
 && [ -z "$(j -n project-orbit pvc orbit-pvc '{.spec.storageClassName}')" ]"""),
        C("Deployment orbit available and mounts the claim at the right path", 3, r"""
eq "$(j -n project-orbit deploy orbit '{.status.availableReplicas}')" 1 \
 && [ -n "$(j -n project-orbit deploy orbit '{.spec.template.spec.volumes[?(@.persistentVolumeClaim.claimName=="orbit-pvc")].name}')" ] \
 && j -n project-orbit deploy orbit '{.spec.template.spec.containers[0].volumeMounts[*].mountPath}' | grep -q '/usr/local/apache2/htdocs/data'"""),
    ],
    solution="""
cat <<EOF | k apply -f -
apiVersion: v1
kind: PersistentVolume
metadata: {name: orbit-pv}
spec:
  capacity: {storage: 3Gi}
  accessModes: [ReadWriteOnce]
  hostPath: {path: /Volumes/orbit}
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: orbit-pvc, namespace: project-orbit}
spec:
  accessModes: [ReadWriteOnce]
  storageClassName: ""          # empty string, NOT omitted: otherwise the default class is used
  resources: {requests: {storage: 3Gi}}
EOF
k -n project-orbit create deploy orbit --image=httpd:2-alpine $do > orbit.yaml
# add to the pod spec:
#   volumes: [{name: data, persistentVolumeClaim: {claimName: orbit-pvc}}]
# and to the container:
#   volumeMounts: [{name: data, mountPath: /usr/local/apache2/htdocs/data}]
k apply -f orbit.yaml
k -n project-orbit get pv,pvc
""",
)

# ------------------------------------------------------------------ 207 ----
task(
    id=207, domain="TSHOOT", title="Resource usage scripts",
    task="""
Write two executable bash scripts that use `kubectl`:

1. `{WS}/q207/usage-nodes.sh` — prints CPU and memory usage of all **nodes**, sorted by CPU
   (highest first).
2. `{WS}/q207/usage-pods.sh` — prints CPU and memory usage of all **Pods in all Namespaces**,
   broken down **per container**.

Use full command names, no aliases. The scripts must be executable.
""",
    setup=r"""rm -rf "$WS/q207"; mkdir -p "$WS/q207" """,
    checks=[
        C("usage-nodes.sh is executable and uses kubectl top node with cpu sorting", 3, r"""
f="$WS/q207/usage-nodes.sh"; [ -x "$f" ] || exit 1
grep -q 'kubectl top node' "$f" && grep -Eq -- '--sort-by[= ]?.?cpu' "$f" && ! grep -qE '(^| )k( |$)' "$f" """),
        C("usage-pods.sh is executable, all namespaces, per container", 3, r"""
f="$WS/q207/usage-pods.sh"; [ -x "$f" ] || exit 1
grep -q 'kubectl top pod' "$f" && grep -Eq -- '(-A|--all-namespaces)' "$f" && grep -q -- '--containers' "$f" """),
    ],
    solution="""
cat > {WS}/q207/usage-nodes.sh <<'EOF'
kubectl top node --sort-by=cpu
EOF
cat > {WS}/q207/usage-pods.sh <<'EOF'
kubectl top pod --all-namespaces --containers=true
EOF
chmod +x {WS}/q207/usage-nodes.sh {WS}/q207/usage-pods.sh
# `kubectl top -h` shows both flags. Aliases (k) don't exist in a fresh shell, so write kubectl.
""",
)

# ------------------------------------------------------------------ 208 ----
# Own namespace (project-vega): it used to share project-atlas with 203, whose "running pods"
# count then dropped by one when this drain deleted the bare pod. And the grader checks the pods
# that were on the node when it was seeded, not "nothing on the node at grading time" - after the
# required uncordon the scheduler is free to put new pods there (212's one-per-node spread always
# does), which failed this task no matter what the candidate did.
task(
    id=208, domain="ARCH", title="Drain a node for maintenance", breaks="w2-drain",
    task="""
Node `{W2}` needs kernel maintenance.

1. Evict every non-DaemonSet Pod from it. Pods with `emptyDir` volumes must not stop you,
   and neither must Pods that aren't managed by a controller.
2. Write the exact command you used to `{WS}/q208/drain.sh`.
3. Deployment `vega-web` in Namespace `project-vega` must still have all 3 replicas Available afterwards.
4. When you're done, make the node schedulable again.

A Pod without a controller is **deleted** by a forced drain, not rescheduled. Before you drain,
look at what else runs on the node (`k get pods -A -o wide --field-selector spec.nodeName={W2}`) —
anything you created there for another task has to be recreated afterwards.
""",
    setup=NS + r"""
mkns project-vega
rm -rf "$WS/q208"; mkdir -p "$WS/q208"
$K -n project-vega delete deploy vega-web --ignore-not-found >/dev/null
$K -n project-vega delete pod vega-orphan --ignore-not-found --force --grace-period=0 --wait=true >/dev/null 2>&1
$K uncordon "$W2" >/dev/null 2>&1 || true
$K apply -f - >/dev/null <<EOF
apiVersion: apps/v1
kind: Deployment
metadata: {name: vega-web, namespace: project-vega}
spec:
  replicas: 3
  selector: {matchLabels: {app: vega-web}}
  template:
    metadata: {labels: {app: vega-web}}
    spec:
      containers:
        - name: web
          image: nginx:1.27
          resources: {requests: {cpu: 10m, memory: 16Mi}}
          volumeMounts: [{name: scratch, mountPath: /scratch}]
      volumes: [{name: scratch, emptyDir: {}}]
      # at least one replica on the node being drained, so the emptyDir flag really matters
      topologySpreadConstraints: [{maxSkew: 1, topologyKey: kubernetes.io/hostname, whenUnsatisfiable: ScheduleAnyway, labelSelector: {matchLabels: {app: vega-web}}}]
EOF
$K -n project-vega rollout status deploy vega-web --timeout=120s >/dev/null 2>&1
$K -n project-vega run vega-orphan --image=busybox:1.36 --restart=Never --overrides="{\"spec\":{\"nodeName\":\"$W2\"}}" -- sleep 86400 >/dev/null 2>&1
$K -n project-vega wait --for=condition=Ready pod/vega-orphan --timeout=90s >/dev/null 2>&1
# what the drain has to clear: every non-DaemonSet, non-mirror, non-kube-system pod on the node now
$K get pods -A -o json | python3 -c '
import sys,json
n=sys.argv[1]
for p in json.load(sys.stdin)["items"]:
    m=p["metadata"]; o=(m.get("ownerReferences") or [{}])[0].get("kind")
    if p["spec"].get("nodeName")==n and m["namespace"]!="kube-system" and o not in ("DaemonSet","Node"):
        print(m["uid"])' "$W2" > "$WS/.q208-uids"
""",
    checks=[
        C("every pod that was on the node when it was seeded has been evicted", 4, r"""
[ -s "$WS/.q208-uids" ] || exit 1
now=$($K get pods -A -o jsonpath='{range .items[*]}{.metadata.uid}{" "}{.spec.nodeName}{"\n"}{end}' 2>/dev/null)
while read -r u; do echo "$now" | grep -q "^$u $W2\$" && exit 1; done < "$WS/.q208-uids"
! $K -n project-vega get pod vega-orphan >/dev/null 2>&1"""),
        C("drain.sh records a drain with --ignore-daemonsets and --delete-emptydir-data", 2, r"""
f="$WS/q208/drain.sh"; grep -q 'drain' "$f" 2>/dev/null && grep -q -- '--ignore-daemonsets' "$f" && grep -q -- '--delete-emptydir-data' "$f" """),
        C("vega-web still has 3 replicas Available", 2, r"""eq "$(j -n project-vega deploy vega-web '{.status.availableReplicas}')" 3"""),
        C("node is schedulable again", 2, r"""[ "$(j node "$W2" '{.spec.unschedulable}')" != true ]"""),
    ],
    solution="""
k get pods -A -o wide --field-selector spec.nodeName={W2}      # what is about to be evicted or deleted
k drain {W2} --ignore-daemonsets --delete-emptydir-data --force
#   --ignore-daemonsets      DaemonSet pods can't be evicted, skip them
#   --delete-emptydir-data   required because vega-web mounts an emptyDir (its data is lost)
#   --force                  required for the bare pod vega-orphan - which is DELETED, not moved
echo 'kubectl drain {W2} --ignore-daemonsets --delete-emptydir-data --force' > {WS}/q208/drain.sh
k get pods -A -o wide --field-selector spec.nodeName={W2}      # only DaemonSet pods left
k -n project-vega rollout status deploy vega-web
k uncordon {W2}
""",
)

# ------------------------------------------------------------------ 209 ----
task(
    id=209, domain="ARCH", title="Query the API from inside a Pod",
    task="""
Namespace `project-orbit` has ServiceAccount `cm-reader`, which may read ConfigMaps cluster-wide.

1. Create Pod `api-probe` in `project-orbit`, image `nginx:1-alpine`, using that ServiceAccount.
2. From **inside** the Pod, query the Kubernetes API for all ConfigMaps in Namespace `project-orbit`
   using `curl` and the Pod's ServiceAccount token, verifying the server certificate with the
   mounted CA (do not use `-k`).
3. Save the JSON response to `{WS}/q209/configmaps.json`.
4. Write the API server URL you used, as seen from inside the Pod, to `{WS}/q209/endpoint.txt`.
""",
    setup=NS + r"""
mkns project-orbit
rm -rf "$WS/q209"; mkdir -p "$WS/q209"
$K -n project-orbit delete pod api-probe --ignore-not-found --wait=false >/dev/null
$K -n project-orbit create sa cm-reader --dry-run=client -o yaml | $K apply -f - >/dev/null
$K delete clusterrole cm-reader --ignore-not-found >/dev/null; $K delete clusterrolebinding cm-reader --ignore-not-found >/dev/null
$K create clusterrole cm-reader --verb=get,list --resource=configmaps >/dev/null
$K create clusterrolebinding cm-reader --clusterrole=cm-reader --serviceaccount=project-orbit:cm-reader >/dev/null
$K -n project-orbit create cm orbit-settings --from-literal=region=us-east --dry-run=client -o yaml | $K apply -f - >/dev/null
""",
    checks=[
        C("Pod api-probe Running with ServiceAccount cm-reader", 2, r"""
eq "$(j -n project-orbit pod api-probe '{.status.phase}')" Running && eq "$(j -n project-orbit pod api-probe '{.spec.serviceAccountName}')" cm-reader"""),
        C("configmaps.json is a real ConfigMapList containing orbit-settings", 4, r"""
python3 -c '
import json,sys
d=json.load(open("'"$WS"'/q209/configmaps.json"))
sys.exit(0 if d.get("kind")=="ConfigMapList" and any(i["metadata"]["name"]=="orbit-settings" for i in d.get("items",[])) else 1)' 2>/dev/null"""),
        C("the response was authorised (not a 403 Status object)", 2, r"""! grep -q '"kind": *"Status"' "$WS/q209/configmaps.json" 2>/dev/null"""),
        C("endpoint.txt names the in-cluster API service", 1, r"""grep -Eq 'kubernetes(\.default)?' "$WS/q209/endpoint.txt" 2>/dev/null"""),
    ],
    solution="""
k -n project-orbit run api-probe --image=nginx:1-alpine $do > p.yaml
# add:  spec.serviceAccountName: cm-reader
k apply -f p.yaml
k -n project-orbit exec -it api-probe -- sh
  TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
  CACERT=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  curl --cacert $CACERT -H "Authorization: Bearer $TOKEN" \\
    https://kubernetes.default/api/v1/namespaces/project-orbit/configmaps > /tmp/cm.json
  exit
k -n project-orbit exec api-probe -- cat /tmp/cm.json > {WS}/q209/configmaps.json
echo https://kubernetes.default > {WS}/q209/endpoint.txt
# Without the token you are system:anonymous and get 403; with -k you skip verification,
# which the task forbids - the CA is mounted next to the token.
""",
)

# ------------------------------------------------------------------ 210 ----
task(
    id=210, domain="ARCH", title="ServiceAccount limited to two resources",
    task="""
In Namespace `project-atlas` create ServiceAccount `pipeline`, plus a Role and a RoleBinding both
named `pipeline`, so that the ServiceAccount may **only**:

- `create` and `update` **secrets** and **configmaps** in that Namespace.

It must not be able to read them back, delete them, or touch anything in another Namespace.
""",
    setup=NS + r"""
mkns project-atlas
$K -n project-atlas delete sa pipeline --ignore-not-found >/dev/null
$K -n project-atlas delete role pipeline --ignore-not-found >/dev/null
$K -n project-atlas delete rolebinding pipeline --ignore-not-found >/dev/null
""",
    checks=[
        C("ServiceAccount, Role and RoleBinding named pipeline exist", 2, r"""$K -n project-atlas get sa pipeline >/dev/null 2>&1 && $K -n project-atlas get role pipeline >/dev/null 2>&1 && $K -n project-atlas get rolebinding pipeline >/dev/null 2>&1"""),
        C("can create and update secrets and configmaps in project-atlas", 3, r"""
for r in secrets configmaps; do for v in create update; do
  eq "$($K auth can-i $v $r -n project-atlas --as=system:serviceaccount:project-atlas:pipeline 2>/dev/null)" yes || exit 1
done; done"""),
        C("cannot get/list/delete them, and has nothing elsewhere", 3, r"""
for t in "get secrets -n project-atlas" "list configmaps -n project-atlas" "delete secrets -n project-atlas" "create secrets -n default"; do
  eq "$($K auth can-i $t --as=system:serviceaccount:project-atlas:pipeline 2>/dev/null || true)" no || exit 1
done"""),
    ],
    solution="""
k -n project-atlas create sa pipeline
k -n project-atlas create role pipeline --verb=create,update --resource=secrets,configmaps
k -n project-atlas create rolebinding pipeline --role=pipeline --serviceaccount=project-atlas:pipeline
k -n project-atlas auth can-i create secret --as=system:serviceaccount:project-atlas:pipeline   # yes
k -n project-atlas auth can-i get    secret --as=system:serviceaccount:project-atlas:pipeline   # no
""",
)

# ------------------------------------------------------------------ 211 ----
task(
    id=211, domain="WORK", title="DaemonSet on every node", breaks=["w1-taint", "w1-block"],
    task="""
In Namespace `project-orion` create a DaemonSet `node-collector`:

- image `httpd:2-alpine`
- labels on the DaemonSet **and** its Pods: `id=node-collector` and `tier=infra`
- each Pod requests 15 millicores of CPU and 15 mebibytes of memory
- a Pod must run on **every** node in the cluster, control-plane included

Every desired Pod must be Ready.
""",
    setup=NS + r"""mkns project-orion; $K -n project-orion delete ds node-collector --ignore-not-found >/dev/null""",
    checks=[
        C("DaemonSet exists with both labels on the object and the pod template", 2, r"""
d="$K -n project-orion get ds node-collector"
eq "$($d -o jsonpath='{.metadata.labels.id}' 2>/dev/null)" node-collector && eq "$($d -o jsonpath='{.metadata.labels.tier}')" infra \
 && eq "$($d -o jsonpath='{.spec.template.metadata.labels.id}')" node-collector && eq "$($d -o jsonpath='{.spec.template.metadata.labels.tier}')" infra"""),
        C("requests are 15m CPU / 15Mi memory", 2, r"""
eq "$(j -n project-orion ds node-collector '{.spec.template.spec.containers[0].resources.requests.cpu}')" 15m \
 && eq "$(j -n project-orion ds node-collector '{.spec.template.spec.containers[0].resources.requests.memory}')" 15Mi"""),
        C("one Ready pod on every node including the control-plane", 4, r"""
n=$($K get nodes --no-headers 2>/dev/null | wc -l)
eq "$(j -n project-orion ds node-collector '{.status.desiredNumberScheduled}')" "$n" && eq "$(j -n project-orion ds node-collector '{.status.numberReady}')" "$n" \
 && j -n project-orion pod -l id=node-collector '{.items[*].spec.nodeName}' | tr ' ' '\n' | grep -q control-plane"""),
    ],
    solution="""
k -n project-orion create deploy node-collector --image=httpd:2-alpine $do > ds.yaml
# change kind Deployment -> DaemonSet, drop replicas/strategy/status, then add:
#   metadata.labels: {id: node-collector, tier: infra}
#   spec.selector.matchLabels + spec.template.metadata.labels: same two labels
#   container resources.requests: {cpu: 15m, memory: 15Mi}
#   spec.template.spec.tolerations:
#     - {key: node-role.kubernetes.io/control-plane, operator: Exists, effect: NoSchedule}
k apply -f ds.yaml
k -n project-orion get ds node-collector -o wide
""",
)

# ------------------------------------------------------------------ 212 ----
task(
    id=212, domain="WORK", title="One Pod per node, on purpose", breaks=["w1-taint", "w1-block"],
    task="""
In Namespace `project-orion` create Deployment `spread-check`:

- 4 replicas, Pods labelled `id=spread-check`
- container `main` with image `nginx:1-alpine`, and a second container `pause` with image
  `registry.k8s.io/pause:3.10`
- at most **one** of its Pods on any worker node (use `topologyKey: kubernetes.io/hostname`)

Because there are only three worker nodes, one Pod will stay `Pending` — that is expected.
Write the name of the Pending Pod to `{WS}/q212/pending.txt`.
""",
    setup=NS + r"""mkns project-orion; rm -rf "$WS/q212"; mkdir -p "$WS/q212"; $K -n project-orion delete deploy spread-check --ignore-not-found >/dev/null""",
    checks=[
        C("Deployment has 4 replicas and both containers", 2, r"""
d="$K -n project-orion get deploy spread-check"
eq "$($d -o jsonpath='{.spec.replicas}' 2>/dev/null)" 4 \
 && $d -o jsonpath='{.spec.template.spec.containers[*].name}' | grep -qw main \
 && $d -o jsonpath='{.spec.template.spec.containers[*].image}' | grep -q 'pause:3.10'"""),
        C("anti-affinity or spread constraint on kubernetes.io/hostname", 3, r"""
t=$(j -n project-orion deploy spread-check '{.spec.template.spec}')
echo "$t" | grep -q '"topologyKey":"kubernetes.io/hostname"' && { echo "$t" | grep -q 'podAntiAffinity' || echo "$t" | grep -q 'topologySpreadConstraints'; }"""),
        C("exactly 3 Pods Running, one per node, 1 Pending", 3, r"""
r=$($K -n project-orion get pod -l id=spread-check --no-headers 2>/dev/null | grep -c Running)
p=$($K -n project-orion get pod -l id=spread-check --no-headers 2>/dev/null | grep -c Pending)
eq "$r" 3 && eq "$p" 1 && [ "$(j -n project-orion pod -l id=spread-check '{.items[*].spec.nodeName}' | tr ' ' '\n' | grep . | sort -u | wc -l)" = 3 ]"""),
        C("pending.txt names the Pending pod", 2, r"""
want=$($K -n project-orion get pod -l id=spread-check --no-headers 2>/dev/null | awk '$3=="Pending"{print $1}')
[ -n "$want" ] && grep -qF "$want" "$WS/q212/pending.txt" 2>/dev/null"""),
    ],
    solution="""
k -n project-orion create deploy spread-check --image=nginx:1-alpine --replicas=4 $do > s.yaml
# set the pod label id=spread-check (selector + template), rename the container to main,
# add the second container, and add either:
#   affinity:
#     podAntiAffinity:
#       requiredDuringSchedulingIgnoredDuringExecution:
#         - labelSelector: {matchLabels: {id: spread-check}}
#           topologyKey: kubernetes.io/hostname
# or:
#   topologySpreadConstraints:
#     - {maxSkew: 1, topologyKey: kubernetes.io/hostname, whenUnsatisfiable: DoNotSchedule,
#        labelSelector: {matchLabels: {id: spread-check}}}
k apply -f s.yaml
k -n project-orion get pod -l id=spread-check -o wide
k -n project-orion get pod -l id=spread-check --field-selector status.phase=Pending -o name > {WS}/q212/pending.txt
k -n project-orion describe pod <pending>     # "didn't match pod anti-affinity rules"
""",
)

# ------------------------------------------------------------------ 213 ----
task(
    id=213, domain="NET", title="Convert an Ingress to Gateway API, then split on a header",
    task="""
Namespace `project-r700` has Services `web-classic` and `web-next` (both port 80) and an existing
Gateway named `edge`. The legacy Ingress is at `{WS}/q213/ingress.yaml`.

Create an HTTPRoute named `router` in that Namespace, attached to the Gateway `edge`, which:

1. reproduces the routes of the old Ingress (host and paths included),
2. adds a path `/beta` that goes to `web-next` **only when** the header `X-Channel` is exactly
   `beta`, and to `web-classic` in every other case.

Path matching is `PathPrefix` throughout. Order matters.
""",
    setup=NS + r"""
mkns project-r700
rm -rf "$WS/q213"; mkdir -p "$WS/q213"
for s in web-classic web-next; do
  $K -n project-r700 create deploy $s --image=nginx:1.27 --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null
  $K -n project-r700 expose deploy $s --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null
done
cat > "$WS/q213/ingress.yaml" <<'EOF'
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata: {name: router, namespace: project-r700}
spec:
  ingressClassName: nginx
  rules:
    - host: r700.gateway
      http:
        paths:
          - {path: /classic, pathType: Prefix, backend: {service: {name: web-classic, port: {number: 80}}}}
          - {path: /next,    pathType: Prefix, backend: {service: {name: web-next,    port: {number: 80}}}}
EOF
$K get crd gateways.gateway.networking.k8s.io >/dev/null 2>&1 || {
  curl -sfL --max-time 60 https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/standard-install.yaml -o /tmp/gwapi.yaml && $K apply -f /tmp/gwapi.yaml >/dev/null 2>&1; } || echo "warn: could not install Gateway API CRDs"
$K -n project-r700 delete httproute router --ignore-not-found >/dev/null 2>&1
$K apply -f - >/dev/null 2>&1 <<'EOF'
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata: {name: edge-class}
spec: {controllerName: example.com/edge}
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata: {name: edge, namespace: project-r700}
spec:
  gatewayClassName: edge-class
  listeners: [{name: http, protocol: HTTP, port: 80, hostname: r700.gateway}]
EOF
""",
    checks=[
        C("HTTPRoute router attached to Gateway edge, hostname r700.gateway", 2, r"""
h="$K -n project-r700 get httproute router"
eq "$($h -o jsonpath='{.spec.parentRefs[0].name}' 2>/dev/null)" edge && $h -o jsonpath='{.spec.hostnames[*]}' | grep -q r700.gateway"""),
        C("/classic -> web-classic:80 and /next -> web-next:80", 3, r"""
$K -n project-r700 get httproute router -o json 2>/dev/null | python3 -c '
import sys,json
rs=json.load(sys.stdin)["spec"]["rules"]
def be(path):
    for r in rs:
        for m in r.get("matches",[]):
            if m.get("path",{}).get("value")==path and m.get("path",{}).get("type")=="PathPrefix" and not m.get("headers"):
                b=r["backendRefs"][0]; return b["name"], b.get("port")
    return None
sys.exit(0 if be("/classic")==("web-classic",80) and be("/next")==("web-next",80) else 1)'"""),
        C("/beta with X-Channel=beta goes to web-next (path AND header in one match)", 3, r"""
$K -n project-r700 get httproute router -o json 2>/dev/null | python3 -c '
import sys,json
rs=json.load(sys.stdin)["spec"]["rules"]
ok=False
for r in rs:
    for m in r.get("matches",[]):
        hs=m.get("headers") or []
        if m.get("path",{}).get("value")=="/beta" and any(h.get("name","").lower()=="x-channel" and h.get("value")=="beta" for h in hs):
            if len(r["backendRefs"])==1 and r["backendRefs"][0]["name"]=="web-next": ok=True
sys.exit(0 if ok else 1)'"""),
        C("/beta catch-all goes to web-classic and comes after the header rule", 3, r"""
$K -n project-r700 get httproute router -o json 2>/dev/null | python3 -c '
import sys,json
rs=json.load(sys.stdin)["spec"]["rules"]
hi=ci=None
for i,r in enumerate(rs):
    for m in r.get("matches",[]):
        if m.get("path",{}).get("value")=="/beta":
            if m.get("headers"):
                if hi is None: hi=i
            else:
                if ci is None and r["backendRefs"][0]["name"]=="web-classic": ci=i
sys.exit(0 if hi is not None and ci is not None and hi<ci else 1)'"""),
    ],
    solution="""
cat <<EOF | k apply -f -
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata: {name: router, namespace: project-r700}
spec:
  parentRefs: [{name: edge}]
  hostnames: ["r700.gateway"]
  rules:
    - matches: [{path: {type: PathPrefix, value: /classic}}]
      backendRefs: [{name: web-classic, port: 80}]
    - matches: [{path: {type: PathPrefix, value: /next}}]
      backendRefs: [{name: web-next, port: 80}]
    - matches:
        - path: {type: PathPrefix, value: /beta}
          headers: [{type: Exact, name: X-Channel, value: beta}]     # path AND header: one match entry
      backendRefs: [{name: web-next, port: 80}]
    - matches: [{path: {type: PathPrefix, value: /beta}}]            # catch-all, must come AFTER
      backendRefs: [{name: web-classic, port: 80}]
EOF
# Writing "- path:" and "- headers:" as two list items would mean OR, not AND - a classic trap.
""",
)

# ------------------------------------------------------------------ 214 ----
task(
    id=214, domain="ARCH", title="Certificate expiry, two ways",
    task="""
On the control-plane node (`{SHELL_CP}`):

1. Using **openssl**, find the expiry of the `kube-apiserver` **etcd client** certificate
   (`/etc/kubernetes/pki/apiserver-etcd-client.crt`) and write the `notAfter` value to
   `{WS}/q214/etcd-client-expiry.txt`.
2. Confirm with `kubeadm certs check-expiration` that both tools agree, and write the whole
   `apiserver-etcd-client` line from kubeadm's output to `{WS}/q214/kubeadm-line.txt`.
3. Write the single command that would renew **only** that certificate to `{WS}/q214/renew.sh`.
""",
    setup=r"""rm -rf "$WS/q214"; mkdir -p "$WS/q214" """,
    checks=[
        C("etcd-client-expiry.txt matches the real certificate", 4, r"""
exp=$(nx "$CP" 'openssl x509 -in /etc/kubernetes/pki/apiserver-etcd-client.crt -noout -enddate' | sed 's/notAfter=//')
yr=$(echo "$exp" | awk '{print $4}'); mon=$(echo "$exp" | awk '{print $1}'); day=$(echo "$exp" | awk '{print $2}')
f="$WS/q214/etcd-client-expiry.txt"; grep -q "$yr" "$f" 2>/dev/null && grep -qi "$mon" "$f" && grep -q "$day" "$f" """),
        C("kubeadm-line.txt is the apiserver-etcd-client row from kubeadm", 3, r"""grep -q 'apiserver-etcd-client' "$WS/q214/kubeadm-line.txt" 2>/dev/null && grep -Eq '[0-9]{4}' "$WS/q214/kubeadm-line.txt" """),
        C("renew.sh renews only that certificate", 2, r"""grep -Eq 'kubeadm +certs +renew +apiserver-etcd-client' "$WS/q214/renew.sh" 2>/dev/null"""),
    ],
    solution="""
{SHELL_CP}
openssl x509 -in /etc/kubernetes/pki/apiserver-etcd-client.crt -noout -enddate
#   notAfter=... -> copy into {WS}/q214/etcd-client-expiry.txt
kubeadm certs check-expiration | grep apiserver-etcd-client
#   -> copy that line into {WS}/q214/kubeadm-line.txt
echo 'kubeadm certs renew apiserver-etcd-client' > {WS}/q214/renew.sh
# openssl -noout -text | grep -A2 Validity  shows both notBefore and notAfter if you prefer.
""",
)

# ------------------------------------------------------------------ 215 ----
task(
    id=215, domain="NET", title="Egress policy with paired destinations",
    task="""
Namespace `project-snake2` runs Pods labelled `app=api`, `app=cache`, `app=metrics` and `app=audit`.

After a breach, the `api` Pods must be restricted. Create a NetworkPolicy `np-api` in that
Namespace so that Pods with `app=api` may open connections **only**:

- to `app=cache` Pods on TCP port 6379,
- to `app=metrics` Pods on TCP port 9090.

Any other egress — including to `cache` on 9090 or to `metrics` on 6379 — must be denied.
Ingress must not be restricted by this policy.
""",
    setup=NS + r"""
mkns project-snake2
for a in api cache metrics audit; do
  $K -n project-snake2 create deploy $a --image=nginx:1.27 --dry-run=client -o yaml | $K apply -f - >/dev/null
  $K -n project-snake2 label deploy $a app=$a --overwrite >/dev/null
  $K -n project-snake2 patch deploy $a -p "{\"spec\":{\"template\":{\"metadata\":{\"labels\":{\"app\":\"$a\"}}}}}" >/dev/null
done
$K -n project-snake2 delete netpol np-api --ignore-not-found >/dev/null
""",
    checks=[
        C("policy selects app=api and is Egress-only", 2, r"""
n="$K -n project-snake2 get netpol np-api"
eq "$($n -o jsonpath='{.spec.podSelector.matchLabels.app}' 2>/dev/null)" api && eq "$($n -o jsonpath='{.spec.policyTypes}')" '["Egress"]'"""),
        C("two separate rules pair each destination with its own port", 5, r"""
$K -n project-snake2 get netpol np-api -o json 2>/dev/null | python3 -c '
import sys,json
e=json.load(sys.stdin)["spec"].get("egress",[])
def pair(app,port):
    for r in e:
        tos=r.get("to",[]); ports=r.get("ports",[])
        if len(tos)==1 and tos[0].get("podSelector",{}).get("matchLabels",{}).get("app")==app \
           and len(ports)==1 and str(ports[0].get("port"))==str(port) and ports[0].get("protocol","TCP")=="TCP":
            return True
    return False
sys.exit(0 if len(e)==2 and pair("cache",6379) and pair("metrics",9090) else 1)'"""),
        C("no rule allows a destination on the wrong port (single combined rule fails here)", 3, r"""
$K -n project-snake2 get netpol np-api -o json 2>/dev/null | python3 -c '
import sys,json
e=json.load(sys.stdin)["spec"].get("egress",[])
for r in e:
    apps={t.get("podSelector",{}).get("matchLabels",{}).get("app") for t in r.get("to",[])}
    ports={str(p.get("port")) for p in r.get("ports",[])}
    if len(apps)>1 or len(ports)>1: sys.exit(1)      # combined rule = cross product = too permissive
sys.exit(0)'"""),
    ],
    solution="""
cat <<EOF | k apply -f -
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: np-api, namespace: project-snake2}
spec:
  podSelector: {matchLabels: {app: api}}
  policyTypes: [Egress]
  egress:
    - to: [{podSelector: {matchLabels: {app: cache}}}]      # rule 1
      ports: [{protocol: TCP, port: 6379}]
    - to: [{podSelector: {matchLabels: {app: metrics}}}]    # rule 2
      ports: [{protocol: TCP, port: 9090}]
EOF
# Two rules = (cache AND 6379) OR (metrics AND 9090).
# One rule with both selectors and both ports = (cache OR metrics) AND (6379 OR 9090),
# which would also allow api -> cache:9090. That is the trap.
""",
)

# ------------------------------------------------------------------ 216 ----
task(
    id=216, domain="NET", title="CoreDNS: back up, extend, verify", breaks="coredns",
    task="""
1. Back up the current CoreDNS configuration to `{WS}/q216/coredns-backup.yaml` in a form you
   could restore from directly with `kubectl apply -f`.
2. Change CoreDNS so that `SERVICE.NAMESPACE.svc.internal.zone` resolves to the same address as
   `SERVICE.NAMESPACE.svc.cluster.local`, **in addition** to the existing name.
3. Roll CoreDNS so the change is live.
4. Prove it: write the output of `nslookup kubernetes.default.svc.internal.zone` from a Pod to
   `{WS}/q216/lookup.txt`.
""",
    setup=r"""
rm -rf "$WS/q216"; mkdir -p "$WS/q216"
cur=$(j -n kube-system cm coredns '{.data.Corefile}')
echo "$cur" | grep -q internal.zone && { echo "$cur" | sed 's/ internal.zone//' > /tmp/cf216; $K -n kube-system create cm coredns --from-file=Corefile=/tmp/cf216 --dry-run=client -o yaml | $K apply -f - >/dev/null; $K -n kube-system rollout restart deploy coredns >/dev/null; }
$K -n default delete pod dns-client --ignore-not-found --wait=false >/dev/null
""",
    checks=[
        C("backup file is a restorable ConfigMap manifest with the original Corefile", 3, r"""
f="$WS/q216/coredns-backup.yaml"; grep -q 'kind: ConfigMap' "$f" 2>/dev/null && grep -q 'name: coredns' "$f" && grep -q 'Corefile' "$f" && grep -q 'kubernetes cluster.local' "$f" """),
        C("Corefile serves internal.zone alongside cluster.local", 3, r"""j -n kube-system cm coredns '{.data.Corefile}' | grep -E '^ *kubernetes ' | grep -q 'internal.zone' """),
        C("CoreDNS rolled out and available", 2, r"""for i in $(seq 1 12); do [ "$(j -n kube-system deploy coredns '{.status.availableReplicas}')" -ge 1 ] 2>/dev/null && exit 0; sleep 5; done; exit 1"""),
        C("lookup.txt resolves the new name to the kubernetes Service IP", 3, r"""
ip=$(j -n default svc kubernetes '{.spec.clusterIP}'); grep -q "$ip" "$WS/q216/lookup.txt" 2>/dev/null && grep -q 'internal.zone' "$WS/q216/lookup.txt" """),
    ],
    solution="""
k -n kube-system get cm coredns -o yaml > {WS}/q216/coredns-backup.yaml
k -n kube-system edit cm coredns
#   kubernetes cluster.local in-addr.arpa ip6.arpa {
# becomes
#   kubernetes cluster.local internal.zone in-addr.arpa ip6.arpa {
k -n kube-system rollout restart deploy coredns
k -n kube-system rollout status deploy coredns
k run dns-client --image=busybox:1.36 --restart=Never -- sleep 3600
k exec dns-client -- nslookup kubernetes.default.svc.internal.zone | tee {WS}/q216/lookup.txt
# Restore if needed:  k delete -f {WS}/q216/coredns-backup.yaml && k apply -f {WS}/q216/coredns-backup.yaml
""",
)

# ------------------------------------------------------------------ 217 ----
task(
    id=217, domain="TSHOOT", title="Inspect a container with crictl",
    task="""
1. In Namespace `project-tiger2` create Pod `tiger-probe`, image `httpd:2-alpine`, with labels
   `pod=probe` and `probe=pod`.
2. Find which node it runs on, connect to that node, and locate its container with `crictl`.
3. Write the container ID and its `runtimeType` (from `crictl inspect`), separated by a space,
   to `{WS}/q217/container.txt`.
4. Write the container's logs to `{WS}/q217/container.log`.
""",
    setup=NS + r"""
mkns project-tiger2
rm -rf "$WS/q217"; mkdir -p "$WS/q217"
$K -n project-tiger2 delete pod tiger-probe --ignore-not-found --wait=false >/dev/null
""",
    checks=[
        C("Pod tiger-probe Running with both labels", 2, r"""
eq "$(j -n project-tiger2 pod tiger-probe '{.status.phase}')" Running && eq "$(j -n project-tiger2 pod tiger-probe '{.metadata.labels.pod}')" probe && eq "$(j -n project-tiger2 pod tiger-probe '{.metadata.labels.probe}')" pod"""),
        C("container.txt holds a real container ID for that Pod", 4, r"""
id=$(awk '{print $1}' "$WS/q217/container.txt" 2>/dev/null); [ -n "$id" ] || exit 1
node=$(j -n project-tiger2 pod tiger-probe '{.spec.nodeName}')
cid=$(j -n project-tiger2 pod tiger-probe '{.status.containerStatuses[0].containerID}' | sed 's|.*//||')
case "$cid" in "$id"*) exit 0;; esac; exit 1"""),
        C("container.txt records the runtimeType", 2, r"""grep -q 'io.containerd.runc' "$WS/q217/container.txt" 2>/dev/null"""),
        C("container.log holds the httpd startup output", 2, r"""grep -qi 'httpd' "$WS/q217/container.log" 2>/dev/null"""),
    ],
    solution="""
k -n project-tiger2 run tiger-probe --image=httpd:2-alpine --labels "pod=probe,probe=pod"
k -n project-tiger2 get pod tiger-probe -o wide        # note the NODE
{SHELL_W1}                                             # ...whichever node it landed on
crictl ps | grep tiger-probe                           # -> container id
crictl inspect <id> | grep runtimeType                 # -> io.containerd.runc.v2
crictl logs <id>
# write "<id> io.containerd.runc.v2" into {WS}/q217/container.txt and the logs into container.log
# crictl ps -a also shows exited containers; crictl pods lists sandboxes.
""",
)

QUESTIONS = Q