kubestronaut-sim/cka/solutions.py

"""Reference solutions, keyed by task id. Rendered with the same %{param} / {WS} / {W1} substitution
as the task text. Shown only after a task has been graded (drills) or the session submitted (exams).
These solutions provide step-by-step instructions, Kubernetes concept explanations, imperative commands,
and verification checks for exam success.
"""

from sim.bank import cmd, hand

S = {}

# ================================================================= ARCH ====
# Domain 1: Cluster Architecture, Installation & Configuration (25%)
# ===========================================================================

S[1] = """# Step 1: Create the ServiceAccount in the target namespace
k -n rbac-app create sa deploy-bot

# Step 2: Create a Role granting specified verbs on deployments.apps
# Verbs: get, list, watch, create, update, patch, delete
k -n rbac-app create role deploy-manager \\
  --verb=get,list,watch,create,update,patch,delete \\
  --resource=deployments.apps

# Step 3: Bind the Role to the ServiceAccount
k -n rbac-app create rolebinding deploy-bot-binding \\
  --role=deploy-manager \\
  --serviceaccount=rbac-app:deploy-bot

# Step 4: Verify permissions using 'kubectl auth can-i'
k auth can-i create deployments -n rbac-app --as=system:serviceaccount:rbac-app:deploy-bot   # Expected: yes
k auth can-i delete pods -n rbac-app --as=system:serviceaccount:rbac-app:deploy-bot          # Expected: no
"""

S[2] = """# Step 1: Access the control plane node
sudo -i                # the task's host is the control-plane node: no hop needed

# Step 2: Take the etcd snapshot with TLS certificates and save to the target path
# Certificates are located in /etc/kubernetes/pki/etcd/
ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 \\
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \\
  --cert=/etc/kubernetes/pki/etcd/server.crt \\
  --key=/etc/kubernetes/pki/etcd/server.key \\
  snapshot save /opt/cka/etcd-snapshot.db

# Step 3: Verify snapshot status and integrity
etcdutl snapshot status /opt/cka/etcd-snapshot.db -w table

# Note: on a cluster where etcdctl is only inside the static pod, run it there. The etcd image
# is distroless (no sh, no cat, no tar), so kubectl cp cannot get the file out - write it into
# /var/lib/etcd instead, which is a hostPath, and move it on the node:
# kubectl -n kube-system exec etcd-{CP} -- etcdctl ... snapshot save /var/lib/etcd/snap.db
# then on the node: mv /var/lib/etcd/snap.db /opt/cka/etcd-snapshot.db
"""

S[3] = [cmd(r"""mkdir -p {WS}/q03/overlays/prod
cd {WS}/q03/overlays/prod
k create ns store-prod --dry-run=client -o yaml > namespace.yaml"""),
        hand(r"""vim kustomization.yaml""",
             r"""The overlay: the base and the Namespace as resources, the namespace, the label and the replica count.
```
resources:
- ../../base
- namespace.yaml
namespace: store-prod
labels:
- pairs:
    env: prod
replicas:
- name: store-api
  count: 3
```""",
             run=r"""cat > kustomization.yaml <<'EOF'
resources:
- ../../base
- namespace.yaml
namespace: store-prod
labels:
- pairs:
    env: prod
replicas:
- name: store-api
  count: 3
EOF"""),
        cmd(r"""k kustomize .
k apply -k .
k -n store-prod get deploy,svc --show-labels
cd""", r"""`k kustomize .` prints what the overlay makes, before anything is applied.""")]

S[4] = [hand(r"""vim crd.yaml""",
             r"""Paste the CustomResourceDefinition from the docs page *Extend the Kubernetes API with CustomResourceDefinitions* and change it to:
```
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  name: backups.data.example.com
spec:
  group: data.example.com
  versions:
  - name: v1
    served: true
    storage: true
    schema:
      openAPIV3Schema:
        type: object
        properties:
          spec:
            type: object
            properties:
              schedule:
                type: string
              target:
                type: string
  scope: Namespaced
  names:
    plural: backups
    singular: backup
    kind: Backup
    shortNames:
    - bk
```""",
             run=r"""cat > crd.yaml <<'EOF'
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  name: backups.data.example.com
spec:
  group: data.example.com
  versions:
  - name: v1
    served: true
    storage: true
    schema:
      openAPIV3Schema:
        type: object
        properties:
          spec:
            type: object
            properties:
              schedule:
                type: string
              target:
                type: string
  scope: Namespaced
  names:
    plural: backups
    singular: backup
    kind: Backup
    shortNames:
    - bk
EOF"""),
        cmd(r"""k apply -f crd.yaml"""),
        hand(r"""vim backup.yaml""",
             r"""The Backup itself - the docs page has one for its example kind just below:
```
apiVersion: data.example.com/v1
kind: Backup
metadata:
  name: nightly
spec:
  schedule: "0 2 * * *"
  target: s3://backups/prod
```""",
             run=r"""kubectl wait --for condition=established crd/backups.data.example.com --timeout=60s
cat > backup.yaml <<'EOF'
apiVersion: data.example.com/v1
kind: Backup
metadata:
  name: nightly
spec:
  schedule: "0 2 * * *"
  target: s3://backups/prod
EOF"""),
        cmd(r"""k apply -f backup.yaml
k get bk""")]

S[5] = [cmd(r"""k run static-web --image=nginx:1.27 --dry-run=client -o yaml""", r"""Copy what it prints: the static Pod's manifest."""),
        hand(r"""{SHELL_W3}
sudo -i
grep staticPodPath /var/lib/kubelet/config.yaml
vim /etc/kubernetes/manifests/static-web.yaml""",
             r"""Paste the Pod into the static Pod path `grep` printed, `:wq`, and `exit` twice - back on the host. The kubelet starts it within seconds.""",
             run=r"""kubectl run static-web --image=nginx:1.27 --dry-run=client -o yaml | ssh {W3} 'sudo tee /etc/kubernetes/manifests/static-web.yaml >/dev/null'"""),
        cmd(r"""k get pod static-web-{W3}""", r"""Again until it is `Running`: the mirror Pod is named after the node.""")]

S[6] = [cmd(r"""sudo kubeadm certs check-expiration | grep apiserver
sudo kubeadm certs check-expiration | grep '^apiserver ' > {WS}/q06/apiserver-expiry.txt""", r"""The host is the control-plane node. `^apiserver ` is the serving certificate's line alone, not `apiserver-etcd-client` or `apiserver-kubelet-client`."""),
        cmd(r"""openssl genrsa -out jane.key 2048
openssl req -new -key jane.key -subj /CN=jane -out jane.csr
base64 -w0 jane.csr""", r"""Copy the line `base64` prints: the CSR as the API wants it."""),
        hand(r"""vim csr.yaml""",
             r"""Paste the CertificateSigningRequest from the docs page *Certificates and Certificate Signing Requests* (under *Create a CertificateSigningRequest*), name it `jane`, and put the copied line in `request:`
```
apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata:
  name: jane
spec:
  request: LS0tLS1CRUdJTi...
  signerName: kubernetes.io/kube-apiserver-client
  usages:
  - client auth
```""",
             run=r"""cat > csr.yaml <<EOF
apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata:
  name: jane
spec:
  request: $(base64 -w0 jane.csr)
  signerName: kubernetes.io/kube-apiserver-client
  usages:
  - client auth
EOF"""),
        cmd(r"""k apply -f csr.yaml
k certificate approve jane
k get csr jane""", r"""`CONDITION` reads `Approved,Issued` once the signer has issued the certificate."""),
        cmd(r"""k -n default create rolebinding jane-view --clusterrole=view --user=jane
k auth can-i list pods -n default --as=jane
k auth can-i delete pods -n default --as=jane""", r"""`yes`, then `no`: `view` reads, and changes nothing.""")]

S[21] = """# Step 1: Create the target namespace
k create ns %{ns}

# Step 2: Install the Helm chart with overridden replicaCount
helm install %{rel} {WS}/q21/chart -n %{ns} --set replicaCount=%{n}

# Step 3: Upgrade the release to NodePort while preserving existing values
helm upgrade %{rel} {WS}/q21/chart -n %{ns} --reuse-values --set service.type=NodePort

# Step 4: Verify the release history shows revision 2
helm -n %{ns} history %{rel}
"""

S[33] = [cmd(r"""k create clusterrole %{cr} --aggregation-rule=rbac.example.com/aggregate-to-%{cr}=true
k create clusterrole %{cr}-pods --verb=get,list,watch --resource=pods
k create clusterrole %{cr}-services --verb=get,list,watch --resource=services
k label clusterrole %{cr}-pods %{cr}-services rbac.example.com/aggregate-to-%{cr}=true
k create clusterrolebinding %{cr}-binding --clusterrole=%{cr} --user=%{user}""", r"""`--aggregation-rule` makes a ClusterRole with no rules of its own: the controller fills them in from every ClusterRole with that label."""),
         cmd(r"""k describe clusterrole %{cr}
k auth can-i list pods -A --as=%{user}
k auth can-i delete pods -A --as=%{user}""", r"""Both resources in its rules; then `yes` and `no`.""")]

S[34] = [cmd(r"""k explain %{kind}.spec --recursive
k explain %{kind}.spec.tuning.%{field}""", r"""The field under `tuning` whose `DESCRIPTION` is *Number of retry attempts*."""),
         cmd(r"""echo spec.tuning.%{field} > {WS}/q34/path.txt"""),
         hand(r"""vim sample.yaml""",
              r"""```
apiVersion: demo.io/v1
kind: %{kind}
metadata:
  name: sample
  namespace: default
spec:
  tuning:
    %{field}: %{n}
```""",
              run=r"""cat > sample.yaml <<'EOF'
apiVersion: demo.io/v1
kind: %{kind}
metadata:
  name: sample
  namespace: default
spec:
  tuning:
    %{field}: %{n}
EOF"""),
         cmd(r"""k apply -f sample.yaml""")]

S[40] = """# Generate a new kubeadm join token with the specified TTL and print-join-command,
# writing the output straight into the required file (the task's host is the control-plane node)
sudo kubeadm token create --ttl %{ttl} --print-join-command > {WS}/q40/join.sh

# Verify active tokens
sudo kubeadm token list
"""

S[41] = """# Step 1: Check Kubernetes Server version (API server)
k version

# Step 2: Extract Kubelet and Container Runtime versions for the target node
k get node %{node} -o jsonpath='{.status.nodeInfo.kubeletVersion}{"\\n"}{.status.nodeInfo.containerRuntimeVersion}{"\\n"}'

# Step 3: Extract etcd image tag from the static pod spec
k -n kube-system get pod -l component=etcd -o jsonpath='{.items[0].spec.containers[0].image}'
# Parse the image tag (everything after the colon)
"""

# Hard Architecture Tasks
S[44] = [cmd(r"""k get clusterrolebinding -o wide | grep %{sa}
k -n %{ns} get rolebinding -o wide | grep %{sa}""", r"""`%{sa}-legacy-admin` binds `cluster-admin` to it: far more than allowed."""),
         cmd(r"""k delete clusterrolebinding %{sa}-legacy-admin
k -n %{ns} create role %{sa}-role --verb=get,list,watch,patch --resource=deployments.apps,replicasets.apps"""),
         hand(r"""k -n %{ns} edit role %{sa}-role""",
              r"""`create role` gives every verb to every resource it names, so the second rule goes in by hand - add it below the first in `rules:`, then `:wq`:
```
- apiGroups:
  - ""
  resources:
  - pods/exec
  verbs:
  - create
```""",
              run=r"""kubectl -n %{ns} patch role %{sa}-role --type=json -p='[{"op":"add","path":"/rules/-","value":{"apiGroups":[""],"resources":["pods/exec"],"verbs":["create"]}}]'"""),
         cmd(r"""k -n %{ns} create rolebinding %{sa}-rb --role=%{sa}-role --serviceaccount=%{ns}:%{sa}
k create clusterrole %{sa}-nodes --verb=list --resource=nodes
k create clusterrolebinding %{sa}-crb --clusterrole=%{sa}-nodes --serviceaccount=%{ns}:%{sa}"""),
         cmd(r"""k auth can-i --list -n %{ns} --as=system:serviceaccount:%{ns}:%{sa}
k auth can-i create pods --subresource=exec -n %{ns} --as=system:serviceaccount:%{ns}:%{sa}""", r"""`yes`. For a subresource, `--subresource`: `can-i create pods/exec` asks about a Pod named `exec`, and says `no` even when the Role is right.""")]

S[45] = [cmd(r"""sudo etcdutl snapshot restore /opt/cka/backup.db --data-dir /var/lib/etcd-restore""", r"""The host is the control-plane node. The restore writes a new data directory; etcd keeps running on the old one until its manifest says otherwise."""),
         hand(r"""sudo vim /etc/kubernetes/manifests/etcd.yaml""",
              r"""In `volumes:` at the bottom, point `etcd-data` at the restored directory, then `:wq`:
```
  - hostPath:
      path: /var/lib/etcd-restore
      type: DirectoryOrCreate
    name: etcd-data
```
The kubelet sees the change and restarts etcd; the API server is gone for a minute or two meanwhile.""",
              run=r"""cpwait() {
  local i
  for i in $(seq 1 80); do
    [ "$(kubectl -n kube-system get pod $1-{CP} -o jsonpath='{.status.containerStatuses[0].ready}' 2>/dev/null)" = true ] && return 0
    sleep 3
  done
  return 1
}
sudo sed -i 's|path: /var/lib/etcd$|path: /var/lib/etcd-restore|' /etc/kubernetes/manifests/etcd.yaml
sleep 30
for i in $(seq 1 80); do kubectl -n default get cm %{cm} >/dev/null 2>&1 && break; sleep 3; done
for c in etcd kube-apiserver kube-scheduler kube-controller-manager; do cpwait $c; done"""),
         cmd(r"""sudo crictl ps | grep -e etcd -e apiserver
k -n default get cm %{cm}
k -n kube-system get pods""", r"""Again until `k` answers: the ConfigMap is back, and every control-plane Pod `Running`.""")]

S[46] = [cmd(r"""k get nodes -l tier=edge
k create ns %{ns}
k -n %{ns} run %{pod} --image=nginx:1.27 --labels=role=edge --dry-run=client -o yaml > static.yaml""", r"""The namespace may be there already; the mirror Pod needs it either way."""),
         hand(r"""vim static.yaml""",
              r"""Replace the container's `resources: {}` with:
```
    resources:
      requests:
        cpu: 50m
        memory: 32Mi
```""",
              run=r"""yq -i '.spec.containers[0].resources = {"requests": {"cpu": "50m", "memory": "32Mi"}}' static.yaml"""),
         cmd(r"""scp static.yaml {W3}:"""),
         hand(r"""{SHELL_W3}
sudo -i
grep staticPodPath /var/lib/kubelet/config.yaml""",
              r"""Not the usual `/etc/kubernetes/manifests`: `/etc/kubernetes/static`.""",
              run=r""""""),
         hand(r"""mv /home/candidate/static.yaml /etc/kubernetes/static/%{pod}.yaml
exit
exit""",
              r"""Into the path the kubelet actually watches; then back on the host.""",
              run=r"""ssh -n {W3} 'sudo mv /home/candidate/static.yaml /etc/kubernetes/static/%{pod}.yaml'"""),
         cmd(r"""k -n %{ns} get pod %{pod}-{W3}""", r"""Again until it is `Running`.""")]

S[47] = [cmd(r"""openssl genrsa -out %{user}.key 2048
openssl req -new -key %{user}.key -subj "/CN=%{user}/O=developers" -out %{user}.csr
base64 -w0 %{user}.csr""", r"""`O=` is the group. Copy the line `base64` prints."""),
         hand(r"""vim csr.yaml""",
              r"""The CertificateSigningRequest from the docs page *Certificates and Certificate Signing Requests*, with the copied line in `request:`
```
apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata:
  name: %{user}
spec:
  request: LS0tLS1CRUdJTi...
  signerName: kubernetes.io/kube-apiserver-client
  usages:
  - client auth
```""",
              run=r"""cat > csr.yaml <<EOF
apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata:
  name: %{user}
spec:
  request: $(base64 -w0 %{user}.csr)
  signerName: kubernetes.io/kube-apiserver-client
  usages:
  - client auth
EOF"""),
         cmd(r"""k apply -f csr.yaml
k certificate approve %{user}
k get csr %{user}""", r"""`Approved,Issued` - the signer issues the certificate a moment after the approval."""),
         hand(r"""k get csr %{user} -o jsonpath='{.status.certificate}' | base64 -d > %{user}.crt""",
              r"""Once it is `Issued`: read too early, the file is empty.""",
              run=r"""until [ -n "$(kubectl get csr %{user} -o jsonpath='{.status.certificate}')" ]; do sleep 1; done
kubectl get csr %{user} -o jsonpath='{.status.certificate}' | base64 -d > %{user}.crt"""),
         cmd(r"""k -n %{ns} create rolebinding %{user}-edit --clusterrole=edit --user=%{user}
k config view --minify | grep server""", r"""The API server's address, for the new kubeconfig."""),
         hand(r"""k --kubeconfig {WS}/q47/%{user}.kubeconfig config set-cluster lab --server=https://... --certificate-authority=/etc/kubernetes/pki/ca.crt --embed-certs""",
              r"""With the address `grep` printed in `--server=`. `--embed-certs` puts the CA itself in the file, not its path.""",
              run=r"""kubectl --kubeconfig {WS}/q47/%{user}.kubeconfig config set-cluster lab --server=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}') --certificate-authority=/etc/kubernetes/pki/ca.crt --embed-certs"""),
         cmd(r"""k --kubeconfig {WS}/q47/%{user}.kubeconfig config set-credentials %{user} --client-certificate=%{user}.crt --client-key=%{user}.key --embed-certs
k --kubeconfig {WS}/q47/%{user}.kubeconfig config set-context %{user}@lab --cluster=lab --user=%{user} --namespace=%{ns}
k --kubeconfig {WS}/q47/%{user}.kubeconfig config use-context %{user}@lab
k --kubeconfig {WS}/q47/%{user}.kubeconfig get pods""", r"""`No resources found in %{ns} namespace.` - no flags, the context's namespace, and `%{user}`'s own rights.""")]

S[48] = [cmd(r"""mkdir -p {WS}/q48/overlays/prod
cd {WS}/q48/overlays/prod"""),
         hand(r"""vim replicas.yaml""",
              r"""A strategic merge patch: enough of the Deployment to name it, and the field to change.
```
apiVersion: apps/v1
kind: Deployment
metadata:
  name: web
spec:
  replicas: %{rep}
```""",
              run=r"""cat > replicas.yaml <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata:
  name: web
spec:
  replicas: %{rep}
EOF"""),
         hand(r"""vim envfrom.yaml""",
              r"""A JSON 6902 patch: a list of operations, each with a path into the object.
```
- op: add
  path: /spec/template/spec/containers/0/envFrom
  value:
  - configMapRef:
      name: web-config
```""",
              run=r"""cat > envfrom.yaml <<'EOF'
- op: add
  path: /spec/template/spec/containers/0/envFrom
  value:
  - configMapRef:
      name: web-config
EOF"""),
         hand(r"""vim kustomization.yaml""",
              r"""A JSON 6902 patch names its target; the strategic merge patch names its own. Kustomize rewrites `web-config` in `envFrom` to the generated name, hash and all.
```
namespace: %{ns}
resources:
- ../../base
labels:
- pairs:
    tier: prod
  includeSelectors: false
configMapGenerator:
- name: web-config
  literals:
  - LOG_LEVEL=%{lvl}
patches:
- path: replicas.yaml
- path: envfrom.yaml
  target:
    kind: Deployment
    name: web
```""",
              run=r"""cat > kustomization.yaml <<'EOF'
namespace: %{ns}
resources:
- ../../base
labels:
- pairs:
    tier: prod
  includeSelectors: false
configMapGenerator:
- name: web-config
  literals:
  - LOG_LEVEL=%{lvl}
patches:
- path: replicas.yaml
- path: envfrom.yaml
  target:
    kind: Deployment
    name: web
EOF"""),
         cmd(r"""k kustomize .
k create ns %{ns}
k apply -k .
k -n %{ns} get deploy,cm
cd""", r"""The namespace first: `namespace:` puts the objects in it, but does not make it.""")]

S[49] = [cmd(r"""k config view --minify | grep namespace""", r"""Not `default`: every command below says `-n default`."""),
         hand(r"""vim crd.yaml""",
              r"""Start from the CRD on the docs page *Extend the Kubernetes API with CustomResourceDefinitions*; its *Validation* and *Defaulting* sections have `minimum`, `enum` and `default`. `required` sits beside `properties`, not inside a field:
```
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  name: %{low}s.deploy.example.com
spec:
  group: deploy.example.com
  scope: Namespaced
  names:
    kind: %{kind}
    plural: %{low}s
    singular: %{low}
    shortNames:
    - %{short}
  versions:
  - name: v1
    served: true
    storage: true
    schema:
      openAPIV3Schema:
        type: object
        properties:
          spec:
            type: object
            required:
            - replicas
            - image
            properties:
              replicas:
                type: integer
                minimum: 1
                maximum: 10
              environment:
                type: string
                enum:
                - prod
                - stage
                - dev
                default: dev
              image:
                type: string
```""",
              run=r"""cat > crd.yaml <<'EOF'
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  name: %{low}s.deploy.example.com
spec:
  group: deploy.example.com
  scope: Namespaced
  names:
    kind: %{kind}
    plural: %{low}s
    singular: %{low}
    shortNames:
    - %{short}
  versions:
  - name: v1
    served: true
    storage: true
    schema:
      openAPIV3Schema:
        type: object
        properties:
          spec:
            type: object
            required:
            - replicas
            - image
            properties:
              replicas:
                type: integer
                minimum: 1
                maximum: 10
              environment:
                type: string
                enum:
                - prod
                - stage
                - dev
                default: dev
              image:
                type: string
EOF"""),
         cmd(r"""k apply -f crd.yaml"""),
         hand(r"""vim good.yaml""",
              r"""```
apiVersion: deploy.example.com/v1
kind: %{kind}
metadata:
  name: good
spec:
  replicas: 3
  image: nginx:1.27
  environment: %{env}
```""",
              run=r"""kubectl wait --for condition=established crd/%{low}s.deploy.example.com --timeout=60s
cat > good.yaml <<'EOF'
apiVersion: deploy.example.com/v1
kind: %{kind}
metadata:
  name: good
spec:
  replicas: 3
  image: nginx:1.27
  environment: %{env}
EOF"""),
         cmd(r"""k -n default apply -f good.yaml"""),
         hand(r"""cp good.yaml bad.yaml
vim bad.yaml""",
              r"""Name it `bad` and set `replicas: 0`.""",
              run=r"""cp good.yaml bad.yaml
yq -i '.metadata.name = "bad" | .spec.replicas = 0' bad.yaml"""),
         cmd(r"""k -n default apply -f bad.yaml 2> {WS}/q49/rejected.txt
cat {WS}/q49/rejected.txt""", r"""`2>`, not `>`: the rejection goes to standard error. It says `spec.replicas` should be greater than or equal to 1."""),
         hand(r"""cp good.yaml defaulted.yaml
vim defaulted.yaml""",
              r"""Name it `defaulted`, `replicas: 2`, and delete the `environment:` line - the API server fills in the default.""",
              run=r"""cp good.yaml defaulted.yaml
yq -i '.metadata.name = "defaulted" | .spec.replicas = 2 | del(.spec.environment)' defaulted.yaml"""),
         cmd(r"""k -n default apply -f defaulted.yaml
k -n default get %{short} defaulted -o yaml | grep environment""", r"""`environment: dev`.""")]

S[65] = """# Step 1: Access control plane node
sudo -i                # the task's host is the control-plane node: no hop needed

# Step 2: Renew all cluster certificates via kubeadm
kubeadm certs renew all

# Step 3: Restart static pods to load renewed certificates
mv /etc/kubernetes/manifests/*.yaml /tmp/ && sleep 20 && mv /tmp/*.yaml /etc/kubernetes/manifests/

# Step 4: Update admin kubeconfig from newly generated credentials
cp /etc/kubernetes/admin.conf ~/.kube/config
exit

# Step 5: Out of the root shell again, write the expiry report into the required file
sudo kubeadm certs check-expiration | grep '^apiserver ' > {WS}/q65/expiry.txt
"""

S[67] = [cmd(r"""sudo kubeadm token create --ttl 2h""", r"""The host is the control-plane node. Copy the token it prints."""),
         cmd(r"""openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex""", r"""The SHA-256 of the CA's public key, as the docs page *kubeadm join* computes it - what `--discovery-token-ca-cert-hash` checks. The hash is everything after `= `."""),
         cmd(r"""k get endpointslices -l kubernetes.io/service-name=kubernetes""", r"""The address and port the API server advertises - not `127.0.0.1:<port>`, how this host's own kubeconfig reaches it."""),
         hand(r"""vim {WS}/q67/node-join.env""",
              r"""The three values you copied:
```
JOIN_TOKEN=abcdef.0123456789abcdef
CA_HASH=sha256:<the 64 hex characters>
API_ENDPOINT=<address>:<port>
```""",
              run=r"""T=$(sudo kubeadm token create --ttl 2h)
H=$(openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //')
E=$(kubectl get endpointslices -l kubernetes.io/service-name=kubernetes -o jsonpath='{.items[0].endpoints[0].addresses[0]}:{.items[0].ports[0].port}')
printf 'JOIN_TOKEN=%s\nCA_HASH=sha256:%s\nAPI_ENDPOINT=%s\n' "$T" "$H" "$E" > {WS}/q67/node-join.env""")]


# ================================================================= WORK ====
# Domain 2: Workloads & Scheduling (15%)
# ===========================================================================

S[7] = [cmd(r"""k -n apps set image deploy web-frontend nginx=nginx:1.27
k -n apps scale deploy web-frontend --replicas=4"""),
        hand(r"""k -n apps edit deploy web-frontend""",
             r"""Under `strategy:`, set the rolling update, then `:wq`:
```
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
    type: RollingUpdate
```""",
             run=r"""kubectl -n apps patch deploy web-frontend -p '{"spec":{"strategy":{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0}}}}'"""),
        cmd(r"""k -n apps rollout status deploy web-frontend
k -n apps get deploy web-frontend""", r"""`READY 4/4`.""")]

S[8] = [cmd(r"""k -n apps run log-shipper --image=busybox:1.36 --dry-run=client -o yaml --command -- sh -c 'while true; do date >> /var/log/app.log; sleep 2; done' > pod.yaml"""),
        hand(r"""vim pod.yaml""",
             r"""Name the container `app`, give it the mount, and copy it as a second container `shipper` with its own command - then the volume, in the Pod's `spec:`. `containers:` ends up as:
```
  containers:
  - command:
    - sh
    - -c
    - while true; do date >> /var/log/app.log; sleep 2; done
    image: busybox:1.36
    name: app
    volumeMounts:
    - name: logs
      mountPath: /var/log
  - command:
    - sh
    - -c
    - tail -f /var/log/app.log
    image: busybox:1.36
    name: shipper
    volumeMounts:
    - name: logs
      mountPath: /var/log
  volumes:
  - name: logs
    emptyDir: {}
```""",
             run=r"""yq -i '.spec.containers[0].name = "app"
  | .spec.containers[0].volumeMounts = [{"name": "logs", "mountPath": "/var/log"}]
  | .spec.containers[1] = .spec.containers[0]
  | .spec.containers[1].name = "shipper"
  | .spec.containers[1].command = ["sh", "-c", "tail -f /var/log/app.log"]
  | .spec.volumes = [{"name": "logs", "emptyDir": {}}]' pod.yaml"""),
        cmd(r"""k apply -f pod.yaml
k -n apps logs log-shipper -c shipper""", r"""Again after a few seconds: a date every 2 seconds, written by `app`, read by `shipper`.""")]

S[9] = [cmd(r"""k get nodes -L disktype
k -n apps create deploy cache --image=redis:7-alpine --replicas=2 --dry-run=client -o yaml > cache.yaml"""),
        hand(r"""vim cache.yaml""",
             r"""Add to the Pod template's `spec:`, above `containers:` (the docs page *Assign Pods to Nodes using Node Affinity* has the block):
```
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: disktype
                operator: In
                values:
                - ssd
```
and replace the container's `resources: {}` with:
```
        resources:
          requests:
            cpu: 100m
            memory: 64Mi
          limits:
            cpu: 200m
            memory: 128Mi
```""",
             run=r"""yq -i '.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms = [{"matchExpressions": [{"key": "disktype", "operator": "In", "values": ["ssd"]}]}]
  | .spec.template.spec.containers[0].resources = {"requests": {"cpu": "100m", "memory": "64Mi"}, "limits": {"cpu": "200m", "memory": "128Mi"}}' cache.yaml"""),
        cmd(r"""k apply -f cache.yaml
k -n apps get pods -o wide""", r"""Both on the node labelled `disktype=ssd`.""")]

S[10] = [cmd(r"""k -n apps create cm app-config --from-literal=APP_MODE=production --from-literal=LOG_LEVEL=debug
k -n apps create secret generic db-creds --from-literal=username=admin --from-literal='password=S3cr3t!'
k -n apps run config-pod --image=busybox:1.36 --dry-run=client -o yaml --command -- sleep 3600 > p.yaml""", r"""The quotes keep the shell off the `!` in the password."""),
         hand(r"""vim p.yaml""",
              r"""Add to the container, below `name: config-pod` (the docs page *Configure a Pod to Use a ConfigMap* has each of these):
```
    env:
    - name: APP_MODE
      valueFrom:
        configMapKeyRef:
          name: app-config
          key: APP_MODE
    envFrom:
    - secretRef:
        name: db-creds
    volumeMounts:
    - name: cfg
      mountPath: /etc/config
```
and to the Pod's `spec:`, right below `spec:` and indented like `containers:`:
```
  volumes:
  - name: cfg
    configMap:
      name: app-config
```""",
              run=r"""yq -i '.spec.containers[0].env = [{"name": "APP_MODE", "valueFrom": {"configMapKeyRef": {"name": "app-config", "key": "APP_MODE"}}}]
  | .spec.containers[0].envFrom = [{"secretRef": {"name": "db-creds"}}]
  | .spec.containers[0].volumeMounts = [{"name": "cfg", "mountPath": "/etc/config"}]
  | .spec.volumes = [{"name": "cfg", "configMap": {"name": "app-config"}}]' p.yaml"""),
         cmd(r"""k apply -f p.yaml
k -n apps exec config-pod -- ls /etc/config
k -n apps exec config-pod -- env""", r"""Once it is `Running`: `APP_MODE`, `LOG_LEVEL` as files, and `username` and `password` in the environment.""")]

S[23] = """# Step 1: Repair metrics-server by adding --kubelet-insecure-tls flag
k -n kube-system edit deploy metrics-server
# Add under spec.template.spec.containers[0].args:
#   - --kubelet-insecure-tls

# Step 2: Wait ~30s and verify metrics reporting
k top nodes

# Step 3: Configure HorizontalPodAutoscaler (HPA) for target deployment
k -n %{ns} autoscale deploy %{dep} --min=2 --max=%{max} --cpu-percent=%{cpu} --name=%{dep}

# Step 4: Verify HPA status
k -n %{ns} get hpa %{dep}
"""

S[24] = [cmd(r"""k create priorityclass %{pc} --value=%{val} --description="%{pc} workloads"
k get pc %{pc}""", r"""`GLOBAL-DEFAULT false`: the default unless `--global-default`."""),
         hand(r"""k -n %{ns} edit deploy payments""",
              r"""In the Pod template's `spec:`, beside `containers:` and indented like it, add the line below, then `:wq` - the change rolls the Pods:
```
      priorityClassName: %{pc}
```""",
              run=r"""kubectl -n %{ns} patch deploy payments -p '{"spec":{"template":{"spec":{"priorityClassName":"%{pc}"}}}}'"""),
         cmd(r"""k -n %{ns} rollout status deploy payments
k -n %{ns} get pods -o custom-columns=NAME:.metadata.name,PRIORITY:.spec.priority""", r"""Every Pod at `%{val}`.""")]

S[25] = [cmd(r"""k taint node {W1} %{key}=%{val}:NoSchedule
k -n %{ns} create deploy %{dep} --image=busybox:1.36 --replicas=2 --dry-run=client -o yaml -- sleep 3600 > d.yaml"""),
         hand(r"""vim d.yaml""",
              r"""In the Pod template's `spec:`, above `containers:` (the docs page *Taints and Tolerations* has the toleration):
```
      nodeSelector:
        kubernetes.io/hostname: {W1}
      tolerations:
      - key: %{key}
        operator: Equal
        value: %{val}
        effect: NoSchedule
```""",
              run=r"""yq -i '.spec.template.spec.nodeSelector = {"kubernetes.io/hostname": "{W1}"}
  | .spec.template.spec.tolerations = [{"key": "%{key}", "operator": "Equal", "value": "%{val}", "effect": "NoSchedule"}]' d.yaml"""),
         cmd(r"""k apply -f d.yaml
k -n %{ns} get pods -o wide""", r"""Both `Running` on `{W1}`.""")]

S[26] = [cmd(r"""k -n %{ns} create deploy %{ds} --image=busybox:1.36 --dry-run=client -o yaml -- sleep 3600 > ds.yaml""", r"""There is no `create daemonset`: a Deployment's YAML is the closest start."""),
         hand(r"""vim ds.yaml""",
              r"""Make it a DaemonSet: `kind: DaemonSet`, and delete `replicas: 1`, `strategy: {}` and `status: {}` - a DaemonSet has none of them. Then, in the Pod template's `spec:`, above `containers:`, the toleration that lets it onto the control plane:
```
      tolerations:
      - key: node-role.kubernetes.io/control-plane
        operator: Exists
        effect: NoSchedule
```
and replace the container's `resources: {}` with:
```
        resources:
          requests:
            cpu: 10m
            memory: 16Mi
```""",
              run=r"""yq -i '.kind = "DaemonSet" | del(.spec.replicas) | del(.spec.strategy) | del(.status)
  | .spec.template.spec.tolerations = [{"key": "node-role.kubernetes.io/control-plane", "operator": "Exists", "effect": "NoSchedule"}]
  | .spec.template.spec.containers[0].resources = {"requests": {"cpu": "10m", "memory": "16Mi"}}' ds.yaml"""),
         cmd(r"""k apply -f ds.yaml
k -n %{ns} get ds,pods -o wide""", r"""`DESIRED` and `READY` equal to the number of nodes, the control plane among them.""")]

S[27] = [cmd(r"""k -n %{ns} create cj %{cj} --image=busybox:1.36 --schedule='*/%{every} * * * *' --dry-run=client -o yaml -- date > cj.yaml"""),
         hand(r"""vim cj.yaml""",
              r"""Add to the CronJob's `spec:`, beside `schedule:`:
```
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 1
```
to the Job's `spec:` (under `jobTemplate:`), above `template:`:
```
      activeDeadlineSeconds: 30
```
and change `restartPolicy: OnFailure` to `restartPolicy: Never`.""",
              run=r"""yq -i '.spec.concurrencyPolicy = "Forbid" | .spec.successfulJobsHistoryLimit = 3 | .spec.failedJobsHistoryLimit = 1
  | .spec.jobTemplate.spec.activeDeadlineSeconds = 30
  | .spec.jobTemplate.spec.template.spec.restartPolicy = "Never"' cj.yaml"""),
         cmd(r"""k apply -f cj.yaml
k -n %{ns} create job %{cj}-manual --from=cronjob/%{cj}
k -n %{ns} get job %{cj}-manual""", r"""Again until `COMPLETIONS 1/1`.""")]

S[28] = [cmd(r"""k -n %{ns} run %{pod} --image=nginx:1.25 --dry-run=client -o yaml > pod.yaml"""),
         hand(r"""vim pod.yaml""",
              r"""Name the container `web` and give it the mount, below its `name:`:
```
    volumeMounts:
    - name: content
      mountPath: /usr/share/nginx/html
```
then add to the Pod's `spec:`, right below `spec:`:
```
  initContainers:
  - name: setup
    image: busybox:1.36
    command: ["sh", "-c", "echo '%{text}' > /work/index.html"]
    volumeMounts:
    - name: content
      mountPath: /work
  volumes:
  - name: content
    emptyDir: {}
```""",
              run=r"""yq -i '.spec.containers[0].name = "web"
  | .spec.containers[0].volumeMounts = [{"name": "content", "mountPath": "/usr/share/nginx/html"}]
  | .spec.initContainers = [{"name": "setup", "image": "busybox:1.36", "command": ["sh", "-c", "echo '"'"'%{text}'"'"' > /work/index.html"], "volumeMounts": [{"name": "content", "mountPath": "/work"}]}]
  | .spec.volumes = [{"name": "content", "emptyDir": {}}]' pod.yaml"""),
         cmd(r"""k apply -f pod.yaml
k -n %{ns} get pod %{pod} -o wide""", r"""Once it is `Running`, `curl` its IP from a node, or:"""),
         cmd(r"""k -n %{ns} exec %{pod} -c web -- curl -s localhost""")]

S[29] = [cmd(r"""k -n %{ns} create cm %{cm}-v2 --from-literal=mode=new --dry-run=client -o yaml > cm.yaml"""),
         hand(r"""vim cm.yaml""",
              r"""Make the new version immutable too: add, at the top level beside `data:`,
```
immutable: true
```""",
              run=r"""yq -i '.immutable = true' cm.yaml"""),
         cmd(r"""k apply -f cm.yaml"""),
         hand(r"""k -n %{ns} edit deploy %{dep}""",
              r"""In the Pod template's `volumes:`, point the volume at the new ConfigMap, then `:wq`:
```
      - configMap:
          name: %{cm}-v2
```""",
              run=r"""kubectl -n %{ns} patch deploy %{dep} --type=json -p='[{"op":"replace","path":"/spec/template/spec/volumes/0/configMap/name","value":"%{cm}-v2"}]'"""),
         cmd(r"""k -n %{ns} rollout status deploy %{dep}""")]

S[31] = [cmd(r"""k -n %{ns} create job %{job} --image=busybox:1.36 --dry-run=client -o yaml -- sh -c "echo done; sleep 2" > job.yaml"""),
         hand(r"""vim job.yaml""",
              r"""Add to the Job's `spec:`, above `template:` (`restartPolicy: Never` is there already):
```
  completions: %{c}
  parallelism: %{p}
  backoffLimit: 2
```""",
              run=r"""yq -i '.spec.completions = %{c} | .spec.parallelism = %{p} | .spec.backoffLimit = 2' job.yaml"""),
         cmd(r"""k apply -f job.yaml"""),
         hand(r"""k -n %{ns} get job %{job}""",
              r"""Again until `COMPLETIONS %{c}/%{c}`.""",
              run=r"""kubectl -n %{ns} wait --for=condition=complete job/%{job} --timeout=120s""")]

S[32] = [cmd(r"""k -n %{ns} create quota %{ns}-quota --hard=pods=%{pods},requests.cpu=1,limits.memory=2Gi"""),
         hand(r"""vim lr.yaml""",
              r"""Paste the LimitRange from the docs page *Configure Default CPU Requests and Limits for a Namespace* and give it both resources:
```
apiVersion: v1
kind: LimitRange
metadata:
  name: %{ns}-defaults
  namespace: %{ns}
spec:
  limits:
  - default:
      cpu: 500m
      memory: 256Mi
    defaultRequest:
      cpu: %{dcpu}
      memory: 64Mi
    type: Container
```""",
              run=r"""cat > lr.yaml <<'EOF'
apiVersion: v1
kind: LimitRange
metadata:
  name: %{ns}-defaults
  namespace: %{ns}
spec:
  limits:
  - default:
      cpu: 500m
      memory: 256Mi
    defaultRequest:
      cpu: %{dcpu}
      memory: 64Mi
    type: Container
EOF"""),
         cmd(r"""k apply -f lr.yaml
k -n %{ns} create deploy worker --image=nginx:1.25 --replicas=2""", r"""The LimitRange first: with the quota on `requests.cpu` and `limits.memory`, a Pod that has neither is refused."""),
         cmd(r"""k -n %{ns} describe quota %{ns}-quota
k -n %{ns} describe pod -l app=worker | grep -A2 -e Limits -e Requests""", r"""The defaults on every Pod, and the quota's `Used` column counting them.""")]

# Hard Workload Tasks
S[50] = [cmd(r"""k describe node {W1} | grep Taints
k -n %{ns} describe quota""", r"""The quota is the trap twice over: 3 Pods do not fit `pods: 2`, nor 3 x 100m CPU in `requests.cpu: 250m`."""),
         hand(r"""k -n %{ns} edit quota %{ns}-quota""",
              r"""Raise both - the quota stays and still caps Pods - then `:wq`:
```
spec:
  hard:
    pods: "3"
    requests.cpu: 300m
```""",
              run=r"""kubectl -n %{ns} patch quota %{ns}-quota -p '{"spec":{"hard":{"pods":"3","requests.cpu":"300m"}}}'"""),
         cmd(r"""k create priorityclass %{dep}-prio --value=50000
k -n %{ns} create deploy %{dep} --image=nginx:1.27 --replicas=3 --dry-run=client -o yaml > d.yaml"""),
         hand(r"""vim d.yaml""",
              r"""In the Pod template's `spec:`, above `containers:` (the docs page *Assigning Pods to Nodes* has the anti-affinity, *Taints and Tolerations* the toleration):
```
      priorityClassName: %{dep}-prio
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchLabels:
                app: %{dep}
            topologyKey: kubernetes.io/hostname
      tolerations:
      - key: %{key}
        operator: Equal
        value: %{val}
        effect: NoSchedule
```
and replace the container's `resources: {}` with:
```
        resources:
          requests:
            cpu: 100m
            memory: 64Mi
```""",
              run=r"""yq -i '.spec.template.spec.priorityClassName = "%{dep}-prio"
  | .spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution = [{"labelSelector": {"matchLabels": {"app": "%{dep}"}}, "topologyKey": "kubernetes.io/hostname"}]
  | .spec.template.spec.tolerations = [{"key": "%{key}", "operator": "Equal", "value": "%{val}", "effect": "NoSchedule"}]
  | .spec.template.spec.containers[0].resources = {"requests": {"cpu": "100m", "memory": "64Mi"}}' d.yaml"""),
         cmd(r"""k apply -f d.yaml
k -n %{ns} get pods -o wide
k -n %{ns} describe quota""", r"""3 `Running` on 3 different workers, and the quota at `3/3`, `300m/300m`. The control plane is tainted too, so there is no slack: a missed quota field or toleration shows as a `Pending` Pod, and `k -n %{ns} describe rs` says which.""")]

S[51] = [cmd(r"""k -n %{ns} rollout history deploy %{dep}
k -n %{ns} rollout history deploy %{dep} --revision=1""", r"""`--revision=N` shows that revision's Pod template. Look at each until one has `nginx:1.25` - here revision 1, which is not the previous one."""),
         cmd(r"""k -n %{ns} rollout undo deploy %{dep} --to-revision=1
k -n %{ns} scale deploy %{dep} --replicas=3
k -n %{ns} rollout status deploy %{dep}"""),
         cmd(r"""k -n %{ns} rollout pause deploy %{dep}
k -n %{ns} set image deploy %{dep} nginx=nginx:1.27
k -n %{ns} set env deploy %{dep} RELEASE=next""", r"""Pause first: while paused, the controller leaves the template alone - no new ReplicaSet, and the Pods stay on `nginx:1.25`."""),
         cmd(r"""k -n %{ns} get deploy %{dep} -o yaml | grep -e paused -e image:
k -n %{ns} get rs -o wide
k -n %{ns} get pods""", r"""`paused: true`, `nginx:1.27` in the template - and no ReplicaSet for it.""")]

S[52] = [cmd(r"""k -n %{ns} run %{pod} --image=nginx:1.27 --dry-run=client -o yaml > pod.yaml"""),
         hand(r"""vim pod.yaml""",
              r"""Name the container `app`, and add below its `name:` the probes and the mount:
```
    readinessProbe:
      httpGet:
        path: /
        port: 80
      periodSeconds: 5
    livenessProbe:
      tcpSocket:
        port: 80
    volumeMounts:
    - name: logs
      mountPath: /var/log/agent
```
then to the Pod's `spec:`, right below `spec:` - the sidecar is an init container that keeps running (`restartPolicy: Always`):
```
  terminationGracePeriodSeconds: 5
  initContainers:
  - name: agent
    image: busybox:1.36
    restartPolicy: Always
    command: ["sh", "-c", "while true; do date >> /var/log/agent/agent.log; sleep 1; done"]
    volumeMounts:
    - name: logs
      mountPath: /var/log/agent
  volumes:
  - name: logs
    emptyDir: {}
```
At `/var/log/agent`, not `/var/log`: an emptyDir there would hide nginx's `/var/log/nginx`, and nginx crash-loops.""",
              run=r"""yq -i '.spec.containers[0].name = "app"
  | .spec.containers[0].readinessProbe = {"httpGet": {"path": "/", "port": 80}, "periodSeconds": 5}
  | .spec.containers[0].livenessProbe = {"tcpSocket": {"port": 80}}
  | .spec.containers[0].volumeMounts = [{"name": "logs", "mountPath": "/var/log/agent"}]
  | .spec.terminationGracePeriodSeconds = 5
  | .spec.initContainers = [{"name": "agent", "image": "busybox:1.36", "restartPolicy": "Always", "command": ["sh", "-c", "while true; do date >> /var/log/agent/agent.log; sleep 1; done"], "volumeMounts": [{"name": "logs", "mountPath": "/var/log/agent"}]}]
  | .spec.volumes = [{"name": "logs", "emptyDir": {}}]' pod.yaml"""),
         cmd(r"""k apply -f pod.yaml"""),
         hand(r"""k -n %{ns} get pod %{pod} -o wide""",
              r"""Again until it is `Running` with `READY 2/2` - it has no IP before that. Note the IP.""",
              run=r"""kubectl -n %{ns} wait --for=condition=Ready pod/%{pod} --timeout=120s"""),
         cmd(r"""k -n %{ns} get pod %{pod} -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}'; echo""", r"""`agent app`: the init container and the regular one."""),
         hand(r"""vim {WS}/q52/containers.txt""",
              r"""Sorted, one per line:
```
agent
app
```""",
              run=r"""printf 'agent\napp\n' > {WS}/q52/containers.txt"""),
         hand(r"""echo 10.244.1.7 > {WS}/q52/ip.txt""",
              r"""With the Pod's IP, as `-o wide` showed it.""",
              run=r"""kubectl -n %{ns} get pod %{pod} -o jsonpath='{.status.podIP}' > {WS}/q52/ip.txt""")]

S[53] = [hand(r"""k -n %{ns} get pods -o wide""",
              r"""Look before you count: which Pods, which phase, which node - again until all but `once-a` are `Running`.""",
              run=r"""kubectl -n %{ns} wait --for=condition=Ready pod web-a web-b batch-a cache-a idle-a --timeout=180s"""),
         cmd(r"""k -n %{ns} get pods -o custom-columns=NAME:.metadata.name,PROBE:.spec.containers[0].readinessProbe""", r"""`<none>`: no readiness probe on the first container - `cache-a` among them, whose probe is on its second; and `once-a` counts though it has `Completed`."""),
         hand(r"""vim {WS}/q53/no-readiness.txt""",
              r"""The names with `<none>`, sorted:
```
batch-a
cache-a
idle-a
once-a
```""",
              run=r"""kubectl -n %{ns} get pods -o jsonpath='{range .items[*]}{.metadata.name} {.spec.containers[0].readinessProbe}{"\n"}{end}' | awk 'NF==1{print $1}' | sort > {WS}/q53/no-readiness.txt"""),
         cmd(r"""k -n %{ns} get pods -o wide --field-selector=status.phase=Running""", r"""`once-a` is gone from the list: it is not `Running`."""),
         hand(r"""vim {WS}/q53/per-node.txt""",
              r"""Count the Pods per node in the `NODE` column, and write the nodes sorted:
```
{W1} 2
{W2} 2
{W3} 1
```""",
              run=r"""kubectl -n %{ns} get pods --field-selector=status.phase=Running -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' | sort | uniq -c | awk '{print $2, $1}' | sort > {WS}/q53/per-node.txt"""),
         cmd(r"""k -n kube-system get pods -o jsonpath='{.items[*].spec.containers[*].image}' | tr ' ' '\n' | sort -u > {WS}/q53/images.txt
cat -A {WS}/q53/images.txt""", r"""Too many to copy by hand. `cat -A` shows a stray space or `\r` as `$` and `^M`.""")]

S[64] = [cmd(r"""k config view --minify | grep namespace""", r"""`namespace: %{ns}` - where everything without `-n` lands."""),
         cmd(r"""k create deploy %{dep} --image=busybox:1.36 --replicas=2 -- sleep 3600
k label deploy %{dep} owner=ac
k expose deploy %{dep} --port=9000 --target-port=9000
echo %{ns} > {WS}/q64/ns.txt""", r"""No `-n` anywhere. `expose` makes a ClusterIP Service unless told otherwise."""),
         cmd(r"""k get deploy,svc %{dep}
k -n default get deploy %{dep}""", r"""Both there - and `NotFound` in `default`, the right answer.""")]


# ================================================================= NET =====
# Domain 3: Services & Networking (20%)
# ===========================================================================

S[11] = [hand(r"""vim netpol.yaml""",
              r"""Paste the example from the docs page *Network Policies* and cut it down to one ingress rule - no `egress`, and `Ingress` alone in `policyTypes`, so egress stays open:
```
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: backend-policy
  namespace: netpol
spec:
  podSelector:
    matchLabels:
      app: backend
  policyTypes:
  - Ingress
  ingress:
  - from:
    - podSelector:
        matchLabels:
          app: frontend
    ports:
    - protocol: TCP
      port: 8080
```""",
              run=r"""cat > netpol.yaml <<'EOF'
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: backend-policy
  namespace: netpol
spec:
  podSelector:
    matchLabels:
      app: backend
  policyTypes:
  - Ingress
  ingress:
  - from:
    - podSelector:
        matchLabels:
          app: frontend
    ports:
    - protocol: TCP
      port: 8080
EOF"""),
         cmd(r"""k apply -f netpol.yaml
k -n netpol describe netpol backend-policy""")]

S[12] = [cmd(r"""k -n svc expose deploy shop --name=shop-svc --type=NodePort --port=80 --target-port=80 --dry-run=client -o yaml > svc.yaml"""),
         hand(r"""vim svc.yaml""",
              r"""Add the fixed node port to the port, below `targetPort: 80`:
```
    nodePort: 30080
```""",
              run=r"""yq -i '.spec.ports[0].nodePort = 30080' svc.yaml"""),
         cmd(r"""k apply -f svc.yaml
k -n svc get svc,endpointslices""", r"""`80:30080/TCP`, and an EndpointSlice with the Pod's address.""")]

S[13] = [cmd(r"""k -n ingress-app create ingress app-ingress --class=nginx --rule='shop.example.com/api*=api:8080' --rule='shop.example.com/*=web:80'
k -n ingress-app describe ingress app-ingress""", r"""A path ending in `*` is `pathType: Prefix` (without, `Exact`); the quotes keep the shell off the `*`.""")]

S[14] = [cmd(r"""k -n dns-check run dns-test --image=busybox:1.36 -- sleep 3600
echo hello.dns-check.svc.cluster.local > {WS}/q14/svc-fqdn.txt""", r"""Not `--rm -it`: the Pod has to be there for the check."""),
         hand(r"""k -n dns-check get pod dns-test""",
              r"""Again until it is `Running`.""",
              run=r"""kubectl -n dns-check wait --for=condition=Ready pod/dns-test --timeout=120s"""),
         cmd(r"""k -n dns-check exec dns-test -- nslookup hello.dns-check.svc.cluster.local > {WS}/q14/nslookup.txt
cat {WS}/q14/nslookup.txt""", r"""The answer's `Address` is the Service's ClusterIP.""")]

S[22] = [cmd(r"""k apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/standard-install.yaml"""),
         hand(r"""k get crd | grep gateway""",
              r"""`gatewayclasses`, `gateways`, `httproutes` and the rest of the standard channel. A moment later the API serves them.""",
              run=r"""kubectl wait --for=condition=Established --timeout=60s crd/gatewayclasses.gateway.networking.k8s.io crd/gateways.gateway.networking.k8s.io crd/httproutes.gateway.networking.k8s.io"""),
         hand(r"""vim gw.yaml""",
              r"""The docs page *Gateway API* has a GatewayClass, a Gateway and an HTTPRoute to start from; one file, `---` between them:
```
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
  name: %{gc}
spec:
  controllerName: example.com/gateway
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: %{gw}
  namespace: %{ns}
spec:
  gatewayClassName: %{gc}
  listeners:
  - name: http
    protocol: HTTP
    port: 80
    hostname: %{host}.example.com
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: %{host}-route
  namespace: %{ns}
spec:
  parentRefs:
  - name: %{gw}
  rules:
  - matches:
    - path:
        type: PathPrefix
        value: /api
    backendRefs:
    - name: api
      port: 8080
  - backendRefs:
    - name: web
      port: 80
```
A rule with no `matches` takes everything else (`PathPrefix /`).""",
              run=r"""cat > gw.yaml <<'EOF'
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
  name: %{gc}
spec:
  controllerName: example.com/gateway
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: %{gw}
  namespace: %{ns}
spec:
  gatewayClassName: %{gc}
  listeners:
  - name: http
    protocol: HTTP
    port: 80
    hostname: %{host}.example.com
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: %{host}-route
  namespace: %{ns}
spec:
  parentRefs:
  - name: %{gw}
  rules:
  - matches:
    - path:
        type: PathPrefix
        value: /api
    backendRefs:
    - name: api
      port: 8080
  - backendRefs:
    - name: web
      port: 80
EOF"""),
         cmd(r"""k apply -f gw.yaml
k -n %{ns} get gateway,httproute""")]

S[30] = [cmd(r"""k -n %{ns} create deploy %{dep} --image=nginx:1.25 --port=80 --dry-run=client -o yaml > d.yaml"""),
         hand(r"""vim d.yaml""",
              r"""Name the port, and add the probe below the container's `ports:`, by that name:
```
        ports:
        - containerPort: 80
          name: %{pname}
        readinessProbe:
          httpGet:
            path: /
            port: %{pname}
```""",
              run=r"""yq -i '.spec.template.spec.containers[0].ports[0].name = "%{pname}"
  | .spec.template.spec.containers[0].readinessProbe = {"httpGet": {"path": "/", "port": "%{pname}"}}' d.yaml"""),
         cmd(r"""k apply -f d.yaml
k -n %{ns} expose deploy %{dep} --port=8080 --target-port=%{pname}
k -n %{ns} get endpointslices -l kubernetes.io/service-name=%{dep}""", r"""`--target-port` takes a port's name as well as its number. An address once the probe passes: empty means the selector or the probe is wrong.""")]

S[42] = [hand(r"""vim netpol.yaml""",
              r"""Paste the example from the docs page *Network Policies* and make it egress only, with two rules. `Egress` alone in `policyTypes` denies all other egress and leaves ingress open; DNS needs both protocols, as resolvers fall back to TCP for long answers:
```
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: %{app}-egress
  namespace: %{ns}
spec:
  podSelector:
    matchLabels:
      app: %{app}
  policyTypes:
  - Egress
  egress:
  - to:
    - namespaceSelector:
        matchLabels:
          kubernetes.io/metadata.name: kube-system
      podSelector:
        matchLabels:
          k8s-app: kube-dns
    ports:
    - protocol: UDP
      port: 53
    - protocol: TCP
      port: 53
  - to:
    - ipBlock:
        cidr: %{cidr}
    ports:
    - protocol: TCP
      port: 443
```
`podSelector` with no `-` before it belongs to the same `to` entry as the `namespaceSelector`: both must match.""",
              run=r"""cat > netpol.yaml <<'EOF'
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: %{app}-egress
  namespace: %{ns}
spec:
  podSelector:
    matchLabels:
      app: %{app}
  policyTypes:
  - Egress
  egress:
  - to:
    - namespaceSelector:
        matchLabels:
          kubernetes.io/metadata.name: kube-system
      podSelector:
        matchLabels:
          k8s-app: kube-dns
    ports:
    - protocol: UDP
      port: 53
    - protocol: TCP
      port: 53
  - to:
    - ipBlock:
        cidr: %{cidr}
    ports:
    - protocol: TCP
      port: 443
EOF"""),
         cmd(r"""k apply -f netpol.yaml
k -n %{ns} describe netpol %{app}-egress""")]

S[43] = [cmd(r"""k -n %{ns} get sts %{sts} -o yaml | grep serviceName""", r"""The headless Service must have exactly this name: it is what makes CoreDNS publish one record per Pod, `<pod>.<service>.<ns>.svc.cluster.local`."""),
         cmd(r"""k -n %{ns} create service clusterip %{sts} --clusterip=None --tcp=5432:5432
k -n %{ns} create service externalname legacy --external-name=%{ext}
k -n %{ns} get svc""", r"""`create service` selects `app=<its name>` - here `app=%{sts}`, the StatefulSet's Pods. `legacy` has no Pods: it is only a CNAME to `%{ext}`."""),
         hand(r"""k -n %{ns} exec %{sts}-0 -- nslookup %{sts}-0.%{sts}.%{ns}.svc.cluster.local > {WS}/q43/pod0.txt
cat {WS}/q43/pod0.txt
k -n %{ns} get pod %{sts}-0 -o wide""",
              r"""The full name: busybox's `nslookup` applies no search domains. The address it gives is the Pod's IP - and if it says `NXDOMAIN`, again in a few seconds: CoreDNS publishes the record once the new Service has endpoints.""",
              run=r"""for i in $(seq 1 20); do
  kubectl -n %{ns} exec %{sts}-0 -- nslookup %{sts}-0.%{sts}.%{ns}.svc.cluster.local > {WS}/q43/pod0.txt 2>&1
  grep -q "$(kubectl -n %{ns} get pod %{sts}-0 -o jsonpath='{.status.podIP}')" {WS}/q43/pod0.txt && break; sleep 3
done""")]

# Hard Networking Tasks
S[54] = [cmd(r"""k -n %{ns} get netpol
k -n %{ns} delete netpol temp-allow-all
k label ns %{from} team=%{from}""", r"""NetworkPolicies add up: one that allows everything cancels every restriction below."""),
         hand(r"""vim zt.yaml""",
              r"""The docs page *Network Policies* has a default deny for both directions to start from. Five policies, `---` between them:
```
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny
  namespace: %{ns}
spec:
  podSelector: {}
  policyTypes:
  - Ingress
  - Egress
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-dns
  namespace: %{ns}
spec:
  podSelector: {}
  policyTypes:
  - Egress
  egress:
  - to:
    - namespaceSelector:
        matchLabels:
          kubernetes.io/metadata.name: kube-system
    ports:
    - protocol: UDP
      port: 53
    - protocol: TCP
      port: 53
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: api-ingress
  namespace: %{ns}
spec:
  podSelector:
    matchLabels:
      app: api
  policyTypes:
  - Ingress
  ingress:
  - from:
    - namespaceSelector:
        matchLabels:
          team: %{from}
    ports:
    - port: 8080
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: api-to-db
  namespace: %{ns}
spec:
  podSelector:
    matchLabels:
      app: api
  policyTypes:
  - Egress
  egress:
  - to:
    - podSelector:
        matchLabels:
          app: db
    ports:
    - port: 5432
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: db-ingress
  namespace: %{ns}
spec:
  podSelector:
    matchLabels:
      app: db
  policyTypes:
  - Ingress
  ingress:
  - from:
    - podSelector:
        matchLabels:
          app: api
    ports:
    - port: 5432
```""",
              run=r"""cat > zt.yaml <<'EOF'
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny
  namespace: %{ns}
spec:
  podSelector: {}
  policyTypes:
  - Ingress
  - Egress
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-dns
  namespace: %{ns}
spec:
  podSelector: {}
  policyTypes:
  - Egress
  egress:
  - to:
    - namespaceSelector:
        matchLabels:
          kubernetes.io/metadata.name: kube-system
    ports:
    - protocol: UDP
      port: 53
    - protocol: TCP
      port: 53
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: api-ingress
  namespace: %{ns}
spec:
  podSelector:
    matchLabels:
      app: api
  policyTypes:
  - Ingress
  ingress:
  - from:
    - namespaceSelector:
        matchLabels:
          team: %{from}
    ports:
    - port: 8080
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: api-to-db
  namespace: %{ns}
spec:
  podSelector:
    matchLabels:
      app: api
  policyTypes:
  - Egress
  egress:
  - to:
    - podSelector:
        matchLabels:
          app: db
    ports:
    - port: 5432
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: db-ingress
  namespace: %{ns}
spec:
  podSelector:
    matchLabels:
      app: db
  policyTypes:
  - Ingress
  ingress:
  - from:
    - podSelector:
        matchLabels:
          app: api
    ports:
    - port: 5432
EOF"""),
         cmd(r"""k apply -f zt.yaml
k -n %{ns} get netpol""")]

S[55] = """# Step 1: Generate self-signed TLS certificate and private key
openssl req -x509 -newkey rsa:2048 -nodes -days 365 -subj "/CN=%{host}" -keyout tls.key -out tls.crt

# Step 2: Create TLS secret in target namespace
k -n %{ns} create secret tls %{host}-tls --cert=tls.crt --key=tls.key

# Step 3: Create Ingress with TLS configuration and SSL redirect annotation
k -n %{ns} create ingress web --class=nginx --rule="%{host}/*=web:80,tls=%{host}-tls" \\
  --annotation nginx.ingress.kubernetes.io/ssl-redirect=true

# Step 4: Extract SHA256 fingerprint to workspace file
openssl x509 -in tls.crt -noout -fingerprint -sha256 | cut -d= -f2 > {WS}/q55/fp.txt
"""

S[56] = [hand(r"""vim canary.yaml""",
              r"""The docs page *Gateway API* has the three kinds to start from; `sectionName` attaches the route to one listener, and a rule without `matches` takes the rest:
```
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
  name: %{host}-gc
spec:
  controllerName: example.com/gw
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: %{host}-gw
  namespace: %{ns}
spec:
  gatewayClassName: %{host}-gc
  listeners:
  - name: web
    protocol: HTTP
    port: 80
    hostname: %{host}.example.com
  - name: api
    protocol: HTTP
    port: 80
    hostname: api.%{host}.example.com
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: %{host}-route
  namespace: %{ns}
spec:
  parentRefs:
  - name: %{host}-gw
    sectionName: web
  rules:
  - matches:
    - headers:
      - name: X-Canary
        value: always
    backendRefs:
    - name: v2
      port: 80
  - backendRefs:
    - name: v1
      port: 80
      weight: %{w2}
    - name: v2
      port: 80
      weight: %{w}
```""",
              run=r"""cat > canary.yaml <<'EOF'
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
  name: %{host}-gc
spec:
  controllerName: example.com/gw
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: %{host}-gw
  namespace: %{ns}
spec:
  gatewayClassName: %{host}-gc
  listeners:
  - name: web
    protocol: HTTP
    port: 80
    hostname: %{host}.example.com
  - name: api
    protocol: HTTP
    port: 80
    hostname: api.%{host}.example.com
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: %{host}-route
  namespace: %{ns}
spec:
  parentRefs:
  - name: %{host}-gw
    sectionName: web
  rules:
  - matches:
    - headers:
      - name: X-Canary
        value: always
    backendRefs:
    - name: v2
      port: 80
  - backendRefs:
    - name: v1
      port: 80
      weight: %{w2}
    - name: v2
      port: 80
      weight: %{w}
EOF"""),
         cmd(r"""k apply -f canary.yaml
k -n %{ns} get gateway,httproute""")]

S[57] = """# Step 1: Remove stray blocking NetworkPolicy in kube-system
k -n kube-system get netpol
k -n kube-system delete netpol maintenance-lockdown

# Step 2: Fix CoreDNS replicas (scale back from 0)
k -n kube-system scale deploy coredns --replicas=2

# Step 3: Correct CoreDNS upstream forwarding in ConfigMap
k -n kube-system edit cm coredns
# Replace: forward . 10.255.255.254 -> forward . /etc/resolv.conf

# Step 4: Restart CoreDNS deployment and test DNS resolution
k -n kube-system rollout restart deploy coredns
# busybox's nslookup applies no search domains, so use FULL names (a short one fails even when DNS works)
k run dns-check --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.default.svc.cluster.local
k run dns-ext --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.io
"""


# ================================================================= STOR ====
# Domain 4: Storage (10%)
# ===========================================================================

S[15] = [hand(r"""vim pv.yaml""",
              r"""Paste the PersistentVolume and the PersistentVolumeClaim from the docs page *Configure a Pod to Use a PersistentVolume for Storage*, one after the other with `---` between them, and change them to:
```
apiVersion: v1
kind: PersistentVolume
metadata:
  name: pv-data
spec:
  storageClassName: manual
  capacity:
    storage: 1Gi
  accessModes:
  - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  hostPath:
    path: /mnt/data
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: pvc-data
  namespace: storage
spec:
  storageClassName: manual
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 500Mi
```""",
              run=r"""cat > pv.yaml <<'EOF'
apiVersion: v1
kind: PersistentVolume
metadata:
  name: pv-data
spec:
  storageClassName: manual
  capacity:
    storage: 1Gi
  accessModes:
  - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  hostPath:
    path: /mnt/data
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: pvc-data
  namespace: storage
spec:
  storageClassName: manual
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 500Mi
EOF"""),
         cmd(r"""k apply -f pv.yaml
k -n storage get pvc pvc-data
k -n storage run data-pod --image=nginx:1.25 --dry-run=client -o yaml > pod.yaml""", r"""`STATUS Bound`, `VOLUME pv-data`."""),
         hand(r"""vim pod.yaml""",
              r"""Add to the container, below `name: data-pod`:
```
    volumeMounts:
    - name: data
      mountPath: /usr/share/data
```
and to the Pod's `spec:`, right below `spec:`:
```
  volumes:
  - name: data
    persistentVolumeClaim:
      claimName: pvc-data
```""",
              run=r"""yq -i '.spec.containers[0].volumeMounts = [{"name": "data", "mountPath": "/usr/share/data"}]
  | .spec.volumes = [{"name": "data", "persistentVolumeClaim": {"claimName": "pvc-data"}}]' pod.yaml"""),
         cmd(r"""k apply -f pod.yaml
k -n storage get pod data-pod""")]

S[16] = [hand(r"""vim sc.yaml""",
              r"""Paste the example from the docs page *Storage Classes* and make it the default with the annotation (the docs page *Change the default StorageClass* names it):
```
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: fast-local
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
provisioner: rancher.io/local-path
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer
```""",
              run=r"""cat > sc.yaml <<'EOF'
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: fast-local
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
provisioner: rancher.io/local-path
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer
EOF"""),
         cmd(r"""k apply -f sc.yaml
k annotate sc standard storageclass.kubernetes.io/is-default-class-
k get sc""", r"""`(default)` beside `fast-local` alone. The `-` at the end removes the annotation."""),
         hand(r"""vim pvc.yaml""",
              r"""A claim with no `storageClassName`, so it gets the default (the docs page *Persistent Volumes* has a claim to start from):
```
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: fast-claim
  namespace: storage
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 200Mi
```""",
              run=r"""cat > pvc.yaml <<'EOF'
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: fast-claim
  namespace: storage
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 200Mi
EOF"""),
         cmd(r"""k apply -f pvc.yaml
k -n storage run fast-pod --image=nginx:1.25 --dry-run=client -o yaml > pod.yaml""", r"""The claim stays `Pending` until a Pod uses it: `WaitForFirstConsumer`."""),
         hand(r"""vim pod.yaml""",
              r"""Add to the container, below `name: fast-pod`:
```
    volumeMounts:
    - name: data
      mountPath: /data
```
and to the Pod's `spec:`, right below `spec:`:
```
  volumes:
  - name: data
    persistentVolumeClaim:
      claimName: fast-claim
```""",
              run=r"""yq -i '.spec.containers[0].volumeMounts = [{"name": "data", "mountPath": "/data"}]
  | .spec.volumes = [{"name": "data", "persistentVolumeClaim": {"claimName": "fast-claim"}}]' pod.yaml"""),
         cmd(r"""k apply -f pod.yaml
k -n storage get pvc,pod""", r"""Again until the claim is `Bound` and the Pod `Running`.""")]

S[35] = [cmd(r"""k get pv %{pv}
k get pv %{pv} -o yaml | grep -A6 claimRef""", r"""`Released`: the PV still names the deleted claim in `claimRef`, so no other claim can have it."""),
         hand(r"""k edit pv %{pv}""",
              r"""Delete the whole `claimRef:` block - `claimRef:` and the lines indented below it - then `:wq`.""",
              run=r"""kubectl patch pv %{pv} --type=json -p='[{"op":"remove","path":"/spec/claimRef"}]'"""),
         cmd(r"""k get pv %{pv}""", r"""`Available`."""),
         hand(r"""vim pvc.yaml""",
              r"""A claim that names the PV, so it binds to that one and no other:
```
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: %{pvc}
  namespace: %{ns}
spec:
  storageClassName: manual
  volumeName: %{pv}
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 500Mi
```""",
              run=r"""cat > pvc.yaml <<'EOF'
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: %{pvc}
  namespace: %{ns}
spec:
  storageClassName: manual
  volumeName: %{pv}
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 500Mi
EOF"""),
         cmd(r"""k apply -f pvc.yaml
k -n %{ns} get pvc %{pvc}""", r"""`Bound` to `%{pv}`.""")]

S[36] = [hand(r"""vim pvc.yaml""",
              r"""No `storageClassName`: the default class takes it.
```
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: %{pvc}
  namespace: %{ns}
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 200Mi
```""",
              run=r"""cat > pvc.yaml <<'EOF'
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: %{pvc}
  namespace: %{ns}
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 200Mi
EOF"""),
         cmd(r"""k apply -f pvc.yaml
k -n %{ns} run writer --image=busybox:1.36 --restart=Never --dry-run=client -o yaml -- sh -c "echo %{text} > /data/marker.txt" > writer.yaml"""),
         hand(r"""vim writer.yaml""",
              r"""Add to the container, below `name: writer`:
```
    volumeMounts:
    - name: data
      mountPath: /data
```
and to the Pod's `spec:`, right below `spec:`:
```
  volumes:
  - name: data
    persistentVolumeClaim:
      claimName: %{pvc}
```""",
              run=r"""yq -i '.spec.containers[0].volumeMounts = [{"name": "data", "mountPath": "/data"}]
  | .spec.volumes = [{"name": "data", "persistentVolumeClaim": {"claimName": "%{pvc}"}}]' writer.yaml"""),
         cmd(r"""k apply -f writer.yaml"""),
         hand(r"""k -n %{ns} get pod writer""",
              r"""Again until `Completed`.""",
              run=r"""kubectl -n %{ns} wait --for=jsonpath='{.status.phase}'=Succeeded pod/writer --timeout=120s"""),
         cmd(r"""k -n %{ns} run reader --image=busybox:1.36 --dry-run=client -o yaml -- sleep 3600 > reader.yaml"""),
         hand(r"""vim reader.yaml""",
              r"""The same two blocks as in `writer.yaml`, below `name: reader` and below `spec:`.""",
              run=r"""yq -i '.spec.containers[0].volumeMounts = [{"name": "data", "mountPath": "/data"}]
  | .spec.volumes = [{"name": "data", "persistentVolumeClaim": {"claimName": "%{pvc}"}}]' reader.yaml"""),
         cmd(r"""k apply -f reader.yaml
k -n %{ns} exec reader -- cat /data/marker.txt""", r"""Once it is `Running`: `%{text}`, written by a Pod that is gone.""")]

# Hard Storage Tasks
S[58] = [cmd(r"""k get pv --show-labels
k get node -l disk=nvme
k describe pv lv-nvme | grep -A3 'Node Affinity'""", r"""`lv-nvme` carries `media=nvme` and is pinned to the nvme node."""),
         hand(r"""vim pvc.yaml""",
              r"""A claim that selects the PV by its label:
```
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: %{pvc}
  namespace: %{ns}
spec:
  storageClassName: local-fast
  selector:
    matchLabels:
      media: nvme
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
```""",
              run=r"""cat > pvc.yaml <<'EOF'
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: %{pvc}
  namespace: %{ns}
spec:
  storageClassName: local-fast
  selector:
    matchLabels:
      media: nvme
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
EOF"""),
         cmd(r"""k apply -f pvc.yaml
k -n %{ns} run consumer --image=busybox:1.36 --dry-run=client -o yaml -- sleep 3600 > pod.yaml"""),
         hand(r"""vim pod.yaml""",
              r"""Add to the container, below `name: consumer`:
```
    volumeMounts:
    - name: data
      mountPath: /data
```
and to the Pod's `spec:`, right below `spec:`:
```
  volumes:
  - name: data
    persistentVolumeClaim:
      claimName: %{pvc}
```
No node selector: the PV's node affinity takes the Pod to the nvme node.""",
              run=r"""yq -i '.spec.containers[0].volumeMounts = [{"name": "data", "mountPath": "/data"}]
  | .spec.volumes = [{"name": "data", "persistentVolumeClaim": {"claimName": "%{pvc}"}}]' pod.yaml"""),
         cmd(r"""k apply -f pod.yaml
k -n %{ns} get pvc,pod -o wide""", r"""The claim `Bound` to `lv-nvme`, and the Pod on the nvme node.""")]

S[59] = [hand(r"""vim new.yaml""",
              r"""The new claim - no `storageClassName`, so the default class:
```
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: new-data
  namespace: %{ns}
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
```""",
              run=r"""cat > new.yaml <<'EOF'
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: new-data
  namespace: %{ns}
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
EOF"""),
         cmd(r"""k apply -f new.yaml
k -n %{ns} create job migrate --image=busybox:1.36 --dry-run=client -o yaml -- sh -c "cp -a /src/. /dst/ && ls /dst" > job.yaml"""),
         hand(r"""vim job.yaml""",
              r"""Both claims in the Job's Pod: add to the container, below `name: migrate`,
```
        volumeMounts:
        - name: src
          mountPath: /src
        - name: dst
          mountPath: /dst
```
and to the Pod template's `spec:`, beside `restartPolicy: Never`:
```
      volumes:
      - name: src
        persistentVolumeClaim:
          claimName: old-data
      - name: dst
        persistentVolumeClaim:
          claimName: new-data
```""",
              run=r"""yq -i '.spec.template.spec.containers[0].volumeMounts = [{"name": "src", "mountPath": "/src"}, {"name": "dst", "mountPath": "/dst"}]
  | .spec.template.spec.volumes = [{"name": "src", "persistentVolumeClaim": {"claimName": "old-data"}}, {"name": "dst", "persistentVolumeClaim": {"claimName": "new-data"}}]' job.yaml"""),
         cmd(r"""k apply -f job.yaml"""),
         hand(r"""k -n %{ns} get job migrate""",
              r"""Again until `COMPLETIONS 1/1`; `k -n %{ns} logs job/migrate` lists the files copied.""",
              run=r"""kubectl -n %{ns} wait --for=condition=complete job/migrate --timeout=240s"""),
         cmd(r"""k get pv | grep old-data
k -n %{ns} get pod legacy -o yaml > legacy.yaml""", r"""`RECLAIM POLICY Retain`: deleting the claim releases the PV and keeps it."""),
         hand(r"""vim legacy.yaml""",
              r"""In `volumes:`, change `claimName: old-data` to `claimName: new-data`.""",
              run=r"""yq -i '(.spec.volumes[] | select(.persistentVolumeClaim.claimName == "old-data") | .persistentVolumeClaim.claimName) = "new-data"' legacy.yaml"""),
         cmd(r"""k replace --force -f legacy.yaml""", r"""A Pod's volumes cannot change: `--force` deletes it and creates it again from the file."""),
         cmd(r"""k -n %{ns} delete pod -l job-name=migrate
k -n %{ns} delete pvc old-data
k get pv | grep old-data
k -n %{ns} get pod legacy""", r"""The Job's finished Pod still holds `old-data` (pvc-protection): delete it first, or `delete pvc` waits forever. The PV is `Released`; the Job stays, still `1/1`.""")]


# =============================================================== TSHOOT ====
# Domain 5: Troubleshooting (30%)
# ===========================================================================

S[17] = """# Step 1: Check node status
k get node {W2}                       # Status: NotReady

# Step 2: SSH into the broken worker node
{SHELL_W2}
sudo -i

# Step 3: Check kubelet service status and journal logs
systemctl status kubelet              # Service inactive/disabled
systemctl enable --now kubelet

# Step 4: Verify kubelet successfully connects and registers node
journalctl -u kubelet -f --no-pager   # Look for: "Successfully registered node"

# Step 5: Exit node shell and verify Ready state from cluster
# k get nodes
"""

S[18] = [cmd(r"""k -n troubleshoot get pods
k -n troubleshoot describe deploy payments | grep -i image""", r"""`ImagePullBackOff`: the tag `nginx:1.99-nope` does not exist."""),
         cmd(r"""k -n troubleshoot set image deploy payments payments=nginx:1.27
k -n troubleshoot get svc payments -o yaml
k -n troubleshoot get pods --show-labels""", r"""The Service selects `app: payment`, the Pods are `app=payments`; and it sends to `targetPort: 8080`, where nginx listens on 80."""),
         hand(r"""k -n troubleshoot edit svc payments""",
              r"""Fix both, then `:wq`:
```
  ports:
  - port: 80
    protocol: TCP
    targetPort: 80
  selector:
    app: payments
```""",
              run=r"""kubectl -n troubleshoot patch svc payments -p '{"spec":{"selector":{"app":"payments"},"ports":[{"port":80,"targetPort":80}]}}'"""),
         cmd(r"""k -n troubleshoot rollout status deploy payments
k -n troubleshoot get endpointslices -l kubernetes.io/service-name=payments""")]

S[19] = """# Step 1: Capture previous container logs to locate crash reason
k -n troubleshoot logs crasher --previous | tee {WS}/q19/crasher.log
# Log output: "FATAL: required environment variable DB_HOST is not set"

# Step 2: Export pod manifest
k -n troubleshoot get pod crasher -o yaml > crasher.yaml

# Step 3: Edit crasher.yaml:
# Add under spec.containers[0]:
#   env:
#     - name: DB_HOST
#       value: db.internal
# (Remove system-managed fields: status, uid, resourceVersion, creationTimestamp)

# Step 4: Recreate pod and verify Running status
k -n troubleshoot delete pod crasher --now && k apply -f crasher.yaml
k -n troubleshoot get pod crasher
"""

S[20] = """# Step 1: Check control-plane pods in kube-system
k -n kube-system get pods
# kube-scheduler static pod is in Error / CrashLoopBackOff

# Step 2: Inspect scheduler pod logs
k -n kube-system logs kube-scheduler-{CP}
# Error: exec: "kube-schedulerr": executable file not found

# Step 3: SSH into control plane node
sudo -i                # the task's host is the control-plane node: no hop needed

# Step 4: Edit scheduler static pod manifest
# In /etc/kubernetes/manifests/kube-scheduler.yaml:
# Change: command: [kube-schedulerr] -> command: [kube-scheduler]
vi /etc/kubernetes/manifests/kube-scheduler.yaml

# Step 5: Wait for kubelet to reload the static pod and verify scheduling
k -n kube-system get pods -l component=kube-scheduler
"""

S[37] = """# Step 1: Inspect CoreDNS pods and logs
k -n kube-system get pods -l k8s-app=kube-dns
k -n kube-system logs -l k8s-app=kube-dns --tail=20

# Step 2: Check CoreDNS ConfigMap
k -n kube-system get cm coredns -o yaml
# Fault: forward . 10.255.255.1 points to an invalid upstream DNS

# Step 3: Edit ConfigMap to use host resolv.conf
k -n kube-system edit cm coredns
# Change: forward . 10.255.255.1 -> forward . /etc/resolv.conf

# Step 4: Restart CoreDNS deployment to apply changes
k -n kube-system rollout restart deploy coredns

# Step 5: Verify DNS resolution from a test pod
# busybox's nslookup applies no search domains, so use FULL names (a short one fails even when DNS works)
k run test-dns --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.default.svc.cluster.local
k run test-ext --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.io
"""

S[38] = """# Step 1: Check control-plane pod status
k -n kube-system get pods
# kube-controller-manager is crashing

# Step 2: Check controller-manager logs
k -n kube-system logs kube-controller-manager-{CP}
# Error: invalid flag: --leader-elect-lease-duration=1s

# Step 3: Access control plane node
sudo -i                # the task's host is the control-plane node: no hop needed

# Step 4: Edit static pod manifest to remove the invalid flag
vi /etc/kubernetes/manifests/kube-controller-manager.yaml

# Step 5: Verify controller-manager restarts and reconciles deployments/replicasets
k create deploy probe --image=busybox:1.36 -- sleep 60
k get rs
k delete deploy probe
"""

S[39] = [hand(r"""{SHELL_W1}
sudo -i
systemctl status kubelet
journalctl -u kubelet -n 20 --no-pager""",
              r"""The kubelet keeps failing on `open /etc/kubernetes/pki/ca-bundle.crt: no such file or directory`.""",
              run=r""""""),
         hand(r"""vim /var/lib/kubelet/config.yaml""",
              r"""Point `clientCAFile` back at the CA that exists, then `:wq`:
```
    clientCAFile: /etc/kubernetes/pki/ca.crt
```""",
              run=r"""ssh -n {W1} 'sudo sed -i "s|ca-bundle.crt|ca.crt|" /var/lib/kubelet/config.yaml'"""),
         hand(r"""systemctl restart kubelet
systemctl enable kubelet
systemctl status kubelet
exit
exit""",
              r"""`active (running)` and `enabled`; then back on the host.""",
              run=r"""ssh -n {W1} 'sudo systemctl restart kubelet; sudo systemctl enable kubelet'"""),
         cmd(r"""k get nodes""", r"""Again until `{W1}` is `Ready`.""")]

# Hard Troubleshooting Tasks
S[60] = [cmd(r"""k get node {W2}""", r"""`NotReady,SchedulingDisabled`: the kubelet is down, and the node cordoned."""),
         hand(r"""{SHELL_W2}
sudo -i
systemctl status kubelet
systemctl is-enabled kubelet""",
              r"""`inactive (dead)` and `disabled`.""",
              run=r""""""),
         hand(r"""systemctl start kubelet
journalctl -u kubelet -n 20 --no-pager""",
              r"""It fails on `open /etc/kubernetes/pki/ca-wrong.crt: no such file or directory`.""",
              run=r""""""),
         hand(r"""vim /var/lib/kubelet/config.yaml""",
              r"""Point `clientCAFile` at the CA that exists, then `:wq`:
```
    clientCAFile: /etc/kubernetes/pki/ca.crt
```""",
              run=r"""ssh -n {W2} 'sudo sed -i "s|ca-wrong.crt|ca.crt|" /var/lib/kubelet/config.yaml'"""),
         hand(r"""systemctl enable --now kubelet
systemctl restart kubelet
systemctl status kubelet
exit
exit""",
              r"""`active (running)`, `enabled`; then back on the host.""",
              run=r"""ssh -n {W2} 'sudo systemctl enable --now kubelet; sudo systemctl restart kubelet'"""),
         cmd(r"""k uncordon {W2}
k get node {W2}""", r"""Again until `Ready`, without `SchedulingDisabled`."""),
         cmd(r"""k run test-pod --image=busybox:1.36 --dry-run=client -o yaml -- sleep 60 > t.yaml"""),
         hand(r"""vim t.yaml""",
              r"""Pin it: in the Pod's `spec:`, right below `spec:`,
```
  nodeName: {W2}
```""",
              run=r"""yq -i '.spec.nodeName = "{W2}"' t.yaml"""),
         cmd(r"""k apply -f t.yaml
k get pod test-pod -o wide""", r"""Again until `Running` on `{W2}`, then clean up: `k delete pod test-pod`.""")]

S[61] = """# Step 1: Check control plane pods in kube-system
k -n kube-system get pods
# scheduler CrashLoop, controller-manager ImagePullBackOff

# Step 2: Access control plane node
sudo -i                # the task's host is the control-plane node: no hop needed

# Step 3: Fix kube-scheduler manifest command typo
vi /etc/kubernetes/manifests/kube-scheduler.yaml
# Replace: kube-schedulerr -> kube-scheduler

# Step 4: Fix kube-controller-manager image tag
vi /etc/kubernetes/manifests/kube-controller-manager.yaml
# Remove broken image tag suffix to match cluster version

# Step 5: Verify both static pods return to Running
k -n kube-system get pods -w
k create deploy probe --image=busybox:1.36 -- sleep 60
k get deploy probe
k delete deploy probe
"""

S[62] = [cmd(r"""k -n %{ns} get all
k -n %{ns} describe rs -l app=%{dep} | grep -i quota
k -n %{ns} describe pod -l app=%{dep} | grep -e Image: -e Error -e Warning""", r"""Four faults: the quota allows 1 Pod; the image `nginx:1.27-nope` does not exist; the env var wants key `DATABASE_URL`, which Secret `%{dep}-env` lacks; and the Service (next) selects the wrong label on the wrong port."""),
         hand(r"""k -n %{ns} edit quota tight""",
              r"""Room for the rolling update, not only the end state: 3 replicas plus the one extra Pod a rollout creates (maxSurge 25% of 3 rounds up to 1; maxUnavailable rounds down to 0). With `pods: "3"` the rollout deadlocks. Set it, then `:wq`:
```
spec:
  hard:
    pods: "4"
```""",
              run=r"""kubectl -n %{ns} patch quota tight -p '{"spec":{"hard":{"pods":"4"}}}'"""),
         cmd(r"""k -n %{ns} create secret generic %{dep}-env --from-literal=API_KEY=abc123 --from-literal=DATABASE_URL=postgres://db --dry-run=client -o yaml | k apply -f -
k -n %{ns} set image deploy %{dep} web=nginx:1.27""", r"""The Secret again, with its old key and the missing one: generated and applied over the old one, no base64 by hand."""),
         cmd(r"""k -n %{ns} get svc %{dep} -o yaml
k -n %{ns} get pods --show-labels""", r"""The Service selects `app=%{dep}-web`, the Pods are `app=%{dep}`; and it sends to 8080, where nginx listens on 80."""),
         hand(r"""k -n %{ns} edit svc %{dep}""",
              r"""Fix both, then `:wq` - an edit keeps the ClusterIP:
```
  ports:
  - port: 80
    protocol: TCP
    targetPort: 80
  selector:
    app: %{dep}
```""",
              run=r"""kubectl -n %{ns} patch svc %{dep} -p '{"spec":{"selector":{"app":"%{dep}"},"ports":[{"port":80,"targetPort":80}]}}'"""),
         cmd(r"""k -n %{ns} rollout status deploy %{dep}
k -n %{ns} get endpointslices -l kubernetes.io/service-name=%{dep}""")]

S[63] = [cmd(r"""k -n kube-system describe pod -l component=kube-controller-manager | grep cluster-cidr
k -n kube-system describe pod -l component=kube-apiserver | grep service-cluster-ip-range
k -n kube-system get svc kube-dns
k -n kube-system get cm kube-proxy -o yaml | grep mode
k get node %{node} -o yaml | grep podCIDR""", r"""Each value from where the task says: the controller manager's and the API server's flags, the Service, the ConfigMap, the node."""),
         hand(r"""vim {WS}/q63/facts.txt""",
              r"""With the values they printed - for example:
```
pod-cidr=10.244.0.0/16
service-cidr=10.96.0.0/16
dns-ip=10.96.0.10
proxy-mode=iptables
%{node}-pod-cidr=10.244.2.0/24
```
`mode: ""` in the ConfigMap means the default: write `iptables`.""",
              run=r"""pc=$(kubectl -n kube-system get pod -l component=kube-controller-manager -o jsonpath='{.items[0].spec.containers[0].command}' | grep -o 'cluster-cidr=[^,"]*' | cut -d= -f2)
sc=$(kubectl -n kube-system get pod -l component=kube-apiserver -o jsonpath='{.items[0].spec.containers[0].command}' | grep -o 'service-cluster-ip-range=[^,"]*' | cut -d= -f2)
d=$(kubectl -n kube-system get svc kube-dns -o jsonpath='{.spec.clusterIP}')
m=$(kubectl -n kube-system get cm kube-proxy -o jsonpath='{.data.config\.conf}' | grep '^mode:' | awk '{print $2}' | tr -d '"')
printf 'pod-cidr=%s\nservice-cidr=%s\ndns-ip=%s\nproxy-mode=%s\n%s-pod-cidr=%s\n' "$pc" "$sc" "$d" "${m:-iptables}" "%{node}" "$(kubectl get node %{node} -o jsonpath='{.spec.podCIDR}')" > {WS}/q63/facts.txt""")]

S[66] = [cmd(r"""k get nodes""", r"""Refused: the API server itself is down, so `k` cannot help - look at the containers on the node."""),
         cmd(r"""sudo crictl ps -a | grep apiserver
sudo journalctl -u kubelet -n 50 --no-pager | grep -i apiserver""", r"""The host is the control-plane node. The container never starts: its command `kube-apiserverr` is not an executable."""),
         hand(r"""sudo vim /etc/kubernetes/manifests/kube-apiserver.yaml""",
              r"""The first line of `command:` reads `- kube-apiserverr`: make it `- kube-apiserver`, then `:wq`. The kubelet starts it again within a minute.""",
              run=r"""cpwait() {
  local i
  for i in $(seq 1 80); do
    [ "$(kubectl -n kube-system get pod $1-{CP} -o jsonpath='{.status.containerStatuses[0].ready}' 2>/dev/null)" = true ] && return 0
    sleep 3
  done
  return 1
}
sudo sed -i 's/kube-apiserverr/kube-apiserver/' /etc/kubernetes/manifests/kube-apiserver.yaml
for i in $(seq 1 80); do kubectl get nodes >/dev/null 2>&1 && break; sleep 3; done
for c in etcd kube-apiserver kube-scheduler kube-controller-manager; do cpwait $c; done"""),
         cmd(r"""k get nodes
k -n kube-system get pods""", r"""Again until `k` answers, and every control-plane Pod is `Running`.""")]