cka-sim/tracks/cka/solutions.py

"""Reference solutions, keyed by task id. Rendered with the same %{param} / {WS} / {W1} substitution
as the task text. Shown only after a task has been graded (drills) or the session submitted (exams).
These solutions provide step-by-step instructions, Kubernetes concept explanations, imperative commands,
and verification checks for exam success.
"""

S = {}

# ================================================================= ARCH ====
# Domain 1: Cluster Architecture, Installation & Configuration (25%)
# ===========================================================================

S[1] = """# Step 1: Create the ServiceAccount in the target namespace
k -n rbac-app create sa deploy-bot

# Step 2: Create a Role granting specified verbs on deployments.apps
# Verbs: get, list, watch, create, update, patch, delete
k -n rbac-app create role deploy-manager \\
  --verb=get,list,watch,create,update,patch,delete \\
  --resource=deployments.apps

# Step 3: Bind the Role to the ServiceAccount
k -n rbac-app create rolebinding deploy-bot-binding \\
  --role=deploy-manager \\
  --serviceaccount=rbac-app:deploy-bot

# Step 4: Verify permissions using 'kubectl auth can-i'
k auth can-i create deployments -n rbac-app --as=system:serviceaccount:rbac-app:deploy-bot   # Expected: yes
k auth can-i delete pods -n rbac-app --as=system:serviceaccount:rbac-app:deploy-bot          # Expected: no
"""

S[2] = """# Step 1: Access the control plane node
{SHELL_CP}

# Step 2: Take the etcd snapshot with TLS certificates and save to the target path
# Certificates are located in /etc/kubernetes/pki/etcd/
ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 \\
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \\
  --cert=/etc/kubernetes/pki/etcd/server.crt \\
  --key=/etc/kubernetes/pki/etcd/server.key \\
  snapshot save /opt/cka/etcd-snapshot.db

# Step 3: Verify snapshot status and integrity
etcdutl snapshot status /opt/cka/etcd-snapshot.db -w table

# Note: on a cluster where etcdctl is only inside the static pod, run it there. The etcd image
# is distroless (no sh, no cat, no tar), so kubectl cp cannot get the file out - write it into
# /var/lib/etcd instead, which is a hostPath, and move it on the node:
# kubectl -n kube-system exec etcd-{CP} -- etcdctl ... snapshot save /var/lib/etcd/snap.db
# then on the node: mv /var/lib/etcd/snap.db /opt/cka/etcd-snapshot.db
# ./etcd-tools.sh installs etcdctl/etcdutl on the control-plane node (setup.sh already does).
"""

S[3] = """# Step 1: Create the overlay directory structure
mkdir -p {WS}/q03/overlays/prod && cd {WS}/q03/overlays/prod

# Step 2: Define the target namespace manifest
cat > namespace.yaml <<EOF
apiVersion: v1
kind: Namespace
metadata: {name: store-prod}
EOF

# Step 3: Write the kustomization.yaml referencing the base and adding namespace, labels, and replicas
cat > kustomization.yaml <<EOF
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: store-prod
resources: [../../base, namespace.yaml]
labels:
  - pairs: {env: prod}
    includeSelectors: false
replicas:
  - name: store-api
    count: 3
EOF

# Step 4: Build and apply the overlay
kubectl apply -k .

# Step 5: Verify resources in the store-prod namespace
k -n store-prod get deploy,pod -l env=prod
"""

S[4] = """# Step 1: Define and create the CustomResourceDefinition (CRD)
cat <<EOF | k apply -f -
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata: {name: backups.data.example.com}
spec:
  group: data.example.com
  scope: Namespaced
  names: {kind: Backup, plural: backups, singular: backup, shortNames: [bk]}
  versions:
    - name: v1
      served: true
      storage: true
      schema:
        openAPIV3Schema:
          type: object
          properties:
            spec:
              type: object
              properties:
                schedule: {type: string}
                target: {type: string}
EOF

# Step 2: Create a custom resource instance of Backup
cat <<EOF | k apply -f -
apiVersion: data.example.com/v1
kind: Backup
metadata: {name: nightly, namespace: default}
spec: {schedule: "0 2 * * *", target: "s3://backups/prod"}
EOF

# Step 3: Verify using the shortname
k get bk
"""

S[5] = """# Step 1: Access the specified worker node
{SHELL_W3}

# Step 2: Verify the kubelet static pod path
grep staticPodPath /var/lib/kubelet/config.yaml        # Default: /etc/kubernetes/manifests

# Step 3: Create the static pod manifest in the static pod directory
cat > /etc/kubernetes/manifests/static-web.yaml <<EOF
apiVersion: v1
kind: Pod
metadata: {name: static-web, namespace: default}
spec:
  containers: [{name: web, image: nginx:1.27}]
EOF

# Step 4: Verify the mirror pod is created and running on the node
# kubelet automatically detects new manifests in the folder within seconds.
# Back on your main terminal: k get pods -o wide | grep static-web
"""

S[6] = """# Part A: Check certificate expiration on control plane
# {WS} is a path on your own machine, so run the remote check and redirect its
# output locally - don't try to write {WS} from inside the node's own shell.
podman exec cka-sim-control-plane kubeadm certs check-expiration | grep '^apiserver ' > {WS}/q06/apiserver-expiry.txt

# Part B: Issue and approve CertificateSigningRequest (CSR) for user Jane
# Step 3: Generate private key and CSR
openssl genrsa -out jane.key 2048
openssl req -new -key jane.key -subj "/CN=jane" -out jane.csr

# Step 4: Submit Kubernetes CSR resource
cat <<EOF | k apply -f -
apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata: {name: jane}
spec:
  request: $(base64 -w0 jane.csr)
  signerName: kubernetes.io/kube-apiserver-client
  usages: [client auth]
EOF

# Step 5: Approve CSR and extract signed certificate
k certificate approve jane
k get csr jane -o jsonpath='{.status.certificate}' | base64 -d > jane.crt

# Step 6: Bind user Jane to the default cluster role 'view'
k -n default create rolebinding jane-view --clusterrole=view --user=jane
"""

S[21] = """# Step 1: Create the target namespace
k create ns %{ns}

# Step 2: Install the Helm chart with overridden replicaCount
helm install %{rel} {WS}/q21/chart -n %{ns} --set replicaCount=%{n}

# Step 3: Upgrade the release to NodePort while preserving existing values
helm upgrade %{rel} {WS}/q21/chart -n %{ns} --reuse-values --set service.type=NodePort

# Step 4: Verify the release history shows revision 2
helm -n %{ns} history %{rel}
"""

S[33] = """# Step 1: Create the aggregating ClusterRole with aggregationRule selector
cat <<EOF | k apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata: {name: %{cr}}
aggregationRule:
  clusterRoleSelectors:
    - matchLabels: {rbac.example.com/aggregate-to-%{cr}: "true"}
rules: []
EOF

# Step 2: Create sub-ClusterRoles for pods and services
k create clusterrole %{cr}-pods --verb=get,list,watch --resource=pods
k create clusterrole %{cr}-services --verb=get,list,watch --resource=services

# Step 3: Apply the matching label so controller aggregates rules into the main ClusterRole
k label clusterrole %{cr}-pods %{cr}-services rbac.example.com/aggregate-to-%{cr}=true

# Step 4: Create ClusterRoleBinding for the user
k create clusterrolebinding %{cr}-binding --clusterrole=%{cr} --user=%{user}

# Step 5: Verify rules are automatically aggregated
k get clusterrole %{cr} -o yaml
"""

S[34] = """# Step 1: Inspect the CRD spec schema recursively to locate the field path
k explain %{kind}.spec --recursive

# Step 2: Check the exact field details and description
k explain %{kind}.spec.tuning.%{field}

# Step 3: Record the full field path to the output file
echo spec.tuning.%{field} > {WS}/q34/path.txt

# Step 4: Create a sample Custom Resource using the discovered field
cat <<EOF | k apply -f -
apiVersion: demo.io/v1
kind: %{kind}
metadata: {name: sample, namespace: default}
spec:
  tuning: {%{field}: %{n}}
EOF
"""

S[40] = """# Generate a new kubeadm join token with the specified TTL and print-join-command,
# writing the output straight into the required workspace file (run from the host, not the node,
# since {WS} is a path on your machine)
podman exec cka-sim-control-plane kubeadm token create --ttl %{ttl} --print-join-command > {WS}/q40/join.sh

# Verify active tokens
podman exec cka-sim-control-plane kubeadm token list
"""

S[41] = """# Step 1: Check Kubernetes Server version (API server)
k version

# Step 2: Extract Kubelet and Container Runtime versions for the target node
k get node %{node} -o jsonpath='{.status.nodeInfo.kubeletVersion}{"\\n"}{.status.nodeInfo.containerRuntimeVersion}{"\\n"}'

# Step 3: Extract etcd image tag from the static pod spec
k -n kube-system get pod -l component=etcd -o jsonpath='{.items[0].spec.containers[0].image}'
# Parse the image tag (everything after the colon)
"""

# Hard Architecture Tasks
S[44] = """# Step 1: Inspect bindings for the ServiceAccount to find and remove over-privileged bindings
k get clusterrolebinding -o wide | grep %{sa}
k delete clusterrolebinding %{sa}-legacy-admin

# Step 2: Create a restricted Role for deployments and replicasets
k -n %{ns} create role %{sa}-role --verb=get,list,watch,patch --resource=deployments.apps,replicasets.apps

# Step 3: Add pods/exec permission rule to the Role
k -n %{ns} patch role %{sa}-role --type=json -p='[{"op":"add","path":"/rules/-","value":{"apiGroups":[""],"resources":["pods/exec"],"verbs":["create"]}}]'

# Step 4: Bind the Role to the ServiceAccount
k -n %{ns} create rolebinding %{sa}-rb --role=%{sa}-role --serviceaccount=%{ns}:%{sa}

# Step 5: Create a ClusterRole with node list privileges and bind it
k create clusterrole %{sa}-nodes --verb=list --resource=nodes
k create clusterrolebinding %{sa}-crb --clusterrole=%{sa}-nodes --serviceaccount=%{ns}:%{sa}

# Step 6: Verify effective permissions. For a SUBRESOURCE use --subresource: `can-i create
# pods/exec` means "create the pod named exec" and says no even when the Role is right.
k auth can-i --list -n %{ns} --as=system:serviceaccount:%{ns}:%{sa}
k auth can-i create pods --subresource=exec -n %{ns} --as=system:serviceaccount:%{ns}:%{sa}   # yes
"""

S[45] = """# Step 1: Access the control plane node
{SHELL_CP}

# Step 2: Restore snapshot into a new data directory
etcdutl snapshot restore /opt/cka/backup.db --data-dir /var/lib/etcd-restore

# Step 3: Update etcd static pod hostPath volume to point to the restored directory
# Edit /etc/kubernetes/manifests/etcd.yaml:
# Change:
#   volumes:
#   - hostPath:
#       path: /var/lib/etcd-restore
#     name: etcd-data
vi /etc/kubernetes/manifests/etcd.yaml

# Step 4: Monitor static pod recreation and verify restored objects
watch crictl ps
kubectl -n default get cm %{cm}
"""

S[46] = """# Step 1: Identify the edge node matching the label selector
k get nodes -l tier=edge

# Step 2: Access the target node
{SHELL_W3}

# Step 3: Check kubelet configuration for custom staticPodPath
grep staticPodPath /var/lib/kubelet/config.yaml

# Step 4: Place the pod manifest directly into the configured staticPodPath
cat > <staticPodPath>/%{pod}.yaml <<EOF
apiVersion: v1
kind: Pod
metadata: {name: %{pod}, namespace: %{ns}, labels: {role: edge}}
spec:
  containers:
    - name: web
      image: nginx:1.27
      resources: {requests: {cpu: 50m, memory: 32Mi}}
EOF

# Step 5: Verify the mirror pod appears in the cluster
# k get pods -n %{ns} -o wide
"""

S[47] = """# Step 1: Generate user private key and Certificate Signing Request
openssl genrsa -out %{user}.key 2048
openssl req -new -key %{user}.key -subj "/CN=%{user}/O=developers" -out %{user}.csr

# Step 2: Create and approve the CSR in Kubernetes
k apply -f - <<EOF
apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata: {name: %{user}}
spec: {request: $(base64 -w0 %{user}.csr), signerName: kubernetes.io/kube-apiserver-client, usages: [client auth]}
EOF
k certificate approve %{user}
k get csr %{user} -o jsonpath='{.status.certificate}' | base64 -d > %{user}.crt

# Step 3: Grant permissions via RoleBinding
k -n %{ns} create rolebinding %{user}-edit --clusterrole=edit --user=%{user}

# Step 4: Construct self-contained kubeconfig with embedded certificates
KC={WS}/q47/%{user}.kubeconfig
k config view --minify --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}' | base64 -d > ca.crt
SERVER=$(k config view --minify -o jsonpath='{.clusters[0].cluster.server}')

kubectl --kubeconfig $KC config set-cluster lab --server=$SERVER --certificate-authority=ca.crt --embed-certs
kubectl --kubeconfig $KC config set-credentials %{user} --client-certificate=%{user}.crt --client-key=%{user}.key --embed-certs
kubectl --kubeconfig $KC config set-context %{user}@lab --cluster=lab --user=%{user} --namespace=%{ns}
kubectl --kubeconfig $KC config use-context %{user}@lab

# Step 5: Test the kubeconfig
kubectl --kubeconfig $KC get pods
"""

S[48] = """# Step 1: Create directory for prod overlay
mkdir -p {WS}/q48/overlays/prod && cd {WS}/q48/overlays/prod

# Step 2: Create replica patch manifest
cat > replicas.yaml <<EOF
apiVersion: apps/v1
kind: Deployment
metadata: {name: web}
spec: {replicas: %{rep}}
EOF

# Step 3: Create JSON patch for envFrom injection
cat > envfrom.yaml <<EOF
- op: add
  path: /spec/template/spec/containers/0/envFrom
  value: [{configMapRef: {name: web-config}}]
EOF

# Step 4: Write kustomization.yaml with generator, labels, and patches
cat > kustomization.yaml <<EOF
namespace: %{ns}
resources: [../../base]
labels:
  - pairs: {tier: prod}
    includeSelectors: false
configMapGenerator:
  - name: web-config
    literals: [LOG_LEVEL=%{lvl}]
patches:
  - path: replicas.yaml
  - path: envfrom.yaml
    target: {kind: Deployment, name: web}
EOF

# Step 5: Create namespace and deploy
k create ns %{ns}
k apply -k .
"""

S[49] = """# Step 0: This context does not default to the `default` namespace - check, then pass -n default
k config view --minify -o jsonpath='{..namespace}{"\\n"}'

# Step 1: CRD with validation (required fields, integer bounds, enum) and a server-side default
cat <<EOF | k apply -f -
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata: {name: $(echo %{kind} | tr A-Z a-z)s.deploy.example.com}
spec:
  group: deploy.example.com
  scope: Namespaced
  names:
    kind: %{kind}
    plural: $(echo %{kind} | tr A-Z a-z)s
    singular: $(echo %{kind} | tr A-Z a-z)
    shortNames: [$(echo %{kind} | tr A-Z a-z | cut -c1-2)]
  versions:
    - name: v1
      served: true
      storage: true
      schema:
        openAPIV3Schema:
          type: object
          properties:
            spec:
              type: object
              required: [replicas, image]          # sits NEXT TO properties, not inside a field
              properties:
                replicas: {type: integer, minimum: 1, maximum: 10}
                environment: {type: string, enum: [prod, stage, dev], default: dev}
                image: {type: string}
EOF

# Step 2: A valid object
k -n default apply -f - <<EOF
apiVersion: deploy.example.com/v1
kind: %{kind}
metadata: {name: good}
spec: {replicas: 3, image: nginx:1.27, environment: %{env}}
EOF

# Step 3: The rejection goes to STDERR - redirect 2>, not >, or the file stays empty
printf 'apiVersion: deploy.example.com/v1\\nkind: %{kind}\\nmetadata: {name: bad}\\nspec: {replicas: 0, image: x}\\n' | k -n default apply -f - 2> {WS}/q49/rejected.txt
cat {WS}/q49/rejected.txt          # ...spec.replicas in body should be greater than or equal to 1

# Step 4: Omit environment and let the API server fill in the default
printf 'apiVersion: deploy.example.com/v1\\nkind: %{kind}\\nmetadata: {name: defaulted}\\nspec: {replicas: 2, image: nginx:1.27}\\n' | k -n default apply -f -
k -n default get $(echo %{kind} | tr A-Z a-z) defaulted -o jsonpath='{.spec.environment}{"\\n"}'   # dev
"""

S[65] = """# Step 1: Access control plane node
{SHELL_CP}

# Step 2: Renew all cluster certificates via kubeadm
kubeadm certs renew all

# Step 3: Restart static pods to load renewed certificates
mv /etc/kubernetes/manifests/*.yaml /tmp/ && sleep 20 && mv /tmp/*.yaml /etc/kubernetes/manifests/

# Step 4: Update admin kubeconfig from newly generated credentials
cp /etc/kubernetes/admin.conf ~/.kube/config
exit

# Step 5: Back on your own machine now - {WS} is a path on your machine, not the node,
# so capture the expiry report from out here and redirect it into the required file
podman exec cka-sim-control-plane kubeadm certs check-expiration | grep '^apiserver ' > {WS}/q65/expiry.txt
"""

S[67] = """# Step 1: On the control-plane node, create a fresh bootstrap token
{SHELL_CP}
kubeadm token create --ttl 2h
# -> prints a token like abcdef.0123456789abcdef

# Step 2: Compute the CA discovery hash yourself. This is exactly what
# `kubeadm join --discovery-token-ca-cert-hash` verifies against, so you should never have to
# trust a value someone else hands you without being able to check it independently.
openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | \\
  openssl rsa -pubin -outform der 2>/dev/null | \\
  openssl dgst -sha256 -hex | sed 's/^.* //'

# Step 3: Find the real in-cluster API endpoint -- NOT what your local kubeconfig uses.
# kind maps the API server to 127.0.0.1:<random-host-port> for YOUR machine only; a node
# joining over the cluster network needs the address the apiserver actually advertises.
exit
kubectl get endpointslices -l kubernetes.io/service-name=kubernetes -o jsonpath='{.items[0].endpoints[0].addresses[0]}:{.items[0].ports[0].port}{"\\n"}'

# Step 4: Assemble the handoff file
cat > {WS}/q67/node-join.env <<EOF
JOIN_TOKEN=<token from step 1>
CA_HASH=sha256:<hash from step 2>
API_ENDPOINT=<ip:port from step 3>
EOF
"""


# ================================================================= WORK ====
# Domain 2: Workloads & Scheduling (15%)
# ===========================================================================

S[7] = """# Step 1: Update container image to nginx:1.27
k -n apps set image deploy web-frontend nginx=nginx:1.27

# Step 2: Scale deployment to 4 replicas
k -n apps scale deploy web-frontend --replicas=4

# Step 3: Patch rolling update strategy (maxSurge=1, maxUnavailable=0)
k -n apps patch deploy web-frontend -p '{"spec":{"strategy":{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0}}}}'

# Step 4: Monitor rollout until complete
k -n apps rollout status deploy web-frontend
"""

S[8] = """# Step 1: Create pod manifest with shared emptyDir volume and sidecar pattern
cat <<EOF | k apply -f -
apiVersion: v1
kind: Pod
metadata: {name: log-shipper, namespace: apps}
spec:
  volumes: [{name: logs, emptyDir: {}}]
  containers:
    - name: app
      image: busybox:1.36
      command: [sh, -c, "while true; do date >> /var/log/app.log; sleep 2; done"]
      volumeMounts: [{name: logs, mountPath: /var/log}]
    - name: shipper
      image: busybox:1.36
      command: [sh, -c, "tail -f /var/log/app.log"]
      volumeMounts: [{name: logs, mountPath: /var/log}]
EOF

# Step 2: Verify sidecar container reads the log
k -n apps logs log-shipper -c shipper
"""

S[9] = """# Step 1: Generate initial deployment manifest with dry-run
k -n apps create deploy cache --image=redis:7-alpine --replicas=2 $do > cache.yaml

# Step 2: Add nodeAffinity (disktype=ssd) and container resources (requests/limits)
# Under spec.template.spec:
#   affinity:
#     nodeAffinity:
#       requiredDuringSchedulingIgnoredDuringExecution:
#         nodeSelectorTerms:
#           - matchExpressions: [{key: disktype, operator: In, values: [ssd]}]
# Under containers[0]:
#   resources:
#     requests: {cpu: 100m, memory: 64Mi}
#     limits: {cpu: 200m, memory: 128Mi}
cat <<EOF | k apply -f -
apiVersion: apps/v1
kind: Deployment
metadata: {name: cache, namespace: apps}
spec:
  replicas: 2
  selector: {matchLabels: {app: cache}}
  template:
    metadata: {labels: {app: cache}}
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions: [{key: disktype, operator: In, values: [ssd]}]
      containers:
        - name: redis
          image: redis:7-alpine
          resources:
            requests: {cpu: 100m, memory: 64Mi}
            limits: {cpu: 200m, memory: 128Mi}
EOF

# Step 3: Verify scheduling on nodes labeled disktype=ssd
k -n apps get pods -o wide
"""

S[10] = """# Step 1: Create ConfigMap with literal values
k -n apps create cm app-config --from-literal=APP_MODE=production --from-literal=LOG_LEVEL=debug

# Step 2: Create Secret with credentials
k -n apps create secret generic db-creds --from-literal=username=admin --from-literal='password=S3cr3t!'

# Step 3: Create Pod mounting ConfigMap as volume, injecting env var from CM, and loading Secret via envFrom
cat <<EOF | k apply -f -
apiVersion: v1
kind: Pod
metadata: {name: config-pod, namespace: apps}
spec:
  volumes: [{name: cfg, configMap: {name: app-config}}]
  containers:
    - name: app
      image: busybox:1.36
      command: [sleep, "3600"]
      env: [{name: APP_MODE, valueFrom: {configMapKeyRef: {name: app-config, key: APP_MODE}}}]
      envFrom: [{secretRef: {name: db-creds}}]
      volumeMounts: [{name: cfg, mountPath: /etc/config}]
EOF

# Step 4: Verify mounted files and environment variables
k -n apps exec config-pod -- ls /etc/config
k -n apps exec config-pod -- env | grep -E 'APP_MODE|username'
"""

S[23] = """# Step 1: Repair metrics-server by adding --kubelet-insecure-tls flag
k -n kube-system edit deploy metrics-server
# Add under spec.template.spec.containers[0].args:
#   - --kubelet-insecure-tls

# Step 2: Wait ~30s and verify metrics reporting
k top nodes

# Step 3: Configure HorizontalPodAutoscaler (HPA) for target deployment
k -n %{ns} autoscale deploy %{dep} --min=2 --max=%{max} --cpu-percent=%{cpu} --name=%{dep}

# Step 4: Verify HPA status
k -n %{ns} get hpa %{dep}
"""

S[24] = """# Step 1: Create PriorityClass with requested integer value
cat <<EOF | k apply -f -
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata: {name: %{pc}}
value: %{val}
globalDefault: false
description: "%{pc} workloads"
EOF

# Step 2: Assign PriorityClass to the Deployment pod template
k -n %{ns} patch deploy payments -p '{"spec":{"template":{"spec":{"priorityClassName":"%{pc}"}}}}'

# Step 3: Monitor rollout
k -n %{ns} rollout status deploy payments
"""

S[25] = """# Step 1: Apply taint to target worker node
k taint node {W1} %{key}=%{val}:NoSchedule

# Step 2: Create Deployment targeting node with matching toleration
cat <<EOF | k apply -f -
apiVersion: apps/v1
kind: Deployment
metadata: {name: %{dep}, namespace: %{ns}}
spec:
  replicas: 2
  selector: {matchLabels: {app: %{dep}}}
  template:
    metadata: {labels: {app: %{dep}}}
    spec:
      nodeSelector:
        kubernetes.io/hostname: {W1}
      tolerations:
        - key: %{key}
          operator: Equal
          value: "%{val}"
          effect: NoSchedule
      containers:
        - name: busybox
          image: busybox:1.36
          command: [sleep, "3600"]
EOF

# Step 3: Verify pods are scheduled on {W1}
k -n %{ns} get pods -o wide
"""

S[26] = """# Step 1: Create DaemonSet manifest tolerating control-plane taints
cat <<EOF | k apply -f -
apiVersion: apps/v1
kind: DaemonSet
metadata: {name: %{ds}, namespace: %{ns}}
spec:
  selector: {matchLabels: {app: %{ds}}}
  template:
    metadata: {labels: {app: %{ds}}}
    spec:
      tolerations:
        - key: node-role.kubernetes.io/control-plane
          operator: Exists
          effect: NoSchedule
      containers:
        - name: agent
          image: busybox:1.36
          command: [sleep, "3600"]
          resources: {requests: {cpu: 10m, memory: 16Mi}}
EOF

# Step 2: Verify DaemonSet runs on all nodes including control plane
k -n %{ns} get ds %{ds}
k -n %{ns} get pods -o wide -l app=%{ds}
"""

S[27] = """# Step 1: Create CronJob with specified schedule and history limits
k -n %{ns} create cj %{cj} --image=busybox:1.36 --schedule='*/%{every} * * * *' $do -- date > cj.yaml

# Step 2: Add concurrencyPolicy, activeDeadlineSeconds, and history limits:
# spec:
#   concurrencyPolicy: Forbid
#   successfulJobsHistoryLimit: 3
#   failedJobsHistoryLimit: 1
#   jobTemplate:
#     spec:
#       activeDeadlineSeconds: 30
#       template:
#         spec:
#           restartPolicy: Never
cat <<EOF | k apply -f -
apiVersion: batch/v1
kind: CronJob
metadata: {name: %{cj}, namespace: %{ns}}
spec:
  schedule: '*/%{every} * * * *'
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 1
  jobTemplate:
    spec:
      activeDeadlineSeconds: 30
      template:
        spec:
          restartPolicy: Never
          containers:
            - name: date
              image: busybox:1.36
              command: ["date"]
EOF

# Step 3: Trigger a manual job run from the CronJob
k -n %{ns} create job %{cj}-manual --from=cronjob/%{cj}
"""

S[28] = """# Step 1: Create Pod with initContainer populating a shared volume
cat <<EOF | k apply -f -
apiVersion: v1
kind: Pod
metadata: {name: %{pod}, namespace: %{ns}}
spec:
  volumes: [{name: content, emptyDir: {}}]
  initContainers:
    - name: setup
      image: busybox:1.36
      command: [sh, -c, "echo '%{text}' > /work/index.html"]
      volumeMounts: [{name: content, mountPath: /work}]
  containers:
    - name: web
      image: nginx:1.25
      volumeMounts: [{name: content, mountPath: /usr/share/nginx/html}]
EOF

# Step 2: Verify web server serves the file created by init container
k -n %{ns} exec %{pod} -c web -- cat /usr/share/nginx/html/index.html
"""

S[29] = """# Step 1: Create the new version of the immutable ConfigMap
k -n %{ns} create cm %{cm}-v2 --from-literal=mode=new
k -n %{ns} patch cm %{cm}-v2 -p '{"immutable":true}'

# Step 2: Update the Deployment volume reference to the new ConfigMap
k -n %{ns} patch deploy %{dep} --type=json -p='[{"op":"replace","path":"/spec/template/spec/volumes/0/configMap/name","value":"%{cm}-v2"}]'

# Step 3: Verify rolling update finishes
k -n %{ns} rollout status deploy %{dep}
"""

S[31] = """# Step 1: Create Job with completions, parallelism, and backoffLimit
cat <<EOF | k apply -f -
apiVersion: batch/v1
kind: Job
metadata: {name: %{job}, namespace: %{ns}}
spec:
  completions: %{c}
  parallelism: %{p}
  backoffLimit: 2
  template:
    spec:
      restartPolicy: Never
      containers:
        - name: runner
          image: busybox:1.36
          command: ["sh", "-c", "echo done; sleep 2"]
EOF

# Step 2: Wait for Job completion
k -n %{ns} wait --for=condition=complete job/%{job} --timeout=120s
"""

S[32] = """# Step 1: Create ResourceQuota for the namespace
k -n %{ns} create quota %{ns}-quota --hard=pods=%{pods},requests.cpu=1,limits.memory=2Gi

# Step 2: Create LimitRange defining default container requests and limits
cat <<EOF | k apply -f -
apiVersion: v1
kind: LimitRange
metadata: {name: %{ns}-defaults, namespace: %{ns}}
spec:
  limits:
    - type: Container
      default: {cpu: 500m, memory: 256Mi}
      defaultRequest: {cpu: %{dcpu}, memory: 64Mi}
EOF

# Step 3: Create the Deployment without specifying resources - it picks up the LimitRange defaults
k -n %{ns} create deploy worker --image=nginx:1.25 --replicas=2

# Step 4: Verify quota, limits and the defaulted pod resources
k -n %{ns} describe quota %{ns}-quota
k -n %{ns} describe limitrange %{ns}-defaults
k -n %{ns} get pod -l app=worker -o jsonpath='{.items[0].spec.containers[0].resources}'
"""

# Hard Workload Tasks
S[50] = """# Step 1: Read the constraints before writing anything
k get node {W1} -o jsonpath='{.spec.taints}{"\\n"}'      # %{key}=%{val}:NoSchedule
k -n %{ns} describe quota                               # pods: 2, requests.cpu: 250m

# Step 2: The quota is the trap twice over: 3 pods do not fit "pods: 2", and 3 x 100m = 300m does
# not fit 250m either. Raise both - the quota has to stay and still cap pods.
k -n %{ns} patch quota %{ns}-quota -p '{"spec":{"hard":{"pods":"3","requests.cpu":"300m"}}}'

# Step 3: The PriorityClass
k create priorityclass %{dep}-prio --value=50000

# Step 4: One pod per worker (required anti-affinity on hostname) that tolerates {W1}'s taint
cat <<EOF | k apply -f -
apiVersion: apps/v1
kind: Deployment
metadata: {name: %{dep}, namespace: %{ns}}
spec:
  replicas: 3
  selector: {matchLabels: {app: %{dep}}}
  template:
    metadata: {labels: {app: %{dep}}}
    spec:
      priorityClassName: %{dep}-prio
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector: {matchLabels: {app: %{dep}}}
              topologyKey: kubernetes.io/hostname
      tolerations:
        - {key: %{key}, operator: Equal, value: "%{val}", effect: NoSchedule}
      containers:
        - name: nginx
          image: nginx:1.27
          resources: {requests: {cpu: 100m, memory: 64Mi}}
EOF

# Step 5: Verify: 3 Running on 3 different nodes. The control-plane is tainted too, so with three
# workers there is zero slack - a missed quota field or toleration shows up as a Pending pod, and
# `k -n %{ns} describe rs` tells you which (FailedCreate: exceeded quota).
k -n %{ns} get pod -l app=%{dep} -o wide
k -n %{ns} describe quota %{ns}-quota      # pods 3/3, requests.cpu 300m/300m
"""

S[51] = """# Step 1: Find WHICH revision used nginx:1.25 - the task asks for that one specifically, which
# is not necessarily the previous one. --revision=N shows a single revision's pod template.
k -n %{ns} rollout history deploy %{dep}
for r in $(k -n %{ns} rollout history deploy %{dep} | awk '/^[0-9]/{print $1}'); do
  echo "revision $r: $(k -n %{ns} rollout history deploy %{dep} --revision=$r | grep -i 'image:')"
done

# Step 2: Roll back to that revision (substitute the number you found for N)
k -n %{ns} rollout undo deploy %{dep} --to-revision=N

# Step 3: Scale to 3 and wait until they are actually available
k -n %{ns} scale deploy %{dep} --replicas=3
k -n %{ns} rollout status deploy %{dep}

# Step 4: Pause BEFORE editing. While paused the controller does not reconcile the template, so
# no new ReplicaSet is created and the running pods stay on nginx:1.25 - that gap between
# spec.template and what is running is exactly what the task checks.
k -n %{ns} rollout pause deploy %{dep}
k -n %{ns} set image deploy/%{dep} $(k -n %{ns} get deploy %{dep} -o jsonpath='{.spec.template.spec.containers[0].name}')=nginx:1.27
k -n %{ns} set env deploy/%{dep} RELEASE=next

# Step 5: Verify - paused true, template 1.27, pods still 1.25, and no RS for the new template
k -n %{ns} get deploy %{dep} -o jsonpath='{.spec.paused}{"\\n"}'
k -n %{ns} get pod -l app=%{dep} -o jsonpath='{.items[*].spec.containers[0].image}{"\\n"}'
k -n %{ns} get rs -o custom-columns=NAME:.metadata.name,IMAGE:.spec.template.spec.containers[0].image
"""

S[52] = """# Step 1: Deploy Pod with a native sidecar (restartPolicy: Always inside initContainers),
# a shared emptyDir the sidecar writes into and the main container reads from. Mount it at
# /var/log/agent, NOT /var/log: an emptyDir over /var/log hides nginx's /var/log/nginx and
# nginx crash-loops with "could not open error log file"
cat <<EOF | k apply -f -
apiVersion: v1
kind: Pod
metadata: {name: %{pod}, namespace: %{ns}}
spec:
  terminationGracePeriodSeconds: 5
  volumes:
    - name: logs
      emptyDir: {}
  initContainers:
    - name: agent
      image: busybox:1.36
      restartPolicy: Always
      command: [sh, -c, "while true; do date >> /var/log/agent/agent.log; sleep 1; done"]
      volumeMounts:
        - {name: logs, mountPath: /var/log/agent}
  containers:
    - name: app
      image: nginx:1.27
      readinessProbe:
        httpGet: {path: /, port: 80}
        periodSeconds: 5
      livenessProbe:
        tcpSocket: {port: 80}
      volumeMounts:
        - {name: logs, mountPath: /var/log/agent}
EOF

# Step 2: Once Running, write the exact report files
k -n %{ns} get pod %{pod} -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}' \\
  | tr ' ' '\\n' | sort > {WS}/q52/containers.txt
k -n %{ns} get pod %{pod} -o jsonpath='{.status.podIP}' > {WS}/q52/ip.txt
"""

S[53] = r"""# Step 0: look before you script - which pods, which phases, which nodes
k -n %{ns} get pods -o wide

# Step 1: FIRST container without a readinessProbe, any phase (the Completed pod counts too).
# jsonpath prints nothing for a missing field, so emit "<name> <probe-or-empty>" and keep the bare names.
k -n %{ns} get pods -o jsonpath='{range .items[*]}{.metadata.name} {.spec.containers[0].readinessProbe}{"\n"}{end}' \
  | awk 'NF==1{print $1}' | sort > {WS}/q53/no-readiness.txt

# Step 2: Running pods per node, "<node> <count>". uniq -c prints "count node", so swap the columns.
k -n %{ns} get pods --field-selector=status.phase=Running -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' \
  | sort | uniq -c | awk '{print $2, $1}' | sort > {WS}/q53/per-node.txt

# Step 3: distinct images in kube-system
k -n kube-system get pods -o jsonpath='{.items[*].spec.containers[*].image}' | tr ' ' '\n' | sort -u > {WS}/q53/images.txt

# Traps: cache-a has a readiness probe - on its SECOND container, so it belongs in file 1;
# once-a is Completed, so it is in file 1 but not in file 2. `cat -A` shows stray spaces or \r.
"""

S[64] = """# Step 1: Switch context and find out which namespace it defaults to
k config use-context %{ctx}
k config view --minify -o jsonpath='{..namespace}{"\\n"}'        # -> %{ns}

# Step 2: Everything WITHOUT -n, so it lands in the context's namespace
k create deploy %{dep} --image=busybox:1.36 --replicas=2 -- sleep 3600
k label deploy %{dep} owner=ac
k expose deploy %{dep} --port=9000 --target-port=9000             # ClusterIP is the default type

# Step 3: Record the namespace exactly as kubectl reports it
k config view --minify -o jsonpath='{..namespace}' > {WS}/q64/ns.txt

# Step 4: Verify it all landed there - and nothing in default
k get deploy,svc %{dep}
k -n default get deploy %{dep}          # NotFound is the right answer
"""


# ================================================================= NET =====
# Domain 3: Services & Networking (20%)
# ===========================================================================

S[11] = """# Step 1: Create NetworkPolicy in namespace 'netpol'
# Allow Ingress on port 8080 only from pods labeled 'app: frontend', no egress restriction
cat <<EOF | k apply -f -
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: backend-policy, namespace: netpol}
spec:
  podSelector: {matchLabels: {app: backend}}
  policyTypes: [Ingress]
  ingress:
    - from:
        - podSelector: {matchLabels: {app: frontend}}
      ports:
        - protocol: TCP
          port: 8080
EOF

# Step 2: Verify NetworkPolicy configuration
k -n netpol describe netpol backend-policy
"""

S[12] = """# Step 1: Expose deployment as NodePort service using dry-run
k -n svc expose deploy shop --name=shop-svc --type=NodePort --port=80 --target-port=80 $do > svc.yaml

# Step 2: Set specific nodePort 30080 in spec.ports[0].nodePort
cat <<EOF | k apply -f -
apiVersion: v1
kind: Service
metadata: {name: shop-svc, namespace: svc}
spec:
  type: NodePort
  selector: {app: shop}
  ports:
    - port: 80
      targetPort: 80
      nodePort: 30080
EOF

# Step 3: Verify NodePort service and endpoints
k -n svc get svc shop-svc
k -n svc get endpointslices -l kubernetes.io/service-name=shop-svc
"""

S[13] = """# Step 1: Create the Ingress with both path rules under one host
cat <<EOF | k apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: app-ingress
  namespace: ingress-app
spec:
  ingressClassName: nginx
  rules:
    - host: shop.example.com
      http:
        paths:
          - path: /api
            pathType: Prefix
            backend:
              service: {name: api, port: {number: 8080}}
          - path: /
            pathType: Prefix
            backend:
              service: {name: web, port: {number: 80}}
EOF

# Step 2: Verify Ingress rules
k -n ingress-app describe ingress app-ingress
"""

S[14] = """# Step 1: Create a long-running pod (not --rm) so it still exists for the check
k -n dns-check run dns-test --image=busybox:1.36 --restart=Never -- sleep 3600

# Step 2: Save the FQDN you're resolving
echo "hello.dns-check.svc.cluster.local" > workspace/q14/svc-fqdn.txt

# Step 3: Resolve it from inside the pod and save the full output
k -n dns-check exec dns-test -- nslookup hello.dns-check.svc.cluster.local > workspace/q14/nslookup.txt
"""

S[22] = """# Step 1: Install the Gateway API standard-channel CRDs
k apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/standard-install.yaml

# Step 2: Apply GatewayClass, Gateway (with the listener hostname) and HTTPRoute
cat <<EOF | k apply -f -
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata: {name: %{gc}}
spec: {controllerName: example.com/gateway}
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata: {name: %{gw}, namespace: %{ns}}
spec:
  gatewayClassName: %{gc}
  listeners:
    - name: http
      protocol: HTTP
      port: 80
      hostname: %{host}.example.com
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata: {name: %{host}-route, namespace: %{ns}}
spec:
  parentRefs: [{name: %{gw}}]
  rules:
    - matches: [{path: {type: PathPrefix, value: /api}}]
      backendRefs: [{name: api, port: 8080}]
    - matches: [{path: {type: PathPrefix, value: /}}]
      backendRefs: [{name: web, port: 80}]
EOF

# Step 3: Verify Gateway and HTTPRoute status
k -n %{ns} get gateway,httproute
"""

S[30] = """# Step 1: Deployment whose container port 80 carries the name %{pname}, with a readiness
# probe that references that port BY NAME rather than by number.
cat <<EOF | k apply -f -
apiVersion: apps/v1
kind: Deployment
metadata: {name: %{dep}, namespace: %{ns}}
spec:
  replicas: 1
  selector: {matchLabels: {app: %{dep}}}
  template:
    metadata: {labels: {app: %{dep}}}
    spec:
      containers:
        - name: web
          image: nginx:1.25
          ports:
            - name: %{pname}
              containerPort: 80
          readinessProbe:
            httpGet: {path: /, port: %{pname}}
EOF

# Step 2: Service whose targetPort is the port NAME, not 80
cat <<EOF | k apply -f -
apiVersion: v1
kind: Service
metadata: {name: %{dep}, namespace: %{ns}}
spec:
  selector: {app: %{dep}}
  ports:
    - port: 8080
      targetPort: %{pname}
EOF

# Step 3: Verify endpoints are registered (empty means the selector or the probe is wrong)
k -n %{ns} get endpointslices -l kubernetes.io/service-name=%{dep}
"""

S[42] = """# Step 1: Egress-only policy with TWO rules. Listing Egress alone in policyTypes is what
# denies everything else - there is no explicit "deny" rule in a NetworkPolicy. Leaving Ingress
# out of policyTypes is what keeps inbound traffic unrestricted.
# The DNS rule needs both UDP and TCP on 53: resolvers fall back to TCP for large answers.
cat <<EOF | k apply -f -
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: %{app}-egress, namespace: %{ns}}
spec:
  podSelector: {matchLabels: {app: %{app}}}
  policyTypes: [Egress]
  egress:
    - to:
        - namespaceSelector:
            matchLabels: {kubernetes.io/metadata.name: kube-system}
          podSelector:
            matchLabels: {k8s-app: kube-dns}
      ports:
        - {protocol: UDP, port: 53}
        - {protocol: TCP, port: 53}
    - to:
        - ipBlock: {cidr: %{cidr}}
      ports:
        - {protocol: TCP, port: 443}
EOF

# Step 2: Describe policy to confirm both egress rules
k -n %{ns} describe netpol %{app}-egress
"""

S[43] = """# Step 1: The headless Service must be named EXACTLY like the StatefulSet's spec.serviceName -
# that name is what makes CoreDNS publish one record per pod: <pod>.<service>.<ns>.svc.cluster.local
k -n %{ns} get sts %{sts} -o jsonpath='{.spec.serviceName}{"\\n"}'      # -> %{sts}
cat <<EOF | k apply -f -
apiVersion: v1
kind: Service
metadata: {name: %{sts}, namespace: %{ns}}
spec:
  clusterIP: None
  selector: {app: %{sts}}
  ports: [{port: 5432, targetPort: 5432}]
---
apiVersion: v1
kind: Service
metadata: {name: legacy, namespace: %{ns}}
spec:
  type: ExternalName
  externalName: %{ext}
EOF

# Step 2: Resolve pod 0 by its FULL name. busybox's nslookup applies no search domains, so the
# short form %{sts}-0.%{sts} returns NXDOMAIN even though DNS is perfectly healthy.
k -n %{ns} exec %{sts}-0 -- nslookup %{sts}-0.%{sts}.%{ns}.svc.cluster.local > {WS}/q43/pod0.txt
cat {WS}/q43/pod0.txt                   # Address: <the pod's IP>
k -n %{ns} get pod %{sts}-0 -o wide     # same IP
k -n %{ns} get svc legacy               # EXTERNAL-IP column shows %{ext} - it is only a CNAME
"""

# Hard Networking Tasks
S[54] = """# Step 1: Label the source namespace for ingress matching
k label ns %{from} team=%{from}

# Step 2: Apply default-deny (ingress + egress), allow-dns egress, and tiered service policies
cat <<EOF | k apply -f -
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: default-deny, namespace: %{ns}}
spec: {podSelector: {}, policyTypes: [Ingress, Egress]}
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: allow-dns, namespace: %{ns}}
spec:
  podSelector: {}
  policyTypes: [Egress]
  egress:
    - to: [{namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: kube-system}}}]
      ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: api-ingress, namespace: %{ns}}
spec:
  podSelector: {matchLabels: {app: api}}
  policyTypes: [Ingress]
  ingress:
    - from: [{namespaceSelector: {matchLabels: {team: %{from}}}}]
      ports: [{port: 8080}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: api-to-db, namespace: %{ns}}
spec:
  podSelector: {matchLabels: {app: api}}
  policyTypes: [Egress]
  egress:
    - to: [{podSelector: {matchLabels: {app: db}}}]
      ports: [{port: 5432}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: db-ingress, namespace: %{ns}}
spec:
  podSelector: {matchLabels: {app: db}}
  policyTypes: [Ingress]
  ingress:
    - from: [{podSelector: {matchLabels: {app: api}}}]
      ports: [{port: 5432}]
EOF
"""

S[55] = """# Step 1: Generate self-signed TLS certificate and private key
openssl req -x509 -newkey rsa:2048 -nodes -days 365 -subj "/CN=%{host}" -keyout tls.key -out tls.crt

# Step 2: Create TLS secret in target namespace
k -n %{ns} create secret tls %{host}-tls --cert=tls.crt --key=tls.key

# Step 3: Create Ingress with TLS configuration and SSL redirect annotation
k -n %{ns} create ingress web --class=nginx --rule="%{host}/*=web:80,tls=%{host}-tls" \\
  --annotation nginx.ingress.kubernetes.io/ssl-redirect=true

# Step 4: Extract SHA256 fingerprint to workspace file
openssl x509 -in tls.crt -noout -fingerprint -sha256 | cut -d= -f2 > {WS}/q55/fp.txt
"""

S[56] = """# Step 1: Create GatewayClass, Gateway with multi-host listeners, and canary HTTPRoute
cat <<EOF | k apply -f -
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata: {name: %{host}-gc}
spec: {controllerName: example.com/gw}
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata: {name: %{host}-gw, namespace: %{ns}}
spec:
  gatewayClassName: %{host}-gc
  listeners:
    - {name: web, protocol: HTTP, port: 80, hostname: %{host}.example.com}
    - {name: api, protocol: HTTP, port: 80, hostname: api.%{host}.example.com}
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata: {name: %{host}-route, namespace: %{ns}}
spec:
  parentRefs: [{name: %{host}-gw, sectionName: web}]
  rules:
    - matches: [{headers: [{name: X-Canary, value: always}]}]
      backendRefs: [{name: v2, port: 80}]
    - backendRefs:
        - {name: v1, port: 80, weight: %{w2}}
        - {name: v2, port: 80, weight: %{w}}
EOF
"""

S[57] = """# Step 1: Remove stray blocking NetworkPolicy in kube-system
k -n kube-system get netpol
k -n kube-system delete netpol maintenance-lockdown

# Step 2: Fix CoreDNS replicas (scale back from 0)
k -n kube-system scale deploy coredns --replicas=2

# Step 3: Correct CoreDNS upstream forwarding in ConfigMap
k -n kube-system edit cm coredns
# Replace: forward . 10.255.255.254 -> forward . /etc/resolv.conf

# Step 4: Restart CoreDNS deployment and test DNS resolution
k -n kube-system rollout restart deploy coredns
# busybox's nslookup applies no search domains, so use FULL names (a short one fails even when DNS works)
k run dns-check --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.default.svc.cluster.local
k run dns-ext --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.io
"""


# ================================================================= STOR ====
# Domain 4: Storage (10%)
# ===========================================================================

S[15] = """# Step 1: Create PersistentVolume with hostPath, Retain policy, and manual storageClass
cat <<EOF | k apply -f -
apiVersion: v1
kind: PersistentVolume
metadata: {name: pv-data}
spec:
  capacity: {storage: 1Gi}
  accessModes: [ReadWriteOnce]
  persistentVolumeReclaimPolicy: Retain
  storageClassName: manual
  hostPath: {path: /mnt/data}
---
# Step 2: Create PersistentVolumeClaim requesting 500Mi
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: pvc-data, namespace: storage}
spec:
  accessModes: [ReadWriteOnce]
  storageClassName: manual
  resources: {requests: {storage: 500Mi}}
---
# Step 3: Create Pod mounting the claim at /usr/share/data
apiVersion: v1
kind: Pod
metadata: {name: data-pod, namespace: storage}
spec:
  volumes: [{name: d, persistentVolumeClaim: {claimName: pvc-data}}]
  containers: [{name: web, image: nginx:1.25, volumeMounts: [{name: d, mountPath: /usr/share/data}]}]
EOF

# Step 4: Verify PV and PVC bound status
k get pv pv-data
k -n storage get pvc pvc-data
"""

S[16] = """# Step 1: Create StorageClass with WaitForFirstConsumer and set as default
cat <<EOF | k apply -f -
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: fast-local
  annotations: {storageclass.kubernetes.io/is-default-class: "true"}
provisioner: rancher.io/local-path
volumeBindingMode: WaitForFirstConsumer
reclaimPolicy: Delete
EOF

# Step 2: Remove default status from the previous default StorageClass
k annotate sc standard storageclass.kubernetes.io/is-default-class-

# Step 3: Create PVC (omitting storageClassName to use new default) and consumer Pod
cat <<EOF | k apply -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: fast-claim, namespace: storage}
spec: {accessModes: [ReadWriteOnce], resources: {requests: {storage: 200Mi}}}
---
apiVersion: v1
kind: Pod
metadata: {name: fast-pod, namespace: storage}
spec:
  volumes: [{name: d, persistentVolumeClaim: {claimName: fast-claim}}]
  containers: [{name: web, image: nginx:1.25, volumeMounts: [{name: d, mountPath: /data}]}]
EOF

# Step 4: Verify volume dynamically binds once pod is scheduled
k -n storage get pvc fast-claim
"""

S[35] = """# Step 1: Inspect Released PV and locate old claimRef
k get pv %{pv} -o yaml | grep -A5 claimRef

# Step 2: Remove the claimRef to make the PV Available again
k patch pv %{pv} --type=json -p='[{"op":"remove","path":"/spec/claimRef"}]'
k get pv %{pv}                                    # Verify status: Available

# Step 3: Create new PVC binding specifically to the released volume
cat <<EOF | k apply -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: %{pvc}, namespace: %{ns}}
spec:
  accessModes: [ReadWriteOnce]
  storageClassName: manual
  volumeName: %{pv}
  resources: {requests: {storage: 500Mi}}
EOF

# Step 4: Confirm PVC binds to the PV
k -n %{ns} get pvc %{pvc}
"""

S[36] = """# Step 1: Create PVC for data persistence
cat <<EOF | k apply -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: %{pvc}, namespace: %{ns}}
spec: {accessModes: [ReadWriteOnce], resources: {requests: {storage: 200Mi}}}
---
# Step 2: Create writer Pod to write marker file and complete
apiVersion: v1
kind: Pod
metadata: {name: writer, namespace: %{ns}}
spec:
  restartPolicy: Never
  volumes: [{name: d, persistentVolumeClaim: {claimName: %{pvc}}}]
  containers: [{name: w, image: busybox:1.36, command: [sh, -c, "echo %{text} > /data/marker.txt"], volumeMounts: [{name: d, mountPath: /data}]}]
EOF

# Step 3: Wait for writer pod completion
k -n %{ns} wait --for=jsonpath='{.status.phase}'=Succeeded pod/writer

# Step 4: Create reader Pod mounting the same claim to verify data persists
cat <<EOF | k apply -f -
apiVersion: v1
kind: Pod
metadata: {name: reader, namespace: %{ns}}
spec:
  volumes: [{name: d, persistentVolumeClaim: {claimName: %{pvc}}}]
  containers: [{name: r, image: busybox:1.36, command: [sleep, "3600"], volumeMounts: [{name: d, mountPath: /data}]}]
EOF

# Step 5: Read data back from reader pod
k -n %{ns} exec reader -- cat /data/marker.txt
"""

# Hard Storage Tasks
S[58] = """# Step 1: Identify PV labels and corresponding node affinity
k get pv -l media --show-labels
k get node -l disk=nvme

# Step 2: Create PVC matching the media=nvme label selector
cat <<EOF | k apply -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: %{pvc}, namespace: %{ns}}
spec:
  accessModes: [ReadWriteOnce]
  storageClassName: local-fast
  selector: {matchLabels: {media: nvme}}
  resources: {requests: {storage: 1Gi}}
---
# Step 3: Create consumer Pod mounting the PVC
apiVersion: v1
kind: Pod
metadata: {name: consumer, namespace: %{ns}}
spec:
  volumes: [{name: d, persistentVolumeClaim: {claimName: %{pvc}}}]
  containers: [{name: c, image: busybox:1.36, command: [sleep, "3600"], volumeMounts: [{name: d, mountPath: /data}]}]
EOF

# Step 4: Verify volume binding and scheduling
k -n %{ns} get pod consumer -o wide
"""

S[59] = """# Step 1: Create target PVC for migration
k -n %{ns} apply -f - <<EOF
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: new-data, namespace: %{ns}}
spec: {accessModes: [ReadWriteOnce], resources: {requests: {storage: 1Gi}}}
---
# Step 2: Run migration Job mounting both old and new PVCs
apiVersion: batch/v1
kind: Job
metadata: {name: migrate, namespace: %{ns}}
spec:
  template:
    spec:
      restartPolicy: Never
      volumes:
        - {name: src, persistentVolumeClaim: {claimName: old-data}}
        - {name: dst, persistentVolumeClaim: {claimName: new-data}}
      containers:
        - name: cp
          image: busybox:1.36
          command: [sh, -c, "cp -a /src/. /dst/ && ls /dst"]
          volumeMounts: [{name: src, mountPath: /src}, {name: dst, mountPath: /dst}]
EOF

# Step 3: Wait for migration job completion
k -n %{ns} wait --for=condition=complete job/migrate --timeout=120s

# Step 4: Re-point legacy pod to new claim
k -n %{ns} get pod legacy -o yaml > legacy.yaml
# Edit legacy.yaml: replace claimName: old-data with claimName: new-data
k -n %{ns} delete pod legacy --now && k apply -f legacy.yaml

# Step 5: Delete the old claim. The Job's finished pod still references old-data, and a pod that
# references a claim - even a Completed one - holds it in pvc-protection: `delete pvc` just sits
# there (describe pvc -> "Used By: migrate-xxxxx"). Delete that pod first; the Job itself stays,
# still reporting succeeded: 1.
k -n %{ns} delete pod -l job-name=migrate
k -n %{ns} delete pvc old-data
k get pv | grep old-data                                    # STATUS Released - Retain kept the data
"""


# =============================================================== TSHOOT ====
# Domain 5: Troubleshooting (30%)
# ===========================================================================

S[17] = """# Step 1: Check node status
k get node {W2}                       # Status: NotReady

# Step 2: SSH into the broken worker node
{SHELL_W2}

# Step 3: Check kubelet service status and journal logs
systemctl status kubelet              # Service inactive/disabled
systemctl enable --now kubelet

# Step 4: Verify kubelet successfully connects and registers node
journalctl -u kubelet -f --no-pager   # Look for: "Successfully registered node"

# Step 5: Exit node shell and verify Ready state from cluster
# k get nodes
"""

S[18] = """# Step 1: Diagnose Deployment failure (ImagePullBackOff)
k -n troubleshoot describe deploy payments | grep -i image
# Notice invalid image tag: nginx:1.99-nope

# Step 2: Fix image on the deployment
k -n troubleshoot set image deploy payments payments=nginx:1.27

# Step 3: Inspect Service definition for mismatched selector and targetPort
k -n troubleshoot get svc payments -o yaml
# Discovered issues: selector is 'app: payment' (typo) and targetPort is 8080 (container port is 80)

# Step 4: Patch the Service selector and targetPort
k -n troubleshoot patch svc payments -p '{"spec":{"selector":{"app":"payments"},"ports":[{"port":80,"targetPort":80}]}}'

# Step 5: Verify Service endpoints
k -n troubleshoot get endpointslices -l kubernetes.io/service-name=payments
"""

S[19] = """# Step 1: Capture previous container logs to locate crash reason
k -n troubleshoot logs crasher --previous | tee {WS}/q19/crasher.log
# Log output: "FATAL: required environment variable DB_HOST is not set"

# Step 2: Export pod manifest
k -n troubleshoot get pod crasher -o yaml > crasher.yaml

# Step 3: Edit crasher.yaml:
# Add under spec.containers[0]:
#   env:
#     - name: DB_HOST
#       value: db.internal
# (Remove system-managed fields: status, uid, resourceVersion, creationTimestamp)

# Step 4: Recreate pod and verify Running status
k -n troubleshoot delete pod crasher --now && k apply -f crasher.yaml
k -n troubleshoot get pod crasher
"""

S[20] = """# Step 1: Check control-plane pods in kube-system
k -n kube-system get pods
# kube-scheduler static pod is in Error / CrashLoopBackOff

# Step 2: Inspect scheduler pod logs
k -n kube-system logs kube-scheduler-{CP}
# Error: exec: "kube-schedulerr": executable file not found

# Step 3: SSH into control plane node
{SHELL_CP}

# Step 4: Edit scheduler static pod manifest
# In /etc/kubernetes/manifests/kube-scheduler.yaml:
# Change: command: [kube-schedulerr] -> command: [kube-scheduler]
vi /etc/kubernetes/manifests/kube-scheduler.yaml

# Step 5: Wait for kubelet to reload the static pod and verify scheduling
k -n kube-system get pods -l component=kube-scheduler
"""

S[37] = """# Step 1: Inspect CoreDNS pods and logs
k -n kube-system get pods -l k8s-app=kube-dns
k -n kube-system logs -l k8s-app=kube-dns --tail=20

# Step 2: Check CoreDNS ConfigMap
k -n kube-system get cm coredns -o yaml
# Fault: forward . 10.255.255.1 points to an invalid upstream DNS

# Step 3: Edit ConfigMap to use host resolv.conf
k -n kube-system edit cm coredns
# Change: forward . 10.255.255.1 -> forward . /etc/resolv.conf

# Step 4: Restart CoreDNS deployment to apply changes
k -n kube-system rollout restart deploy coredns

# Step 5: Verify DNS resolution from a test pod
# busybox's nslookup applies no search domains, so use FULL names (a short one fails even when DNS works)
k run test-dns --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.default.svc.cluster.local
k run test-ext --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.io
"""

S[38] = """# Step 1: Check control-plane pod status
k -n kube-system get pods
# kube-controller-manager is crashing

# Step 2: Check controller-manager logs
k -n kube-system logs kube-controller-manager-{CP}
# Error: invalid flag: --leader-elect-lease-duration=1s

# Step 3: Access control plane node
{SHELL_CP}

# Step 4: Edit static pod manifest to remove the invalid flag
vi /etc/kubernetes/manifests/kube-controller-manager.yaml

# Step 5: Verify controller-manager restarts and reconciles deployments/replicasets
k create deploy probe --image=busybox:1.36 -- sleep 60
k get rs
k delete deploy probe
"""

S[39] = """# Step 1: Access the failing worker node
{SHELL_W1}

# Step 2: Inspect kubelet status and journal logs
systemctl status kubelet
journalctl -u kubelet -n 30 --no-pager
# Error: "open /etc/kubernetes/pki/ca-bundle.crt: no such file or directory"

# Step 3: Check /var/lib/kubelet/config.yaml for incorrect certificate path
grep -n clientCAFile /var/lib/kubelet/config.yaml
# Fix path: replace /etc/kubernetes/pki/ca-bundle.crt with /etc/kubernetes/pki/ca.crt
sed -i 's|ca-bundle.crt|ca.crt|' /var/lib/kubelet/config.yaml

# Step 4: Restart and enable kubelet service
systemctl restart kubelet && systemctl enable kubelet
systemctl status kubelet
"""

# Hard Troubleshooting Tasks
S[60] = """# Step 1: Check node status in cluster
k get node {W2}                                   # NotReady,SchedulingDisabled

# Step 2: Access the node
{SHELL_W2}

# Step 3: Fix Fault 1 (kubelet service disabled) and Fault 2 (wrong CA certificate file)
systemctl status kubelet
sed -i 's|ca-wrong.crt|ca.crt|' /var/lib/kubelet/config.yaml
systemctl enable --now kubelet
systemctl status kubelet
exit

# Step 4: Fix Fault 3 (node was cordoned in cluster)
k uncordon {W2}

# Step 5: Verify scheduling on the repaired node
k run test-pod --image=busybox:1.36 --restart=Never --overrides='{"spec":{"nodeName":"{W2}"}}' -- sleep 60
k get pod test-pod -o wide
k delete pod test-pod --now
"""

S[61] = """# Step 1: Check control plane pods in kube-system
k -n kube-system get pods
# scheduler CrashLoop, controller-manager ImagePullBackOff

# Step 2: Access control plane node
{SHELL_CP}

# Step 3: Fix kube-scheduler manifest command typo
vi /etc/kubernetes/manifests/kube-scheduler.yaml
# Replace: kube-schedulerr -> kube-scheduler

# Step 4: Fix kube-controller-manager image tag
vi /etc/kubernetes/manifests/kube-controller-manager.yaml
# Remove broken image tag suffix to match cluster version

# Step 5: Verify both static pods return to Running
k -n kube-system get pods -w
k create deploy probe --image=busybox:1.36 -- sleep 60
k get deploy probe
k delete deploy probe
"""

S[62] = """# Step 1: Diagnose namespace quota and deployment events
k -n %{ns} get all
k -n %{ns} describe deploy %{dep}
k -n %{ns} describe rs -l app=%{dep}               # Quota exceeded: pods limit is 1

# Step 2: Fix the quota - with room for the rolling update, not just the end state. The default
# strategy (maxSurge 25% -> 1, maxUnavailable 25% of 3 -> 0) needs a 4th pod mid-rollout, and with
# maxUnavailable 0 the controller will not remove an old pod first either: at pods=3 the rollout
# deadlocks at "0 out of 3 new replicas have been updated".
k -n %{ns} patch quota tight -p '{"spec":{"hard":{"pods":"4"}}}'

# Step 3: Fix deployment image tag
k -n %{ns} set image deploy %{dep} web=nginx:1.27

# Step 4: Fix Secret missing key referenced by container environment variables
k -n %{ns} patch secret %{dep}-env -p '{"stringData":{"DATABASE_URL":"postgres://db"}}'
k -n %{ns} rollout restart deploy %{dep}

# Step 5: Fix Service selector and targetPort
k -n %{ns} patch svc %{dep} -p '{"spec":{"selector":{"app":"%{dep}"},"ports":[{"port":80,"targetPort":80}]}}'

# Step 6: Verify all deployment replicas are Available with endpoints
k -n %{ns} get deploy %{dep}
k -n %{ns} get endpointslices -l kubernetes.io/service-name=%{dep}
"""

S[63] = r"""# Gather each fact and append it directly to the report file, one key=value line at a time
pc=$(k -n kube-system get pod -l component=kube-controller-manager -o jsonpath='{.items[0].spec.containers[0].command}' | grep -o 'cluster-cidr=[^,"]*' | cut -d= -f2)
echo "pod-cidr=$pc" > {WS}/q63/facts.txt

sc=$(k -n kube-system get pod -l component=kube-apiserver -o jsonpath='{.items[0].spec.containers[0].command}' | grep -o 'service-cluster-ip-range=[^,"]*' | cut -d= -f2)
echo "service-cidr=$sc" >> {WS}/q63/facts.txt

d=$(k -n kube-system get svc kube-dns -o jsonpath='{.spec.clusterIP}')
echo "dns-ip=$d" >> {WS}/q63/facts.txt

# Empty mode in the ConfigMap means kube-proxy defaults to iptables
m=$(k -n kube-system get cm kube-proxy -o jsonpath='{.data.config\.conf}' | grep '^mode:' | awk '{print $2}' | tr -d '"')
echo "proxy-mode=${m:-iptables}" >> {WS}/q63/facts.txt

echo "%{node}-pod-cidr=$(k get node %{node} -o jsonpath='{.spec.podCIDR}')" >> {WS}/q63/facts.txt
"""

S[66] = """# Step 1: Confirm nothing works through kubectl at all
kubectl get nodes    # connection refused / timeout -- the API server itself is down

# Step 2: Access the control-plane node directly (kubectl can't help you here)
{SHELL_CP}

# Step 3: Check what containerd/kubelet is trying (and failing) to run
crictl ps -a | grep apiserver
journalctl -u kubelet -n 50 --no-pager | grep -i apiserver

# Step 4: Fix the static pod manifest
vi /etc/kubernetes/manifests/kube-apiserver.yaml
# kube-apiserverr -> kube-apiserver

# Step 5: Confirm recovery from your own terminal
exit
kubectl get nodes
kubectl -n kube-system get pods -l component=kube-apiserver
"""