"""Reference solutions, keyed by task id. Rendered with the same %{param} / {WS} / {W1} substitution
as the task text. Shown only after a task has been graded (drills) or the session submitted (exams).
These solutions provide step-by-step instructions, Kubernetes concept explanations, imperative commands,
and verification checks for exam success.
"""

S = {}

# ================================================================= ARCH ====
# Domain 1: Cluster Architecture, Installation & Configuration (25%)
# ===========================================================================

S[1] = """# Step 1: Create the ServiceAccount in the target namespace
k -n rbac-app create sa deploy-bot

# Step 2: Create a Role granting specified verbs on deployments.apps
# Verbs: get, list, watch, create, update, patch, delete
k -n rbac-app create role deploy-manager \\
  --verb=get,list,watch,create,update,patch,delete \\
  --resource=deployments.apps

# Step 3: Bind the Role to the ServiceAccount
k -n rbac-app create rolebinding deploy-bot-binding \\
  --role=deploy-manager \\
  --serviceaccount=rbac-app:deploy-bot

# Step 4: Verify permissions using 'kubectl auth can-i'
k auth can-i create deployments -n rbac-app --as=system:serviceaccount:rbac-app:deploy-bot   # Expected: yes
k auth can-i delete pods -n rbac-app --as=system:serviceaccount:rbac-app:deploy-bot          # Expected: no
"""

S[2] = """# Step 1: Access the control plane node
{SHELL_CP}

# Step 2: Take the etcd snapshot with TLS certificates and save to the target path
# Certificates are located in /etc/kubernetes/pki/etcd/
ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 \\
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \\
  --cert=/etc/kubernetes/pki/etcd/server.crt \\
  --key=/etc/kubernetes/pki/etcd/server.key \\
  snapshot save /opt/cka/etcd-snapshot.db

# Step 3: Verify snapshot status and integrity
etcdutl snapshot status /opt/cka/etcd-snapshot.db -w table

# Note: on a cluster where etcdctl is only inside the static pod, run it there. The etcd image
# is distroless (no sh, no cat, no tar), so kubectl cp cannot get the file out - write it into
# /var/lib/etcd instead, which is a hostPath, and move it on the node:
# kubectl -n kube-system exec etcd-{CP} -- etcdctl ... snapshot save /var/lib/etcd/snap.db
# then on the node: mv /var/lib/etcd/snap.db /opt/cka/etcd-snapshot.db
# ./etcd-tools.sh installs etcdctl/etcdutl on the control-plane node (setup.sh already does).
"""

S[3] = """# Step 1: Create the overlay directory structure
mkdir -p {WS}/q03/overlays/prod && cd {WS}/q03/overlays/prod

# Step 2: Define the target namespace manifest
cat > namespace.yaml <<EOF
apiVersion: v1
kind: Namespace
metadata: {name: store-prod}
EOF

# Step 3: Write the kustomization.yaml referencing the base and adding namespace, labels, and replicas
cat > kustomization.yaml <<EOF
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: store-prod
resources: [../../base, namespace.yaml]
labels:
  - pairs: {env: prod}
    includeSelectors: false
replicas:
  - name: store-api
    count: 3
EOF

# Step 4: Build and apply the overlay
kubectl apply -k .

# Step 5: Verify resources in the store-prod namespace
k -n store-prod get deploy,pod -l env=prod
"""

S[4] = """# Step 1: Define and create the CustomResourceDefinition (CRD)
cat <<EOF | k apply -f -
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata: {name: backups.data.example.com}
spec:
  group: data.example.com
  scope: Namespaced
  names: {kind: Backup, plural: backups, singular: backup, shortNames: [bk]}
  versions:
    - name: v1
      served: true
      storage: true
      schema:
        openAPIV3Schema:
          type: object
          properties:
            spec:
              type: object
              properties:
                schedule: {type: string}
                target: {type: string}
EOF

# Step 2: Create a custom resource instance of Backup
cat <<EOF | k apply -f -
apiVersion: data.example.com/v1
kind: Backup
metadata: {name: nightly, namespace: default}
spec: {schedule: "0 2 * * *", target: "s3://backups/prod"}
EOF

# Step 3: Verify using the shortname
k get bk
"""

S[5] = """# Step 1: Access the specified worker node
{SHELL_W3}

# Step 2: Verify the kubelet static pod path
grep staticPodPath /var/lib/kubelet/config.yaml        # Default: /etc/kubernetes/manifests

# Step 3: Create the static pod manifest in the static pod directory
cat > /etc/kubernetes/manifests/static-web.yaml <<EOF
apiVersion: v1
kind: Pod
metadata: {name: static-web, namespace: default}
spec:
  containers: [{name: web, image: nginx:1.27}]
EOF

# Step 4: Verify the mirror pod is created and running on the node
# kubelet automatically detects new manifests in the folder within seconds.
# Back on your main terminal: k get pods -o wide | grep static-web
"""

S[6] = """# Part A: Check certificate expiration on control plane
# Step 1: Access control plane
{SHELL_CP}

# Step 2: Inspect kubeadm cert expiration and extract the apiserver expiry line
kubeadm certs check-expiration | grep '^apiserver ' > {WS}/q06/apiserver-expiry.txt

# Part B: Issue and approve CertificateSigningRequest (CSR) for user Jane
# Step 3: Generate private key and CSR
openssl genrsa -out jane.key 2048
openssl req -new -key jane.key -subj "/CN=jane" -out jane.csr

# Step 4: Submit Kubernetes CSR resource
cat <<EOF | k apply -f -
apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata: {name: jane}
spec:
  request: $(base64 -w0 jane.csr)
  signerName: kubernetes.io/kube-apiserver-client
  usages: [client auth]
EOF

# Step 5: Approve CSR and extract signed certificate
k certificate approve jane
k get csr jane -o jsonpath='{.status.certificate}' | base64 -d > jane.crt

# Step 6: Bind user Jane to the default cluster role 'view'
k -n default create rolebinding jane-view --clusterrole=view --user=jane
"""

S[21] = """# Step 1: Create the target namespace
k create ns %{ns}

# Step 2: Install the Helm chart with overridden replicaCount
helm install %{rel} {WS}/q21/chart -n %{ns} --set replicaCount=%{n}

# Step 3: Upgrade the release to NodePort while preserving existing values
helm upgrade %{rel} {WS}/q21/chart -n %{ns} --reuse-values --set service.type=NodePort

# Step 4: Verify the release history shows revision 2
helm -n %{ns} history %{rel}
"""

S[33] = """# Step 1: Create the aggregating ClusterRole with aggregationRule selector
cat <<EOF | k apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata: {name: %{cr}}
aggregationRule:
  clusterRoleSelectors:
    - matchLabels: {rbac.example.com/aggregate-to-%{cr}: "true"}
rules: []
EOF

# Step 2: Create sub-ClusterRoles for pods and services
k create clusterrole %{cr}-pods --verb=get,list,watch --resource=pods
k create clusterrole %{cr}-services --verb=get,list,watch --resource=services

# Step 3: Apply the matching label so controller aggregates rules into the main ClusterRole
k label clusterrole %{cr}-pods %{cr}-services rbac.example.com/aggregate-to-%{cr}=true

# Step 4: Create ClusterRoleBinding for the user
k create clusterrolebinding %{cr}-binding --clusterrole=%{cr} --user=%{user}

# Step 5: Verify rules are automatically aggregated
k get clusterrole %{cr} -o yaml
"""

S[34] = """# Step 1: Inspect the CRD spec schema recursively to locate the field path
k explain %{kind}.spec --recursive

# Step 2: Check the exact field details and description
k explain %{kind}.spec.tuning.%{field}

# Step 3: Record the full field path to the output file
echo spec.tuning.%{field} > {WS}/q34/path.txt

# Step 4: Create a sample Custom Resource using the discovered field
cat <<EOF | k apply -f -
apiVersion: demo.io/v1
kind: %{kind}
metadata: {name: sample, namespace: default}
spec:
  tuning: {%{field}: %{n}}
EOF
"""

S[40] = """# Step 1: Access the control plane node
{SHELL_CP}

# Step 2: Generate a new kubeadm join token with specified TTL and print join command
kubeadm token create --ttl %{ttl} --print-join-command

# Step 3: Save the join command into the required workspace script
# (e.g., kubeadm join <control-plane-ip>:6443 --token <token> --discovery-token-ca-cert-hash sha256:<hash>)
# Copy the printed command into {WS}/q40/join.sh

# Step 4: Verify active tokens
kubeadm token list
"""

S[41] = """# Step 1: Check Kubernetes Server version (API server)
k version

# Step 2: Extract Kubelet and Container Runtime versions for the target node
k get node %{node} -o jsonpath='{.status.nodeInfo.kubeletVersion}{"\\n"}{.status.nodeInfo.containerRuntimeVersion}{"\\n"}'

# Step 3: Extract etcd image tag from the static pod spec
k -n kube-system get pod -l component=etcd -o jsonpath='{.items[0].spec.containers[0].image}'
# Parse the image tag (everything after the colon)
"""

# Hard Architecture Tasks
S[101] = """# Step 1: Inspect bindings for the ServiceAccount to find and remove over-privileged bindings
k get clusterrolebinding -o wide | grep %{sa}
k delete clusterrolebinding %{sa}-legacy-admin

# Step 2: Create a restricted Role for deployments and replicasets
k -n %{ns} create role %{sa}-role --verb=get,list,watch,patch --resource=deployments.apps,replicasets.apps

# Step 3: Add pods/exec permission rule to the Role
k -n %{ns} patch role %{sa}-role --type=json -p='[{"op":"add","path":"/rules/-","value":{"apiGroups":[""],"resources":["pods/exec"],"verbs":["create"]}}]'

# Step 4: Bind the Role to the ServiceAccount
k -n %{ns} create rolebinding %{sa}-rb --role=%{sa}-role --serviceaccount=%{ns}:%{sa}

# Step 5: Create a ClusterRole with node list privileges and bind it
k create clusterrole %{sa}-nodes --verb=list --resource=nodes
k create clusterrolebinding %{sa}-crb --clusterrole=%{sa}-nodes --serviceaccount=%{ns}:%{sa}

# Step 6: Verify effective permissions
k auth can-i --list -n %{ns} --as=system:serviceaccount:%{ns}:%{sa}
"""

S[102] = """# Step 1: Access the control plane node
{SHELL_CP}

# Step 2: Restore snapshot into a new data directory
etcdutl snapshot restore /opt/cka/backup.db --data-dir /var/lib/etcd-restore

# Step 3: Update etcd static pod hostPath volume to point to the restored directory
# Edit /etc/kubernetes/manifests/etcd.yaml:
# Change:
#   volumes:
#   - hostPath:
#       path: /var/lib/etcd-restore
#     name: etcd-data
vi /etc/kubernetes/manifests/etcd.yaml

# Step 4: Monitor static pod recreation and verify restored objects
watch crictl ps
kubectl -n default get cm %{cm}
"""

S[103] = """# Step 1: Identify the edge node matching the label selector
k get nodes -l tier=edge

# Step 2: Access the target node
{SHELL_W3}

# Step 3: Check kubelet configuration for custom staticPodPath
grep staticPodPath /var/lib/kubelet/config.yaml

# Step 4: Place the pod manifest directly into the configured staticPodPath
cat > <staticPodPath>/%{pod}.yaml <<EOF
apiVersion: v1
kind: Pod
metadata: {name: %{pod}, namespace: %{ns}, labels: {role: edge}}
spec:
  containers:
    - name: web
      image: nginx:1.27
      resources: {requests: {cpu: 50m, memory: 32Mi}}
EOF

# Step 5: Verify the mirror pod appears in the cluster
# k get pods -n %{ns} -o wide
"""

S[104] = """# Step 1: Generate user private key and Certificate Signing Request
openssl genrsa -out %{user}.key 2048
openssl req -new -key %{user}.key -subj "/CN=%{user}/O=developers" -out %{user}.csr

# Step 2: Create and approve the CSR in Kubernetes
k apply -f - <<EOF
apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata: {name: %{user}}
spec: {request: $(base64 -w0 %{user}.csr), signerName: kubernetes.io/kube-apiserver-client, usages: [client auth]}
EOF
k certificate approve %{user}
k get csr %{user} -o jsonpath='{.status.certificate}' | base64 -d > %{user}.crt

# Step 3: Grant permissions via RoleBinding
k -n %{ns} create rolebinding %{user}-edit --clusterrole=edit --user=%{user}

# Step 4: Construct self-contained kubeconfig with embedded certificates
KC={WS}/q104/%{user}.kubeconfig
k config view --minify --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}' | base64 -d > ca.crt
SERVER=$(k config view --minify -o jsonpath='{.clusters[0].cluster.server}')

kubectl --kubeconfig $KC config set-cluster lab --server=$SERVER --certificate-authority=ca.crt --embed-certs
kubectl --kubeconfig $KC config set-credentials %{user} --client-certificate=%{user}.crt --client-key=%{user}.key --embed-certs
kubectl --kubeconfig $KC config set-context %{user}@lab --cluster=lab --user=%{user} --namespace=%{ns}
kubectl --kubeconfig $KC config use-context %{user}@lab

# Step 5: Test the kubeconfig
kubectl --kubeconfig $KC get pods
"""

S[105] = """# Step 1: Create directory for prod overlay
mkdir -p {WS}/q105/overlays/prod && cd {WS}/q105/overlays/prod

# Step 2: Create replica patch manifest
cat > replicas.yaml <<EOF
apiVersion: apps/v1
kind: Deployment
metadata: {name: web}
spec: {replicas: %{rep}}
EOF

# Step 3: Create JSON patch for envFrom injection
cat > envfrom.yaml <<EOF
- op: add
  path: /spec/template/spec/containers/0/envFrom
  value: [{configMapRef: {name: web-config}}]
EOF

# Step 4: Write kustomization.yaml with generator, labels, and patches
cat > kustomization.yaml <<EOF
namespace: %{ns}
resources: [../../base]
labels:
  - pairs: {tier: prod}
    includeSelectors: false
configMapGenerator:
  - name: web-config
    literals: [LOG_LEVEL=%{lvl}]
patches:
  - path: replicas.yaml
  - path: envfrom.yaml
    target: {kind: Deployment, name: web}
EOF

# Step 5: Create namespace and deploy
k create ns %{ns}
k apply -k .
"""

S[106] = """# Step 1: Create CRD with validation rules (required fields, enum, integer bounds)
cat <<EOF | k apply -f -
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata: {name: $(echo %{kind} | tr A-Z a-z)s.deploy.example.com}
spec:
  group: deploy.example.com
  scope: Namespaced
  names: {kind: %{kind}, plural: $(echo %{kind} | tr A-Z a-z)s, singular: $(echo %{kind} | tr A-Z a-z)}
  versions:
    - name: v1
      served: true
      storage: true
      schema:
        openAPIV3Schema:
          type: object
          properties:
            spec:
              type: object
              required: [replicas, image]
              properties:
                replicas: {type: integer, minimum: 1, maximum: 10}
                environment: {type: string, enum: [prod, stage, dev], default: dev}
                image: {type: string}
EOF

# Step 2: Apply valid custom resource
k apply -f - <<EOF
apiVersion: deploy.example.com/v1
kind: %{kind}
metadata: {name: good}
spec: {replicas: 3, image: nginx:1.27, environment: %{env}}
EOF

# Step 3: Test schema rejection of invalid resource (0 replicas)
printf 'apiVersion: deploy.example.com/v1\\nkind: %{kind}\\nmetadata: {name: bad}\\nspec: {replicas: 0, image: x}\\n' | k apply -f - 2> {WS}/q106/rejected.txt

# Step 4: Verify default values on omitted fields
printf 'apiVersion: deploy.example.com/v1\\nkind: %{kind}\\nmetadata: {name: defaulted}\\nspec: {replicas: 2, image: nginx:1.27}\\n' | k apply -f -
"""

S[122] = """# Step 1: Access control plane node
{SHELL_CP}

# Step 2: Renew all cluster certificates via kubeadm
kubeadm certs renew all

# Step 3: Restart static pods to load renewed certificates
mv /etc/kubernetes/manifests/*.yaml /tmp/ && sleep 20 && mv /tmp/*.yaml /etc/kubernetes/manifests/

# Step 4: Update admin kubeconfig from newly generated credentials
cp /etc/kubernetes/admin.conf ~/.kube/config

# Step 5: Confirm new expiration dates
kubeadm certs check-expiration | grep '^apiserver '
"""


# ================================================================= WORK ====
# Domain 2: Workloads & Scheduling (15%)
# ===========================================================================

S[7] = """# Step 1: Update container image to nginx:1.27
k -n apps set image deploy web-frontend nginx=nginx:1.27

# Step 2: Scale deployment to 4 replicas
k -n apps scale deploy web-frontend --replicas=4

# Step 3: Patch rolling update strategy (maxSurge=1, maxUnavailable=0)
k -n apps patch deploy web-frontend -p '{"spec":{"strategy":{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0}}}}'

# Step 4: Monitor rollout until complete
k -n apps rollout status deploy web-frontend
"""

S[8] = """# Step 1: Create pod manifest with shared emptyDir volume and sidecar pattern
cat <<EOF | k apply -f -
apiVersion: v1
kind: Pod
metadata: {name: log-shipper, namespace: apps}
spec:
  volumes: [{name: logs, emptyDir: {}}]
  containers:
    - name: app
      image: busybox:1.36
      command: [sh, -c, "while true; do date >> /var/log/app.log; sleep 2; done"]
      volumeMounts: [{name: logs, mountPath: /var/log}]
    - name: shipper
      image: busybox:1.36
      command: [sh, -c, "tail -f /var/log/app.log"]
      volumeMounts: [{name: logs, mountPath: /var/log}]
EOF

# Step 2: Verify sidecar container reads the log
k -n apps logs log-shipper -c shipper
"""

S[9] = """# Step 1: Generate initial deployment manifest with dry-run
k -n apps create deploy cache --image=redis:7-alpine --replicas=2 $do > cache.yaml

# Step 2: Add nodeAffinity (disktype=ssd) and container resources (requests/limits)
# Under spec.template.spec:
#   affinity:
#     nodeAffinity:
#       requiredDuringSchedulingIgnoredDuringExecution:
#         nodeSelectorTerms:
#           - matchExpressions: [{key: disktype, operator: In, values: [ssd]}]
# Under containers[0]:
#   resources:
#     requests: {cpu: 100m, memory: 64Mi}
#     limits: {cpu: 200m, memory: 128Mi}
cat <<EOF | k apply -f -
apiVersion: apps/v1
kind: Deployment
metadata: {name: cache, namespace: apps}
spec:
  replicas: 2
  selector: {matchLabels: {app: cache}}
  template:
    metadata: {labels: {app: cache}}
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions: [{key: disktype, operator: In, values: [ssd]}]
      containers:
        - name: redis
          image: redis:7-alpine
          resources:
            requests: {cpu: 100m, memory: 64Mi}
            limits: {cpu: 200m, memory: 128Mi}
EOF

# Step 3: Verify scheduling on nodes labeled disktype=ssd
k -n apps get pods -o wide
"""

S[10] = """# Step 1: Create ConfigMap with literal values
k -n apps create cm app-config --from-literal=APP_MODE=production --from-literal=LOG_LEVEL=debug

# Step 2: Create Secret with credentials
k -n apps create secret generic db-creds --from-literal=username=admin --from-literal='password=S3cr3t!'

# Step 3: Create Pod mounting ConfigMap as volume, injecting env var from CM, and loading Secret via envFrom
cat <<EOF | k apply -f -
apiVersion: v1
kind: Pod
metadata: {name: config-pod, namespace: apps}
spec:
  volumes: [{name: cfg, configMap: {name: app-config}}]
  containers:
    - name: app
      image: busybox:1.36
      command: [sleep, "3600"]
      env: [{name: APP_MODE, valueFrom: {configMapKeyRef: {name: app-config, key: APP_MODE}}}]
      envFrom: [{secretRef: {name: db-creds}}]
      volumeMounts: [{name: cfg, mountPath: /etc/config}]
EOF

# Step 4: Verify mounted files and environment variables
k -n apps exec config-pod -- ls /etc/config
k -n apps exec config-pod -- env | grep -E 'APP_MODE|username'
"""

S[23] = """# Step 1: Repair metrics-server by adding --kubelet-insecure-tls flag
k -n kube-system edit deploy metrics-server
# Add under spec.template.spec.containers[0].args:
#   - --kubelet-insecure-tls

# Step 2: Wait ~30s and verify metrics reporting
k top nodes

# Step 3: Configure HorizontalPodAutoscaler (HPA) for target deployment
k -n %{ns} autoscale deploy %{dep} --min=2 --max=%{max} --cpu-percent=%{cpu} --name=%{dep}

# Step 4: Verify HPA status
k -n %{ns} get hpa %{dep}
"""

S[24] = """# Step 1: Create PriorityClass with requested integer value
cat <<EOF | k apply -f -
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata: {name: %{pc}}
value: %{val}
globalDefault: false
description: "%{pc} workloads"
EOF

# Step 2: Assign PriorityClass to the Deployment pod template
k -n %{ns} patch deploy payments -p '{"spec":{"template":{"spec":{"priorityClassName":"%{pc}"}}}}'

# Step 3: Monitor rollout
k -n %{ns} rollout status deploy payments
"""

S[25] = """# Step 1: Apply taint to target worker node
k taint node {W1} %{key}=%{val}:NoSchedule

# Step 2: Create Deployment targeting node with matching toleration
cat <<EOF | k apply -f -
apiVersion: apps/v1
kind: Deployment
metadata: {name: %{dep}, namespace: %{ns}}
spec:
  replicas: 2
  selector: {matchLabels: {app: %{dep}}}
  template:
    metadata: {labels: {app: %{dep}}}
    spec:
      nodeSelector:
        kubernetes.io/hostname: {W1}
      tolerations:
        - key: %{key}
          operator: Equal
          value: "%{val}"
          effect: NoSchedule
      containers:
        - name: busybox
          image: busybox:1.36
          command: [sleep, "3600"]
EOF

# Step 3: Verify pods are scheduled on {W1}
k -n %{ns} get pods -o wide
"""

S[26] = """# Step 1: Create DaemonSet manifest tolerating control-plane taints
cat <<EOF | k apply -f -
apiVersion: apps/v1
kind: DaemonSet
metadata: {name: %{ds}, namespace: %{ns}}
spec:
  selector: {matchLabels: {app: %{ds}}}
  template:
    metadata: {labels: {app: %{ds}}}
    spec:
      tolerations:
        - key: node-role.kubernetes.io/control-plane
          operator: Exists
          effect: NoSchedule
      containers:
        - name: agent
          image: busybox:1.36
          command: [sleep, "3600"]
          resources: {requests: {cpu: 10m, memory: 16Mi}}
EOF

# Step 2: Verify DaemonSet runs on all nodes including control plane
k -n %{ns} get ds %{ds}
k -n %{ns} get pods -o wide -l app=%{ds}
"""

S[27] = """# Step 1: Create CronJob with specified schedule and history limits
k -n %{ns} create cj %{cj} --image=busybox:1.36 --schedule='*/%{every} * * * *' $do -- date > cj.yaml

# Step 2: Add concurrencyPolicy, activeDeadlineSeconds, and history limits:
# spec:
#   concurrencyPolicy: Forbid
#   successfulJobsHistoryLimit: 3
#   failedJobsHistoryLimit: 1
#   jobTemplate:
#     spec:
#       activeDeadlineSeconds: 30
#       template:
#         spec:
#           restartPolicy: Never
cat <<EOF | k apply -f -
apiVersion: batch/v1
kind: CronJob
metadata: {name: %{cj}, namespace: %{ns}}
spec:
  schedule: '*/%{every} * * * *'
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 1
  jobTemplate:
    spec:
      activeDeadlineSeconds: 30
      template:
        spec:
          restartPolicy: Never
          containers:
            - name: date
              image: busybox:1.36
              command: ["date"]
EOF

# Step 3: Trigger a manual job run from the CronJob
k -n %{ns} create job %{cj}-manual --from=cronjob/%{cj}
"""

S[28] = """# Step 1: Create Pod with initContainer populating a shared volume
cat <<EOF | k apply -f -
apiVersion: v1
kind: Pod
metadata: {name: %{pod}, namespace: %{ns}}
spec:
  volumes: [{name: content, emptyDir: {}}]
  initContainers:
    - name: setup
      image: busybox:1.36
      command: [sh, -c, "echo '%{text}' > /work/index.html"]
      volumeMounts: [{name: content, mountPath: /work}]
  containers:
    - name: web
      image: nginx:1.25
      volumeMounts: [{name: content, mountPath: /usr/share/nginx/html}]
EOF

# Step 2: Verify web server serves the file created by init container
k -n %{ns} exec %{pod} -c web -- cat /usr/share/nginx/html/index.html
"""

S[29] = """# Step 1: Create the new version of the immutable ConfigMap
k -n %{ns} create cm %{cm}-v2 --from-literal=mode=new
k -n %{ns} patch cm %{cm}-v2 -p '{"immutable":true}'

# Step 2: Update the Deployment volume reference to the new ConfigMap
k -n %{ns} patch deploy %{dep} --type=json -p='[{"op":"replace","path":"/spec/template/spec/volumes/0/configMap/name","value":"%{cm}-v2"}]'

# Step 3: Verify rolling update finishes
k -n %{ns} rollout status deploy %{dep}
"""

S[31] = """# Step 1: Create Job with completions, parallelism, and backoffLimit
cat <<EOF | k apply -f -
apiVersion: batch/v1
kind: Job
metadata: {name: %{job}, namespace: %{ns}}
spec:
  completions: %{c}
  parallelism: %{p}
  backoffLimit: 2
  template:
    spec:
      restartPolicy: Never
      containers:
        - name: runner
          image: busybox:1.36
          command: ["sh", "-c", "echo done; sleep 2"]
EOF

# Step 2: Wait for Job completion
k -n %{ns} wait --for=condition=complete job/%{job} --timeout=120s
"""

S[32] = """# Step 1: Create ResourceQuota for the namespace
k -n %{ns} create quota %{ns}-quota --hard=pods=%{pods},requests.cpu=1,limits.memory=2Gi

# Step 2: Create LimitRange defining default container requests and limits
cat <<EOF | k apply -f -
apiVersion: v1
kind: LimitRange
metadata: {name: %{ns}-limitrange, namespace: %{ns}}
spec:
  limits:
    - type: Container
      default: {cpu: 200m, memory: 256Mi}
      defaultRequest: {cpu: 100m, memory: 128Mi}
      max: {cpu: 500m, memory: 512Mi}
      min: {cpu: 50m, memory: 64Mi}
EOF

# Step 3: Verify quota and limits
k -n %{ns} describe quota %{ns}-quota
k -n %{ns} describe limitrange %{ns}-limitrange
"""

# Hard Workload Tasks
S[107] = """# Step 1: Inspect namespace quota constraints
k -n %{ns} get quota

# Step 2: Deploy pods with podAntiAffinity, tolerations, PriorityClass, and fitted resources
cat <<EOF | k apply -f -
apiVersion: apps/v1
kind: Deployment
metadata: {name: anti-workload, namespace: %{ns}}
spec:
  replicas: 3
  selector: {matchLabels: {app: anti-workload}}
  template:
    metadata: {labels: {app: anti-workload}}
    spec:
      priorityClassName: high-priority
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector: {matchLabels: {app: anti-workload}}
              topologyKey: kubernetes.io/hostname
      tolerations:
        - key: dedicated
          operator: Equal
          value: special
          effect: NoSchedule
      containers:
        - name: app
          image: busybox:1.36
          command: [sleep, "3600"]
          resources: {requests: {cpu: 50m, memory: 32Mi}, limits: {cpu: 100m, memory: 64Mi}}
EOF

# Step 3: Confirm each replica landed on a distinct node
k -n %{ns} get pods -o wide -l app=anti-workload
"""

S[108] = """# Step 1: Inspect rollout revision history
k -n %{ns} rollout history deploy %{dep}

# Step 2: Roll back Deployment to specified target revision
k -n %{ns} rollout undo deploy %{dep} --to-revision=%{rev}

# Step 3: Immediately pause the rollout
k -n %{ns} rollout pause deploy %{dep}

# Step 4: Verify deployment status is paused
k -n %{ns} get deploy %{dep}
"""

S[109] = """# Step 1: Deploy Pod with native sidecar (restartPolicy: Always inside initContainers)
cat <<EOF | k apply -f -
apiVersion: v1
kind: Pod
metadata: {name: %{pod}, namespace: %{ns}}
spec:
  initContainers:
    - name: proxy
      image: busybox:1.36
      restartPolicy: Always
      command: [sh, -c, "while true; do sleep 5; done"]
      readinessProbe:
        exec: {command: [true]}
        initialDelaySeconds: 1
  containers:
    - name: main
      image: nginx:1.27
      livenessProbe:
        httpGet: {path: /, port: 80}
        initialDelaySeconds: 5
        periodSeconds: 10
EOF

# Step 2: Verify both containers are running
k -n %{ns} get pod %{pod}
"""

S[110] = r"""# Step 1: Filter pods outside kube-system with custom output
k get pods -A -o json | jq -r '.items[] | select(.metadata.namespace!="kube-system") | select(.spec.restartPolicy=="Always") | "\(.metadata.namespace) \(.metadata.name)"' > {WS}/q110/always.txt

# Step 2: Count Running pods per node
k get pods -A --field-selector=status.phase=Running -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' | sort | uniq -c > {WS}/q110/per-node.txt

# Step 3: Extract sorted unique container images used across kube-system
k -n kube-system get pods -o jsonpath='{.items[*].spec.containers[*].image}' | tr ' ' '\n' | sort -u > {WS}/q110/images.txt
"""

S[121] = """# Step 1: Confirm current context namespace
k config view --minify -o jsonpath='{.contexts[0].context.namespace}'

# Step 2: Perform all required operations without specifying -n, relying on context namespace
k create deploy %{dep} --image=nginx:1.27 --replicas=2
k set env deploy/%{dep} APP_ENV=production
k expose deploy %{dep} --port=80 --target-port=80
"""


# ================================================================= NET =====
# Domain 3: Services & Networking (20%)
# ===========================================================================

S[11] = """# Step 1: Create NetworkPolicy in namespace 'marketing'
# Allow Ingress on port 80 only from pods labeled 'role: frontend'
cat <<EOF | k apply -f -
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: allow-frontend, namespace: marketing}
spec:
  podSelector: {matchLabels: {app: api}}
  policyTypes: [Ingress]
  ingress:
    - from:
        - podSelector: {matchLabels: {role: frontend}}
      ports:
        - protocol: TCP
          port: 80
EOF

# Step 2: Verify NetworkPolicy configuration
k -n marketing describe netpol allow-frontend
"""

S[12] = """# Step 1: Expose deployment as NodePort service using dry-run
k -n web expose deploy web-server --type=NodePort --port=80 --target-port=80 $do > svc.yaml

# Step 2: Set specific nodePort 30080 in spec.ports[0].nodePort
cat <<EOF | k apply -f -
apiVersion: v1
kind: Service
metadata: {name: web-server, namespace: web}
spec:
  type: NodePort
  selector: {app: web-server}
  ports:
    - port: 80
      targetPort: 80
      nodePort: 30080
EOF

# Step 3: Verify NodePort service and endpoints
k -n web get svc web-server
k -n web get endpoints web-server
"""

S[13] = """# Step 1: Create Ingress resource routing host to backend service
k -n web create ingress web-ingress \\
  --class=nginx \\
  --rule="order.example.com/api*=order-service:8080"

# Step 2: Verify Ingress rules
k -n web describe ingress web-ingress
"""

S[14] = """# Step 1: Run temporary debug pod to query DNS
# Query A-record for a Service: <service>.<namespace>.svc.cluster.local
k -n default run dns-test --image=busybox:1.36 --rm -it --restart=Never -- \\
  nslookup web-service.apps.svc.cluster.local

# Step 2: Query Pod DNS (IP with dashes): <ip-with-dashes>.<namespace>.pod.cluster.local
k -n default run dns-test --image=busybox:1.36 --rm -it --restart=Never -- \\
  nslookup 10-244-1-5.apps.pod.cluster.local
"""

S[22] = """# Step 1: Apply GatewayClass and Gateway resources
cat <<EOF | k apply -f -
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata: {name: %{gc}}
spec: {controllerName: example.com/gateway-controller}
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata: {name: %{gw}, namespace: %{ns}}
spec:
  gatewayClassName: %{gc}
  listeners:
    - name: http
      protocol: HTTP
      port: 80
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata: {name: %{route}, namespace: %{ns}}
spec:
  parentRefs: [{name: %{gw}}]
  rules:
    - matches: [{path: {type: PathPrefix, value: /api}}]
      backendRefs: [{name: %{svc}, port: 8080}]
EOF

# Step 2: Verify Gateway and HTTPRoute status
k -n %{ns} get gateway,httproute
"""

S[30] = """# Step 1: Update pod/deployment container definition to include named ports
# spec.containers[0].ports: [{name: http-web, containerPort: 80}]
# Step 2: Create Service referencing the container port by name
cat <<EOF | k apply -f -
apiVersion: v1
kind: Service
metadata: {name: %{svc}, namespace: %{ns}}
spec:
  selector: {app: %{app}}
  ports:
    - name: http
      port: 8080
      targetPort: http-web
EOF

# Step 3: Verify endpoints are registered
k -n %{ns} get endpoints %{svc}
"""

S[42] = """# Step 1: Create egress-only NetworkPolicy restricting outbound traffic to external CIDR
cat <<EOF | k apply -f -
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: %{np}, namespace: %{ns}}
spec:
  podSelector: {matchLabels: {role: crawler}}
  policyTypes: [Egress]
  egress:
    - to:
        - ipBlock: {cidr: %{cidr}}
      ports:
        - protocol: TCP
          port: %{port}
EOF

# Step 2: Describe policy to confirm egress rules
k -n %{ns} describe netpol %{np}
"""

S[43] = """# Step 1: Create Headless Service (clusterIP: None)
cat <<EOF | k apply -f -
apiVersion: v1
kind: Service
metadata: {name: db-headless, namespace: %{ns}}
spec:
  clusterIP: None
  selector: {app: db}
  ports: [{port: 5432, targetPort: 5432}]
---
# Step 2: Create ExternalName Service pointing to external domain
apiVersion: v1
kind: Service
metadata: {name: external-api, namespace: %{ns}}
spec:
  type: ExternalName
  externalName: api.partner.com
EOF

# Step 3: Verify DNS resolution for headless endpoints and external CNAME
k -n %{ns} get svc
"""

# Hard Networking Tasks
S[111] = """# Step 1: Label the source namespace for ingress matching
k label ns %{from} team=%{from}

# Step 2: Apply default-deny (ingress + egress), allow-dns egress, and tiered service policies
cat <<EOF | k apply -f -
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: default-deny, namespace: %{ns}}
spec: {podSelector: {}, policyTypes: [Ingress, Egress]}
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: allow-dns, namespace: %{ns}}
spec:
  podSelector: {}
  policyTypes: [Egress]
  egress:
    - to: [{namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: kube-system}}}]
      ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: api-ingress, namespace: %{ns}}
spec:
  podSelector: {matchLabels: {app: api}}
  policyTypes: [Ingress]
  ingress:
    - from: [{namespaceSelector: {matchLabels: {team: %{from}}}}]
      ports: [{port: 8080}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: api-to-db, namespace: %{ns}}
spec:
  podSelector: {matchLabels: {app: api}}
  policyTypes: [Egress]
  egress:
    - to: [{podSelector: {matchLabels: {app: db}}}]
      ports: [{port: 5432}]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: db-ingress, namespace: %{ns}}
spec:
  podSelector: {matchLabels: {app: db}}
  policyTypes: [Ingress]
  ingress:
    - from: [{podSelector: {matchLabels: {app: api}}}]
      ports: [{port: 5432}]
EOF
"""

S[112] = """# Step 1: Generate self-signed TLS certificate and private key
openssl req -x509 -newkey rsa:2048 -nodes -days 365 -subj "/CN=%{host}" -keyout tls.key -out tls.crt

# Step 2: Create TLS secret in target namespace
k -n %{ns} create secret tls %{host}-tls --cert=tls.crt --key=tls.key

# Step 3: Create Ingress with TLS configuration and SSL redirect annotation
k -n %{ns} create ingress web --class=nginx --rule="%{host}/*=web:80,tls=%{host}-tls" \\
  --annotation nginx.ingress.kubernetes.io/ssl-redirect=true

# Step 4: Extract SHA256 fingerprint to workspace file
openssl x509 -in tls.crt -noout -fingerprint -sha256 | cut -d= -f2 > {WS}/q112/fp.txt
"""

S[113] = """# Step 1: Create GatewayClass, Gateway with multi-host listeners, and canary HTTPRoute
cat <<EOF | k apply -f -
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata: {name: %{host}-gc}
spec: {controllerName: example.com/gw}
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata: {name: %{host}-gw, namespace: %{ns}}
spec:
  gatewayClassName: %{host}-gc
  listeners:
    - {name: web, protocol: HTTP, port: 80, hostname: %{host}.example.com}
    - {name: api, protocol: HTTP, port: 80, hostname: api.%{host}.example.com}
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata: {name: %{host}-route, namespace: %{ns}}
spec:
  parentRefs: [{name: %{host}-gw, sectionName: web}]
  rules:
    - matches: [{headers: [{name: X-Canary, value: always}]}]
      backendRefs: [{name: v2, port: 80}]
    - backendRefs:
        - {name: v1, port: 80, weight: %{w2}}
        - {name: v2, port: 80, weight: %{w}}
EOF
"""

S[114] = """# Step 1: Remove stray blocking NetworkPolicy in kube-system
k -n kube-system get netpol
k -n kube-system delete netpol maintenance-lockdown

# Step 2: Fix CoreDNS replicas (scale back from 0)
k -n kube-system scale deploy coredns --replicas=2

# Step 3: Correct CoreDNS upstream forwarding in ConfigMap
k -n kube-system edit cm coredns
# Replace: forward . 10.255.255.254 -> forward . /etc/resolv.conf

# Step 4: Restart CoreDNS deployment and test DNS resolution
k -n kube-system rollout restart deploy coredns
k run dns-check --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.default
"""


# ================================================================= STOR ====
# Domain 4: Storage (10%)
# ===========================================================================

S[15] = """# Step 1: Create PersistentVolume with hostPath, Retain policy, and manual storageClass
cat <<EOF | k apply -f -
apiVersion: v1
kind: PersistentVolume
metadata: {name: pv-data}
spec:
  capacity: {storage: 1Gi}
  accessModes: [ReadWriteOnce]
  persistentVolumeReclaimPolicy: Retain
  storageClassName: manual
  hostPath: {path: /mnt/data}
---
# Step 2: Create PersistentVolumeClaim requesting 500Mi
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: pvc-data, namespace: storage}
spec:
  accessModes: [ReadWriteOnce]
  storageClassName: manual
  resources: {requests: {storage: 500Mi}}
---
# Step 3: Create Pod mounting the claim at /usr/share/data
apiVersion: v1
kind: Pod
metadata: {name: data-pod, namespace: storage}
spec:
  volumes: [{name: d, persistentVolumeClaim: {claimName: pvc-data}}]
  containers: [{name: web, image: nginx:1.25, volumeMounts: [{name: d, mountPath: /usr/share/data}]}]
EOF

# Step 4: Verify PV and PVC bound status
k get pv pv-data
k -n storage get pvc pvc-data
"""

S[16] = """# Step 1: Create StorageClass with WaitForFirstConsumer and set as default
cat <<EOF | k apply -f -
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: fast-local
  annotations: {storageclass.kubernetes.io/is-default-class: "true"}
provisioner: rancher.io/local-path
volumeBindingMode: WaitForFirstConsumer
reclaimPolicy: Delete
EOF

# Step 2: Remove default status from the previous default StorageClass
k annotate sc standard storageclass.kubernetes.io/is-default-class-

# Step 3: Create PVC (omitting storageClassName to use new default) and consumer Pod
cat <<EOF | k apply -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: fast-claim, namespace: storage}
spec: {accessModes: [ReadWriteOnce], resources: {requests: {storage: 200Mi}}}
---
apiVersion: v1
kind: Pod
metadata: {name: fast-pod, namespace: storage}
spec:
  volumes: [{name: d, persistentVolumeClaim: {claimName: fast-claim}}]
  containers: [{name: web, image: nginx:1.25, volumeMounts: [{name: d, mountPath: /data}]}]
EOF

# Step 4: Verify volume dynamically binds once pod is scheduled
k -n storage get pvc fast-claim
"""

S[35] = """# Step 1: Inspect Released PV and locate old claimRef
k get pv %{pv} -o yaml | grep -A5 claimRef

# Step 2: Remove the claimRef to make the PV Available again
k patch pv %{pv} --type=json -p='[{"op":"remove","path":"/spec/claimRef"}]'
k get pv %{pv}                                    # Verify status: Available

# Step 3: Create new PVC binding specifically to the released volume
cat <<EOF | k apply -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: %{pvc}, namespace: %{ns}}
spec:
  accessModes: [ReadWriteOnce]
  storageClassName: manual
  volumeName: %{pv}
  resources: {requests: {storage: 500Mi}}
EOF

# Step 4: Confirm PVC binds to the PV
k -n %{ns} get pvc %{pvc}
"""

S[36] = """# Step 1: Create PVC for data persistence
cat <<EOF | k apply -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: %{pvc}, namespace: %{ns}}
spec: {accessModes: [ReadWriteOnce], resources: {requests: {storage: 200Mi}}}
---
# Step 2: Create writer Pod to write marker file and complete
apiVersion: v1
kind: Pod
metadata: {name: writer, namespace: %{ns}}
spec:
  restartPolicy: Never
  volumes: [{name: d, persistentVolumeClaim: {claimName: %{pvc}}}]
  containers: [{name: w, image: busybox:1.36, command: [sh, -c, "echo %{text} > /data/marker.txt"], volumeMounts: [{name: d, mountPath: /data}]}]
EOF

# Step 3: Wait for writer pod completion
k -n %{ns} wait --for=jsonpath='{.status.phase}'=Succeeded pod/writer

# Step 4: Create reader Pod mounting the same claim to verify data persists
cat <<EOF | k apply -f -
apiVersion: v1
kind: Pod
metadata: {name: reader, namespace: %{ns}}
spec:
  volumes: [{name: d, persistentVolumeClaim: {claimName: %{pvc}}}]
  containers: [{name: r, image: busybox:1.36, command: [sleep, "3600"], volumeMounts: [{name: d, mountPath: /data}]}]
EOF

# Step 5: Read data back from reader pod
k -n %{ns} exec reader -- cat /data/marker.txt
"""

# Hard Storage Tasks
S[115] = """# Step 1: Identify PV labels and corresponding node affinity
k get pv -l media --show-labels
k get node -l disk=nvme

# Step 2: Create PVC matching the media=nvme label selector
cat <<EOF | k apply -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: %{pvc}, namespace: %{ns}}
spec:
  accessModes: [ReadWriteOnce]
  storageClassName: local-fast
  selector: {matchLabels: {media: nvme}}
  resources: {requests: {storage: 1Gi}}
---
# Step 3: Create consumer Pod mounting the PVC
apiVersion: v1
kind: Pod
metadata: {name: consumer, namespace: %{ns}}
spec:
  volumes: [{name: d, persistentVolumeClaim: {claimName: %{pvc}}}]
  containers: [{name: c, image: busybox:1.36, command: [sleep, "3600"], volumeMounts: [{name: d, mountPath: /data}]}]
EOF

# Step 4: Verify volume binding and scheduling
k -n %{ns} get pod consumer -o wide
"""

S[116] = """# Step 1: Create target PVC for migration
k -n %{ns} apply -f - <<EOF
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: new-data, namespace: %{ns}}
spec: {accessModes: [ReadWriteOnce], resources: {requests: {storage: 1Gi}}}
---
# Step 2: Run migration Job mounting both old and new PVCs
apiVersion: batch/v1
kind: Job
metadata: {name: migrate, namespace: %{ns}}
spec:
  template:
    spec:
      restartPolicy: Never
      volumes:
        - {name: src, persistentVolumeClaim: {claimName: old-data}}
        - {name: dst, persistentVolumeClaim: {claimName: new-data}}
      containers:
        - name: cp
          image: busybox:1.36
          command: [sh, -c, "cp -a /src/. /dst/ && ls /dst"]
          volumeMounts: [{name: src, mountPath: /src}, {name: dst, mountPath: /dst}]
EOF

# Step 3: Wait for migration job completion
k -n %{ns} wait --for=condition=complete job/migrate --timeout=120s

# Step 4: Re-point legacy pod to new claim
k -n %{ns} get pod legacy -o yaml > legacy.yaml
# Edit legacy.yaml: replace claimName: old-data with claimName: new-data
k -n %{ns} delete pod legacy --now && k apply -f legacy.yaml

# Step 5: Clean up old PVC (PV remains Retained)
k -n %{ns} delete pvc old-data
"""


# =============================================================== TSHOOT ====
# Domain 5: Troubleshooting (30%)
# ===========================================================================

S[17] = """# Step 1: Check node status
k get node {W2}                       # Status: NotReady

# Step 2: SSH into the broken worker node
{SHELL_W2}

# Step 3: Check kubelet service status and journal logs
systemctl status kubelet              # Service inactive/disabled
systemctl enable --now kubelet

# Step 4: Verify kubelet successfully connects and registers node
journalctl -u kubelet -f --no-pager   # Look for: "Successfully registered node"

# Step 5: Exit node shell and verify Ready state from cluster
# k get nodes
"""

S[18] = """# Step 1: Diagnose Deployment failure (ImagePullBackOff)
k -n troubleshoot describe deploy payments | grep -i image
# Notice invalid image tag: nginx:1.99-nope

# Step 2: Fix image on the deployment
k -n troubleshoot set image deploy payments payments=nginx:1.27

# Step 3: Inspect Service definition for mismatched selector and targetPort
k -n troubleshoot get svc payments -o yaml
# Discovered issues: selector is 'app: payment' (typo) and targetPort is 8080 (container port is 80)

# Step 4: Patch the Service selector and targetPort
k -n troubleshoot patch svc payments -p '{"spec":{"selector":{"app":"payments"},"ports":[{"port":80,"targetPort":80}]}}'

# Step 5: Verify Service endpoints
k -n troubleshoot get endpointslices -l kubernetes.io/service-name=payments
"""

S[19] = """# Step 1: Capture previous container logs to locate crash reason
k -n troubleshoot logs crasher --previous | tee {WS}/q19/crasher.log
# Log output: "FATAL: required environment variable DB_HOST is not set"

# Step 2: Export pod manifest
k -n troubleshoot get pod crasher -o yaml > crasher.yaml

# Step 3: Edit crasher.yaml:
# Add under spec.containers[0]:
#   env:
#     - name: DB_HOST
#       value: db.internal
# (Remove system-managed fields: status, uid, resourceVersion, creationTimestamp)

# Step 4: Recreate pod and verify Running status
k -n troubleshoot delete pod crasher --now && k apply -f crasher.yaml
k -n troubleshoot get pod crasher
"""

S[20] = """# Step 1: Check control-plane pods in kube-system
k -n kube-system get pods
# kube-scheduler static pod is in Error / CrashLoopBackOff

# Step 2: Inspect scheduler pod logs
k -n kube-system logs kube-scheduler-{CP}
# Error: exec: "kube-schedulerr": executable file not found

# Step 3: SSH into control plane node
{SHELL_CP}

# Step 4: Edit scheduler static pod manifest
# In /etc/kubernetes/manifests/kube-scheduler.yaml:
# Change: command: [kube-schedulerr] -> command: [kube-scheduler]
vi /etc/kubernetes/manifests/kube-scheduler.yaml

# Step 5: Wait for kubelet to reload the static pod and verify scheduling
k -n kube-system get pods -l component=kube-scheduler
"""

S[37] = """# Step 1: Inspect CoreDNS pods and logs
k -n kube-system get pods -l k8s-app=kube-dns
k -n kube-system logs -l k8s-app=kube-dns --tail=20

# Step 2: Check CoreDNS ConfigMap
k -n kube-system get cm coredns -o yaml
# Fault: forward . 10.255.255.1 points to an invalid upstream DNS

# Step 3: Edit ConfigMap to use host resolv.conf
k -n kube-system edit cm coredns
# Change: forward . 10.255.255.1 -> forward . /etc/resolv.conf

# Step 4: Restart CoreDNS deployment to apply changes
k -n kube-system rollout restart deploy coredns

# Step 5: Verify DNS resolution from a test pod
k run test-dns --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.default
"""

S[38] = """# Step 1: Check control-plane pod status
k -n kube-system get pods
# kube-controller-manager is crashing

# Step 2: Check controller-manager logs
k -n kube-system logs kube-controller-manager-{CP}
# Error: invalid flag: --leader-elect-lease-duration=1s

# Step 3: Access control plane node
{SHELL_CP}

# Step 4: Edit static pod manifest to remove the invalid flag
vi /etc/kubernetes/manifests/kube-controller-manager.yaml

# Step 5: Verify controller-manager restarts and reconciles deployments/replicasets
k create deploy probe --image=busybox:1.36 -- sleep 60
k get rs
k delete deploy probe
"""

S[39] = """# Step 1: Access the failing worker node
{SHELL_W1}

# Step 2: Inspect kubelet status and journal logs
systemctl status kubelet
journalctl -u kubelet -n 30 --no-pager
# Error: "open /etc/kubernetes/pki/ca-bundle.crt: no such file or directory"

# Step 3: Check /var/lib/kubelet/config.yaml for incorrect certificate path
grep -n clientCAFile /var/lib/kubelet/config.yaml
# Fix path: replace /etc/kubernetes/pki/ca-bundle.crt with /etc/kubernetes/pki/ca.crt
sed -i 's|ca-bundle.crt|ca.crt|' /var/lib/kubelet/config.yaml

# Step 4: Restart and enable kubelet service
systemctl restart kubelet && systemctl enable kubelet
systemctl status kubelet
"""

# Hard Troubleshooting Tasks
S[117] = """# Step 1: Check node status in cluster
k get node {W2}                                   # NotReady,SchedulingDisabled

# Step 2: Access the node
{SHELL_W2}

# Step 3: Fix Fault 1 (kubelet service disabled) and Fault 2 (wrong CA certificate file)
systemctl status kubelet
sed -i 's|ca-wrong.crt|ca.crt|' /var/lib/kubelet/config.yaml
systemctl enable --now kubelet
systemctl status kubelet
exit

# Step 4: Fix Fault 3 (node was cordoned in cluster)
k uncordon {W2}

# Step 5: Verify scheduling on the repaired node
k run test-pod --image=busybox:1.36 --restart=Never --overrides='{"spec":{"nodeName":"{W2}"}}' -- sleep 60
k get pod test-pod -o wide
k delete pod test-pod --now
"""

S[118] = """# Step 1: Check control plane pods in kube-system
k -n kube-system get pods
# scheduler CrashLoop, controller-manager ImagePullBackOff

# Step 2: Access control plane node
{SHELL_CP}

# Step 3: Fix kube-scheduler manifest command typo
vi /etc/kubernetes/manifests/kube-scheduler.yaml
# Replace: kube-schedulerr -> kube-scheduler

# Step 4: Fix kube-controller-manager image tag
vi /etc/kubernetes/manifests/kube-controller-manager.yaml
# Remove broken image tag suffix to match cluster version

# Step 5: Verify both static pods return to Running
k -n kube-system get pods -w
k create deploy probe --image=busybox:1.36 -- sleep 60
k get deploy probe
k delete deploy probe
"""

S[119] = """# Step 1: Diagnose namespace quota and deployment events
k -n %{ns} get all
k -n %{ns} describe deploy %{dep}
k -n %{ns} describe rs -l app=%{dep}               # Quota exceeded: pods limit is 1

# Step 2: Fix quota to accommodate required replicas
k -n %{ns} patch quota tight -p '{"spec":{"hard":{"pods":"3"}}}'

# Step 3: Fix deployment image tag
k -n %{ns} set image deploy %{dep} web=nginx:1.27

# Step 4: Fix Secret missing key referenced by container environment variables
k -n %{ns} patch secret %{dep}-env -p '{"stringData":{"DATABASE_URL":"postgres://db"}}'
k -n %{ns} rollout restart deploy %{dep}

# Step 5: Fix Service selector and targetPort
k -n %{ns} patch svc %{dep} -p '{"spec":{"selector":{"app":"%{dep}"},"ports":[{"port":80,"targetPort":80}]}}'

# Step 6: Verify all deployment replicas are Available with endpoints
k -n %{ns} get deploy %{dep}
k -n %{ns} get endpoints %{dep}
"""

S[120] = """# Step 1: Query cluster CIDR from kube-controller-manager pod spec
k -n kube-system get pod -l component=kube-controller-manager -o yaml | grep cluster-cidr

# Step 2: Query service CIDR from kube-apiserver pod spec
k -n kube-system get pod -l component=kube-apiserver -o yaml | grep service-cluster-ip-range

# Step 3: Query CoreDNS ClusterIP
k -n kube-system get svc kube-dns -o jsonpath='{.spec.clusterIP}'

# Step 4: Query kube-proxy operating mode from ConfigMap
k -n kube-system get cm kube-proxy -o yaml | grep '^ *mode:'      # Empty string indicates default: iptables

# Step 5: Query individual node Pod CIDR
k get node %{node} -o jsonpath='{.spec.podCIDR}'
"""