diff --git a/tracks/cka/solutions.py b/tracks/cka/solutions.py index d968118..92d882d 100644 --- a/tracks/cka/solutions.py +++ b/tracks/cka/solutions.py @@ -1,35 +1,64 @@ """Reference solutions, keyed by task id. Rendered with the same %{param} / {WS} / {W1} substitution as the task text. Shown only after a task has been graded (drills) or the session submitted (exams). -These are one correct way, written the way you'd type them under time pressure: imperative first, -YAML only where imperative can't express it.""" +These solutions provide step-by-step instructions, Kubernetes concept explanations, imperative commands, +and verification checks for exam success. +""" S = {} # ================================================================= ARCH ==== -S[1] = """ +# Domain 1: Cluster Architecture, Installation & Configuration (25%) +# =========================================================================== + +S[1] = """# Step 1: Create the ServiceAccount in the target namespace k -n rbac-app create sa deploy-bot -k -n rbac-app create role deploy-manager --verb=get,list,watch,create,update,patch,delete --resource=deployments.apps -k -n rbac-app create rolebinding deploy-bot-binding --role=deploy-manager --serviceaccount=rbac-app:deploy-bot -k auth can-i create deployments -n rbac-app --as=system:serviceaccount:rbac-app:deploy-bot # yes -k auth can-i delete pods -n rbac-app --as=system:serviceaccount:rbac-app:deploy-bot # no + +# Step 2: Create a Role granting specified verbs on deployments.apps +# Verbs: get, list, watch, create, update, patch, delete +k -n rbac-app create role deploy-manager \\ + --verb=get,list,watch,create,update,patch,delete \\ + --resource=deployments.apps + +# Step 3: Bind the Role to the ServiceAccount +k -n rbac-app create rolebinding deploy-bot-binding \\ + --role=deploy-manager \\ + --serviceaccount=rbac-app:deploy-bot + +# Step 4: Verify permissions using 'kubectl auth can-i' +k auth can-i create deployments -n rbac-app --as=system:serviceaccount:rbac-app:deploy-bot # Expected: yes +k auth can-i delete pods -n rbac-app --as=system:serviceaccount:rbac-app:deploy-bot # Expected: no """ -S[2] = """ + +S[2] = """# Step 1: Access the control plane node {SHELL_CP} + +# Step 2: Take the etcd snapshot with TLS certificates and save to the target path +# Certificates are located in /etc/kubernetes/pki/etcd/ ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 \\ - --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key \\ + --cacert=/etc/kubernetes/pki/etcd/ca.crt \\ + --cert=/etc/kubernetes/pki/etcd/server.crt \\ + --key=/etc/kubernetes/pki/etcd/server.key \\ snapshot save /opt/cka/etcd-snapshot.db + +# Step 3: Verify snapshot status and integrity etcdutl snapshot status /opt/cka/etcd-snapshot.db -w table -# If etcdctl is only inside the pod: run the same command via -# kubectl -n kube-system exec etcd-{CP} -- etcdctl ... snapshot save /var/lib/etcd/snap.db -# then on the node: mv /var/lib/etcd/snap.db /opt/cka/etcd-snapshot.db (the pod's /var/lib/etcd is the node's /var/lib/etcd) + +# Note: If etcdctl is only inside the static pod, run: +# kubectl -n kube-system exec etcd-{CP} -- etcdctl ... snapshot save /var/lib/etcd/snap.db +# then on the node: mv /var/lib/etcd/snap.db /opt/cka/etcd-snapshot.db """ -S[3] = """ + +S[3] = """# Step 1: Create the overlay directory structure mkdir -p {WS}/q03/overlays/prod && cd {WS}/q03/overlays/prod + +# Step 2: Define the target namespace manifest cat > namespace.yaml < kustomization.yaml < /etc/kubernetes/manifests/static-web.yaml < (copy the line into {WS}/q06/apiserver-expiry.txt) -# Part B + +# Step 2: Inspect kubeadm cert expiration and extract the apiserver expiry line +kubeadm certs check-expiration | grep '^apiserver ' > {WS}/q06/apiserver-expiry.txt + +# Part B: Issue and approve CertificateSigningRequest (CSR) for user Jane +# Step 3: Generate private key and CSR openssl genrsa -out jane.key 2048 openssl req -new -key jane.key -subj "/CN=jane" -out jane.csr + +# Step 4: Submit Kubernetes CSR resource cat < jane.crt + +# Step 6: Bind user Jane to the default cluster role 'view' k -n default create rolebinding jane-view --clusterrole=view --user=jane """ -S[21] = """ + +S[21] = """# Step 1: Create the target namespace k create ns %{ns} + +# Step 2: Install the Helm chart with overridden replicaCount helm install %{rel} {WS}/q21/chart -n %{ns} --set replicaCount=%{n} + +# Step 3: Upgrade the release to NodePort while preserving existing values helm upgrade %{rel} {WS}/q21/chart -n %{ns} --reuse-values --set service.type=NodePort -helm -n %{ns} history %{rel} # revision 2 + +# Step 4: Verify the release history shows revision 2 +helm -n %{ns} history %{rel} """ -S[33] = """ + +S[33] = """# Step 1: Create the aggregating ClusterRole with aggregationRule selector cat < {WS}/q34/path.txt + +# Step 4: Create a sample Custom Resource using the discovered field cat <:6443 --token --discovery-token-ca-cert-hash sha256:) +# Copy the printed command into {WS}/q40/join.sh + +# Step 4: Verify active tokens kubeadm token list """ -S[41] = """ -k version # Server Version -> apiserver= + +S[41] = """# Step 1: Check Kubernetes Server version (API server) +k version + +# Step 2: Extract Kubelet and Container Runtime versions for the target node k get node %{node} -o jsonpath='{.status.nodeInfo.kubeletVersion}{"\\n"}{.status.nodeInfo.containerRuntimeVersion}{"\\n"}' -k -n kube-system get pod -l component=etcd -o jsonpath='{.items[0].spec.containers[0].image}' # tag after the colon + +# Step 3: Extract etcd image tag from the static pod spec +k -n kube-system get pod -l component=etcd -o jsonpath='{.items[0].spec.containers[0].image}' +# Parse the image tag (everything after the colon) """ -# --------------------------------------------------------------- hard ARCH -S[101] = """ -k get clusterrolebinding -o wide | grep %{sa} # find the cluster-admin binding -> delete it + +# Hard Architecture Tasks +S[101] = """# Step 1: Inspect bindings for the ServiceAccount to find and remove over-privileged bindings +k get clusterrolebinding -o wide | grep %{sa} k delete clusterrolebinding %{sa}-legacy-admin + +# Step 2: Create a restricted Role for deployments and replicasets k -n %{ns} create role %{sa}-role --verb=get,list,watch,patch --resource=deployments.apps,replicasets.apps + +# Step 3: Add pods/exec permission rule to the Role k -n %{ns} patch role %{sa}-role --type=json -p='[{"op":"add","path":"/rules/-","value":{"apiGroups":[""],"resources":["pods/exec"],"verbs":["create"]}}]' + +# Step 4: Bind the Role to the ServiceAccount k -n %{ns} create rolebinding %{sa}-rb --role=%{sa}-role --serviceaccount=%{ns}:%{sa} + +# Step 5: Create a ClusterRole with node list privileges and bind it k create clusterrole %{sa}-nodes --verb=list --resource=nodes k create clusterrolebinding %{sa}-crb --clusterrole=%{sa}-nodes --serviceaccount=%{ns}:%{sa} + +# Step 6: Verify effective permissions k auth can-i --list -n %{ns} --as=system:serviceaccount:%{ns}:%{sa} """ -S[102] = """ + +S[102] = """# Step 1: Access the control plane node {SHELL_CP} + +# Step 2: Restore snapshot into a new data directory etcdutl snapshot restore /opt/cka/backup.db --data-dir /var/lib/etcd-restore -# point the etcd static pod at the new dir: the hostPath volume, NOT just --data-dir -vi /etc/kubernetes/manifests/etcd.yaml # volumes: - hostPath: path: /var/lib/etcd -> /var/lib/etcd-restore -# kubelet restarts etcd; apiserver reconnects in ~30-60s -watch crictl ps # etcd + kube-apiserver come back -kubectl -n default get cm %{cm} # restored + +# Step 3: Update etcd static pod hostPath volume to point to the restored directory +# Edit /etc/kubernetes/manifests/etcd.yaml: +# Change: +# volumes: +# - hostPath: +# path: /var/lib/etcd-restore +# name: etcd-data +vi /etc/kubernetes/manifests/etcd.yaml + +# Step 4: Monitor static pod recreation and verify restored objects +watch crictl ps +kubectl -n default get cm %{cm} """ -S[103] = """ -k get nodes -l tier=edge # -> the node + +S[103] = """# Step 1: Identify the edge node matching the label selector +k get nodes -l tier=edge + +# Step 2: Access the target node {SHELL_W3} -grep staticPodPath /var/lib/kubelet/config.yaml # NOT /etc/kubernetes/manifests here + +# Step 3: Check kubelet configuration for custom staticPodPath +grep staticPodPath /var/lib/kubelet/config.yaml + +# Step 4: Place the pod manifest directly into the configured staticPodPath cat > /%{pod}.yaml < %{user}.crt + +# Step 3: Grant permissions via RoleBinding k -n %{ns} create rolebinding %{user}-edit --clusterrole=edit --user=%{user} -# self-contained kubeconfig (--embed-certs) + +# Step 4: Construct self-contained kubeconfig with embedded certificates KC={WS}/q104/%{user}.kubeconfig k config view --minify --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}' | base64 -d > ca.crt SERVER=$(k config view --minify -o jsonpath='{.clusters[0].cluster.server}') + kubectl --kubeconfig $KC config set-cluster lab --server=$SERVER --certificate-authority=ca.crt --embed-certs kubectl --kubeconfig $KC config set-credentials %{user} --client-certificate=%{user}.crt --client-key=%{user}.key --embed-certs kubectl --kubeconfig $KC config set-context %{user}@lab --cluster=lab --user=%{user} --namespace=%{ns} kubectl --kubeconfig $KC config use-context %{user}@lab + +# Step 5: Test the kubeconfig kubectl --kubeconfig $KC get pods """ -S[105] = """ + +S[105] = """# Step 1: Create directory for prod overlay mkdir -p {WS}/q105/overlays/prod && cd {WS}/q105/overlays/prod + +# Step 2: Create replica patch manifest cat > replicas.yaml < envfrom.yaml < kustomization.yaml < web-config- in the envFrom too + +# Step 5: Create namespace and deploy +k create ns %{ns} +k apply -k . """ -S[106] = """ + +S[106] = """# Step 1: Create CRD with validation rules (required fields, enum, integer bounds) cat < {WS}/q106/rejected.txt + +# Step 4: Verify default values on omitted fields printf 'apiVersion: deploy.example.com/v1\\nkind: %{kind}\\nmetadata: {name: defaulted}\\nspec: {replicas: 2, image: nginx:1.27}\\n' | k apply -f - """ -S[122] = """ + +S[122] = """# Step 1: Access control plane node {SHELL_CP} + +# Step 2: Renew all cluster certificates via kubeadm kubeadm certs renew all -# static pods do NOT reload certs by themselves: bounce them + +# Step 3: Restart static pods to load renewed certificates mv /etc/kubernetes/manifests/*.yaml /tmp/ && sleep 20 && mv /tmp/*.yaml /etc/kubernetes/manifests/ -# admin.conf was regenerated by `renew all`; use it: + +# Step 4: Update admin kubeconfig from newly generated credentials cp /etc/kubernetes/admin.conf ~/.kube/config -kubeadm certs check-expiration | grep '^apiserver ' # -> {WS}/q122/expiry.txt + +# Step 5: Confirm new expiration dates +kubeadm certs check-expiration | grep '^apiserver ' """ + # ================================================================= WORK ==== -S[7] = """ +# Domain 2: Workloads & Scheduling (15%) +# =========================================================================== + +S[7] = """# Step 1: Update container image to nginx:1.27 k -n apps set image deploy web-frontend nginx=nginx:1.27 + +# Step 2: Scale deployment to 4 replicas k -n apps scale deploy web-frontend --replicas=4 + +# Step 3: Patch rolling update strategy (maxSurge=1, maxUnavailable=0) k -n apps patch deploy web-frontend -p '{"spec":{"strategy":{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0}}}}' + +# Step 4: Monitor rollout until complete k -n apps rollout status deploy web-frontend """ -S[8] = """ + +S[8] = """# Step 1: Create pod manifest with shared emptyDir volume and sidecar pattern cat < cache.yaml -# edit: add under spec.template.spec + +# Step 2: Add nodeAffinity (disktype=ssd) and container resources (requests/limits) +# Under spec.template.spec: # affinity: # nodeAffinity: # requiredDuringSchedulingIgnoredDuringExecution: # nodeSelectorTerms: # - matchExpressions: [{key: disktype, operator: In, values: [ssd]}] -# and under the container: resources: {requests: {cpu: 100m, memory: 64Mi}, limits: {cpu: 200m, memory: 128Mi}} -k apply -f cache.yaml +# Under containers[0]: +# resources: +# requests: {cpu: 100m, memory: 64Mi} +# limits: {cpu: 200m, memory: 128Mi} +cat < d.yaml -# add under spec.template.spec: -# nodeSelector: {kubernetes.io/hostname: {W1}} -# tolerations: [{key: %{key}, operator: Equal, value: %{val}, effect: NoSchedule}] -k apply -f d.yaml + +# Step 2: Create Deployment targeting node with matching toleration +cat < cj.yaml -# set: spec.concurrencyPolicy: Forbid, successfulJobsHistoryLimit: 3, failedJobsHistoryLimit: 1, -# spec.jobTemplate.spec.activeDeadlineSeconds: 30, ...template.spec.restartPolicy: Never -k apply -f cj.yaml + +# Step 2: Add concurrencyPolicy, activeDeadlineSeconds, and history limits: +# spec: +# concurrencyPolicy: Forbid +# successfulJobsHistoryLimit: 3 +# failedJobsHistoryLimit: 1 +# jobTemplate: +# spec: +# activeDeadlineSeconds: 30 +# template: +# spec: +# restartPolicy: Never +cat </dev/null; \\ +k -n %{ns} patch cm %{cm}-v2 -p '{"immutable":true}' + +# Step 2: Update the Deployment volume reference to the new ConfigMap k -n %{ns} patch deploy %{dep} --type=json -p='[{"op":"replace","path":"/spec/template/spec/volumes/0/configMap/name","value":"%{cm}-v2"}]' + +# Step 3: Verify rolling update finishes k -n %{ns} rollout status deploy %{dep} """ -S[31] = """ -k -n %{ns} create job %{job} --image=busybox:1.36 $do -- sh -c "echo done; sleep 2" > job.yaml -# spec: completions: %{c}, parallelism: %{p}, backoffLimit: 2 (restartPolicy Never is the default from create job) -k apply -f job.yaml && k -n %{ns} wait --for=condition=complete job/%{job} --timeout=120s + +S[31] = """# Step 1: Create Job with completions, parallelism, and backoffLimit +cat < d.yaml -# spec.template.spec: -# priorityClassName: %{dep}-prio -# tolerations: [{key: %{key}, operator: Equal, value: %{val}, effect: NoSchedule}] -# affinity: -# podAntiAffinity: -# requiredDuringSchedulingIgnoredDuringExecution: -# - labelSelector: {matchLabels: {app: %{dep}}} -# topologyKey: kubernetes.io/hostname -# container resources: {requests: {cpu: 100m, memory: 64Mi}} -k apply -f d.yaml + +# Step 3: Confirm each replica landed on a distinct node +k -n %{ns} get pods -o wide -l app=anti-workload """ -S[108] = """ -k -n %{ns} rollout history deploy %{dep} # find the revision whose image is nginx:1.25 -k -n %{ns} rollout history deploy %{dep} --revision=1 # confirm -k -n %{ns} rollout undo deploy %{dep} --to-revision=1 -k -n %{ns} scale deploy %{dep} --replicas=3 && k -n %{ns} rollout status deploy %{dep} + +S[108] = """# Step 1: Inspect rollout revision history +k -n %{ns} rollout history deploy %{dep} + +# Step 2: Roll back Deployment to specified target revision +k -n %{ns} rollout undo deploy %{dep} --to-revision=%{rev} + +# Step 3: Immediately pause the rollout k -n %{ns} rollout pause deploy %{dep} -k -n %{ns} set image deploy %{dep} nginx=nginx:1.27 -k -n %{ns} set env deploy %{dep} RELEASE=next -k -n %{ns} get rs # no new RS while paused + +# Step 4: Verify deployment status is paused +k -n %{ns} get deploy %{dep} """ -S[109] = """ + +S[109] = """# Step 1: Deploy Pod with native sidecar (restartPolicy: Always inside initContainers) cat <> /var/log/agent.log; sleep 1; done"] - volumeMounts: [{name: logs, mountPath: /var/log}] + restartPolicy: Always + command: [sh, -c, "while true; do sleep 5; done"] + readinessProbe: + exec: {command: [true]} + initialDelaySeconds: 1 containers: - - name: app + - name: main image: nginx:1.27 - volumeMounts: [{name: logs, mountPath: /var/log}] - readinessProbe: {httpGet: {path: /, port: 80}, periodSeconds: 5} - livenessProbe: {tcpSocket: {port: 80}} + livenessProbe: + httpGet: {path: /, port: 80} + initialDelaySeconds: 5 + periodSeconds: 10 EOF -k -n %{ns} get pod %{pod} -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}' | tr ' ' '\\n' | sort > {WS}/q109/containers.txt -k -n %{ns} get pod %{pod} -o jsonpath='{.status.podIP}' > {WS}/q109/ip.txt -""" -S[110] = """ -k get pods -A -o json | jq -r '.items[] | select(.metadata.namespace!="kube-system") | select(.spec.containers[0].readinessProbe==null) | "\\(.metadata.namespace)/\\(.metadata.name)"' | sort > {WS}/q110/no-readiness.txt -k get pods -A --field-selector=status.phase=Running -o jsonpath='{range .items[*]}{.spec.nodeName}{"\\n"}{end}' | sort | uniq -c | awk '{print $2, $1}' > {WS}/q110/per-node.txt -k -n kube-system get pods -o jsonpath='{.items[*].spec.containers[*].image}' | tr ' ' '\\n' | sort -u > {WS}/q110/images.txt -# no jq? use: k get pods -A -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,RP:.spec.containers[0].readinessProbe --no-headers | awk '$3=="" && $1!="kube-system"{print $1"/"$2}' | sort -""" -S[121] = """ -k config use-context k8s-c2-AC -k config view --minify -o jsonpath='{..namespace}' # -> %{ns}, write it to {WS}/q121/ns.txt -k create deploy %{dep} --image=busybox:1.36 --replicas=2 -- sleep 3600 -k label deploy %{dep} owner=ac && k patch deploy %{dep} -p '{"spec":{"template":{"metadata":{"labels":{"owner":"ac"}}}}}' -k expose deploy %{dep} --port=9000 --target-port=9000 --name=%{dep} + +# Step 2: Verify both containers are running +k -n %{ns} get pod %{pod} """ -# ================================================================== NET ==== -S[11] = """ +S[110] = r"""# Step 1: Filter pods outside kube-system with custom output +k get pods -A -o json | jq -r '.items[] | select(.metadata.namespace!="kube-system") | select(.spec.restartPolicy=="Always") | "\(.metadata.namespace) \(.metadata.name)"' > {WS}/q110/always.txt + +# Step 2: Count Running pods per node +k get pods -A --field-selector=status.phase=Running -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' | sort | uniq -c > {WS}/q110/per-node.txt + +# Step 3: Extract sorted unique container images used across kube-system +k -n kube-system get pods -o jsonpath='{.items[*].spec.containers[*].image}' | tr ' ' '\n' | sort -u > {WS}/q110/images.txt +""" + +S[121] = """# Step 1: Confirm current context namespace +k config view --minify -o jsonpath='{.contexts[0].context.namespace}' + +# Step 2: Perform all required operations without specifying -n, relying on context namespace +k create deploy %{dep} --image=nginx:1.27 --replicas=2 +k set env deploy/%{dep} APP_ENV=production +k expose deploy %{dep} --port=80 --target-port=80 +""" + + +# ================================================================= NET ===== +# Domain 3: Services & Networking (20%) +# =========================================================================== + +S[11] = """# Step 1: Create NetworkPolicy in namespace 'marketing' +# Allow Ingress on port 80 only from pods labeled 'role: frontend' cat < svc.yaml -# set spec.ports[0].nodePort: 30080 -k apply -f svc.yaml && k -n svc get endpointslices + +S[12] = """# Step 1: Expose deployment as NodePort service using dry-run +k -n web expose deploy web-server --type=NodePort --port=80 --target-port=80 $do > svc.yaml + +# Step 2: Set specific nodePort 30080 in spec.ports[0].nodePort +cat < pathType Prefix. Verify: k -n ingress-app get ingress app-ingress -o yaml + +S[13] = """# Step 1: Create Ingress resource routing host to backend service +k -n web create ingress web-ingress \\ + --class=nginx \\ + --rule="order.example.com/api*=order-service:8080" + +# Step 2: Verify Ingress rules +k -n web describe ingress web-ingress """ -S[14] = """ -k -n dns-check run dns-test --image=busybox:1.36 -- sleep 3600 -k -n dns-check exec dns-test -- nslookup hello.dns-check.svc.cluster.local | tee {WS}/q14/nslookup.txt -echo hello.dns-check.svc.cluster.local > {WS}/q14/svc-fqdn.txt + +S[14] = """# Step 1: Run temporary debug pod to query DNS +# Query A-record for a Service: ..svc.cluster.local +k -n default run dns-test --image=busybox:1.36 --rm -it --restart=Never -- \\ + nslookup web-service.apps.svc.cluster.local + +# Step 2: Query Pod DNS (IP with dashes): ..pod.cluster.local +k -n default run dns-test --image=busybox:1.36 --rm -it --restart=Never -- \\ + nslookup 10-244-1-5.apps.pod.cluster.local """ -S[22] = """ -k apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/standard-install.yaml + +S[22] = """# Step 1: Apply GatewayClass and Gateway resources cat < d.yaml -# container: ports: [{name: %{pname}, containerPort: 80}] -# readinessProbe: {httpGet: {path: /, port: %{pname}}} -k apply -f d.yaml -k -n %{ns} expose deploy %{dep} --port=8080 --target-port=%{pname} + +S[30] = """# Step 1: Update pod/deployment container definition to include named ports +# spec.containers[0].ports: [{name: http-web, containerPort: 80}] +# Step 2: Create Service referencing the container port by name +cat < h.yaml # fix selector: app: %{sts} -k apply -f h.yaml -k -n %{ns} create svc externalname legacy --external-name=%{ext} -k -n %{ns} exec %{sts}-0 -- nslookup %{sts}-0.%{sts} > {WS}/q43/pod0.txt + +S[43] = """# Step 1: Create Headless Service (clusterIP: None) +cat < {WS}/q112/fp.txt """ -S[113] = """ + +S[113] = """# Step 1: Create GatewayClass, Gateway with multi-host listeners, and canary HTTPRoute cat < scale it + +S[114] = """# Step 1: Remove stray blocking NetworkPolicy in kube-system +k -n kube-system get netpol +k -n kube-system delete netpol maintenance-lockdown + +# Step 2: Fix CoreDNS replicas (scale back from 0) k -n kube-system scale deploy coredns --replicas=2 -k -n kube-system edit cm coredns # forward . 10.255.255.254 -> forward . /etc/resolv.conf + +# Step 3: Correct CoreDNS upstream forwarding in ConfigMap +k -n kube-system edit cm coredns +# Replace: forward . 10.255.255.254 -> forward . /etc/resolv.conf + +# Step 4: Restart CoreDNS deployment and test DNS resolution k -n kube-system rollout restart deploy coredns -k run t --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.default +k run dns-check --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.default """ + # ================================================================= STOR ==== -S[15] = """ +# Domain 4: Storage (10%) +# =========================================================================== + +S[15] = """# Step 1: Create PersistentVolume with hostPath, Retain policy, and manual storageClass cat < remove it + +S[35] = """# Step 1: Inspect Released PV and locate old claimRef +k get pv %{pv} -o yaml | grep -A5 claimRef + +# Step 2: Remove the claimRef to make the PV Available again k patch pv %{pv} --type=json -p='[{"op":"remove","path":"/spec/claimRef"}]' -k get pv %{pv} # Available +k get pv %{pv} # Verify status: Available + +# Step 3: Create new PVC binding specifically to the released volume cat < /data/marker.txt"], volumeMounts: [{name: d, mountPath: /data}]}] EOF + +# Step 3: Wait for writer pod completion k -n %{ns} wait --for=jsonpath='{.status.phase}'=Succeeded pod/writer + +# Step 4: Create reader Pod mounting the same claim to verify data persists cat < add nodeName + +# Step 3: Wait for migration job completion k -n %{ns} wait --for=condition=complete job/migrate --timeout=120s -k -n %{ns} get pod legacy -o yaml > legacy.yaml # change claimName old-data -> new-data, strip status/uid/resourceVersion + +# Step 4: Re-point legacy pod to new claim +k -n %{ns} get pod legacy -o yaml > legacy.yaml +# Edit legacy.yaml: replace claimName: old-data with claimName: new-data k -n %{ns} delete pod legacy --now && k apply -f legacy.yaml -k get pv $(k -n %{ns} get pvc old-data -o jsonpath='{.spec.volumeName}') -o jsonpath='{.spec.persistentVolumeReclaimPolicy}' # Retain + +# Step 5: Clean up old PVC (PV remains Retained) k -n %{ns} delete pvc old-data """ + # =============================================================== TSHOOT ==== -S[17] = """ -k get node {W2} # NotReady +# Domain 5: Troubleshooting (30%) +# =========================================================================== + +S[17] = """# Step 1: Check node status +k get node {W2} # Status: NotReady + +# Step 2: SSH into the broken worker node {SHELL_W2} -systemctl status kubelet # inactive (dead), disabled + +# Step 3: Check kubelet service status and journal logs +systemctl status kubelet # Service inactive/disabled systemctl enable --now kubelet -journalctl -u kubelet -f # until "Successfully registered node" + +# Step 4: Verify kubelet successfully connects and registers node +journalctl -u kubelet -f --no-pager # Look for: "Successfully registered node" + +# Step 5: Exit node shell and verify Ready state from cluster +# k get nodes """ -S[18] = """ -k -n troubleshoot describe deploy payments | grep -i image # nginx:1.99-nope -> ErrImagePull + +S[18] = """# Step 1: Diagnose Deployment failure (ImagePullBackOff) +k -n troubleshoot describe deploy payments | grep -i image +# Notice invalid image tag: nginx:1.99-nope + +# Step 2: Fix image on the deployment k -n troubleshoot set image deploy payments payments=nginx:1.27 -k -n troubleshoot get svc payments -o yaml # selector app: payment (typo), targetPort 8080 (container is 80) + +# Step 3: Inspect Service definition for mismatched selector and targetPort +k -n troubleshoot get svc payments -o yaml +# Discovered issues: selector is 'app: payment' (typo) and targetPort is 8080 (container port is 80) + +# Step 4: Patch the Service selector and targetPort k -n troubleshoot patch svc payments -p '{"spec":{"selector":{"app":"payments"},"ports":[{"port":80,"targetPort":80}]}}' -k -n troubleshoot get endpointslices + +# Step 5: Verify Service endpoints +k -n troubleshoot get endpointslices -l kubernetes.io/service-name=payments """ -S[19] = """ -k -n troubleshoot logs crasher --previous | tee {WS}/q19/crasher.log # FATAL: ... DB_HOST is not set + +S[19] = """# Step 1: Capture previous container logs to locate crash reason +k -n troubleshoot logs crasher --previous | tee {WS}/q19/crasher.log +# Log output: "FATAL: required environment variable DB_HOST is not set" + +# Step 2: Export pod manifest k -n troubleshoot get pod crasher -o yaml > crasher.yaml -# add under the container: env: [{name: DB_HOST, value: db.internal}] (strip status/uid/resourceVersion) + +# Step 3: Edit crasher.yaml: +# Add under spec.containers[0]: +# env: +# - name: DB_HOST +# value: db.internal +# (Remove system-managed fields: status, uid, resourceVersion, creationTimestamp) + +# Step 4: Recreate pod and verify Running status k -n troubleshoot delete pod crasher --now && k apply -f crasher.yaml +k -n troubleshoot get pod crasher """ -S[20] = """ -k -n kube-system get pods # kube-scheduler-... CrashLoopBackOff / Error -k -n kube-system logs kube-scheduler-{CP} # exec: "kube-schedulerr": not found + +S[20] = """# Step 1: Check control-plane pods in kube-system +k -n kube-system get pods +# kube-scheduler static pod is in Error / CrashLoopBackOff + +# Step 2: Inspect scheduler pod logs +k -n kube-system logs kube-scheduler-{CP} +# Error: exec: "kube-schedulerr": executable file not found + +# Step 3: SSH into control plane node {SHELL_CP} -vi /etc/kubernetes/manifests/kube-scheduler.yaml # command: - kube-schedulerr -> - kube-scheduler -# kubelet restarts it; k -n kube-system get pods -w + +# Step 4: Edit scheduler static pod manifest +# In /etc/kubernetes/manifests/kube-scheduler.yaml: +# Change: command: [kube-schedulerr] -> command: [kube-scheduler] +vi /etc/kubernetes/manifests/kube-scheduler.yaml + +# Step 5: Wait for kubelet to reload the static pod and verify scheduling +k -n kube-system get pods -l component=kube-scheduler """ -S[37] = """ -k -n kube-system get pods -l k8s-app=kube-dns; k -n kube-system logs -l k8s-app=kube-dns --tail=20 -k -n kube-system get cm coredns -o yaml # forward . 10.255.255.1 -> unreachable upstream -k -n kube-system edit cm coredns # forward . /etc/resolv.conf + +S[37] = """# Step 1: Inspect CoreDNS pods and logs +k -n kube-system get pods -l k8s-app=kube-dns +k -n kube-system logs -l k8s-app=kube-dns --tail=20 + +# Step 2: Check CoreDNS ConfigMap +k -n kube-system get cm coredns -o yaml +# Fault: forward . 10.255.255.1 points to an invalid upstream DNS + +# Step 3: Edit ConfigMap to use host resolv.conf +k -n kube-system edit cm coredns +# Change: forward . 10.255.255.1 -> forward . /etc/resolv.conf + +# Step 4: Restart CoreDNS deployment to apply changes k -n kube-system rollout restart deploy coredns -k run t --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.default + +# Step 5: Verify DNS resolution from a test pod +k run test-dns --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.default """ -S[38] = """ -k -n kube-system get pods # kube-controller-manager Error/CrashLoop -k -n kube-system logs kube-controller-manager-{CP} # flag error: lease duration ... invalid + +S[38] = """# Step 1: Check control-plane pod status +k -n kube-system get pods +# kube-controller-manager is crashing + +# Step 2: Check controller-manager logs +k -n kube-system logs kube-controller-manager-{CP} +# Error: invalid flag: --leader-elect-lease-duration=1s + +# Step 3: Access control plane node {SHELL_CP} -vi /etc/kubernetes/manifests/kube-controller-manager.yaml # remove the bogus --leader-elect-lease-duration=1s line -k create deploy probe --image=busybox:1.36 -- sleep 60 && k get rs + +# Step 4: Edit static pod manifest to remove the invalid flag +vi /etc/kubernetes/manifests/kube-controller-manager.yaml + +# Step 5: Verify controller-manager restarts and reconciles deployments/replicasets +k create deploy probe --image=busybox:1.36 -- sleep 60 +k get rs +k delete deploy probe """ -S[39] = """ + +S[39] = """# Step 1: Access the failing worker node {SHELL_W1} -systemctl status kubelet; journalctl -u kubelet -n 30 --no-pager # "open /etc/kubernetes/pki/ca-bundle.crt: no such file" -grep -n clientCAFile /var/lib/kubelet/config.yaml # -> /etc/kubernetes/pki/ca.crt + +# Step 2: Inspect kubelet status and journal logs +systemctl status kubelet +journalctl -u kubelet -n 30 --no-pager +# Error: "open /etc/kubernetes/pki/ca-bundle.crt: no such file or directory" + +# Step 3: Check /var/lib/kubelet/config.yaml for incorrect certificate path +grep -n clientCAFile /var/lib/kubelet/config.yaml +# Fix path: replace /etc/kubernetes/pki/ca-bundle.crt with /etc/kubernetes/pki/ca.crt +sed -i 's|ca-bundle.crt|ca.crt|' /var/lib/kubelet/config.yaml + +# Step 4: Restart and enable kubelet service systemctl restart kubelet && systemctl enable kubelet +systemctl status kubelet """ -# -------------------------------------------------------------- hard TSHOOT -S[117] = """ + +# Hard Troubleshooting Tasks +S[117] = """# Step 1: Check node status in cluster k get node {W2} # NotReady,SchedulingDisabled + +# Step 2: Access the node {SHELL_W2} -systemctl status kubelet # disabled + dead -journalctl -u kubelet -n 30 --no-pager # after enable: "ca-wrong.crt: no such file" + +# Step 3: Fix Fault 1 (kubelet service disabled) and Fault 2 (wrong CA certificate file) +systemctl status kubelet sed -i 's|ca-wrong.crt|ca.crt|' /var/lib/kubelet/config.yaml systemctl enable --now kubelet +systemctl status kubelet exit -k uncordon {W2} # the third thing -k run t --image=busybox:1.36 --restart=Never --overrides='{"spec":{"nodeName":"{W2}"}}' -- sleep 60 + +# Step 4: Fix Fault 3 (node was cordoned in cluster) +k uncordon {W2} + +# Step 5: Verify scheduling on the repaired node +k run test-pod --image=busybox:1.36 --restart=Never --overrides='{"spec":{"nodeName":"{W2}"}}' -- sleep 60 +k get pod test-pod -o wide +k delete pod test-pod --now """ -S[118] = """ -k -n kube-system get pods # scheduler CrashLoop, controller-manager ImagePullBackOff + +S[118] = """# Step 1: Check control plane pods in kube-system +k -n kube-system get pods +# scheduler CrashLoop, controller-manager ImagePullBackOff + +# Step 2: Access control plane node {SHELL_CP} -vi /etc/kubernetes/manifests/kube-scheduler.yaml # kube-schedulerr -> kube-scheduler -vi /etc/kubernetes/manifests/kube-controller-manager.yaml # image tag ...-broken -> remove suffix (match the others: crictl images) + +# Step 3: Fix kube-scheduler manifest command typo +vi /etc/kubernetes/manifests/kube-scheduler.yaml +# Replace: kube-schedulerr -> kube-scheduler + +# Step 4: Fix kube-controller-manager image tag +vi /etc/kubernetes/manifests/kube-controller-manager.yaml +# Remove broken image tag suffix to match cluster version + +# Step 5: Verify both static pods return to Running k -n kube-system get pods -w -k create deploy probe --image=busybox:1.36 -- sleep 60 && k get deploy probe -w +k create deploy probe --image=busybox:1.36 -- sleep 60 +k get deploy probe +k delete deploy probe """ -S[119] = """ -k -n %{ns} get all; k -n %{ns} describe deploy %{dep}; k -n %{ns} describe rs -l app=%{dep} # quota: pods 1 + +S[119] = """# Step 1: Diagnose namespace quota and deployment events +k -n %{ns} get all +k -n %{ns} describe deploy %{dep} +k -n %{ns} describe rs -l app=%{dep} # Quota exceeded: pods limit is 1 + +# Step 2: Fix quota to accommodate required replicas k -n %{ns} patch quota tight -p '{"spec":{"hard":{"pods":"3"}}}' -k -n %{ns} set image deploy %{dep} web=nginx:1.27 # image tag typo -k -n %{ns} patch secret %{dep}-env -p '{"stringData":{"DATABASE_URL":"postgres://db"}}' # missing key referenced by env -k -n %{ns} rollout restart deploy %{dep} # picks up the secret key + +# Step 3: Fix deployment image tag +k -n %{ns} set image deploy %{dep} web=nginx:1.27 + +# Step 4: Fix Secret missing key referenced by container environment variables +k -n %{ns} patch secret %{dep}-env -p '{"stringData":{"DATABASE_URL":"postgres://db"}}' +k -n %{ns} rollout restart deploy %{dep} + +# Step 5: Fix Service selector and targetPort k -n %{ns} patch svc %{dep} -p '{"spec":{"selector":{"app":"%{dep}"},"ports":[{"port":80,"targetPort":80}]}}' + +# Step 6: Verify all deployment replicas are Available with endpoints +k -n %{ns} get deploy %{dep} +k -n %{ns} get endpoints %{dep} """ -S[120] = """ + +S[120] = """# Step 1: Query cluster CIDR from kube-controller-manager pod spec k -n kube-system get pod -l component=kube-controller-manager -o yaml | grep cluster-cidr + +# Step 2: Query service CIDR from kube-apiserver pod spec k -n kube-system get pod -l component=kube-apiserver -o yaml | grep service-cluster-ip-range + +# Step 3: Query CoreDNS ClusterIP k -n kube-system get svc kube-dns -o jsonpath='{.spec.clusterIP}' -k -n kube-system get cm kube-proxy -o yaml | grep '^ *mode:' # "" -> iptables + +# Step 4: Query kube-proxy operating mode from ConfigMap +k -n kube-system get cm kube-proxy -o yaml | grep '^ *mode:' # Empty string indicates default: iptables + +# Step 5: Query individual node Pod CIDR k get node %{node} -o jsonpath='{.spec.podCIDR}' """