"""Reference solutions, keyed by task id. Rendered with the same %{param} / {WS} / {W1} substitution as the task text. Shown only after a task has been graded (drills) or the session submitted (exams). These solutions provide step-by-step instructions, Kubernetes concept explanations, imperative commands, and verification checks for exam success. """ from sim.bank import cmd, hand S = {} # ================================================================= ARCH ==== # Domain 1: Cluster Architecture, Installation & Configuration (25%) # =========================================================================== S[1] = """# Step 1: Create the ServiceAccount in the target namespace k -n rbac-app create sa deploy-bot # Step 2: Create a Role granting specified verbs on deployments.apps # Verbs: get, list, watch, create, update, patch, delete k -n rbac-app create role deploy-manager \\ --verb=get,list,watch,create,update,patch,delete \\ --resource=deployments.apps # Step 3: Bind the Role to the ServiceAccount k -n rbac-app create rolebinding deploy-bot-binding \\ --role=deploy-manager \\ --serviceaccount=rbac-app:deploy-bot # Step 4: Verify permissions using 'kubectl auth can-i' k auth can-i create deployments -n rbac-app --as=system:serviceaccount:rbac-app:deploy-bot # Expected: yes k auth can-i delete pods -n rbac-app --as=system:serviceaccount:rbac-app:deploy-bot # Expected: no """ S[2] = """# Step 1: Access the control plane node sudo -i # the task's host is the control-plane node: no hop needed # Step 2: Take the etcd snapshot with TLS certificates and save to the target path # Certificates are located in /etc/kubernetes/pki/etcd/ ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 \\ --cacert=/etc/kubernetes/pki/etcd/ca.crt \\ --cert=/etc/kubernetes/pki/etcd/server.crt \\ --key=/etc/kubernetes/pki/etcd/server.key \\ snapshot save /opt/cka/etcd-snapshot.db # Step 3: Verify snapshot status and integrity etcdutl snapshot status /opt/cka/etcd-snapshot.db -w table # Note: on a cluster where etcdctl is only inside the static pod, run it there. The etcd image # is distroless (no sh, no cat, no tar), so kubectl cp cannot get the file out - write it into # /var/lib/etcd instead, which is a hostPath, and move it on the node: # kubectl -n kube-system exec etcd-{CP} -- etcdctl ... snapshot save /var/lib/etcd/snap.db # then on the node: mv /var/lib/etcd/snap.db /opt/cka/etcd-snapshot.db """ S[3] = [cmd(r"""mkdir -p {WS}/q03/overlays/prod cd {WS}/q03/overlays/prod k create ns store-prod --dry-run=client -o yaml > namespace.yaml"""), hand(r"""vim kustomization.yaml""", r"""The overlay: the base and the Namespace as resources, the namespace, the label and the replica count. ``` resources: - ../../base - namespace.yaml namespace: store-prod labels: - pairs: env: prod replicas: - name: store-api count: 3 ```""", run=r"""cat > kustomization.yaml <<'EOF' resources: - ../../base - namespace.yaml namespace: store-prod labels: - pairs: env: prod replicas: - name: store-api count: 3 EOF"""), cmd(r"""k kustomize . k apply -k . k -n store-prod get deploy,svc --show-labels cd""", r"""`k kustomize .` prints what the overlay makes, before anything is applied.""")] S[4] = [hand(r"""vim crd.yaml""", r"""Paste the CustomResourceDefinition from the docs page *Extend the Kubernetes API with CustomResourceDefinitions* and change it to: ``` apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: backups.data.example.com spec: group: data.example.com versions: - name: v1 served: true storage: true schema: openAPIV3Schema: type: object properties: spec: type: object properties: schedule: type: string target: type: string scope: Namespaced names: plural: backups singular: backup kind: Backup shortNames: - bk ```""", run=r"""cat > crd.yaml <<'EOF' apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: backups.data.example.com spec: group: data.example.com versions: - name: v1 served: true storage: true schema: openAPIV3Schema: type: object properties: spec: type: object properties: schedule: type: string target: type: string scope: Namespaced names: plural: backups singular: backup kind: Backup shortNames: - bk EOF"""), cmd(r"""k apply -f crd.yaml"""), hand(r"""vim backup.yaml""", r"""The Backup itself - the docs page has one for its example kind just below: ``` apiVersion: data.example.com/v1 kind: Backup metadata: name: nightly spec: schedule: "0 2 * * *" target: s3://backups/prod ```""", run=r"""kubectl wait --for condition=established crd/backups.data.example.com --timeout=60s cat > backup.yaml <<'EOF' apiVersion: data.example.com/v1 kind: Backup metadata: name: nightly spec: schedule: "0 2 * * *" target: s3://backups/prod EOF"""), cmd(r"""k apply -f backup.yaml k get bk""")] S[5] = [cmd(r"""k run static-web --image=nginx:1.27 --dry-run=client -o yaml""", r"""Copy what it prints: the static Pod's manifest."""), hand(r"""{SHELL_W3} sudo -i grep staticPodPath /var/lib/kubelet/config.yaml vim /etc/kubernetes/manifests/static-web.yaml""", r"""Paste the Pod into the static Pod path `grep` printed, `:wq`, and `exit` twice - back on the host. The kubelet starts it within seconds.""", run=r"""kubectl run static-web --image=nginx:1.27 --dry-run=client -o yaml | ssh {W3} 'sudo tee /etc/kubernetes/manifests/static-web.yaml >/dev/null'"""), cmd(r"""k get pod static-web-{W3}""", r"""Again until it is `Running`: the mirror Pod is named after the node.""")] S[6] = [cmd(r"""sudo kubeadm certs check-expiration | grep apiserver sudo kubeadm certs check-expiration | grep '^apiserver ' > {WS}/q06/apiserver-expiry.txt""", r"""The host is the control-plane node. `^apiserver ` is the serving certificate's line alone, not `apiserver-etcd-client` or `apiserver-kubelet-client`."""), cmd(r"""openssl genrsa -out jane.key 2048 openssl req -new -key jane.key -subj /CN=jane -out jane.csr base64 -w0 jane.csr""", r"""Copy the line `base64` prints: the CSR as the API wants it."""), hand(r"""vim csr.yaml""", r"""Paste the CertificateSigningRequest from the docs page *Certificates and Certificate Signing Requests* (under *Create a CertificateSigningRequest*), name it `jane`, and put the copied line in `request:` ``` apiVersion: certificates.k8s.io/v1 kind: CertificateSigningRequest metadata: name: jane spec: request: LS0tLS1CRUdJTi... signerName: kubernetes.io/kube-apiserver-client usages: - client auth ```""", run=r"""cat > csr.yaml < {WS}/q34/path.txt"""), hand(r"""vim sample.yaml""", r"""``` apiVersion: demo.io/v1 kind: %{kind} metadata: name: sample namespace: default spec: tuning: %{field}: %{n} ```""", run=r"""cat > sample.yaml <<'EOF' apiVersion: demo.io/v1 kind: %{kind} metadata: name: sample namespace: default spec: tuning: %{field}: %{n} EOF"""), cmd(r"""k apply -f sample.yaml""")] S[40] = """# Generate a new kubeadm join token with the specified TTL and print-join-command, # writing the output straight into the required file (the task's host is the control-plane node) sudo kubeadm token create --ttl %{ttl} --print-join-command > {WS}/q40/join.sh # Verify active tokens sudo kubeadm token list """ S[41] = """# Step 1: Check Kubernetes Server version (API server) k version # Step 2: Extract Kubelet and Container Runtime versions for the target node k get node %{node} -o jsonpath='{.status.nodeInfo.kubeletVersion}{"\\n"}{.status.nodeInfo.containerRuntimeVersion}{"\\n"}' # Step 3: Extract etcd image tag from the static pod spec k -n kube-system get pod -l component=etcd -o jsonpath='{.items[0].spec.containers[0].image}' # Parse the image tag (everything after the colon) """ # Hard Architecture Tasks S[44] = [cmd(r"""k get clusterrolebinding -o wide | grep %{sa} k -n %{ns} get rolebinding -o wide | grep %{sa}""", r"""`%{sa}-legacy-admin` binds `cluster-admin` to it: far more than allowed."""), cmd(r"""k delete clusterrolebinding %{sa}-legacy-admin k -n %{ns} create role %{sa}-role --verb=get,list,watch,patch --resource=deployments.apps,replicasets.apps"""), hand(r"""k -n %{ns} edit role %{sa}-role""", r"""`create role` gives every verb to every resource it names, so the second rule goes in by hand - add it below the first in `rules:`, then `:wq`: ``` - apiGroups: - "" resources: - pods/exec verbs: - create ```""", run=r"""kubectl -n %{ns} patch role %{sa}-role --type=json -p='[{"op":"add","path":"/rules/-","value":{"apiGroups":[""],"resources":["pods/exec"],"verbs":["create"]}}]'"""), cmd(r"""k -n %{ns} create rolebinding %{sa}-rb --role=%{sa}-role --serviceaccount=%{ns}:%{sa} k create clusterrole %{sa}-nodes --verb=list --resource=nodes k create clusterrolebinding %{sa}-crb --clusterrole=%{sa}-nodes --serviceaccount=%{ns}:%{sa}"""), cmd(r"""k auth can-i --list -n %{ns} --as=system:serviceaccount:%{ns}:%{sa} k auth can-i create pods --subresource=exec -n %{ns} --as=system:serviceaccount:%{ns}:%{sa}""", r"""`yes`. For a subresource, `--subresource`: `can-i create pods/exec` asks about a Pod named `exec`, and says `no` even when the Role is right.""")] S[45] = [cmd(r"""sudo etcdutl snapshot restore /opt/cka/backup.db --data-dir /var/lib/etcd-restore""", r"""The host is the control-plane node. The restore writes a new data directory; etcd keeps running on the old one until its manifest says otherwise."""), hand(r"""sudo vim /etc/kubernetes/manifests/etcd.yaml""", r"""In `volumes:` at the bottom, point `etcd-data` at the restored directory, then `:wq`: ``` - hostPath: path: /var/lib/etcd-restore type: DirectoryOrCreate name: etcd-data ``` The kubelet sees the change and restarts etcd; the API server is gone for a minute or two meanwhile.""", run=r"""cpwait() { local i for i in $(seq 1 80); do [ "$(kubectl -n kube-system get pod $1-{CP} -o jsonpath='{.status.containerStatuses[0].ready}' 2>/dev/null)" = true ] && return 0 sleep 3 done return 1 } sudo sed -i 's|path: /var/lib/etcd$|path: /var/lib/etcd-restore|' /etc/kubernetes/manifests/etcd.yaml sleep 30 for i in $(seq 1 80); do kubectl -n default get cm %{cm} >/dev/null 2>&1 && break; sleep 3; done for c in etcd kube-apiserver kube-scheduler kube-controller-manager; do cpwait $c; done"""), cmd(r"""sudo crictl ps | grep -e etcd -e apiserver k -n default get cm %{cm} k -n kube-system get pods""", r"""Again until `k` answers: the ConfigMap is back, and every control-plane Pod `Running`.""")] S[46] = [cmd(r"""k get nodes -l tier=edge k create ns %{ns} k -n %{ns} run %{pod} --image=nginx:1.27 --labels=role=edge --dry-run=client -o yaml > static.yaml""", r"""The namespace may be there already; the mirror Pod needs it either way."""), hand(r"""vim static.yaml""", r"""Replace the container's `resources: {}` with: ``` resources: requests: cpu: 50m memory: 32Mi ```""", run=r"""yq -i '.spec.containers[0].resources = {"requests": {"cpu": "50m", "memory": "32Mi"}}' static.yaml"""), cmd(r"""scp static.yaml {W3}:"""), hand(r"""{SHELL_W3} sudo -i grep staticPodPath /var/lib/kubelet/config.yaml""", r"""Not the usual `/etc/kubernetes/manifests`: `/etc/kubernetes/static`.""", run=r""""""), hand(r"""mv /home/candidate/static.yaml /etc/kubernetes/static/%{pod}.yaml exit exit""", r"""Into the path the kubelet actually watches; then back on the host.""", run=r"""ssh -n {W3} 'sudo mv /home/candidate/static.yaml /etc/kubernetes/static/%{pod}.yaml'"""), cmd(r"""k -n %{ns} get pod %{pod}-{W3}""", r"""Again until it is `Running`.""")] S[47] = [cmd(r"""openssl genrsa -out %{user}.key 2048 openssl req -new -key %{user}.key -subj "/CN=%{user}/O=developers" -out %{user}.csr base64 -w0 %{user}.csr""", r"""`O=` is the group. Copy the line `base64` prints."""), hand(r"""vim csr.yaml""", r"""The CertificateSigningRequest from the docs page *Certificates and Certificate Signing Requests*, with the copied line in `request:` ``` apiVersion: certificates.k8s.io/v1 kind: CertificateSigningRequest metadata: name: %{user} spec: request: LS0tLS1CRUdJTi... signerName: kubernetes.io/kube-apiserver-client usages: - client auth ```""", run=r"""cat > csr.yaml < %{user}.crt""", r"""Once it is `Issued`: read too early, the file is empty.""", run=r"""until [ -n "$(kubectl get csr %{user} -o jsonpath='{.status.certificate}')" ]; do sleep 1; done kubectl get csr %{user} -o jsonpath='{.status.certificate}' | base64 -d > %{user}.crt"""), cmd(r"""k -n %{ns} create rolebinding %{user}-edit --clusterrole=edit --user=%{user} k config view --minify | grep server""", r"""The API server's address, for the new kubeconfig."""), hand(r"""k --kubeconfig {WS}/q47/%{user}.kubeconfig config set-cluster lab --server=https://... --certificate-authority=/etc/kubernetes/pki/ca.crt --embed-certs""", r"""With the address `grep` printed in `--server=`. `--embed-certs` puts the CA itself in the file, not its path.""", run=r"""kubectl --kubeconfig {WS}/q47/%{user}.kubeconfig config set-cluster lab --server=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}') --certificate-authority=/etc/kubernetes/pki/ca.crt --embed-certs"""), cmd(r"""k --kubeconfig {WS}/q47/%{user}.kubeconfig config set-credentials %{user} --client-certificate=%{user}.crt --client-key=%{user}.key --embed-certs k --kubeconfig {WS}/q47/%{user}.kubeconfig config set-context %{user}@lab --cluster=lab --user=%{user} --namespace=%{ns} k --kubeconfig {WS}/q47/%{user}.kubeconfig config use-context %{user}@lab k --kubeconfig {WS}/q47/%{user}.kubeconfig get pods""", r"""`No resources found in %{ns} namespace.` - no flags, the context's namespace, and `%{user}`'s own rights.""")] S[48] = [cmd(r"""mkdir -p {WS}/q48/overlays/prod cd {WS}/q48/overlays/prod"""), hand(r"""vim replicas.yaml""", r"""A strategic merge patch: enough of the Deployment to name it, and the field to change. ``` apiVersion: apps/v1 kind: Deployment metadata: name: web spec: replicas: %{rep} ```""", run=r"""cat > replicas.yaml <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: name: web spec: replicas: %{rep} EOF"""), hand(r"""vim envfrom.yaml""", r"""A JSON 6902 patch: a list of operations, each with a path into the object. ``` - op: add path: /spec/template/spec/containers/0/envFrom value: - configMapRef: name: web-config ```""", run=r"""cat > envfrom.yaml <<'EOF' - op: add path: /spec/template/spec/containers/0/envFrom value: - configMapRef: name: web-config EOF"""), hand(r"""vim kustomization.yaml""", r"""A JSON 6902 patch names its target; the strategic merge patch names its own. Kustomize rewrites `web-config` in `envFrom` to the generated name, hash and all. ``` namespace: %{ns} resources: - ../../base labels: - pairs: tier: prod includeSelectors: false configMapGenerator: - name: web-config literals: - LOG_LEVEL=%{lvl} patches: - path: replicas.yaml - path: envfrom.yaml target: kind: Deployment name: web ```""", run=r"""cat > kustomization.yaml <<'EOF' namespace: %{ns} resources: - ../../base labels: - pairs: tier: prod includeSelectors: false configMapGenerator: - name: web-config literals: - LOG_LEVEL=%{lvl} patches: - path: replicas.yaml - path: envfrom.yaml target: kind: Deployment name: web EOF"""), cmd(r"""k kustomize . k create ns %{ns} k apply -k . k -n %{ns} get deploy,cm cd""", r"""The namespace first: `namespace:` puts the objects in it, but does not make it.""")] S[49] = [cmd(r"""k config view --minify | grep namespace""", r"""Not `default`: every command below says `-n default`."""), hand(r"""vim crd.yaml""", r"""Start from the CRD on the docs page *Extend the Kubernetes API with CustomResourceDefinitions*; its *Validation* and *Defaulting* sections have `minimum`, `enum` and `default`. `required` sits beside `properties`, not inside a field: ``` apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: %{low}s.deploy.example.com spec: group: deploy.example.com scope: Namespaced names: kind: %{kind} plural: %{low}s singular: %{low} shortNames: - %{short} versions: - name: v1 served: true storage: true schema: openAPIV3Schema: type: object properties: spec: type: object required: - replicas - image properties: replicas: type: integer minimum: 1 maximum: 10 environment: type: string enum: - prod - stage - dev default: dev image: type: string ```""", run=r"""cat > crd.yaml <<'EOF' apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: %{low}s.deploy.example.com spec: group: deploy.example.com scope: Namespaced names: kind: %{kind} plural: %{low}s singular: %{low} shortNames: - %{short} versions: - name: v1 served: true storage: true schema: openAPIV3Schema: type: object properties: spec: type: object required: - replicas - image properties: replicas: type: integer minimum: 1 maximum: 10 environment: type: string enum: - prod - stage - dev default: dev image: type: string EOF"""), cmd(r"""k apply -f crd.yaml"""), hand(r"""vim good.yaml""", r"""``` apiVersion: deploy.example.com/v1 kind: %{kind} metadata: name: good spec: replicas: 3 image: nginx:1.27 environment: %{env} ```""", run=r"""kubectl wait --for condition=established crd/%{low}s.deploy.example.com --timeout=60s cat > good.yaml <<'EOF' apiVersion: deploy.example.com/v1 kind: %{kind} metadata: name: good spec: replicas: 3 image: nginx:1.27 environment: %{env} EOF"""), cmd(r"""k -n default apply -f good.yaml"""), hand(r"""cp good.yaml bad.yaml vim bad.yaml""", r"""Name it `bad` and set `replicas: 0`.""", run=r"""cp good.yaml bad.yaml yq -i '.metadata.name = "bad" | .spec.replicas = 0' bad.yaml"""), cmd(r"""k -n default apply -f bad.yaml 2> {WS}/q49/rejected.txt cat {WS}/q49/rejected.txt""", r"""`2>`, not `>`: the rejection goes to standard error. It says `spec.replicas` should be greater than or equal to 1."""), hand(r"""cp good.yaml defaulted.yaml vim defaulted.yaml""", r"""Name it `defaulted`, `replicas: 2`, and delete the `environment:` line - the API server fills in the default.""", run=r"""cp good.yaml defaulted.yaml yq -i '.metadata.name = "defaulted" | .spec.replicas = 2 | del(.spec.environment)' defaulted.yaml"""), cmd(r"""k -n default apply -f defaulted.yaml k -n default get %{short} defaulted -o yaml | grep environment""", r"""`environment: dev`.""")] S[65] = """# Step 1: Access control plane node sudo -i # the task's host is the control-plane node: no hop needed # Step 2: Renew all cluster certificates via kubeadm kubeadm certs renew all # Step 3: Restart static pods to load renewed certificates mv /etc/kubernetes/manifests/*.yaml /tmp/ && sleep 20 && mv /tmp/*.yaml /etc/kubernetes/manifests/ # Step 4: Update admin kubeconfig from newly generated credentials cp /etc/kubernetes/admin.conf ~/.kube/config exit # Step 5: Out of the root shell again, write the expiry report into the required file sudo kubeadm certs check-expiration | grep '^apiserver ' > {WS}/q65/expiry.txt """ S[67] = [cmd(r"""sudo kubeadm token create --ttl 2h""", r"""The host is the control-plane node. Copy the token it prints."""), cmd(r"""openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex""", r"""The SHA-256 of the CA's public key, as the docs page *kubeadm join* computes it - what `--discovery-token-ca-cert-hash` checks. The hash is everything after `= `."""), cmd(r"""k get endpointslices -l kubernetes.io/service-name=kubernetes""", r"""The address and port the API server advertises - not `127.0.0.1:`, how this host's own kubeconfig reaches it."""), hand(r"""vim {WS}/q67/node-join.env""", r"""The three values you copied: ``` JOIN_TOKEN=abcdef.0123456789abcdef CA_HASH=sha256: API_ENDPOINT=
: ```""", run=r"""T=$(sudo kubeadm token create --ttl 2h) H=$(openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //') E=$(kubectl get endpointslices -l kubernetes.io/service-name=kubernetes -o jsonpath='{.items[0].endpoints[0].addresses[0]}:{.items[0].ports[0].port}') printf 'JOIN_TOKEN=%s\nCA_HASH=sha256:%s\nAPI_ENDPOINT=%s\n' "$T" "$H" "$E" > {WS}/q67/node-join.env""")] # ================================================================= WORK ==== # Domain 2: Workloads & Scheduling (15%) # =========================================================================== S[7] = [cmd(r"""k -n apps set image deploy web-frontend nginx=nginx:1.27 k -n apps scale deploy web-frontend --replicas=4"""), hand(r"""k -n apps edit deploy web-frontend""", r"""Under `strategy:`, set the rolling update, then `:wq`: ``` strategy: rollingUpdate: maxSurge: 1 maxUnavailable: 0 type: RollingUpdate ```""", run=r"""kubectl -n apps patch deploy web-frontend -p '{"spec":{"strategy":{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0}}}}'"""), cmd(r"""k -n apps rollout status deploy web-frontend k -n apps get deploy web-frontend""", r"""`READY 4/4`.""")] S[8] = [cmd(r"""k -n apps run log-shipper --image=busybox:1.36 --dry-run=client -o yaml --command -- sh -c 'while true; do date >> /var/log/app.log; sleep 2; done' > pod.yaml"""), hand(r"""vim pod.yaml""", r"""Name the container `app`, give it the mount, and copy it as a second container `shipper` with its own command - then the volume, in the Pod's `spec:`. `containers:` ends up as: ``` containers: - command: - sh - -c - while true; do date >> /var/log/app.log; sleep 2; done image: busybox:1.36 name: app volumeMounts: - name: logs mountPath: /var/log - command: - sh - -c - tail -f /var/log/app.log image: busybox:1.36 name: shipper volumeMounts: - name: logs mountPath: /var/log volumes: - name: logs emptyDir: {} ```""", run=r"""yq -i '.spec.containers[0].name = "app" | .spec.containers[0].volumeMounts = [{"name": "logs", "mountPath": "/var/log"}] | .spec.containers[1] = .spec.containers[0] | .spec.containers[1].name = "shipper" | .spec.containers[1].command = ["sh", "-c", "tail -f /var/log/app.log"] | .spec.volumes = [{"name": "logs", "emptyDir": {}}]' pod.yaml"""), cmd(r"""k apply -f pod.yaml k -n apps logs log-shipper -c shipper""", r"""Again after a few seconds: a date every 2 seconds, written by `app`, read by `shipper`.""")] S[9] = [cmd(r"""k get nodes -L disktype k -n apps create deploy cache --image=redis:7-alpine --replicas=2 --dry-run=client -o yaml > cache.yaml"""), hand(r"""vim cache.yaml""", r"""Add to the Pod template's `spec:`, above `containers:` (the docs page *Assign Pods to Nodes using Node Affinity* has the block): ``` affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: disktype operator: In values: - ssd ``` and replace the container's `resources: {}` with: ``` resources: requests: cpu: 100m memory: 64Mi limits: cpu: 200m memory: 128Mi ```""", run=r"""yq -i '.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms = [{"matchExpressions": [{"key": "disktype", "operator": "In", "values": ["ssd"]}]}] | .spec.template.spec.containers[0].resources = {"requests": {"cpu": "100m", "memory": "64Mi"}, "limits": {"cpu": "200m", "memory": "128Mi"}}' cache.yaml"""), cmd(r"""k apply -f cache.yaml k -n apps get pods -o wide""", r"""Both on the node labelled `disktype=ssd`.""")] S[10] = [cmd(r"""k -n apps create cm app-config --from-literal=APP_MODE=production --from-literal=LOG_LEVEL=debug k -n apps create secret generic db-creds --from-literal=username=admin --from-literal='password=S3cr3t!' k -n apps run config-pod --image=busybox:1.36 --dry-run=client -o yaml --command -- sleep 3600 > p.yaml""", r"""The quotes keep the shell off the `!` in the password."""), hand(r"""vim p.yaml""", r"""Add to the container, below `name: config-pod` (the docs page *Configure a Pod to Use a ConfigMap* has each of these): ``` env: - name: APP_MODE valueFrom: configMapKeyRef: name: app-config key: APP_MODE envFrom: - secretRef: name: db-creds volumeMounts: - name: cfg mountPath: /etc/config ``` and to the Pod's `spec:`, right below `spec:` and indented like `containers:`: ``` volumes: - name: cfg configMap: name: app-config ```""", run=r"""yq -i '.spec.containers[0].env = [{"name": "APP_MODE", "valueFrom": {"configMapKeyRef": {"name": "app-config", "key": "APP_MODE"}}}] | .spec.containers[0].envFrom = [{"secretRef": {"name": "db-creds"}}] | .spec.containers[0].volumeMounts = [{"name": "cfg", "mountPath": "/etc/config"}] | .spec.volumes = [{"name": "cfg", "configMap": {"name": "app-config"}}]' p.yaml"""), cmd(r"""k apply -f p.yaml k -n apps exec config-pod -- ls /etc/config k -n apps exec config-pod -- env""", r"""Once it is `Running`: `APP_MODE`, `LOG_LEVEL` as files, and `username` and `password` in the environment.""")] S[23] = """# Step 1: Repair metrics-server by adding --kubelet-insecure-tls flag k -n kube-system edit deploy metrics-server # Add under spec.template.spec.containers[0].args: # - --kubelet-insecure-tls # Step 2: Wait ~30s and verify metrics reporting k top nodes # Step 3: Configure HorizontalPodAutoscaler (HPA) for target deployment k -n %{ns} autoscale deploy %{dep} --min=2 --max=%{max} --cpu-percent=%{cpu} --name=%{dep} # Step 4: Verify HPA status k -n %{ns} get hpa %{dep} """ S[24] = [cmd(r"""k create priorityclass %{pc} --value=%{val} --description="%{pc} workloads" k get pc %{pc}""", r"""`GLOBAL-DEFAULT false`: the default unless `--global-default`."""), hand(r"""k -n %{ns} edit deploy payments""", r"""In the Pod template's `spec:`, beside `containers:` and indented like it, add the line below, then `:wq` - the change rolls the Pods: ``` priorityClassName: %{pc} ```""", run=r"""kubectl -n %{ns} patch deploy payments -p '{"spec":{"template":{"spec":{"priorityClassName":"%{pc}"}}}}'"""), cmd(r"""k -n %{ns} rollout status deploy payments k -n %{ns} get pods -o custom-columns=NAME:.metadata.name,PRIORITY:.spec.priority""", r"""Every Pod at `%{val}`.""")] S[25] = [cmd(r"""k taint node {W1} %{key}=%{val}:NoSchedule k -n %{ns} create deploy %{dep} --image=busybox:1.36 --replicas=2 --dry-run=client -o yaml -- sleep 3600 > d.yaml"""), hand(r"""vim d.yaml""", r"""In the Pod template's `spec:`, above `containers:` (the docs page *Taints and Tolerations* has the toleration): ``` nodeSelector: kubernetes.io/hostname: {W1} tolerations: - key: %{key} operator: Equal value: %{val} effect: NoSchedule ```""", run=r"""yq -i '.spec.template.spec.nodeSelector = {"kubernetes.io/hostname": "{W1}"} | .spec.template.spec.tolerations = [{"key": "%{key}", "operator": "Equal", "value": "%{val}", "effect": "NoSchedule"}]' d.yaml"""), cmd(r"""k apply -f d.yaml k -n %{ns} get pods -o wide""", r"""Both `Running` on `{W1}`.""")] S[26] = [cmd(r"""k -n %{ns} create deploy %{ds} --image=busybox:1.36 --dry-run=client -o yaml -- sleep 3600 > ds.yaml""", r"""There is no `create daemonset`: a Deployment's YAML is the closest start."""), hand(r"""vim ds.yaml""", r"""Make it a DaemonSet: `kind: DaemonSet`, and delete `replicas: 1`, `strategy: {}` and `status: {}` - a DaemonSet has none of them. Then, in the Pod template's `spec:`, above `containers:`, the toleration that lets it onto the control plane: ``` tolerations: - key: node-role.kubernetes.io/control-plane operator: Exists effect: NoSchedule ``` and replace the container's `resources: {}` with: ``` resources: requests: cpu: 10m memory: 16Mi ```""", run=r"""yq -i '.kind = "DaemonSet" | del(.spec.replicas) | del(.spec.strategy) | del(.status) | .spec.template.spec.tolerations = [{"key": "node-role.kubernetes.io/control-plane", "operator": "Exists", "effect": "NoSchedule"}] | .spec.template.spec.containers[0].resources = {"requests": {"cpu": "10m", "memory": "16Mi"}}' ds.yaml"""), cmd(r"""k apply -f ds.yaml k -n %{ns} get ds,pods -o wide""", r"""`DESIRED` and `READY` equal to the number of nodes, the control plane among them.""")] S[27] = [cmd(r"""k -n %{ns} create cj %{cj} --image=busybox:1.36 --schedule='*/%{every} * * * *' --dry-run=client -o yaml -- date > cj.yaml"""), hand(r"""vim cj.yaml""", r"""Add to the CronJob's `spec:`, beside `schedule:`: ``` concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 1 ``` to the Job's `spec:` (under `jobTemplate:`), above `template:`: ``` activeDeadlineSeconds: 30 ``` and change `restartPolicy: OnFailure` to `restartPolicy: Never`.""", run=r"""yq -i '.spec.concurrencyPolicy = "Forbid" | .spec.successfulJobsHistoryLimit = 3 | .spec.failedJobsHistoryLimit = 1 | .spec.jobTemplate.spec.activeDeadlineSeconds = 30 | .spec.jobTemplate.spec.template.spec.restartPolicy = "Never"' cj.yaml"""), cmd(r"""k apply -f cj.yaml k -n %{ns} create job %{cj}-manual --from=cronjob/%{cj} k -n %{ns} get job %{cj}-manual""", r"""Again until `COMPLETIONS 1/1`.""")] S[28] = [cmd(r"""k -n %{ns} run %{pod} --image=nginx:1.25 --dry-run=client -o yaml > pod.yaml"""), hand(r"""vim pod.yaml""", r"""Name the container `web` and give it the mount, below its `name:`: ``` volumeMounts: - name: content mountPath: /usr/share/nginx/html ``` then add to the Pod's `spec:`, right below `spec:`: ``` initContainers: - name: setup image: busybox:1.36 command: ["sh", "-c", "echo '%{text}' > /work/index.html"] volumeMounts: - name: content mountPath: /work volumes: - name: content emptyDir: {} ```""", run=r"""yq -i '.spec.containers[0].name = "web" | .spec.containers[0].volumeMounts = [{"name": "content", "mountPath": "/usr/share/nginx/html"}] | .spec.initContainers = [{"name": "setup", "image": "busybox:1.36", "command": ["sh", "-c", "echo '"'"'%{text}'"'"' > /work/index.html"], "volumeMounts": [{"name": "content", "mountPath": "/work"}]}] | .spec.volumes = [{"name": "content", "emptyDir": {}}]' pod.yaml"""), cmd(r"""k apply -f pod.yaml k -n %{ns} get pod %{pod} -o wide""", r"""Once it is `Running`, `curl` its IP from a node, or:"""), cmd(r"""k -n %{ns} exec %{pod} -c web -- curl -s localhost""")] S[29] = [cmd(r"""k -n %{ns} create cm %{cm}-v2 --from-literal=mode=new --dry-run=client -o yaml > cm.yaml"""), hand(r"""vim cm.yaml""", r"""Make the new version immutable too: add, at the top level beside `data:`, ``` immutable: true ```""", run=r"""yq -i '.immutable = true' cm.yaml"""), cmd(r"""k apply -f cm.yaml"""), hand(r"""k -n %{ns} edit deploy %{dep}""", r"""In the Pod template's `volumes:`, point the volume at the new ConfigMap, then `:wq`: ``` - configMap: name: %{cm}-v2 ```""", run=r"""kubectl -n %{ns} patch deploy %{dep} --type=json -p='[{"op":"replace","path":"/spec/template/spec/volumes/0/configMap/name","value":"%{cm}-v2"}]'"""), cmd(r"""k -n %{ns} rollout status deploy %{dep}""")] S[31] = [cmd(r"""k -n %{ns} create job %{job} --image=busybox:1.36 --dry-run=client -o yaml -- sh -c "echo done; sleep 2" > job.yaml"""), hand(r"""vim job.yaml""", r"""Add to the Job's `spec:`, above `template:` (`restartPolicy: Never` is there already): ``` completions: %{c} parallelism: %{p} backoffLimit: 2 ```""", run=r"""yq -i '.spec.completions = %{c} | .spec.parallelism = %{p} | .spec.backoffLimit = 2' job.yaml"""), cmd(r"""k apply -f job.yaml"""), hand(r"""k -n %{ns} get job %{job}""", r"""Again until `COMPLETIONS %{c}/%{c}`.""", run=r"""kubectl -n %{ns} wait --for=condition=complete job/%{job} --timeout=120s""")] S[32] = [cmd(r"""k -n %{ns} create quota %{ns}-quota --hard=pods=%{pods},requests.cpu=1,limits.memory=2Gi"""), hand(r"""vim lr.yaml""", r"""Paste the LimitRange from the docs page *Configure Default CPU Requests and Limits for a Namespace* and give it both resources: ``` apiVersion: v1 kind: LimitRange metadata: name: %{ns}-defaults namespace: %{ns} spec: limits: - default: cpu: 500m memory: 256Mi defaultRequest: cpu: %{dcpu} memory: 64Mi type: Container ```""", run=r"""cat > lr.yaml <<'EOF' apiVersion: v1 kind: LimitRange metadata: name: %{ns}-defaults namespace: %{ns} spec: limits: - default: cpu: 500m memory: 256Mi defaultRequest: cpu: %{dcpu} memory: 64Mi type: Container EOF"""), cmd(r"""k apply -f lr.yaml k -n %{ns} create deploy worker --image=nginx:1.25 --replicas=2""", r"""The LimitRange first: with the quota on `requests.cpu` and `limits.memory`, a Pod that has neither is refused."""), cmd(r"""k -n %{ns} describe quota %{ns}-quota k -n %{ns} describe pod -l app=worker | grep -A2 -e Limits -e Requests""", r"""The defaults on every Pod, and the quota's `Used` column counting them.""")] # Hard Workload Tasks S[50] = [cmd(r"""k describe node {W1} | grep Taints k -n %{ns} describe quota""", r"""The quota is the trap twice over: 3 Pods do not fit `pods: 2`, nor 3 x 100m CPU in `requests.cpu: 250m`."""), hand(r"""k -n %{ns} edit quota %{ns}-quota""", r"""Raise both - the quota stays and still caps Pods - then `:wq`: ``` spec: hard: pods: "3" requests.cpu: 300m ```""", run=r"""kubectl -n %{ns} patch quota %{ns}-quota -p '{"spec":{"hard":{"pods":"3","requests.cpu":"300m"}}}'"""), cmd(r"""k create priorityclass %{dep}-prio --value=50000 k -n %{ns} create deploy %{dep} --image=nginx:1.27 --replicas=3 --dry-run=client -o yaml > d.yaml"""), hand(r"""vim d.yaml""", r"""In the Pod template's `spec:`, above `containers:` (the docs page *Assigning Pods to Nodes* has the anti-affinity, *Taints and Tolerations* the toleration): ``` priorityClassName: %{dep}-prio affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app: %{dep} topologyKey: kubernetes.io/hostname tolerations: - key: %{key} operator: Equal value: %{val} effect: NoSchedule ``` and replace the container's `resources: {}` with: ``` resources: requests: cpu: 100m memory: 64Mi ```""", run=r"""yq -i '.spec.template.spec.priorityClassName = "%{dep}-prio" | .spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution = [{"labelSelector": {"matchLabels": {"app": "%{dep}"}}, "topologyKey": "kubernetes.io/hostname"}] | .spec.template.spec.tolerations = [{"key": "%{key}", "operator": "Equal", "value": "%{val}", "effect": "NoSchedule"}] | .spec.template.spec.containers[0].resources = {"requests": {"cpu": "100m", "memory": "64Mi"}}' d.yaml"""), cmd(r"""k apply -f d.yaml k -n %{ns} get pods -o wide k -n %{ns} describe quota""", r"""3 `Running` on 3 different workers, and the quota at `3/3`, `300m/300m`. The control plane is tainted too, so there is no slack: a missed quota field or toleration shows as a `Pending` Pod, and `k -n %{ns} describe rs` says which.""")] S[51] = [cmd(r"""k -n %{ns} rollout history deploy %{dep} k -n %{ns} rollout history deploy %{dep} --revision=1""", r"""`--revision=N` shows that revision's Pod template. Look at each until one has `nginx:1.25` - here revision 1, which is not the previous one."""), cmd(r"""k -n %{ns} rollout undo deploy %{dep} --to-revision=1 k -n %{ns} scale deploy %{dep} --replicas=3 k -n %{ns} rollout status deploy %{dep}"""), cmd(r"""k -n %{ns} rollout pause deploy %{dep} k -n %{ns} set image deploy %{dep} nginx=nginx:1.27 k -n %{ns} set env deploy %{dep} RELEASE=next""", r"""Pause first: while paused, the controller leaves the template alone - no new ReplicaSet, and the Pods stay on `nginx:1.25`."""), cmd(r"""k -n %{ns} get deploy %{dep} -o yaml | grep -e paused -e image: k -n %{ns} get rs -o wide k -n %{ns} get pods""", r"""`paused: true`, `nginx:1.27` in the template - and no ReplicaSet for it.""")] S[52] = [cmd(r"""k -n %{ns} run %{pod} --image=nginx:1.27 --dry-run=client -o yaml > pod.yaml"""), hand(r"""vim pod.yaml""", r"""Name the container `app`, and add below its `name:` the probes and the mount: ``` readinessProbe: httpGet: path: / port: 80 periodSeconds: 5 livenessProbe: tcpSocket: port: 80 volumeMounts: - name: logs mountPath: /var/log/agent ``` then to the Pod's `spec:`, right below `spec:` - the sidecar is an init container that keeps running (`restartPolicy: Always`): ``` terminationGracePeriodSeconds: 5 initContainers: - name: agent image: busybox:1.36 restartPolicy: Always command: ["sh", "-c", "while true; do date >> /var/log/agent/agent.log; sleep 1; done"] volumeMounts: - name: logs mountPath: /var/log/agent volumes: - name: logs emptyDir: {} ``` At `/var/log/agent`, not `/var/log`: an emptyDir there would hide nginx's `/var/log/nginx`, and nginx crash-loops.""", run=r"""yq -i '.spec.containers[0].name = "app" | .spec.containers[0].readinessProbe = {"httpGet": {"path": "/", "port": 80}, "periodSeconds": 5} | .spec.containers[0].livenessProbe = {"tcpSocket": {"port": 80}} | .spec.containers[0].volumeMounts = [{"name": "logs", "mountPath": "/var/log/agent"}] | .spec.terminationGracePeriodSeconds = 5 | .spec.initContainers = [{"name": "agent", "image": "busybox:1.36", "restartPolicy": "Always", "command": ["sh", "-c", "while true; do date >> /var/log/agent/agent.log; sleep 1; done"], "volumeMounts": [{"name": "logs", "mountPath": "/var/log/agent"}]}] | .spec.volumes = [{"name": "logs", "emptyDir": {}}]' pod.yaml"""), cmd(r"""k apply -f pod.yaml"""), hand(r"""k -n %{ns} get pod %{pod} -o wide""", r"""Again until it is `Running` with `READY 2/2` - it has no IP before that. Note the IP.""", run=r"""kubectl -n %{ns} wait --for=condition=Ready pod/%{pod} --timeout=120s"""), cmd(r"""k -n %{ns} get pod %{pod} -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}'; echo""", r"""`agent app`: the init container and the regular one."""), hand(r"""vim {WS}/q52/containers.txt""", r"""Sorted, one per line: ``` agent app ```""", run=r"""printf 'agent\napp\n' > {WS}/q52/containers.txt"""), hand(r"""echo 10.244.1.7 > {WS}/q52/ip.txt""", r"""With the Pod's IP, as `-o wide` showed it.""", run=r"""kubectl -n %{ns} get pod %{pod} -o jsonpath='{.status.podIP}' > {WS}/q52/ip.txt""")] S[53] = [hand(r"""k -n %{ns} get pods -o wide""", r"""Look before you count: which Pods, which phase, which node - again until all but `once-a` are `Running`.""", run=r"""kubectl -n %{ns} wait --for=condition=Ready pod web-a web-b batch-a cache-a idle-a --timeout=180s"""), cmd(r"""k -n %{ns} get pods -o custom-columns=NAME:.metadata.name,PROBE:.spec.containers[0].readinessProbe""", r"""``: no readiness probe on the first container - `cache-a` among them, whose probe is on its second; and `once-a` counts though it has `Completed`."""), hand(r"""vim {WS}/q53/no-readiness.txt""", r"""The names with ``, sorted: ``` batch-a cache-a idle-a once-a ```""", run=r"""kubectl -n %{ns} get pods -o jsonpath='{range .items[*]}{.metadata.name} {.spec.containers[0].readinessProbe}{"\n"}{end}' | awk 'NF==1{print $1}' | sort > {WS}/q53/no-readiness.txt"""), cmd(r"""k -n %{ns} get pods -o wide --field-selector=status.phase=Running""", r"""`once-a` is gone from the list: it is not `Running`."""), hand(r"""vim {WS}/q53/per-node.txt""", r"""Count the Pods per node in the `NODE` column, and write the nodes sorted: ``` {W1} 2 {W2} 2 {W3} 1 ```""", run=r"""kubectl -n %{ns} get pods --field-selector=status.phase=Running -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' | sort | uniq -c | awk '{print $2, $1}' | sort > {WS}/q53/per-node.txt"""), cmd(r"""k -n kube-system get pods -o jsonpath='{.items[*].spec.containers[*].image}' | tr ' ' '\n' | sort -u > {WS}/q53/images.txt cat -A {WS}/q53/images.txt""", r"""Too many to copy by hand. `cat -A` shows a stray space or `\r` as `$` and `^M`.""")] S[64] = [cmd(r"""k config view --minify | grep namespace""", r"""`namespace: %{ns}` - where everything without `-n` lands."""), cmd(r"""k create deploy %{dep} --image=busybox:1.36 --replicas=2 -- sleep 3600 k label deploy %{dep} owner=ac k expose deploy %{dep} --port=9000 --target-port=9000 echo %{ns} > {WS}/q64/ns.txt""", r"""No `-n` anywhere. `expose` makes a ClusterIP Service unless told otherwise."""), cmd(r"""k get deploy,svc %{dep} k -n default get deploy %{dep}""", r"""Both there - and `NotFound` in `default`, the right answer.""")] # ================================================================= NET ===== # Domain 3: Services & Networking (20%) # =========================================================================== S[11] = [hand(r"""vim netpol.yaml""", r"""Paste the example from the docs page *Network Policies* and cut it down to one ingress rule - no `egress`, and `Ingress` alone in `policyTypes`, so egress stays open: ``` apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: backend-policy namespace: netpol spec: podSelector: matchLabels: app: backend policyTypes: - Ingress ingress: - from: - podSelector: matchLabels: app: frontend ports: - protocol: TCP port: 8080 ```""", run=r"""cat > netpol.yaml <<'EOF' apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: backend-policy namespace: netpol spec: podSelector: matchLabels: app: backend policyTypes: - Ingress ingress: - from: - podSelector: matchLabels: app: frontend ports: - protocol: TCP port: 8080 EOF"""), cmd(r"""k apply -f netpol.yaml k -n netpol describe netpol backend-policy""")] S[12] = [cmd(r"""k -n svc expose deploy shop --name=shop-svc --type=NodePort --port=80 --target-port=80 --dry-run=client -o yaml > svc.yaml"""), hand(r"""vim svc.yaml""", r"""Add the fixed node port to the port, below `targetPort: 80`: ``` nodePort: 30080 ```""", run=r"""yq -i '.spec.ports[0].nodePort = 30080' svc.yaml"""), cmd(r"""k apply -f svc.yaml k -n svc get svc,endpointslices""", r"""`80:30080/TCP`, and an EndpointSlice with the Pod's address.""")] S[13] = [cmd(r"""k -n ingress-app create ingress app-ingress --class=nginx --rule='shop.example.com/api*=api:8080' --rule='shop.example.com/*=web:80' k -n ingress-app describe ingress app-ingress""", r"""A path ending in `*` is `pathType: Prefix` (without, `Exact`); the quotes keep the shell off the `*`.""")] S[14] = [cmd(r"""k -n dns-check run dns-test --image=busybox:1.36 -- sleep 3600 echo hello.dns-check.svc.cluster.local > {WS}/q14/svc-fqdn.txt""", r"""Not `--rm -it`: the Pod has to be there for the check."""), hand(r"""k -n dns-check get pod dns-test""", r"""Again until it is `Running`.""", run=r"""kubectl -n dns-check wait --for=condition=Ready pod/dns-test --timeout=120s"""), cmd(r"""k -n dns-check exec dns-test -- nslookup hello.dns-check.svc.cluster.local > {WS}/q14/nslookup.txt cat {WS}/q14/nslookup.txt""", r"""The answer's `Address` is the Service's ClusterIP.""")] S[22] = [cmd(r"""k apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/standard-install.yaml"""), hand(r"""k get crd | grep gateway""", r"""`gatewayclasses`, `gateways`, `httproutes` and the rest of the standard channel. A moment later the API serves them.""", run=r"""kubectl wait --for=condition=Established --timeout=60s crd/gatewayclasses.gateway.networking.k8s.io crd/gateways.gateway.networking.k8s.io crd/httproutes.gateway.networking.k8s.io"""), hand(r"""vim gw.yaml""", r"""The docs page *Gateway API* has a GatewayClass, a Gateway and an HTTPRoute to start from; one file, `---` between them: ``` apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: name: %{gc} spec: controllerName: example.com/gateway --- apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: name: %{gw} namespace: %{ns} spec: gatewayClassName: %{gc} listeners: - name: http protocol: HTTP port: 80 hostname: %{host}.example.com --- apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: %{host}-route namespace: %{ns} spec: parentRefs: - name: %{gw} rules: - matches: - path: type: PathPrefix value: /api backendRefs: - name: api port: 8080 - backendRefs: - name: web port: 80 ``` A rule with no `matches` takes everything else (`PathPrefix /`).""", run=r"""cat > gw.yaml <<'EOF' apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: name: %{gc} spec: controllerName: example.com/gateway --- apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: name: %{gw} namespace: %{ns} spec: gatewayClassName: %{gc} listeners: - name: http protocol: HTTP port: 80 hostname: %{host}.example.com --- apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: %{host}-route namespace: %{ns} spec: parentRefs: - name: %{gw} rules: - matches: - path: type: PathPrefix value: /api backendRefs: - name: api port: 8080 - backendRefs: - name: web port: 80 EOF"""), cmd(r"""k apply -f gw.yaml k -n %{ns} get gateway,httproute""")] S[30] = [cmd(r"""k -n %{ns} create deploy %{dep} --image=nginx:1.25 --port=80 --dry-run=client -o yaml > d.yaml"""), hand(r"""vim d.yaml""", r"""Name the port, and add the probe below the container's `ports:`, by that name: ``` ports: - containerPort: 80 name: %{pname} readinessProbe: httpGet: path: / port: %{pname} ```""", run=r"""yq -i '.spec.template.spec.containers[0].ports[0].name = "%{pname}" | .spec.template.spec.containers[0].readinessProbe = {"httpGet": {"path": "/", "port": "%{pname}"}}' d.yaml"""), cmd(r"""k apply -f d.yaml k -n %{ns} expose deploy %{dep} --port=8080 --target-port=%{pname} k -n %{ns} get endpointslices -l kubernetes.io/service-name=%{dep}""", r"""`--target-port` takes a port's name as well as its number. An address once the probe passes: empty means the selector or the probe is wrong.""")] S[42] = [hand(r"""vim netpol.yaml""", r"""Paste the example from the docs page *Network Policies* and make it egress only, with two rules. `Egress` alone in `policyTypes` denies all other egress and leaves ingress open; DNS needs both protocols, as resolvers fall back to TCP for long answers: ``` apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: %{app}-egress namespace: %{ns} spec: podSelector: matchLabels: app: %{app} policyTypes: - Egress egress: - to: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: kube-system podSelector: matchLabels: k8s-app: kube-dns ports: - protocol: UDP port: 53 - protocol: TCP port: 53 - to: - ipBlock: cidr: %{cidr} ports: - protocol: TCP port: 443 ``` `podSelector` with no `-` before it belongs to the same `to` entry as the `namespaceSelector`: both must match.""", run=r"""cat > netpol.yaml <<'EOF' apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: %{app}-egress namespace: %{ns} spec: podSelector: matchLabels: app: %{app} policyTypes: - Egress egress: - to: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: kube-system podSelector: matchLabels: k8s-app: kube-dns ports: - protocol: UDP port: 53 - protocol: TCP port: 53 - to: - ipBlock: cidr: %{cidr} ports: - protocol: TCP port: 443 EOF"""), cmd(r"""k apply -f netpol.yaml k -n %{ns} describe netpol %{app}-egress""")] S[43] = [cmd(r"""k -n %{ns} get sts %{sts} -o yaml | grep serviceName""", r"""The headless Service must have exactly this name: it is what makes CoreDNS publish one record per Pod, `...svc.cluster.local`."""), cmd(r"""k -n %{ns} create service clusterip %{sts} --clusterip=None --tcp=5432:5432 k -n %{ns} create service externalname legacy --external-name=%{ext} k -n %{ns} get svc""", r"""`create service` selects `app=` - here `app=%{sts}`, the StatefulSet's Pods. `legacy` has no Pods: it is only a CNAME to `%{ext}`."""), hand(r"""k -n %{ns} exec %{sts}-0 -- nslookup %{sts}-0.%{sts}.%{ns}.svc.cluster.local > {WS}/q43/pod0.txt cat {WS}/q43/pod0.txt k -n %{ns} get pod %{sts}-0 -o wide""", r"""The full name: busybox's `nslookup` applies no search domains. The address it gives is the Pod's IP - and if it says `NXDOMAIN`, again in a few seconds: CoreDNS publishes the record once the new Service has endpoints.""", run=r"""for i in $(seq 1 20); do kubectl -n %{ns} exec %{sts}-0 -- nslookup %{sts}-0.%{sts}.%{ns}.svc.cluster.local > {WS}/q43/pod0.txt 2>&1 grep -q "$(kubectl -n %{ns} get pod %{sts}-0 -o jsonpath='{.status.podIP}')" {WS}/q43/pod0.txt && break; sleep 3 done""")] # Hard Networking Tasks S[54] = [cmd(r"""k -n %{ns} get netpol k -n %{ns} delete netpol temp-allow-all k label ns %{from} team=%{from}""", r"""NetworkPolicies add up: one that allows everything cancels every restriction below."""), hand(r"""vim zt.yaml""", r"""The docs page *Network Policies* has a default deny for both directions to start from. Five policies, `---` between them: ``` apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: default-deny namespace: %{ns} spec: podSelector: {} policyTypes: - Ingress - Egress --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-dns namespace: %{ns} spec: podSelector: {} policyTypes: - Egress egress: - to: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: kube-system ports: - protocol: UDP port: 53 - protocol: TCP port: 53 --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: api-ingress namespace: %{ns} spec: podSelector: matchLabels: app: api policyTypes: - Ingress ingress: - from: - namespaceSelector: matchLabels: team: %{from} ports: - port: 8080 --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: api-to-db namespace: %{ns} spec: podSelector: matchLabels: app: api policyTypes: - Egress egress: - to: - podSelector: matchLabels: app: db ports: - port: 5432 --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: db-ingress namespace: %{ns} spec: podSelector: matchLabels: app: db policyTypes: - Ingress ingress: - from: - podSelector: matchLabels: app: api ports: - port: 5432 ```""", run=r"""cat > zt.yaml <<'EOF' apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: default-deny namespace: %{ns} spec: podSelector: {} policyTypes: - Ingress - Egress --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-dns namespace: %{ns} spec: podSelector: {} policyTypes: - Egress egress: - to: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: kube-system ports: - protocol: UDP port: 53 - protocol: TCP port: 53 --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: api-ingress namespace: %{ns} spec: podSelector: matchLabels: app: api policyTypes: - Ingress ingress: - from: - namespaceSelector: matchLabels: team: %{from} ports: - port: 8080 --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: api-to-db namespace: %{ns} spec: podSelector: matchLabels: app: api policyTypes: - Egress egress: - to: - podSelector: matchLabels: app: db ports: - port: 5432 --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: db-ingress namespace: %{ns} spec: podSelector: matchLabels: app: db policyTypes: - Ingress ingress: - from: - podSelector: matchLabels: app: api ports: - port: 5432 EOF"""), cmd(r"""k apply -f zt.yaml k -n %{ns} get netpol""")] S[55] = """# Step 1: Generate self-signed TLS certificate and private key openssl req -x509 -newkey rsa:2048 -nodes -days 365 -subj "/CN=%{host}" -keyout tls.key -out tls.crt # Step 2: Create TLS secret in target namespace k -n %{ns} create secret tls %{host}-tls --cert=tls.crt --key=tls.key # Step 3: Create Ingress with TLS configuration and SSL redirect annotation k -n %{ns} create ingress web --class=nginx --rule="%{host}/*=web:80,tls=%{host}-tls" \\ --annotation nginx.ingress.kubernetes.io/ssl-redirect=true # Step 4: Extract SHA256 fingerprint to workspace file openssl x509 -in tls.crt -noout -fingerprint -sha256 | cut -d= -f2 > {WS}/q55/fp.txt """ S[56] = [hand(r"""vim canary.yaml""", r"""The docs page *Gateway API* has the three kinds to start from; `sectionName` attaches the route to one listener, and a rule without `matches` takes the rest: ``` apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: name: %{host}-gc spec: controllerName: example.com/gw --- apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: name: %{host}-gw namespace: %{ns} spec: gatewayClassName: %{host}-gc listeners: - name: web protocol: HTTP port: 80 hostname: %{host}.example.com - name: api protocol: HTTP port: 80 hostname: api.%{host}.example.com --- apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: %{host}-route namespace: %{ns} spec: parentRefs: - name: %{host}-gw sectionName: web rules: - matches: - headers: - name: X-Canary value: always backendRefs: - name: v2 port: 80 - backendRefs: - name: v1 port: 80 weight: %{w2} - name: v2 port: 80 weight: %{w} ```""", run=r"""cat > canary.yaml <<'EOF' apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: name: %{host}-gc spec: controllerName: example.com/gw --- apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: name: %{host}-gw namespace: %{ns} spec: gatewayClassName: %{host}-gc listeners: - name: web protocol: HTTP port: 80 hostname: %{host}.example.com - name: api protocol: HTTP port: 80 hostname: api.%{host}.example.com --- apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: %{host}-route namespace: %{ns} spec: parentRefs: - name: %{host}-gw sectionName: web rules: - matches: - headers: - name: X-Canary value: always backendRefs: - name: v2 port: 80 - backendRefs: - name: v1 port: 80 weight: %{w2} - name: v2 port: 80 weight: %{w} EOF"""), cmd(r"""k apply -f canary.yaml k -n %{ns} get gateway,httproute""")] S[57] = """# Step 1: Remove stray blocking NetworkPolicy in kube-system k -n kube-system get netpol k -n kube-system delete netpol maintenance-lockdown # Step 2: Fix CoreDNS replicas (scale back from 0) k -n kube-system scale deploy coredns --replicas=2 # Step 3: Correct CoreDNS upstream forwarding in ConfigMap k -n kube-system edit cm coredns # Replace: forward . 10.255.255.254 -> forward . /etc/resolv.conf # Step 4: Restart CoreDNS deployment and test DNS resolution k -n kube-system rollout restart deploy coredns # busybox's nslookup applies no search domains, so use FULL names (a short one fails even when DNS works) k run dns-check --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.default.svc.cluster.local k run dns-ext --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.io """ # ================================================================= STOR ==== # Domain 4: Storage (10%) # =========================================================================== S[15] = [hand(r"""vim pv.yaml""", r"""Paste the PersistentVolume and the PersistentVolumeClaim from the docs page *Configure a Pod to Use a PersistentVolume for Storage*, one after the other with `---` between them, and change them to: ``` apiVersion: v1 kind: PersistentVolume metadata: name: pv-data spec: storageClassName: manual capacity: storage: 1Gi accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Retain hostPath: path: /mnt/data --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-data namespace: storage spec: storageClassName: manual accessModes: - ReadWriteOnce resources: requests: storage: 500Mi ```""", run=r"""cat > pv.yaml <<'EOF' apiVersion: v1 kind: PersistentVolume metadata: name: pv-data spec: storageClassName: manual capacity: storage: 1Gi accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Retain hostPath: path: /mnt/data --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-data namespace: storage spec: storageClassName: manual accessModes: - ReadWriteOnce resources: requests: storage: 500Mi EOF"""), cmd(r"""k apply -f pv.yaml k -n storage get pvc pvc-data k -n storage run data-pod --image=nginx:1.25 --dry-run=client -o yaml > pod.yaml""", r"""`STATUS Bound`, `VOLUME pv-data`."""), hand(r"""vim pod.yaml""", r"""Add to the container, below `name: data-pod`: ``` volumeMounts: - name: data mountPath: /usr/share/data ``` and to the Pod's `spec:`, right below `spec:`: ``` volumes: - name: data persistentVolumeClaim: claimName: pvc-data ```""", run=r"""yq -i '.spec.containers[0].volumeMounts = [{"name": "data", "mountPath": "/usr/share/data"}] | .spec.volumes = [{"name": "data", "persistentVolumeClaim": {"claimName": "pvc-data"}}]' pod.yaml"""), cmd(r"""k apply -f pod.yaml k -n storage get pod data-pod""")] S[16] = [hand(r"""vim sc.yaml""", r"""Paste the example from the docs page *Storage Classes* and make it the default with the annotation (the docs page *Change the default StorageClass* names it): ``` apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: fast-local annotations: storageclass.kubernetes.io/is-default-class: "true" provisioner: rancher.io/local-path reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer ```""", run=r"""cat > sc.yaml <<'EOF' apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: fast-local annotations: storageclass.kubernetes.io/is-default-class: "true" provisioner: rancher.io/local-path reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer EOF"""), cmd(r"""k apply -f sc.yaml k annotate sc standard storageclass.kubernetes.io/is-default-class- k get sc""", r"""`(default)` beside `fast-local` alone. The `-` at the end removes the annotation."""), hand(r"""vim pvc.yaml""", r"""A claim with no `storageClassName`, so it gets the default (the docs page *Persistent Volumes* has a claim to start from): ``` apiVersion: v1 kind: PersistentVolumeClaim metadata: name: fast-claim namespace: storage spec: accessModes: - ReadWriteOnce resources: requests: storage: 200Mi ```""", run=r"""cat > pvc.yaml <<'EOF' apiVersion: v1 kind: PersistentVolumeClaim metadata: name: fast-claim namespace: storage spec: accessModes: - ReadWriteOnce resources: requests: storage: 200Mi EOF"""), cmd(r"""k apply -f pvc.yaml k -n storage run fast-pod --image=nginx:1.25 --dry-run=client -o yaml > pod.yaml""", r"""The claim stays `Pending` until a Pod uses it: `WaitForFirstConsumer`."""), hand(r"""vim pod.yaml""", r"""Add to the container, below `name: fast-pod`: ``` volumeMounts: - name: data mountPath: /data ``` and to the Pod's `spec:`, right below `spec:`: ``` volumes: - name: data persistentVolumeClaim: claimName: fast-claim ```""", run=r"""yq -i '.spec.containers[0].volumeMounts = [{"name": "data", "mountPath": "/data"}] | .spec.volumes = [{"name": "data", "persistentVolumeClaim": {"claimName": "fast-claim"}}]' pod.yaml"""), cmd(r"""k apply -f pod.yaml k -n storage get pvc,pod""", r"""Again until the claim is `Bound` and the Pod `Running`.""")] S[35] = [cmd(r"""k get pv %{pv} k get pv %{pv} -o yaml | grep -A6 claimRef""", r"""`Released`: the PV still names the deleted claim in `claimRef`, so no other claim can have it."""), hand(r"""k edit pv %{pv}""", r"""Delete the whole `claimRef:` block - `claimRef:` and the lines indented below it - then `:wq`.""", run=r"""kubectl patch pv %{pv} --type=json -p='[{"op":"remove","path":"/spec/claimRef"}]'"""), cmd(r"""k get pv %{pv}""", r"""`Available`."""), hand(r"""vim pvc.yaml""", r"""A claim that names the PV, so it binds to that one and no other: ``` apiVersion: v1 kind: PersistentVolumeClaim metadata: name: %{pvc} namespace: %{ns} spec: storageClassName: manual volumeName: %{pv} accessModes: - ReadWriteOnce resources: requests: storage: 500Mi ```""", run=r"""cat > pvc.yaml <<'EOF' apiVersion: v1 kind: PersistentVolumeClaim metadata: name: %{pvc} namespace: %{ns} spec: storageClassName: manual volumeName: %{pv} accessModes: - ReadWriteOnce resources: requests: storage: 500Mi EOF"""), cmd(r"""k apply -f pvc.yaml k -n %{ns} get pvc %{pvc}""", r"""`Bound` to `%{pv}`.""")] S[36] = [hand(r"""vim pvc.yaml""", r"""No `storageClassName`: the default class takes it. ``` apiVersion: v1 kind: PersistentVolumeClaim metadata: name: %{pvc} namespace: %{ns} spec: accessModes: - ReadWriteOnce resources: requests: storage: 200Mi ```""", run=r"""cat > pvc.yaml <<'EOF' apiVersion: v1 kind: PersistentVolumeClaim metadata: name: %{pvc} namespace: %{ns} spec: accessModes: - ReadWriteOnce resources: requests: storage: 200Mi EOF"""), cmd(r"""k apply -f pvc.yaml k -n %{ns} run writer --image=busybox:1.36 --restart=Never --dry-run=client -o yaml -- sh -c "echo %{text} > /data/marker.txt" > writer.yaml"""), hand(r"""vim writer.yaml""", r"""Add to the container, below `name: writer`: ``` volumeMounts: - name: data mountPath: /data ``` and to the Pod's `spec:`, right below `spec:`: ``` volumes: - name: data persistentVolumeClaim: claimName: %{pvc} ```""", run=r"""yq -i '.spec.containers[0].volumeMounts = [{"name": "data", "mountPath": "/data"}] | .spec.volumes = [{"name": "data", "persistentVolumeClaim": {"claimName": "%{pvc}"}}]' writer.yaml"""), cmd(r"""k apply -f writer.yaml"""), hand(r"""k -n %{ns} get pod writer""", r"""Again until `Completed`.""", run=r"""kubectl -n %{ns} wait --for=jsonpath='{.status.phase}'=Succeeded pod/writer --timeout=120s"""), cmd(r"""k -n %{ns} run reader --image=busybox:1.36 --dry-run=client -o yaml -- sleep 3600 > reader.yaml"""), hand(r"""vim reader.yaml""", r"""The same two blocks as in `writer.yaml`, below `name: reader` and below `spec:`.""", run=r"""yq -i '.spec.containers[0].volumeMounts = [{"name": "data", "mountPath": "/data"}] | .spec.volumes = [{"name": "data", "persistentVolumeClaim": {"claimName": "%{pvc}"}}]' reader.yaml"""), cmd(r"""k apply -f reader.yaml k -n %{ns} exec reader -- cat /data/marker.txt""", r"""Once it is `Running`: `%{text}`, written by a Pod that is gone.""")] # Hard Storage Tasks S[58] = [cmd(r"""k get pv --show-labels k get node -l disk=nvme k describe pv lv-nvme | grep -A3 'Node Affinity'""", r"""`lv-nvme` carries `media=nvme` and is pinned to the nvme node."""), hand(r"""vim pvc.yaml""", r"""A claim that selects the PV by its label: ``` apiVersion: v1 kind: PersistentVolumeClaim metadata: name: %{pvc} namespace: %{ns} spec: storageClassName: local-fast selector: matchLabels: media: nvme accessModes: - ReadWriteOnce resources: requests: storage: 1Gi ```""", run=r"""cat > pvc.yaml <<'EOF' apiVersion: v1 kind: PersistentVolumeClaim metadata: name: %{pvc} namespace: %{ns} spec: storageClassName: local-fast selector: matchLabels: media: nvme accessModes: - ReadWriteOnce resources: requests: storage: 1Gi EOF"""), cmd(r"""k apply -f pvc.yaml k -n %{ns} run consumer --image=busybox:1.36 --dry-run=client -o yaml -- sleep 3600 > pod.yaml"""), hand(r"""vim pod.yaml""", r"""Add to the container, below `name: consumer`: ``` volumeMounts: - name: data mountPath: /data ``` and to the Pod's `spec:`, right below `spec:`: ``` volumes: - name: data persistentVolumeClaim: claimName: %{pvc} ``` No node selector: the PV's node affinity takes the Pod to the nvme node.""", run=r"""yq -i '.spec.containers[0].volumeMounts = [{"name": "data", "mountPath": "/data"}] | .spec.volumes = [{"name": "data", "persistentVolumeClaim": {"claimName": "%{pvc}"}}]' pod.yaml"""), cmd(r"""k apply -f pod.yaml k -n %{ns} get pvc,pod -o wide""", r"""The claim `Bound` to `lv-nvme`, and the Pod on the nvme node.""")] S[59] = [hand(r"""vim new.yaml""", r"""The new claim - no `storageClassName`, so the default class: ``` apiVersion: v1 kind: PersistentVolumeClaim metadata: name: new-data namespace: %{ns} spec: accessModes: - ReadWriteOnce resources: requests: storage: 1Gi ```""", run=r"""cat > new.yaml <<'EOF' apiVersion: v1 kind: PersistentVolumeClaim metadata: name: new-data namespace: %{ns} spec: accessModes: - ReadWriteOnce resources: requests: storage: 1Gi EOF"""), cmd(r"""k apply -f new.yaml k -n %{ns} create job migrate --image=busybox:1.36 --dry-run=client -o yaml -- sh -c "cp -a /src/. /dst/ && ls /dst" > job.yaml"""), hand(r"""vim job.yaml""", r"""Both claims in the Job's Pod: add to the container, below `name: migrate`, ``` volumeMounts: - name: src mountPath: /src - name: dst mountPath: /dst ``` and to the Pod template's `spec:`, beside `restartPolicy: Never`: ``` volumes: - name: src persistentVolumeClaim: claimName: old-data - name: dst persistentVolumeClaim: claimName: new-data ```""", run=r"""yq -i '.spec.template.spec.containers[0].volumeMounts = [{"name": "src", "mountPath": "/src"}, {"name": "dst", "mountPath": "/dst"}] | .spec.template.spec.volumes = [{"name": "src", "persistentVolumeClaim": {"claimName": "old-data"}}, {"name": "dst", "persistentVolumeClaim": {"claimName": "new-data"}}]' job.yaml"""), cmd(r"""k apply -f job.yaml"""), hand(r"""k -n %{ns} get job migrate""", r"""Again until `COMPLETIONS 1/1`; `k -n %{ns} logs job/migrate` lists the files copied.""", run=r"""kubectl -n %{ns} wait --for=condition=complete job/migrate --timeout=240s"""), cmd(r"""k get pv | grep old-data k -n %{ns} get pod legacy -o yaml > legacy.yaml""", r"""`RECLAIM POLICY Retain`: deleting the claim releases the PV and keeps it."""), hand(r"""vim legacy.yaml""", r"""In `volumes:`, change `claimName: old-data` to `claimName: new-data`.""", run=r"""yq -i '(.spec.volumes[] | select(.persistentVolumeClaim.claimName == "old-data") | .persistentVolumeClaim.claimName) = "new-data"' legacy.yaml"""), cmd(r"""k replace --force -f legacy.yaml""", r"""A Pod's volumes cannot change: `--force` deletes it and creates it again from the file."""), cmd(r"""k -n %{ns} delete pod -l job-name=migrate k -n %{ns} delete pvc old-data k get pv | grep old-data k -n %{ns} get pod legacy""", r"""The Job's finished Pod still holds `old-data` (pvc-protection): delete it first, or `delete pvc` waits forever. The PV is `Released`; the Job stays, still `1/1`.""")] # =============================================================== TSHOOT ==== # Domain 5: Troubleshooting (30%) # =========================================================================== S[17] = """# Step 1: Check node status k get node {W2} # Status: NotReady # Step 2: SSH into the broken worker node {SHELL_W2} sudo -i # Step 3: Check kubelet service status and journal logs systemctl status kubelet # Service inactive/disabled systemctl enable --now kubelet # Step 4: Verify kubelet successfully connects and registers node journalctl -u kubelet -f --no-pager # Look for: "Successfully registered node" # Step 5: Exit node shell and verify Ready state from cluster # k get nodes """ S[18] = [cmd(r"""k -n troubleshoot get pods k -n troubleshoot describe deploy payments | grep -i image""", r"""`ImagePullBackOff`: the tag `nginx:1.99-nope` does not exist."""), cmd(r"""k -n troubleshoot set image deploy payments payments=nginx:1.27 k -n troubleshoot get svc payments -o yaml k -n troubleshoot get pods --show-labels""", r"""The Service selects `app: payment`, the Pods are `app=payments`; and it sends to `targetPort: 8080`, where nginx listens on 80."""), hand(r"""k -n troubleshoot edit svc payments""", r"""Fix both, then `:wq`: ``` ports: - port: 80 protocol: TCP targetPort: 80 selector: app: payments ```""", run=r"""kubectl -n troubleshoot patch svc payments -p '{"spec":{"selector":{"app":"payments"},"ports":[{"port":80,"targetPort":80}]}}'"""), cmd(r"""k -n troubleshoot rollout status deploy payments k -n troubleshoot get endpointslices -l kubernetes.io/service-name=payments""")] S[19] = """# Step 1: Capture previous container logs to locate crash reason k -n troubleshoot logs crasher --previous | tee {WS}/q19/crasher.log # Log output: "FATAL: required environment variable DB_HOST is not set" # Step 2: Export pod manifest k -n troubleshoot get pod crasher -o yaml > crasher.yaml # Step 3: Edit crasher.yaml: # Add under spec.containers[0]: # env: # - name: DB_HOST # value: db.internal # (Remove system-managed fields: status, uid, resourceVersion, creationTimestamp) # Step 4: Recreate pod and verify Running status k -n troubleshoot delete pod crasher --now && k apply -f crasher.yaml k -n troubleshoot get pod crasher """ S[20] = """# Step 1: Check control-plane pods in kube-system k -n kube-system get pods # kube-scheduler static pod is in Error / CrashLoopBackOff # Step 2: Inspect scheduler pod logs k -n kube-system logs kube-scheduler-{CP} # Error: exec: "kube-schedulerr": executable file not found # Step 3: SSH into control plane node sudo -i # the task's host is the control-plane node: no hop needed # Step 4: Edit scheduler static pod manifest # In /etc/kubernetes/manifests/kube-scheduler.yaml: # Change: command: [kube-schedulerr] -> command: [kube-scheduler] vi /etc/kubernetes/manifests/kube-scheduler.yaml # Step 5: Wait for kubelet to reload the static pod and verify scheduling k -n kube-system get pods -l component=kube-scheduler """ S[37] = """# Step 1: Inspect CoreDNS pods and logs k -n kube-system get pods -l k8s-app=kube-dns k -n kube-system logs -l k8s-app=kube-dns --tail=20 # Step 2: Check CoreDNS ConfigMap k -n kube-system get cm coredns -o yaml # Fault: forward . 10.255.255.1 points to an invalid upstream DNS # Step 3: Edit ConfigMap to use host resolv.conf k -n kube-system edit cm coredns # Change: forward . 10.255.255.1 -> forward . /etc/resolv.conf # Step 4: Restart CoreDNS deployment to apply changes k -n kube-system rollout restart deploy coredns # Step 5: Verify DNS resolution from a test pod # busybox's nslookup applies no search domains, so use FULL names (a short one fails even when DNS works) k run test-dns --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.default.svc.cluster.local k run test-ext --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.io """ S[38] = """# Step 1: Check control-plane pod status k -n kube-system get pods # kube-controller-manager is crashing # Step 2: Check controller-manager logs k -n kube-system logs kube-controller-manager-{CP} # Error: invalid flag: --leader-elect-lease-duration=1s # Step 3: Access control plane node sudo -i # the task's host is the control-plane node: no hop needed # Step 4: Edit static pod manifest to remove the invalid flag vi /etc/kubernetes/manifests/kube-controller-manager.yaml # Step 5: Verify controller-manager restarts and reconciles deployments/replicasets k create deploy probe --image=busybox:1.36 -- sleep 60 k get rs k delete deploy probe """ S[39] = [hand(r"""{SHELL_W1} sudo -i systemctl status kubelet journalctl -u kubelet -n 20 --no-pager""", r"""The kubelet keeps failing on `open /etc/kubernetes/pki/ca-bundle.crt: no such file or directory`.""", run=r""""""), hand(r"""vim /var/lib/kubelet/config.yaml""", r"""Point `clientCAFile` back at the CA that exists, then `:wq`: ``` clientCAFile: /etc/kubernetes/pki/ca.crt ```""", run=r"""ssh -n {W1} 'sudo sed -i "s|ca-bundle.crt|ca.crt|" /var/lib/kubelet/config.yaml'"""), hand(r"""systemctl restart kubelet systemctl enable kubelet systemctl status kubelet exit exit""", r"""`active (running)` and `enabled`; then back on the host.""", run=r"""ssh -n {W1} 'sudo systemctl restart kubelet; sudo systemctl enable kubelet'"""), cmd(r"""k get nodes""", r"""Again until `{W1}` is `Ready`.""")] # Hard Troubleshooting Tasks S[60] = [cmd(r"""k get node {W2}""", r"""`NotReady,SchedulingDisabled`: the kubelet is down, and the node cordoned."""), hand(r"""{SHELL_W2} sudo -i systemctl status kubelet systemctl is-enabled kubelet""", r"""`inactive (dead)` and `disabled`.""", run=r""""""), hand(r"""systemctl start kubelet journalctl -u kubelet -n 20 --no-pager""", r"""It fails on `open /etc/kubernetes/pki/ca-wrong.crt: no such file or directory`.""", run=r""""""), hand(r"""vim /var/lib/kubelet/config.yaml""", r"""Point `clientCAFile` at the CA that exists, then `:wq`: ``` clientCAFile: /etc/kubernetes/pki/ca.crt ```""", run=r"""ssh -n {W2} 'sudo sed -i "s|ca-wrong.crt|ca.crt|" /var/lib/kubelet/config.yaml'"""), hand(r"""systemctl enable --now kubelet systemctl restart kubelet systemctl status kubelet exit exit""", r"""`active (running)`, `enabled`; then back on the host.""", run=r"""ssh -n {W2} 'sudo systemctl enable --now kubelet; sudo systemctl restart kubelet'"""), cmd(r"""k uncordon {W2} k get node {W2}""", r"""Again until `Ready`, without `SchedulingDisabled`."""), cmd(r"""k run test-pod --image=busybox:1.36 --dry-run=client -o yaml -- sleep 60 > t.yaml"""), hand(r"""vim t.yaml""", r"""Pin it: in the Pod's `spec:`, right below `spec:`, ``` nodeName: {W2} ```""", run=r"""yq -i '.spec.nodeName = "{W2}"' t.yaml"""), cmd(r"""k apply -f t.yaml k get pod test-pod -o wide""", r"""Again until `Running` on `{W2}`, then clean up: `k delete pod test-pod`.""")] S[61] = """# Step 1: Check control plane pods in kube-system k -n kube-system get pods # scheduler CrashLoop, controller-manager ImagePullBackOff # Step 2: Access control plane node sudo -i # the task's host is the control-plane node: no hop needed # Step 3: Fix kube-scheduler manifest command typo vi /etc/kubernetes/manifests/kube-scheduler.yaml # Replace: kube-schedulerr -> kube-scheduler # Step 4: Fix kube-controller-manager image tag vi /etc/kubernetes/manifests/kube-controller-manager.yaml # Remove broken image tag suffix to match cluster version # Step 5: Verify both static pods return to Running k -n kube-system get pods -w k create deploy probe --image=busybox:1.36 -- sleep 60 k get deploy probe k delete deploy probe """ S[62] = [cmd(r"""k -n %{ns} get all k -n %{ns} describe rs -l app=%{dep} | grep -i quota k -n %{ns} describe pod -l app=%{dep} | grep -e Image: -e Error -e Warning""", r"""Four faults: the quota allows 1 Pod; the image `nginx:1.27-nope` does not exist; the env var wants key `DATABASE_URL`, which Secret `%{dep}-env` lacks; and the Service (next) selects the wrong label on the wrong port."""), hand(r"""k -n %{ns} edit quota tight""", r"""Room for the rolling update, not only the end state: 3 replicas plus the one extra Pod a rollout creates (maxSurge 25% of 3 rounds up to 1; maxUnavailable rounds down to 0). With `pods: "3"` the rollout deadlocks. Set it, then `:wq`: ``` spec: hard: pods: "4" ```""", run=r"""kubectl -n %{ns} patch quota tight -p '{"spec":{"hard":{"pods":"4"}}}'"""), cmd(r"""k -n %{ns} create secret generic %{dep}-env --from-literal=API_KEY=abc123 --from-literal=DATABASE_URL=postgres://db --dry-run=client -o yaml | k apply -f - k -n %{ns} set image deploy %{dep} web=nginx:1.27""", r"""The Secret again, with its old key and the missing one: generated and applied over the old one, no base64 by hand."""), cmd(r"""k -n %{ns} get svc %{dep} -o yaml k -n %{ns} get pods --show-labels""", r"""The Service selects `app=%{dep}-web`, the Pods are `app=%{dep}`; and it sends to 8080, where nginx listens on 80."""), hand(r"""k -n %{ns} edit svc %{dep}""", r"""Fix both, then `:wq` - an edit keeps the ClusterIP: ``` ports: - port: 80 protocol: TCP targetPort: 80 selector: app: %{dep} ```""", run=r"""kubectl -n %{ns} patch svc %{dep} -p '{"spec":{"selector":{"app":"%{dep}"},"ports":[{"port":80,"targetPort":80}]}}'"""), cmd(r"""k -n %{ns} rollout status deploy %{dep} k -n %{ns} get endpointslices -l kubernetes.io/service-name=%{dep}""")] S[63] = [cmd(r"""k -n kube-system describe pod -l component=kube-controller-manager | grep cluster-cidr k -n kube-system describe pod -l component=kube-apiserver | grep service-cluster-ip-range k -n kube-system get svc kube-dns k -n kube-system get cm kube-proxy -o yaml | grep mode k get node %{node} -o yaml | grep podCIDR""", r"""Each value from where the task says: the controller manager's and the API server's flags, the Service, the ConfigMap, the node."""), hand(r"""vim {WS}/q63/facts.txt""", r"""With the values they printed - for example: ``` pod-cidr=10.244.0.0/16 service-cidr=10.96.0.0/16 dns-ip=10.96.0.10 proxy-mode=iptables %{node}-pod-cidr=10.244.2.0/24 ``` `mode: ""` in the ConfigMap means the default: write `iptables`.""", run=r"""pc=$(kubectl -n kube-system get pod -l component=kube-controller-manager -o jsonpath='{.items[0].spec.containers[0].command}' | grep -o 'cluster-cidr=[^,"]*' | cut -d= -f2) sc=$(kubectl -n kube-system get pod -l component=kube-apiserver -o jsonpath='{.items[0].spec.containers[0].command}' | grep -o 'service-cluster-ip-range=[^,"]*' | cut -d= -f2) d=$(kubectl -n kube-system get svc kube-dns -o jsonpath='{.spec.clusterIP}') m=$(kubectl -n kube-system get cm kube-proxy -o jsonpath='{.data.config\.conf}' | grep '^mode:' | awk '{print $2}' | tr -d '"') printf 'pod-cidr=%s\nservice-cidr=%s\ndns-ip=%s\nproxy-mode=%s\n%s-pod-cidr=%s\n' "$pc" "$sc" "$d" "${m:-iptables}" "%{node}" "$(kubectl get node %{node} -o jsonpath='{.spec.podCIDR}')" > {WS}/q63/facts.txt""")] S[66] = [cmd(r"""k get nodes""", r"""Refused: the API server itself is down, so `k` cannot help - look at the containers on the node."""), cmd(r"""sudo crictl ps -a | grep apiserver sudo journalctl -u kubelet -n 50 --no-pager | grep -i apiserver""", r"""The host is the control-plane node. The container never starts: its command `kube-apiserverr` is not an executable."""), hand(r"""sudo vim /etc/kubernetes/manifests/kube-apiserver.yaml""", r"""The first line of `command:` reads `- kube-apiserverr`: make it `- kube-apiserver`, then `:wq`. The kubelet starts it again within a minute.""", run=r"""cpwait() { local i for i in $(seq 1 80); do [ "$(kubectl -n kube-system get pod $1-{CP} -o jsonpath='{.status.containerStatuses[0].ready}' 2>/dev/null)" = true ] && return 0 sleep 3 done return 1 } sudo sed -i 's/kube-apiserverr/kube-apiserver/' /etc/kubernetes/manifests/kube-apiserver.yaml for i in $(seq 1 80); do kubectl get nodes >/dev/null 2>&1 && break; sleep 3; done for c in etcd kube-apiserver kube-scheduler kube-controller-manager; do cpwait $c; done"""), cmd(r"""k get nodes k -n kube-system get pods""", r"""Again until `k` answers, and every control-plane Pod is `Running`.""")]