f3d0e536b6
kubestronaut-sim runs each exam task on a host of its own - a kind (kubeadm) cluster in rootless podman, or a QEMU/KVM Ubuntu VM for the CKS tasks that need their own kernel (Falco, AppArmor) - and puts a browser desktop in front of them, the way killer.sh and sailor.sh do. - ksim, sim/: the engine (sessions, drills, study mode, grading, coverage, audit), host and VM lifecycle, the desktop - images/: exam host images (common, CKS variant with its tools, VM guest) - cka/ (131 tasks), ckad/ (105), cks/ (110): task banks, curricula with study notes, reference solutions and scripted solutions Every domain of every track has at least 17 pool tasks. `./ksim audit` proves a task three ways: it seeds cleanly on its own host, scores nothing before anyone has worked on it, and reaches full marks when its scripted solution is run on the host as `candidate`. All 346 tasks pass (CKA audited 2026-10-13, CKAD and CKS 2026-10-12). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01JZPd3GGWyxTgAkL6a4gAJz
1736 lines
61 KiB
Python
1736 lines
61 KiB
Python
"""Reference solutions, keyed by task id. Rendered with the same %{param} / {WS} / {W1} substitution
|
|
as the task text. Shown only after a task has been graded (drills) or the session submitted (exams).
|
|
These solutions provide step-by-step instructions, Kubernetes concept explanations, imperative commands,
|
|
and verification checks for exam success.
|
|
"""
|
|
|
|
S = {}
|
|
|
|
# ================================================================= ARCH ====
|
|
# Domain 1: Cluster Architecture, Installation & Configuration (25%)
|
|
# ===========================================================================
|
|
|
|
S[1] = """# Step 1: Create the ServiceAccount in the target namespace
|
|
k -n rbac-app create sa deploy-bot
|
|
|
|
# Step 2: Create a Role granting specified verbs on deployments.apps
|
|
# Verbs: get, list, watch, create, update, patch, delete
|
|
k -n rbac-app create role deploy-manager \\
|
|
--verb=get,list,watch,create,update,patch,delete \\
|
|
--resource=deployments.apps
|
|
|
|
# Step 3: Bind the Role to the ServiceAccount
|
|
k -n rbac-app create rolebinding deploy-bot-binding \\
|
|
--role=deploy-manager \\
|
|
--serviceaccount=rbac-app:deploy-bot
|
|
|
|
# Step 4: Verify permissions using 'kubectl auth can-i'
|
|
k auth can-i create deployments -n rbac-app --as=system:serviceaccount:rbac-app:deploy-bot # Expected: yes
|
|
k auth can-i delete pods -n rbac-app --as=system:serviceaccount:rbac-app:deploy-bot # Expected: no
|
|
"""
|
|
|
|
S[2] = """# Step 1: Access the control plane node
|
|
sudo -i # the task's host is the control-plane node: no hop needed
|
|
|
|
# Step 2: Take the etcd snapshot with TLS certificates and save to the target path
|
|
# Certificates are located in /etc/kubernetes/pki/etcd/
|
|
ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 \\
|
|
--cacert=/etc/kubernetes/pki/etcd/ca.crt \\
|
|
--cert=/etc/kubernetes/pki/etcd/server.crt \\
|
|
--key=/etc/kubernetes/pki/etcd/server.key \\
|
|
snapshot save /opt/cka/etcd-snapshot.db
|
|
|
|
# Step 3: Verify snapshot status and integrity
|
|
etcdutl snapshot status /opt/cka/etcd-snapshot.db -w table
|
|
|
|
# Note: on a cluster where etcdctl is only inside the static pod, run it there. The etcd image
|
|
# is distroless (no sh, no cat, no tar), so kubectl cp cannot get the file out - write it into
|
|
# /var/lib/etcd instead, which is a hostPath, and move it on the node:
|
|
# kubectl -n kube-system exec etcd-{CP} -- etcdctl ... snapshot save /var/lib/etcd/snap.db
|
|
# then on the node: mv /var/lib/etcd/snap.db /opt/cka/etcd-snapshot.db
|
|
"""
|
|
|
|
S[3] = """# Step 1: Create the overlay directory structure
|
|
mkdir -p {WS}/q03/overlays/prod && cd {WS}/q03/overlays/prod
|
|
|
|
# Step 2: Define the target namespace manifest
|
|
cat > namespace.yaml <<EOF
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata: {name: store-prod}
|
|
EOF
|
|
|
|
# Step 3: Write the kustomization.yaml referencing the base and adding namespace, labels, and replicas
|
|
cat > kustomization.yaml <<EOF
|
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
|
kind: Kustomization
|
|
namespace: store-prod
|
|
resources: [../../base, namespace.yaml]
|
|
labels:
|
|
- pairs: {env: prod}
|
|
includeSelectors: false
|
|
replicas:
|
|
- name: store-api
|
|
count: 3
|
|
EOF
|
|
|
|
# Step 4: Build and apply the overlay
|
|
kubectl apply -k .
|
|
|
|
# Step 5: Verify resources in the store-prod namespace
|
|
k -n store-prod get deploy,pod -l env=prod
|
|
"""
|
|
|
|
S[4] = """# Step 1: Define and create the CustomResourceDefinition (CRD)
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: apiextensions.k8s.io/v1
|
|
kind: CustomResourceDefinition
|
|
metadata: {name: backups.data.example.com}
|
|
spec:
|
|
group: data.example.com
|
|
scope: Namespaced
|
|
names: {kind: Backup, plural: backups, singular: backup, shortNames: [bk]}
|
|
versions:
|
|
- name: v1
|
|
served: true
|
|
storage: true
|
|
schema:
|
|
openAPIV3Schema:
|
|
type: object
|
|
properties:
|
|
spec:
|
|
type: object
|
|
properties:
|
|
schedule: {type: string}
|
|
target: {type: string}
|
|
EOF
|
|
|
|
# Step 2: Create a custom resource instance of Backup
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: data.example.com/v1
|
|
kind: Backup
|
|
metadata: {name: nightly, namespace: default}
|
|
spec: {schedule: "0 2 * * *", target: "s3://backups/prod"}
|
|
EOF
|
|
|
|
# Step 3: Verify using the shortname
|
|
k get bk
|
|
"""
|
|
|
|
S[5] = """# Step 1: Access the specified worker node
|
|
{SHELL_W3}
|
|
sudo -i
|
|
|
|
# Step 2: Verify the kubelet static pod path
|
|
grep staticPodPath /var/lib/kubelet/config.yaml # Default: /etc/kubernetes/manifests
|
|
|
|
# Step 3: Create the static pod manifest in the static pod directory
|
|
cat > /etc/kubernetes/manifests/static-web.yaml <<EOF
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: static-web, namespace: default}
|
|
spec:
|
|
containers: [{name: web, image: nginx:1.27}]
|
|
EOF
|
|
|
|
# Step 4: Verify the mirror pod is created and running on the node
|
|
# kubelet automatically detects new manifests in the folder within seconds.
|
|
# Back on your main terminal: k get pods -o wide | grep static-web
|
|
"""
|
|
|
|
S[6] = """# Part A: Check certificate expiration on control plane
|
|
# The task's host IS the control-plane node; kubeadm reads the certificates as root.
|
|
sudo kubeadm certs check-expiration | grep '^apiserver ' > {WS}/q06/apiserver-expiry.txt
|
|
|
|
# Part B: Issue and approve CertificateSigningRequest (CSR) for user Jane
|
|
# Step 3: Generate private key and CSR
|
|
openssl genrsa -out jane.key 2048
|
|
openssl req -new -key jane.key -subj "/CN=jane" -out jane.csr
|
|
|
|
# Step 4: Submit Kubernetes CSR resource
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: certificates.k8s.io/v1
|
|
kind: CertificateSigningRequest
|
|
metadata: {name: jane}
|
|
spec:
|
|
request: $(base64 -w0 jane.csr)
|
|
signerName: kubernetes.io/kube-apiserver-client
|
|
usages: [client auth]
|
|
EOF
|
|
|
|
# Step 5: Approve CSR and extract signed certificate
|
|
k certificate approve jane
|
|
# the signer issues the certificate a moment AFTER the approval - read it too early and the file is empty
|
|
until [ -n "$(k get csr jane -o jsonpath='{.status.certificate}')" ]; do sleep 1; done
|
|
k get csr jane -o jsonpath='{.status.certificate}' | base64 -d > jane.crt
|
|
|
|
# Step 6: Bind user Jane to the default cluster role 'view'
|
|
k -n default create rolebinding jane-view --clusterrole=view --user=jane
|
|
"""
|
|
|
|
S[21] = """# Step 1: Create the target namespace
|
|
k create ns %{ns}
|
|
|
|
# Step 2: Install the Helm chart with overridden replicaCount
|
|
helm install %{rel} {WS}/q21/chart -n %{ns} --set replicaCount=%{n}
|
|
|
|
# Step 3: Upgrade the release to NodePort while preserving existing values
|
|
helm upgrade %{rel} {WS}/q21/chart -n %{ns} --reuse-values --set service.type=NodePort
|
|
|
|
# Step 4: Verify the release history shows revision 2
|
|
helm -n %{ns} history %{rel}
|
|
"""
|
|
|
|
S[33] = """# Step 1: Create the aggregating ClusterRole with aggregationRule selector
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRole
|
|
metadata: {name: %{cr}}
|
|
aggregationRule:
|
|
clusterRoleSelectors:
|
|
- matchLabels: {rbac.example.com/aggregate-to-%{cr}: "true"}
|
|
rules: []
|
|
EOF
|
|
|
|
# Step 2: Create sub-ClusterRoles for pods and services
|
|
k create clusterrole %{cr}-pods --verb=get,list,watch --resource=pods
|
|
k create clusterrole %{cr}-services --verb=get,list,watch --resource=services
|
|
|
|
# Step 3: Apply the matching label so controller aggregates rules into the main ClusterRole
|
|
k label clusterrole %{cr}-pods %{cr}-services rbac.example.com/aggregate-to-%{cr}=true
|
|
|
|
# Step 4: Create ClusterRoleBinding for the user
|
|
k create clusterrolebinding %{cr}-binding --clusterrole=%{cr} --user=%{user}
|
|
|
|
# Step 5: Verify rules are automatically aggregated
|
|
k get clusterrole %{cr} -o yaml
|
|
"""
|
|
|
|
S[34] = """# Step 1: Inspect the CRD spec schema recursively to locate the field path
|
|
k explain %{kind}.spec --recursive
|
|
|
|
# Step 2: Check the exact field details and description
|
|
k explain %{kind}.spec.tuning.%{field}
|
|
|
|
# Step 3: Record the full field path to the output file
|
|
echo spec.tuning.%{field} > {WS}/q34/path.txt
|
|
|
|
# Step 4: Create a sample Custom Resource using the discovered field
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: demo.io/v1
|
|
kind: %{kind}
|
|
metadata: {name: sample, namespace: default}
|
|
spec:
|
|
tuning: {%{field}: %{n}}
|
|
EOF
|
|
"""
|
|
|
|
S[40] = """# Generate a new kubeadm join token with the specified TTL and print-join-command,
|
|
# writing the output straight into the required file (the task's host is the control-plane node)
|
|
sudo kubeadm token create --ttl %{ttl} --print-join-command > {WS}/q40/join.sh
|
|
|
|
# Verify active tokens
|
|
sudo kubeadm token list
|
|
"""
|
|
|
|
S[41] = """# Step 1: Check Kubernetes Server version (API server)
|
|
k version
|
|
|
|
# Step 2: Extract Kubelet and Container Runtime versions for the target node
|
|
k get node %{node} -o jsonpath='{.status.nodeInfo.kubeletVersion}{"\\n"}{.status.nodeInfo.containerRuntimeVersion}{"\\n"}'
|
|
|
|
# Step 3: Extract etcd image tag from the static pod spec
|
|
k -n kube-system get pod -l component=etcd -o jsonpath='{.items[0].spec.containers[0].image}'
|
|
# Parse the image tag (everything after the colon)
|
|
"""
|
|
|
|
# Hard Architecture Tasks
|
|
S[44] = """# Step 1: Inspect bindings for the ServiceAccount to find and remove over-privileged bindings
|
|
k get clusterrolebinding -o wide | grep %{sa}
|
|
k delete clusterrolebinding %{sa}-legacy-admin
|
|
|
|
# Step 2: Create a restricted Role for deployments and replicasets
|
|
k -n %{ns} create role %{sa}-role --verb=get,list,watch,patch --resource=deployments.apps,replicasets.apps
|
|
|
|
# Step 3: Add pods/exec permission rule to the Role
|
|
k -n %{ns} patch role %{sa}-role --type=json -p='[{"op":"add","path":"/rules/-","value":{"apiGroups":[""],"resources":["pods/exec"],"verbs":["create"]}}]'
|
|
|
|
# Step 4: Bind the Role to the ServiceAccount
|
|
k -n %{ns} create rolebinding %{sa}-rb --role=%{sa}-role --serviceaccount=%{ns}:%{sa}
|
|
|
|
# Step 5: Create a ClusterRole with node list privileges and bind it
|
|
k create clusterrole %{sa}-nodes --verb=list --resource=nodes
|
|
k create clusterrolebinding %{sa}-crb --clusterrole=%{sa}-nodes --serviceaccount=%{ns}:%{sa}
|
|
|
|
# Step 6: Verify effective permissions. For a SUBRESOURCE use --subresource: `can-i create
|
|
# pods/exec` means "create the pod named exec" and says no even when the Role is right.
|
|
k auth can-i --list -n %{ns} --as=system:serviceaccount:%{ns}:%{sa}
|
|
k auth can-i create pods --subresource=exec -n %{ns} --as=system:serviceaccount:%{ns}:%{sa} # yes
|
|
"""
|
|
|
|
S[45] = """# Step 1: Access the control plane node
|
|
sudo -i # the task's host is the control-plane node: no hop needed
|
|
|
|
# Step 2: Restore snapshot into a new data directory
|
|
etcdutl snapshot restore /opt/cka/backup.db --data-dir /var/lib/etcd-restore
|
|
|
|
# Step 3: Update etcd static pod hostPath volume to point to the restored directory
|
|
# Edit /etc/kubernetes/manifests/etcd.yaml:
|
|
# Change:
|
|
# volumes:
|
|
# - hostPath:
|
|
# path: /var/lib/etcd-restore
|
|
# name: etcd-data
|
|
vi /etc/kubernetes/manifests/etcd.yaml
|
|
|
|
# Step 4: Monitor static pod recreation and verify restored objects
|
|
watch crictl ps
|
|
kubectl -n default get cm %{cm}
|
|
"""
|
|
|
|
S[46] = """# Step 1: Identify the edge node matching the label selector
|
|
k get nodes -l tier=edge
|
|
|
|
# Step 2: Access the target node
|
|
{SHELL_W3}
|
|
sudo -i
|
|
|
|
# Step 3: Check kubelet configuration for custom staticPodPath
|
|
grep staticPodPath /var/lib/kubelet/config.yaml
|
|
|
|
# Step 4: Place the pod manifest directly into the configured staticPodPath
|
|
cat > <staticPodPath>/%{pod}.yaml <<EOF
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: %{pod}, namespace: %{ns}, labels: {role: edge}}
|
|
spec:
|
|
containers:
|
|
- name: web
|
|
image: nginx:1.27
|
|
resources: {requests: {cpu: 50m, memory: 32Mi}}
|
|
EOF
|
|
|
|
# Step 5: Verify the mirror pod appears in the cluster
|
|
# k get pods -n %{ns} -o wide
|
|
"""
|
|
|
|
S[47] = """# Step 1: Generate user private key and Certificate Signing Request
|
|
openssl genrsa -out %{user}.key 2048
|
|
openssl req -new -key %{user}.key -subj "/CN=%{user}/O=developers" -out %{user}.csr
|
|
|
|
# Step 2: Create and approve the CSR in Kubernetes
|
|
k apply -f - <<EOF
|
|
apiVersion: certificates.k8s.io/v1
|
|
kind: CertificateSigningRequest
|
|
metadata: {name: %{user}}
|
|
spec: {request: $(base64 -w0 %{user}.csr), signerName: kubernetes.io/kube-apiserver-client, usages: [client auth]}
|
|
EOF
|
|
k certificate approve %{user}
|
|
# the signer issues the certificate a moment AFTER the approval - read it too early and the file is empty
|
|
until [ -n "$(k get csr %{user} -o jsonpath='{.status.certificate}')" ]; do sleep 1; done
|
|
k get csr %{user} -o jsonpath='{.status.certificate}' | base64 -d > %{user}.crt
|
|
|
|
# Step 3: Grant permissions via RoleBinding
|
|
k -n %{ns} create rolebinding %{user}-edit --clusterrole=edit --user=%{user}
|
|
|
|
# Step 4: Construct self-contained kubeconfig with embedded certificates
|
|
KC={WS}/q47/%{user}.kubeconfig
|
|
k config view --minify --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}' | base64 -d > ca.crt
|
|
SERVER=$(k config view --minify -o jsonpath='{.clusters[0].cluster.server}')
|
|
|
|
kubectl --kubeconfig $KC config set-cluster lab --server=$SERVER --certificate-authority=ca.crt --embed-certs
|
|
kubectl --kubeconfig $KC config set-credentials %{user} --client-certificate=%{user}.crt --client-key=%{user}.key --embed-certs
|
|
kubectl --kubeconfig $KC config set-context %{user}@lab --cluster=lab --user=%{user} --namespace=%{ns}
|
|
kubectl --kubeconfig $KC config use-context %{user}@lab
|
|
|
|
# Step 5: Test the kubeconfig
|
|
kubectl --kubeconfig $KC get pods
|
|
"""
|
|
|
|
S[48] = """# Step 1: Create directory for prod overlay
|
|
mkdir -p {WS}/q48/overlays/prod && cd {WS}/q48/overlays/prod
|
|
|
|
# Step 2: Create replica patch manifest
|
|
cat > replicas.yaml <<EOF
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: web}
|
|
spec: {replicas: %{rep}}
|
|
EOF
|
|
|
|
# Step 3: Create JSON patch for envFrom injection
|
|
cat > envfrom.yaml <<EOF
|
|
- op: add
|
|
path: /spec/template/spec/containers/0/envFrom
|
|
value: [{configMapRef: {name: web-config}}]
|
|
EOF
|
|
|
|
# Step 4: Write kustomization.yaml with generator, labels, and patches
|
|
cat > kustomization.yaml <<EOF
|
|
namespace: %{ns}
|
|
resources: [../../base]
|
|
labels:
|
|
- pairs: {tier: prod}
|
|
includeSelectors: false
|
|
configMapGenerator:
|
|
- name: web-config
|
|
literals: [LOG_LEVEL=%{lvl}]
|
|
patches:
|
|
- path: replicas.yaml
|
|
- path: envfrom.yaml
|
|
target: {kind: Deployment, name: web}
|
|
EOF
|
|
|
|
# Step 5: Create namespace and deploy
|
|
k create ns %{ns}
|
|
k apply -k .
|
|
"""
|
|
|
|
S[49] = """# Step 0: This context does not default to the `default` namespace - check, then pass -n default
|
|
k config view --minify -o jsonpath='{..namespace}{"\\n"}'
|
|
|
|
# Step 1: CRD with validation (required fields, integer bounds, enum) and a server-side default
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: apiextensions.k8s.io/v1
|
|
kind: CustomResourceDefinition
|
|
metadata: {name: $(echo %{kind} | tr A-Z a-z)s.deploy.example.com}
|
|
spec:
|
|
group: deploy.example.com
|
|
scope: Namespaced
|
|
names:
|
|
kind: %{kind}
|
|
plural: $(echo %{kind} | tr A-Z a-z)s
|
|
singular: $(echo %{kind} | tr A-Z a-z)
|
|
shortNames: [$(echo %{kind} | tr A-Z a-z | cut -c1-2)]
|
|
versions:
|
|
- name: v1
|
|
served: true
|
|
storage: true
|
|
schema:
|
|
openAPIV3Schema:
|
|
type: object
|
|
properties:
|
|
spec:
|
|
type: object
|
|
required: [replicas, image] # sits NEXT TO properties, not inside a field
|
|
properties:
|
|
replicas: {type: integer, minimum: 1, maximum: 10}
|
|
environment: {type: string, enum: [prod, stage, dev], default: dev}
|
|
image: {type: string}
|
|
EOF
|
|
|
|
# Step 2: A valid object
|
|
k -n default apply -f - <<EOF
|
|
apiVersion: deploy.example.com/v1
|
|
kind: %{kind}
|
|
metadata: {name: good}
|
|
spec: {replicas: 3, image: nginx:1.27, environment: %{env}}
|
|
EOF
|
|
|
|
# Step 3: The rejection goes to STDERR - redirect 2>, not >, or the file stays empty
|
|
printf 'apiVersion: deploy.example.com/v1\\nkind: %{kind}\\nmetadata: {name: bad}\\nspec: {replicas: 0, image: x}\\n' | k -n default apply -f - 2> {WS}/q49/rejected.txt
|
|
cat {WS}/q49/rejected.txt # ...spec.replicas in body should be greater than or equal to 1
|
|
|
|
# Step 4: Omit environment and let the API server fill in the default
|
|
printf 'apiVersion: deploy.example.com/v1\\nkind: %{kind}\\nmetadata: {name: defaulted}\\nspec: {replicas: 2, image: nginx:1.27}\\n' | k -n default apply -f -
|
|
k -n default get $(echo %{kind} | tr A-Z a-z) defaulted -o jsonpath='{.spec.environment}{"\\n"}' # dev
|
|
"""
|
|
|
|
S[65] = """# Step 1: Access control plane node
|
|
sudo -i # the task's host is the control-plane node: no hop needed
|
|
|
|
# Step 2: Renew all cluster certificates via kubeadm
|
|
kubeadm certs renew all
|
|
|
|
# Step 3: Restart static pods to load renewed certificates
|
|
mv /etc/kubernetes/manifests/*.yaml /tmp/ && sleep 20 && mv /tmp/*.yaml /etc/kubernetes/manifests/
|
|
|
|
# Step 4: Update admin kubeconfig from newly generated credentials
|
|
cp /etc/kubernetes/admin.conf ~/.kube/config
|
|
exit
|
|
|
|
# Step 5: Out of the root shell again, write the expiry report into the required file
|
|
sudo kubeadm certs check-expiration | grep '^apiserver ' > {WS}/q65/expiry.txt
|
|
"""
|
|
|
|
S[67] = """# Step 1: On the control-plane node, create a fresh bootstrap token
|
|
sudo -i # the task's host is the control-plane node: no hop needed
|
|
kubeadm token create --ttl 2h
|
|
# -> prints a token like abcdef.0123456789abcdef
|
|
|
|
# Step 2: Compute the CA discovery hash yourself. This is exactly what
|
|
# `kubeadm join --discovery-token-ca-cert-hash` verifies against, so you should never have to
|
|
# trust a value someone else hands you without being able to check it independently.
|
|
openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | \\
|
|
openssl rsa -pubin -outform der 2>/dev/null | \\
|
|
openssl dgst -sha256 -hex | sed 's/^.* //'
|
|
|
|
# Step 3: Find the real in-cluster API endpoint -- NOT what your local kubeconfig uses.
|
|
# A kubeconfig on the control-plane node may well say 127.0.0.1 or a name only this host
|
|
# resolves; a node joining over the cluster network needs the address the apiserver advertises.
|
|
exit
|
|
kubectl get endpointslices -l kubernetes.io/service-name=kubernetes -o jsonpath='{.items[0].endpoints[0].addresses[0]}:{.items[0].ports[0].port}{"\\n"}'
|
|
|
|
# Step 4: Assemble the handoff file
|
|
cat > {WS}/q67/node-join.env <<EOF
|
|
JOIN_TOKEN=<token from step 1>
|
|
CA_HASH=sha256:<hash from step 2>
|
|
API_ENDPOINT=<ip:port from step 3>
|
|
EOF
|
|
"""
|
|
|
|
|
|
# ================================================================= WORK ====
|
|
# Domain 2: Workloads & Scheduling (15%)
|
|
# ===========================================================================
|
|
|
|
S[7] = """# Step 1: Update container image to nginx:1.27
|
|
k -n apps set image deploy web-frontend nginx=nginx:1.27
|
|
|
|
# Step 2: Scale deployment to 4 replicas
|
|
k -n apps scale deploy web-frontend --replicas=4
|
|
|
|
# Step 3: Patch rolling update strategy (maxSurge=1, maxUnavailable=0)
|
|
k -n apps patch deploy web-frontend -p '{"spec":{"strategy":{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0}}}}'
|
|
|
|
# Step 4: Monitor rollout until complete
|
|
k -n apps rollout status deploy web-frontend
|
|
"""
|
|
|
|
S[8] = """# Step 1: Create pod manifest with shared emptyDir volume and sidecar pattern
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: log-shipper, namespace: apps}
|
|
spec:
|
|
volumes: [{name: logs, emptyDir: {}}]
|
|
containers:
|
|
- name: app
|
|
image: busybox:1.36
|
|
command: [sh, -c, "while true; do date >> /var/log/app.log; sleep 2; done"]
|
|
volumeMounts: [{name: logs, mountPath: /var/log}]
|
|
- name: shipper
|
|
image: busybox:1.36
|
|
command: [sh, -c, "tail -f /var/log/app.log"]
|
|
volumeMounts: [{name: logs, mountPath: /var/log}]
|
|
EOF
|
|
|
|
# Step 2: Verify sidecar container reads the log
|
|
k -n apps logs log-shipper -c shipper
|
|
"""
|
|
|
|
S[9] = """# Step 1: Generate initial deployment manifest with dry-run
|
|
k -n apps create deploy cache --image=redis:7-alpine --replicas=2 $do > cache.yaml
|
|
|
|
# Step 2: Add nodeAffinity (disktype=ssd) and container resources (requests/limits)
|
|
# Under spec.template.spec:
|
|
# affinity:
|
|
# nodeAffinity:
|
|
# requiredDuringSchedulingIgnoredDuringExecution:
|
|
# nodeSelectorTerms:
|
|
# - matchExpressions: [{key: disktype, operator: In, values: [ssd]}]
|
|
# Under containers[0]:
|
|
# resources:
|
|
# requests: {cpu: 100m, memory: 64Mi}
|
|
# limits: {cpu: 200m, memory: 128Mi}
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: cache, namespace: apps}
|
|
spec:
|
|
replicas: 2
|
|
selector: {matchLabels: {app: cache}}
|
|
template:
|
|
metadata: {labels: {app: cache}}
|
|
spec:
|
|
affinity:
|
|
nodeAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
nodeSelectorTerms:
|
|
- matchExpressions: [{key: disktype, operator: In, values: [ssd]}]
|
|
containers:
|
|
- name: redis
|
|
image: redis:7-alpine
|
|
resources:
|
|
requests: {cpu: 100m, memory: 64Mi}
|
|
limits: {cpu: 200m, memory: 128Mi}
|
|
EOF
|
|
|
|
# Step 3: Verify scheduling on nodes labeled disktype=ssd
|
|
k -n apps get pods -o wide
|
|
"""
|
|
|
|
S[10] = """# Step 1: Create ConfigMap with literal values
|
|
k -n apps create cm app-config --from-literal=APP_MODE=production --from-literal=LOG_LEVEL=debug
|
|
|
|
# Step 2: Create Secret with credentials
|
|
k -n apps create secret generic db-creds --from-literal=username=admin --from-literal='password=S3cr3t!'
|
|
|
|
# Step 3: Create Pod mounting ConfigMap as volume, injecting env var from CM, and loading Secret via envFrom
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: config-pod, namespace: apps}
|
|
spec:
|
|
volumes: [{name: cfg, configMap: {name: app-config}}]
|
|
containers:
|
|
- name: app
|
|
image: busybox:1.36
|
|
command: [sleep, "3600"]
|
|
env: [{name: APP_MODE, valueFrom: {configMapKeyRef: {name: app-config, key: APP_MODE}}}]
|
|
envFrom: [{secretRef: {name: db-creds}}]
|
|
volumeMounts: [{name: cfg, mountPath: /etc/config}]
|
|
EOF
|
|
|
|
# Step 4: Verify mounted files and environment variables
|
|
k -n apps exec config-pod -- ls /etc/config
|
|
k -n apps exec config-pod -- env | grep -E 'APP_MODE|username'
|
|
"""
|
|
|
|
S[23] = """# Step 1: Repair metrics-server by adding --kubelet-insecure-tls flag
|
|
k -n kube-system edit deploy metrics-server
|
|
# Add under spec.template.spec.containers[0].args:
|
|
# - --kubelet-insecure-tls
|
|
|
|
# Step 2: Wait ~30s and verify metrics reporting
|
|
k top nodes
|
|
|
|
# Step 3: Configure HorizontalPodAutoscaler (HPA) for target deployment
|
|
k -n %{ns} autoscale deploy %{dep} --min=2 --max=%{max} --cpu-percent=%{cpu} --name=%{dep}
|
|
|
|
# Step 4: Verify HPA status
|
|
k -n %{ns} get hpa %{dep}
|
|
"""
|
|
|
|
S[24] = """# Step 1: Create PriorityClass with requested integer value
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: scheduling.k8s.io/v1
|
|
kind: PriorityClass
|
|
metadata: {name: %{pc}}
|
|
value: %{val}
|
|
globalDefault: false
|
|
description: "%{pc} workloads"
|
|
EOF
|
|
|
|
# Step 2: Assign PriorityClass to the Deployment pod template
|
|
k -n %{ns} patch deploy payments -p '{"spec":{"template":{"spec":{"priorityClassName":"%{pc}"}}}}'
|
|
|
|
# Step 3: Monitor rollout
|
|
k -n %{ns} rollout status deploy payments
|
|
"""
|
|
|
|
S[25] = """# Step 1: Apply taint to target worker node
|
|
k taint node {W1} %{key}=%{val}:NoSchedule
|
|
|
|
# Step 2: Create Deployment targeting node with matching toleration
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: %{dep}, namespace: %{ns}}
|
|
spec:
|
|
replicas: 2
|
|
selector: {matchLabels: {app: %{dep}}}
|
|
template:
|
|
metadata: {labels: {app: %{dep}}}
|
|
spec:
|
|
nodeSelector:
|
|
kubernetes.io/hostname: {W1}
|
|
tolerations:
|
|
- key: %{key}
|
|
operator: Equal
|
|
value: "%{val}"
|
|
effect: NoSchedule
|
|
containers:
|
|
- name: busybox
|
|
image: busybox:1.36
|
|
command: [sleep, "3600"]
|
|
EOF
|
|
|
|
# Step 3: Verify pods are scheduled on {W1}
|
|
k -n %{ns} get pods -o wide
|
|
"""
|
|
|
|
S[26] = """# Step 1: Create DaemonSet manifest tolerating control-plane taints
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: apps/v1
|
|
kind: DaemonSet
|
|
metadata: {name: %{ds}, namespace: %{ns}}
|
|
spec:
|
|
selector: {matchLabels: {app: %{ds}}}
|
|
template:
|
|
metadata: {labels: {app: %{ds}}}
|
|
spec:
|
|
tolerations:
|
|
- key: node-role.kubernetes.io/control-plane
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
containers:
|
|
- name: agent
|
|
image: busybox:1.36
|
|
command: [sleep, "3600"]
|
|
resources: {requests: {cpu: 10m, memory: 16Mi}}
|
|
EOF
|
|
|
|
# Step 2: Verify DaemonSet runs on all nodes including control plane
|
|
k -n %{ns} get ds %{ds}
|
|
k -n %{ns} get pods -o wide -l app=%{ds}
|
|
"""
|
|
|
|
S[27] = """# Step 1: Create CronJob with specified schedule and history limits
|
|
k -n %{ns} create cj %{cj} --image=busybox:1.36 --schedule='*/%{every} * * * *' $do -- date > cj.yaml
|
|
|
|
# Step 2: Add concurrencyPolicy, activeDeadlineSeconds, and history limits:
|
|
# spec:
|
|
# concurrencyPolicy: Forbid
|
|
# successfulJobsHistoryLimit: 3
|
|
# failedJobsHistoryLimit: 1
|
|
# jobTemplate:
|
|
# spec:
|
|
# activeDeadlineSeconds: 30
|
|
# template:
|
|
# spec:
|
|
# restartPolicy: Never
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata: {name: %{cj}, namespace: %{ns}}
|
|
spec:
|
|
schedule: '*/%{every} * * * *'
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 1
|
|
jobTemplate:
|
|
spec:
|
|
activeDeadlineSeconds: 30
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
containers:
|
|
- name: date
|
|
image: busybox:1.36
|
|
command: ["date"]
|
|
EOF
|
|
|
|
# Step 3: Trigger a manual job run from the CronJob
|
|
k -n %{ns} create job %{cj}-manual --from=cronjob/%{cj}
|
|
"""
|
|
|
|
S[28] = """# Step 1: Create Pod with initContainer populating a shared volume
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: %{pod}, namespace: %{ns}}
|
|
spec:
|
|
volumes: [{name: content, emptyDir: {}}]
|
|
initContainers:
|
|
- name: setup
|
|
image: busybox:1.36
|
|
command: [sh, -c, "echo '%{text}' > /work/index.html"]
|
|
volumeMounts: [{name: content, mountPath: /work}]
|
|
containers:
|
|
- name: web
|
|
image: nginx:1.25
|
|
volumeMounts: [{name: content, mountPath: /usr/share/nginx/html}]
|
|
EOF
|
|
|
|
# Step 2: Verify web server serves the file created by init container
|
|
k -n %{ns} exec %{pod} -c web -- cat /usr/share/nginx/html/index.html
|
|
"""
|
|
|
|
S[29] = """# Step 1: Create the new version of the immutable ConfigMap
|
|
k -n %{ns} create cm %{cm}-v2 --from-literal=mode=new
|
|
k -n %{ns} patch cm %{cm}-v2 -p '{"immutable":true}'
|
|
|
|
# Step 2: Update the Deployment volume reference to the new ConfigMap
|
|
k -n %{ns} patch deploy %{dep} --type=json -p='[{"op":"replace","path":"/spec/template/spec/volumes/0/configMap/name","value":"%{cm}-v2"}]'
|
|
|
|
# Step 3: Verify rolling update finishes
|
|
k -n %{ns} rollout status deploy %{dep}
|
|
"""
|
|
|
|
S[31] = """# Step 1: Create Job with completions, parallelism, and backoffLimit
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata: {name: %{job}, namespace: %{ns}}
|
|
spec:
|
|
completions: %{c}
|
|
parallelism: %{p}
|
|
backoffLimit: 2
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
containers:
|
|
- name: runner
|
|
image: busybox:1.36
|
|
command: ["sh", "-c", "echo done; sleep 2"]
|
|
EOF
|
|
|
|
# Step 2: Wait for Job completion
|
|
k -n %{ns} wait --for=condition=complete job/%{job} --timeout=120s
|
|
"""
|
|
|
|
S[32] = """# Step 1: Create ResourceQuota for the namespace
|
|
k -n %{ns} create quota %{ns}-quota --hard=pods=%{pods},requests.cpu=1,limits.memory=2Gi
|
|
|
|
# Step 2: Create LimitRange defining default container requests and limits
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: LimitRange
|
|
metadata: {name: %{ns}-defaults, namespace: %{ns}}
|
|
spec:
|
|
limits:
|
|
- type: Container
|
|
default: {cpu: 500m, memory: 256Mi}
|
|
defaultRequest: {cpu: %{dcpu}, memory: 64Mi}
|
|
EOF
|
|
|
|
# Step 3: Create the Deployment without specifying resources - it picks up the LimitRange defaults
|
|
k -n %{ns} create deploy worker --image=nginx:1.25 --replicas=2
|
|
|
|
# Step 4: Verify quota, limits and the defaulted pod resources
|
|
k -n %{ns} describe quota %{ns}-quota
|
|
k -n %{ns} describe limitrange %{ns}-defaults
|
|
k -n %{ns} get pod -l app=worker -o jsonpath='{.items[0].spec.containers[0].resources}'
|
|
"""
|
|
|
|
# Hard Workload Tasks
|
|
S[50] = """# Step 1: Read the constraints before writing anything
|
|
k get node {W1} -o jsonpath='{.spec.taints}{"\\n"}' # %{key}=%{val}:NoSchedule
|
|
k -n %{ns} describe quota # pods: 2, requests.cpu: 250m
|
|
|
|
# Step 2: The quota is the trap twice over: 3 pods do not fit "pods: 2", and 3 x 100m = 300m does
|
|
# not fit 250m either. Raise both - the quota has to stay and still cap pods.
|
|
k -n %{ns} patch quota %{ns}-quota -p '{"spec":{"hard":{"pods":"3","requests.cpu":"300m"}}}'
|
|
|
|
# Step 3: The PriorityClass
|
|
k create priorityclass %{dep}-prio --value=50000
|
|
|
|
# Step 4: One pod per worker (required anti-affinity on hostname) that tolerates {W1}'s taint
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: %{dep}, namespace: %{ns}}
|
|
spec:
|
|
replicas: 3
|
|
selector: {matchLabels: {app: %{dep}}}
|
|
template:
|
|
metadata: {labels: {app: %{dep}}}
|
|
spec:
|
|
priorityClassName: %{dep}-prio
|
|
affinity:
|
|
podAntiAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
- labelSelector: {matchLabels: {app: %{dep}}}
|
|
topologyKey: kubernetes.io/hostname
|
|
tolerations:
|
|
- {key: %{key}, operator: Equal, value: "%{val}", effect: NoSchedule}
|
|
containers:
|
|
- name: nginx
|
|
image: nginx:1.27
|
|
resources: {requests: {cpu: 100m, memory: 64Mi}}
|
|
EOF
|
|
|
|
# Step 5: Verify: 3 Running on 3 different nodes. The control-plane is tainted too, so with three
|
|
# workers there is zero slack - a missed quota field or toleration shows up as a Pending pod, and
|
|
# `k -n %{ns} describe rs` tells you which (FailedCreate: exceeded quota).
|
|
k -n %{ns} get pod -l app=%{dep} -o wide
|
|
k -n %{ns} describe quota %{ns}-quota # pods 3/3, requests.cpu 300m/300m
|
|
"""
|
|
|
|
S[51] = """# Step 1: Find WHICH revision used nginx:1.25 - the task asks for that one specifically, which
|
|
# is not necessarily the previous one. --revision=N shows a single revision's pod template.
|
|
k -n %{ns} rollout history deploy %{dep}
|
|
for r in $(k -n %{ns} rollout history deploy %{dep} | awk '/^[0-9]/{print $1}'); do
|
|
echo "revision $r: $(k -n %{ns} rollout history deploy %{dep} --revision=$r | grep -i 'image:')"
|
|
done
|
|
|
|
# Step 2: Roll back to that revision (substitute the number you found for N)
|
|
k -n %{ns} rollout undo deploy %{dep} --to-revision=N
|
|
|
|
# Step 3: Scale to 3 and wait until they are actually available
|
|
k -n %{ns} scale deploy %{dep} --replicas=3
|
|
k -n %{ns} rollout status deploy %{dep}
|
|
|
|
# Step 4: Pause BEFORE editing. While paused the controller does not reconcile the template, so
|
|
# no new ReplicaSet is created and the running pods stay on nginx:1.25 - that gap between
|
|
# spec.template and what is running is exactly what the task checks.
|
|
k -n %{ns} rollout pause deploy %{dep}
|
|
k -n %{ns} set image deploy/%{dep} $(k -n %{ns} get deploy %{dep} -o jsonpath='{.spec.template.spec.containers[0].name}')=nginx:1.27
|
|
k -n %{ns} set env deploy/%{dep} RELEASE=next
|
|
|
|
# Step 5: Verify - paused true, template 1.27, pods still 1.25, and no RS for the new template
|
|
k -n %{ns} get deploy %{dep} -o jsonpath='{.spec.paused}{"\\n"}'
|
|
k -n %{ns} get pod -l app=%{dep} -o jsonpath='{.items[*].spec.containers[0].image}{"\\n"}'
|
|
k -n %{ns} get rs -o custom-columns=NAME:.metadata.name,IMAGE:.spec.template.spec.containers[0].image
|
|
"""
|
|
|
|
S[52] = """# Step 1: Deploy Pod with a native sidecar (restartPolicy: Always inside initContainers),
|
|
# a shared emptyDir the sidecar writes into and the main container reads from. Mount it at
|
|
# /var/log/agent, NOT /var/log: an emptyDir over /var/log hides nginx's /var/log/nginx and
|
|
# nginx crash-loops with "could not open error log file"
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: %{pod}, namespace: %{ns}}
|
|
spec:
|
|
terminationGracePeriodSeconds: 5
|
|
volumes:
|
|
- name: logs
|
|
emptyDir: {}
|
|
initContainers:
|
|
- name: agent
|
|
image: busybox:1.36
|
|
restartPolicy: Always
|
|
command: [sh, -c, "while true; do date >> /var/log/agent/agent.log; sleep 1; done"]
|
|
volumeMounts:
|
|
- {name: logs, mountPath: /var/log/agent}
|
|
containers:
|
|
- name: app
|
|
image: nginx:1.27
|
|
readinessProbe:
|
|
httpGet: {path: /, port: 80}
|
|
periodSeconds: 5
|
|
livenessProbe:
|
|
tcpSocket: {port: 80}
|
|
volumeMounts:
|
|
- {name: logs, mountPath: /var/log/agent}
|
|
EOF
|
|
|
|
# Step 2: Once Running - the Pod has no IP before that - write the exact report files
|
|
k -n %{ns} wait --for=condition=Ready pod/%{pod} --timeout=120s
|
|
k -n %{ns} get pod %{pod} -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}' \\
|
|
| tr ' ' '\\n' | sort > {WS}/q52/containers.txt
|
|
k -n %{ns} get pod %{pod} -o jsonpath='{.status.podIP}' > {WS}/q52/ip.txt
|
|
"""
|
|
|
|
S[53] = r"""# Step 0: look before you script - which pods, which phases, which nodes
|
|
k -n %{ns} get pods -o wide
|
|
|
|
# Step 1: FIRST container without a readinessProbe, any phase (the Completed pod counts too).
|
|
# jsonpath prints nothing for a missing field, so emit "<name> <probe-or-empty>" and keep the bare names.
|
|
k -n %{ns} get pods -o jsonpath='{range .items[*]}{.metadata.name} {.spec.containers[0].readinessProbe}{"\n"}{end}' \
|
|
| awk 'NF==1{print $1}' | sort > {WS}/q53/no-readiness.txt
|
|
|
|
# Step 2: Running pods per node, "<node> <count>". uniq -c prints "count node", so swap the columns.
|
|
k -n %{ns} get pods --field-selector=status.phase=Running -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' \
|
|
| sort | uniq -c | awk '{print $2, $1}' | sort > {WS}/q53/per-node.txt
|
|
|
|
# Step 3: distinct images in kube-system
|
|
k -n kube-system get pods -o jsonpath='{.items[*].spec.containers[*].image}' | tr ' ' '\n' | sort -u > {WS}/q53/images.txt
|
|
|
|
# Traps: cache-a has a readiness probe - on its SECOND container, so it belongs in file 1;
|
|
# once-a is Completed, so it is in file 1 but not in file 2. `cat -A` shows stray spaces or \r.
|
|
"""
|
|
|
|
S[64] = """# Step 1: Switch context and find out which namespace it defaults to
|
|
k config use-context %{ctx}
|
|
k config view --minify -o jsonpath='{..namespace}{"\\n"}' # -> %{ns}
|
|
|
|
# Step 2: Everything WITHOUT -n, so it lands in the context's namespace
|
|
k create deploy %{dep} --image=busybox:1.36 --replicas=2 -- sleep 3600
|
|
k label deploy %{dep} owner=ac
|
|
k expose deploy %{dep} --port=9000 --target-port=9000 # ClusterIP is the default type
|
|
|
|
# Step 3: Record the namespace exactly as kubectl reports it
|
|
k config view --minify -o jsonpath='{..namespace}' > {WS}/q64/ns.txt
|
|
|
|
# Step 4: Verify it all landed there - and nothing in default
|
|
k get deploy,svc %{dep}
|
|
k -n default get deploy %{dep} # NotFound is the right answer
|
|
"""
|
|
|
|
|
|
# ================================================================= NET =====
|
|
# Domain 3: Services & Networking (20%)
|
|
# ===========================================================================
|
|
|
|
S[11] = """# Step 1: Create NetworkPolicy in namespace 'netpol'
|
|
# Allow Ingress on port 8080 only from pods labeled 'app: frontend', no egress restriction
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata: {name: backend-policy, namespace: netpol}
|
|
spec:
|
|
podSelector: {matchLabels: {app: backend}}
|
|
policyTypes: [Ingress]
|
|
ingress:
|
|
- from:
|
|
- podSelector: {matchLabels: {app: frontend}}
|
|
ports:
|
|
- protocol: TCP
|
|
port: 8080
|
|
EOF
|
|
|
|
# Step 2: Verify NetworkPolicy configuration
|
|
k -n netpol describe netpol backend-policy
|
|
"""
|
|
|
|
S[12] = """# Step 1: Expose deployment as NodePort service using dry-run
|
|
k -n svc expose deploy shop --name=shop-svc --type=NodePort --port=80 --target-port=80 $do > svc.yaml
|
|
|
|
# Step 2: Set specific nodePort 30080 in spec.ports[0].nodePort
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata: {name: shop-svc, namespace: svc}
|
|
spec:
|
|
type: NodePort
|
|
selector: {app: shop}
|
|
ports:
|
|
- port: 80
|
|
targetPort: 80
|
|
nodePort: 30080
|
|
EOF
|
|
|
|
# Step 3: Verify NodePort service and endpoints
|
|
k -n svc get svc shop-svc
|
|
k -n svc get endpointslices -l kubernetes.io/service-name=shop-svc
|
|
"""
|
|
|
|
S[13] = """# Step 1: Create the Ingress with both path rules under one host
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: app-ingress
|
|
namespace: ingress-app
|
|
spec:
|
|
ingressClassName: nginx
|
|
rules:
|
|
- host: shop.example.com
|
|
http:
|
|
paths:
|
|
- path: /api
|
|
pathType: Prefix
|
|
backend:
|
|
service: {name: api, port: {number: 8080}}
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service: {name: web, port: {number: 80}}
|
|
EOF
|
|
|
|
# Step 2: Verify Ingress rules
|
|
k -n ingress-app describe ingress app-ingress
|
|
"""
|
|
|
|
S[14] = """# Step 1: Create a long-running pod (not --rm) so it still exists for the check
|
|
k -n dns-check run dns-test --image=busybox:1.36 --restart=Never -- sleep 3600
|
|
|
|
# Step 2: Save the FQDN you're resolving
|
|
echo "hello.dns-check.svc.cluster.local" > {WS}/q14/svc-fqdn.txt
|
|
|
|
# Step 3: Resolve it from inside the pod and save the full output
|
|
k -n dns-check wait --for=condition=Ready pod/dns-test --timeout=120s
|
|
k -n dns-check exec dns-test -- nslookup hello.dns-check.svc.cluster.local > {WS}/q14/nslookup.txt
|
|
"""
|
|
|
|
S[22] = """# Step 1: Install the Gateway API standard-channel CRDs
|
|
k apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/standard-install.yaml
|
|
# a CRD is only usable once the API server serves it: applying objects in the same breath can
|
|
# fail with "no matches for kind"
|
|
k wait --for=condition=Established --timeout=60s crd/gatewayclasses.gateway.networking.k8s.io \\
|
|
crd/gateways.gateway.networking.k8s.io crd/httproutes.gateway.networking.k8s.io
|
|
|
|
# Step 2: Apply GatewayClass, Gateway (with the listener hostname) and HTTPRoute
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: GatewayClass
|
|
metadata: {name: %{gc}}
|
|
spec: {controllerName: example.com/gateway}
|
|
---
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: Gateway
|
|
metadata: {name: %{gw}, namespace: %{ns}}
|
|
spec:
|
|
gatewayClassName: %{gc}
|
|
listeners:
|
|
- name: http
|
|
protocol: HTTP
|
|
port: 80
|
|
hostname: %{host}.example.com
|
|
---
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata: {name: %{host}-route, namespace: %{ns}}
|
|
spec:
|
|
parentRefs: [{name: %{gw}}]
|
|
rules:
|
|
- matches: [{path: {type: PathPrefix, value: /api}}]
|
|
backendRefs: [{name: api, port: 8080}]
|
|
- matches: [{path: {type: PathPrefix, value: /}}]
|
|
backendRefs: [{name: web, port: 80}]
|
|
EOF
|
|
|
|
# Step 3: Verify Gateway and HTTPRoute status
|
|
k -n %{ns} get gateway,httproute
|
|
"""
|
|
|
|
S[30] = """# Step 1: Deployment whose container port 80 carries the name %{pname}, with a readiness
|
|
# probe that references that port BY NAME rather than by number.
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: %{dep}, namespace: %{ns}}
|
|
spec:
|
|
replicas: 1
|
|
selector: {matchLabels: {app: %{dep}}}
|
|
template:
|
|
metadata: {labels: {app: %{dep}}}
|
|
spec:
|
|
containers:
|
|
- name: web
|
|
image: nginx:1.25
|
|
ports:
|
|
- name: %{pname}
|
|
containerPort: 80
|
|
readinessProbe:
|
|
httpGet: {path: /, port: %{pname}}
|
|
EOF
|
|
|
|
# Step 2: Service whose targetPort is the port NAME, not 80
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata: {name: %{dep}, namespace: %{ns}}
|
|
spec:
|
|
selector: {app: %{dep}}
|
|
ports:
|
|
- port: 8080
|
|
targetPort: %{pname}
|
|
EOF
|
|
|
|
# Step 3: Verify endpoints are registered (empty means the selector or the probe is wrong)
|
|
k -n %{ns} get endpointslices -l kubernetes.io/service-name=%{dep}
|
|
"""
|
|
|
|
S[42] = """# Step 1: Egress-only policy with TWO rules. Listing Egress alone in policyTypes is what
|
|
# denies everything else - there is no explicit "deny" rule in a NetworkPolicy. Leaving Ingress
|
|
# out of policyTypes is what keeps inbound traffic unrestricted.
|
|
# The DNS rule needs both UDP and TCP on 53: resolvers fall back to TCP for large answers.
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata: {name: %{app}-egress, namespace: %{ns}}
|
|
spec:
|
|
podSelector: {matchLabels: {app: %{app}}}
|
|
policyTypes: [Egress]
|
|
egress:
|
|
- to:
|
|
- namespaceSelector:
|
|
matchLabels: {kubernetes.io/metadata.name: kube-system}
|
|
podSelector:
|
|
matchLabels: {k8s-app: kube-dns}
|
|
ports:
|
|
- {protocol: UDP, port: 53}
|
|
- {protocol: TCP, port: 53}
|
|
- to:
|
|
- ipBlock: {cidr: %{cidr}}
|
|
ports:
|
|
- {protocol: TCP, port: 443}
|
|
EOF
|
|
|
|
# Step 2: Describe policy to confirm both egress rules
|
|
k -n %{ns} describe netpol %{app}-egress
|
|
"""
|
|
|
|
S[43] = """# Step 1: The headless Service must be named EXACTLY like the StatefulSet's spec.serviceName -
|
|
# that name is what makes CoreDNS publish one record per pod: <pod>.<service>.<ns>.svc.cluster.local
|
|
k -n %{ns} get sts %{sts} -o jsonpath='{.spec.serviceName}{"\\n"}' # -> %{sts}
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata: {name: %{sts}, namespace: %{ns}}
|
|
spec:
|
|
clusterIP: None
|
|
selector: {app: %{sts}}
|
|
ports: [{port: 5432, targetPort: 5432}]
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata: {name: legacy, namespace: %{ns}}
|
|
spec:
|
|
type: ExternalName
|
|
externalName: %{ext}
|
|
EOF
|
|
|
|
# Step 2: Resolve pod 0 by its FULL name. busybox's nslookup applies no search domains, so the
|
|
# short form %{sts}-0.%{sts} returns NXDOMAIN even though DNS is perfectly healthy.
|
|
k -n %{ns} exec %{sts}-0 -- nslookup %{sts}-0.%{sts}.%{ns}.svc.cluster.local > {WS}/q43/pod0.txt
|
|
cat {WS}/q43/pod0.txt # Address: <the pod's IP>
|
|
k -n %{ns} get pod %{sts}-0 -o wide # same IP
|
|
k -n %{ns} get svc legacy # EXTERNAL-IP column shows %{ext} - it is only a CNAME
|
|
"""
|
|
|
|
# Hard Networking Tasks
|
|
S[54] = """# Step 0: See what is there already. NetworkPolicies add up, so one that allows everything
|
|
# cancels every restriction you are about to write - "remove anything that contradicts this".
|
|
k -n %{ns} get netpol
|
|
k -n %{ns} delete netpol temp-allow-all
|
|
|
|
# Step 1: Label the source namespace for ingress matching
|
|
k label ns %{from} team=%{from}
|
|
|
|
# Step 2: Apply default-deny (ingress + egress), allow-dns egress, and tiered service policies
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata: {name: default-deny, namespace: %{ns}}
|
|
spec: {podSelector: {}, policyTypes: [Ingress, Egress]}
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata: {name: allow-dns, namespace: %{ns}}
|
|
spec:
|
|
podSelector: {}
|
|
policyTypes: [Egress]
|
|
egress:
|
|
- to: [{namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: kube-system}}}]
|
|
ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}]
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata: {name: api-ingress, namespace: %{ns}}
|
|
spec:
|
|
podSelector: {matchLabels: {app: api}}
|
|
policyTypes: [Ingress]
|
|
ingress:
|
|
- from: [{namespaceSelector: {matchLabels: {team: %{from}}}}]
|
|
ports: [{port: 8080}]
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata: {name: api-to-db, namespace: %{ns}}
|
|
spec:
|
|
podSelector: {matchLabels: {app: api}}
|
|
policyTypes: [Egress]
|
|
egress:
|
|
- to: [{podSelector: {matchLabels: {app: db}}}]
|
|
ports: [{port: 5432}]
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata: {name: db-ingress, namespace: %{ns}}
|
|
spec:
|
|
podSelector: {matchLabels: {app: db}}
|
|
policyTypes: [Ingress]
|
|
ingress:
|
|
- from: [{podSelector: {matchLabels: {app: api}}}]
|
|
ports: [{port: 5432}]
|
|
EOF
|
|
"""
|
|
|
|
S[55] = """# Step 1: Generate self-signed TLS certificate and private key
|
|
openssl req -x509 -newkey rsa:2048 -nodes -days 365 -subj "/CN=%{host}" -keyout tls.key -out tls.crt
|
|
|
|
# Step 2: Create TLS secret in target namespace
|
|
k -n %{ns} create secret tls %{host}-tls --cert=tls.crt --key=tls.key
|
|
|
|
# Step 3: Create Ingress with TLS configuration and SSL redirect annotation
|
|
k -n %{ns} create ingress web --class=nginx --rule="%{host}/*=web:80,tls=%{host}-tls" \\
|
|
--annotation nginx.ingress.kubernetes.io/ssl-redirect=true
|
|
|
|
# Step 4: Extract SHA256 fingerprint to workspace file
|
|
openssl x509 -in tls.crt -noout -fingerprint -sha256 | cut -d= -f2 > {WS}/q55/fp.txt
|
|
"""
|
|
|
|
S[56] = """# Step 1: Create GatewayClass, Gateway with multi-host listeners, and canary HTTPRoute
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: GatewayClass
|
|
metadata: {name: %{host}-gc}
|
|
spec: {controllerName: example.com/gw}
|
|
---
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: Gateway
|
|
metadata: {name: %{host}-gw, namespace: %{ns}}
|
|
spec:
|
|
gatewayClassName: %{host}-gc
|
|
listeners:
|
|
- {name: web, protocol: HTTP, port: 80, hostname: %{host}.example.com}
|
|
- {name: api, protocol: HTTP, port: 80, hostname: api.%{host}.example.com}
|
|
---
|
|
apiVersion: gateway.networking.k8s.io/v1
|
|
kind: HTTPRoute
|
|
metadata: {name: %{host}-route, namespace: %{ns}}
|
|
spec:
|
|
parentRefs: [{name: %{host}-gw, sectionName: web}]
|
|
rules:
|
|
- matches: [{headers: [{name: X-Canary, value: always}]}]
|
|
backendRefs: [{name: v2, port: 80}]
|
|
- backendRefs:
|
|
- {name: v1, port: 80, weight: %{w2}}
|
|
- {name: v2, port: 80, weight: %{w}}
|
|
EOF
|
|
"""
|
|
|
|
S[57] = """# Step 1: Remove stray blocking NetworkPolicy in kube-system
|
|
k -n kube-system get netpol
|
|
k -n kube-system delete netpol maintenance-lockdown
|
|
|
|
# Step 2: Fix CoreDNS replicas (scale back from 0)
|
|
k -n kube-system scale deploy coredns --replicas=2
|
|
|
|
# Step 3: Correct CoreDNS upstream forwarding in ConfigMap
|
|
k -n kube-system edit cm coredns
|
|
# Replace: forward . 10.255.255.254 -> forward . /etc/resolv.conf
|
|
|
|
# Step 4: Restart CoreDNS deployment and test DNS resolution
|
|
k -n kube-system rollout restart deploy coredns
|
|
# busybox's nslookup applies no search domains, so use FULL names (a short one fails even when DNS works)
|
|
k run dns-check --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.default.svc.cluster.local
|
|
k run dns-ext --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.io
|
|
"""
|
|
|
|
|
|
# ================================================================= STOR ====
|
|
# Domain 4: Storage (10%)
|
|
# ===========================================================================
|
|
|
|
S[15] = """# Step 1: Create PersistentVolume with hostPath, Retain policy, and manual storageClass
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: PersistentVolume
|
|
metadata: {name: pv-data}
|
|
spec:
|
|
capacity: {storage: 1Gi}
|
|
accessModes: [ReadWriteOnce]
|
|
persistentVolumeReclaimPolicy: Retain
|
|
storageClassName: manual
|
|
hostPath: {path: /mnt/data}
|
|
---
|
|
# Step 2: Create PersistentVolumeClaim requesting 500Mi
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: pvc-data, namespace: storage}
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: manual
|
|
resources: {requests: {storage: 500Mi}}
|
|
---
|
|
# Step 3: Create Pod mounting the claim at /usr/share/data
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: data-pod, namespace: storage}
|
|
spec:
|
|
volumes: [{name: d, persistentVolumeClaim: {claimName: pvc-data}}]
|
|
containers: [{name: web, image: nginx:1.25, volumeMounts: [{name: d, mountPath: /usr/share/data}]}]
|
|
EOF
|
|
|
|
# Step 4: Verify PV and PVC bound status
|
|
k get pv pv-data
|
|
k -n storage get pvc pvc-data
|
|
"""
|
|
|
|
S[16] = """# Step 1: Create StorageClass with WaitForFirstConsumer and set as default
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: storage.k8s.io/v1
|
|
kind: StorageClass
|
|
metadata:
|
|
name: fast-local
|
|
annotations: {storageclass.kubernetes.io/is-default-class: "true"}
|
|
provisioner: rancher.io/local-path
|
|
volumeBindingMode: WaitForFirstConsumer
|
|
reclaimPolicy: Delete
|
|
EOF
|
|
|
|
# Step 2: Remove default status from the previous default StorageClass
|
|
k annotate sc standard storageclass.kubernetes.io/is-default-class-
|
|
|
|
# Step 3: Create PVC (omitting storageClassName to use new default) and consumer Pod
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: fast-claim, namespace: storage}
|
|
spec: {accessModes: [ReadWriteOnce], resources: {requests: {storage: 200Mi}}}
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: fast-pod, namespace: storage}
|
|
spec:
|
|
volumes: [{name: d, persistentVolumeClaim: {claimName: fast-claim}}]
|
|
containers: [{name: web, image: nginx:1.25, volumeMounts: [{name: d, mountPath: /data}]}]
|
|
EOF
|
|
|
|
# Step 4: Verify volume dynamically binds once pod is scheduled
|
|
k -n storage get pvc fast-claim
|
|
"""
|
|
|
|
S[35] = """# Step 1: Inspect Released PV and locate old claimRef
|
|
k get pv %{pv} -o yaml | grep -A5 claimRef
|
|
|
|
# Step 2: Remove the claimRef to make the PV Available again
|
|
k patch pv %{pv} --type=json -p='[{"op":"remove","path":"/spec/claimRef"}]'
|
|
k get pv %{pv} # Verify status: Available
|
|
|
|
# Step 3: Create new PVC binding specifically to the released volume
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: %{pvc}, namespace: %{ns}}
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: manual
|
|
volumeName: %{pv}
|
|
resources: {requests: {storage: 500Mi}}
|
|
EOF
|
|
|
|
# Step 4: Confirm PVC binds to the PV
|
|
k -n %{ns} get pvc %{pvc}
|
|
"""
|
|
|
|
S[36] = """# Step 1: Create PVC for data persistence
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: %{pvc}, namespace: %{ns}}
|
|
spec: {accessModes: [ReadWriteOnce], resources: {requests: {storage: 200Mi}}}
|
|
---
|
|
# Step 2: Create writer Pod to write marker file and complete
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: writer, namespace: %{ns}}
|
|
spec:
|
|
restartPolicy: Never
|
|
volumes: [{name: d, persistentVolumeClaim: {claimName: %{pvc}}}]
|
|
containers: [{name: w, image: busybox:1.36, command: [sh, -c, "echo %{text} > /data/marker.txt"], volumeMounts: [{name: d, mountPath: /data}]}]
|
|
EOF
|
|
|
|
# Step 3: Wait for writer pod completion
|
|
k -n %{ns} wait --for=jsonpath='{.status.phase}'=Succeeded pod/writer
|
|
|
|
# Step 4: Create reader Pod mounting the same claim to verify data persists
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: reader, namespace: %{ns}}
|
|
spec:
|
|
volumes: [{name: d, persistentVolumeClaim: {claimName: %{pvc}}}]
|
|
containers: [{name: r, image: busybox:1.36, command: [sleep, "3600"], volumeMounts: [{name: d, mountPath: /data}]}]
|
|
EOF
|
|
|
|
# Step 5: Read data back from reader pod
|
|
k -n %{ns} exec reader -- cat /data/marker.txt
|
|
"""
|
|
|
|
# Hard Storage Tasks
|
|
S[58] = """# Step 1: Identify PV labels and corresponding node affinity
|
|
k get pv -l media --show-labels
|
|
k get node -l disk=nvme
|
|
|
|
# Step 2: Create PVC matching the media=nvme label selector
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: %{pvc}, namespace: %{ns}}
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: local-fast
|
|
selector: {matchLabels: {media: nvme}}
|
|
resources: {requests: {storage: 1Gi}}
|
|
---
|
|
# Step 3: Create consumer Pod mounting the PVC
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: consumer, namespace: %{ns}}
|
|
spec:
|
|
volumes: [{name: d, persistentVolumeClaim: {claimName: %{pvc}}}]
|
|
containers: [{name: c, image: busybox:1.36, command: [sleep, "3600"], volumeMounts: [{name: d, mountPath: /data}]}]
|
|
EOF
|
|
|
|
# Step 4: Verify volume binding and scheduling
|
|
k -n %{ns} get pod consumer -o wide
|
|
"""
|
|
|
|
S[59] = """# Step 1: Create target PVC for migration
|
|
k -n %{ns} apply -f - <<EOF
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: new-data, namespace: %{ns}}
|
|
spec: {accessModes: [ReadWriteOnce], resources: {requests: {storage: 1Gi}}}
|
|
---
|
|
# Step 2: Run migration Job mounting both old and new PVCs
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata: {name: migrate, namespace: %{ns}}
|
|
spec:
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
volumes:
|
|
- {name: src, persistentVolumeClaim: {claimName: old-data}}
|
|
- {name: dst, persistentVolumeClaim: {claimName: new-data}}
|
|
containers:
|
|
- name: cp
|
|
image: busybox:1.36
|
|
command: [sh, -c, "cp -a /src/. /dst/ && ls /dst"]
|
|
volumeMounts: [{name: src, mountPath: /src}, {name: dst, mountPath: /dst}]
|
|
EOF
|
|
|
|
# Step 3: Wait for migration job completion
|
|
k -n %{ns} wait --for=condition=complete job/migrate --timeout=120s
|
|
|
|
# Step 4: Re-point legacy pod to new claim
|
|
k -n %{ns} get pod legacy -o yaml > legacy.yaml
|
|
# Edit legacy.yaml: replace claimName: old-data with claimName: new-data
|
|
k -n %{ns} delete pod legacy --now && k apply -f legacy.yaml
|
|
|
|
# Step 5: Delete the old claim. The Job's finished pod still references old-data, and a pod that
|
|
# references a claim - even a Completed one - holds it in pvc-protection: `delete pvc` just sits
|
|
# there (describe pvc -> "Used By: migrate-xxxxx"). Delete that pod first; the Job itself stays,
|
|
# still reporting succeeded: 1.
|
|
k -n %{ns} delete pod -l job-name=migrate
|
|
k -n %{ns} delete pvc old-data
|
|
k get pv | grep old-data # STATUS Released - Retain kept the data
|
|
"""
|
|
|
|
|
|
# =============================================================== TSHOOT ====
|
|
# Domain 5: Troubleshooting (30%)
|
|
# ===========================================================================
|
|
|
|
S[17] = """# Step 1: Check node status
|
|
k get node {W2} # Status: NotReady
|
|
|
|
# Step 2: SSH into the broken worker node
|
|
{SHELL_W2}
|
|
sudo -i
|
|
|
|
# Step 3: Check kubelet service status and journal logs
|
|
systemctl status kubelet # Service inactive/disabled
|
|
systemctl enable --now kubelet
|
|
|
|
# Step 4: Verify kubelet successfully connects and registers node
|
|
journalctl -u kubelet -f --no-pager # Look for: "Successfully registered node"
|
|
|
|
# Step 5: Exit node shell and verify Ready state from cluster
|
|
# k get nodes
|
|
"""
|
|
|
|
S[18] = """# Step 1: Diagnose Deployment failure (ImagePullBackOff)
|
|
k -n troubleshoot describe deploy payments | grep -i image
|
|
# Notice invalid image tag: nginx:1.99-nope
|
|
|
|
# Step 2: Fix image on the deployment
|
|
k -n troubleshoot set image deploy payments payments=nginx:1.27
|
|
|
|
# Step 3: Inspect Service definition for mismatched selector and targetPort
|
|
k -n troubleshoot get svc payments -o yaml
|
|
# Discovered issues: selector is 'app: payment' (typo) and targetPort is 8080 (container port is 80)
|
|
|
|
# Step 4: Patch the Service selector and targetPort
|
|
k -n troubleshoot patch svc payments -p '{"spec":{"selector":{"app":"payments"},"ports":[{"port":80,"targetPort":80}]}}'
|
|
|
|
# Step 5: Verify Service endpoints
|
|
k -n troubleshoot get endpointslices -l kubernetes.io/service-name=payments
|
|
"""
|
|
|
|
S[19] = """# Step 1: Capture previous container logs to locate crash reason
|
|
k -n troubleshoot logs crasher --previous | tee {WS}/q19/crasher.log
|
|
# Log output: "FATAL: required environment variable DB_HOST is not set"
|
|
|
|
# Step 2: Export pod manifest
|
|
k -n troubleshoot get pod crasher -o yaml > crasher.yaml
|
|
|
|
# Step 3: Edit crasher.yaml:
|
|
# Add under spec.containers[0]:
|
|
# env:
|
|
# - name: DB_HOST
|
|
# value: db.internal
|
|
# (Remove system-managed fields: status, uid, resourceVersion, creationTimestamp)
|
|
|
|
# Step 4: Recreate pod and verify Running status
|
|
k -n troubleshoot delete pod crasher --now && k apply -f crasher.yaml
|
|
k -n troubleshoot get pod crasher
|
|
"""
|
|
|
|
S[20] = """# Step 1: Check control-plane pods in kube-system
|
|
k -n kube-system get pods
|
|
# kube-scheduler static pod is in Error / CrashLoopBackOff
|
|
|
|
# Step 2: Inspect scheduler pod logs
|
|
k -n kube-system logs kube-scheduler-{CP}
|
|
# Error: exec: "kube-schedulerr": executable file not found
|
|
|
|
# Step 3: SSH into control plane node
|
|
sudo -i # the task's host is the control-plane node: no hop needed
|
|
|
|
# Step 4: Edit scheduler static pod manifest
|
|
# In /etc/kubernetes/manifests/kube-scheduler.yaml:
|
|
# Change: command: [kube-schedulerr] -> command: [kube-scheduler]
|
|
vi /etc/kubernetes/manifests/kube-scheduler.yaml
|
|
|
|
# Step 5: Wait for kubelet to reload the static pod and verify scheduling
|
|
k -n kube-system get pods -l component=kube-scheduler
|
|
"""
|
|
|
|
S[37] = """# Step 1: Inspect CoreDNS pods and logs
|
|
k -n kube-system get pods -l k8s-app=kube-dns
|
|
k -n kube-system logs -l k8s-app=kube-dns --tail=20
|
|
|
|
# Step 2: Check CoreDNS ConfigMap
|
|
k -n kube-system get cm coredns -o yaml
|
|
# Fault: forward . 10.255.255.1 points to an invalid upstream DNS
|
|
|
|
# Step 3: Edit ConfigMap to use host resolv.conf
|
|
k -n kube-system edit cm coredns
|
|
# Change: forward . 10.255.255.1 -> forward . /etc/resolv.conf
|
|
|
|
# Step 4: Restart CoreDNS deployment to apply changes
|
|
k -n kube-system rollout restart deploy coredns
|
|
|
|
# Step 5: Verify DNS resolution from a test pod
|
|
# busybox's nslookup applies no search domains, so use FULL names (a short one fails even when DNS works)
|
|
k run test-dns --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.default.svc.cluster.local
|
|
k run test-ext --image=busybox:1.36 --rm -it --restart=Never -- nslookup kubernetes.io
|
|
"""
|
|
|
|
S[38] = """# Step 1: Check control-plane pod status
|
|
k -n kube-system get pods
|
|
# kube-controller-manager is crashing
|
|
|
|
# Step 2: Check controller-manager logs
|
|
k -n kube-system logs kube-controller-manager-{CP}
|
|
# Error: invalid flag: --leader-elect-lease-duration=1s
|
|
|
|
# Step 3: Access control plane node
|
|
sudo -i # the task's host is the control-plane node: no hop needed
|
|
|
|
# Step 4: Edit static pod manifest to remove the invalid flag
|
|
vi /etc/kubernetes/manifests/kube-controller-manager.yaml
|
|
|
|
# Step 5: Verify controller-manager restarts and reconciles deployments/replicasets
|
|
k create deploy probe --image=busybox:1.36 -- sleep 60
|
|
k get rs
|
|
k delete deploy probe
|
|
"""
|
|
|
|
S[39] = """# Step 1: Access the failing worker node
|
|
{SHELL_W1}
|
|
sudo -i
|
|
|
|
# Step 2: Inspect kubelet status and journal logs
|
|
systemctl status kubelet
|
|
journalctl -u kubelet -n 30 --no-pager
|
|
# Error: "open /etc/kubernetes/pki/ca-bundle.crt: no such file or directory"
|
|
|
|
# Step 3: Check /var/lib/kubelet/config.yaml for incorrect certificate path
|
|
grep -n clientCAFile /var/lib/kubelet/config.yaml
|
|
# Fix path: replace /etc/kubernetes/pki/ca-bundle.crt with /etc/kubernetes/pki/ca.crt
|
|
sed -i 's|ca-bundle.crt|ca.crt|' /var/lib/kubelet/config.yaml
|
|
|
|
# Step 4: Restart and enable kubelet service
|
|
systemctl restart kubelet && systemctl enable kubelet
|
|
systemctl status kubelet
|
|
"""
|
|
|
|
# Hard Troubleshooting Tasks
|
|
S[60] = """# Step 1: Check node status in cluster
|
|
k get node {W2} # NotReady,SchedulingDisabled
|
|
|
|
# Step 2: Access the node
|
|
{SHELL_W2}
|
|
sudo -i
|
|
|
|
# Step 3: Fix Fault 1 (kubelet service disabled) and Fault 2 (wrong CA certificate file)
|
|
systemctl status kubelet
|
|
sed -i 's|ca-wrong.crt|ca.crt|' /var/lib/kubelet/config.yaml
|
|
systemctl enable --now kubelet
|
|
systemctl status kubelet
|
|
exit; exit
|
|
|
|
# Step 4: Fix Fault 3 (node was cordoned in cluster)
|
|
k uncordon {W2}
|
|
|
|
# Step 5: Verify scheduling on the repaired node
|
|
k run test-pod --image=busybox:1.36 --restart=Never --overrides='{"spec":{"nodeName":"{W2}"}}' -- sleep 60
|
|
k get pod test-pod -o wide
|
|
k delete pod test-pod --now
|
|
"""
|
|
|
|
S[61] = """# Step 1: Check control plane pods in kube-system
|
|
k -n kube-system get pods
|
|
# scheduler CrashLoop, controller-manager ImagePullBackOff
|
|
|
|
# Step 2: Access control plane node
|
|
sudo -i # the task's host is the control-plane node: no hop needed
|
|
|
|
# Step 3: Fix kube-scheduler manifest command typo
|
|
vi /etc/kubernetes/manifests/kube-scheduler.yaml
|
|
# Replace: kube-schedulerr -> kube-scheduler
|
|
|
|
# Step 4: Fix kube-controller-manager image tag
|
|
vi /etc/kubernetes/manifests/kube-controller-manager.yaml
|
|
# Remove broken image tag suffix to match cluster version
|
|
|
|
# Step 5: Verify both static pods return to Running
|
|
k -n kube-system get pods -w
|
|
k create deploy probe --image=busybox:1.36 -- sleep 60
|
|
k get deploy probe
|
|
k delete deploy probe
|
|
"""
|
|
|
|
S[62] = """# Step 1: Diagnose namespace quota and deployment events
|
|
k -n %{ns} get all
|
|
k -n %{ns} describe deploy %{dep}
|
|
k -n %{ns} describe rs -l app=%{dep} # Quota exceeded: pods limit is 1
|
|
|
|
# Step 2: Fix the quota - with room for the rolling update, not just the end state. The default
|
|
# strategy (maxSurge 25% -> 1, maxUnavailable 25% of 3 -> 0) needs a 4th pod mid-rollout, and with
|
|
# maxUnavailable 0 the controller will not remove an old pod first either: at pods=3 the rollout
|
|
# deadlocks at "0 out of 3 new replicas have been updated".
|
|
k -n %{ns} patch quota tight -p '{"spec":{"hard":{"pods":"4"}}}'
|
|
|
|
# Step 3: Fix deployment image tag
|
|
k -n %{ns} set image deploy %{dep} web=nginx:1.27
|
|
|
|
# Step 4: Fix Secret missing key referenced by container environment variables
|
|
k -n %{ns} patch secret %{dep}-env -p '{"stringData":{"DATABASE_URL":"postgres://db"}}'
|
|
k -n %{ns} rollout restart deploy %{dep}
|
|
|
|
# Step 5: Fix Service selector and targetPort
|
|
k -n %{ns} patch svc %{dep} -p '{"spec":{"selector":{"app":"%{dep}"},"ports":[{"port":80,"targetPort":80}]}}'
|
|
|
|
# Step 6: Verify all deployment replicas are Available with endpoints
|
|
k -n %{ns} get deploy %{dep}
|
|
k -n %{ns} get endpointslices -l kubernetes.io/service-name=%{dep}
|
|
"""
|
|
|
|
S[63] = r"""# Gather each fact and append it directly to the report file, one key=value line at a time
|
|
pc=$(k -n kube-system get pod -l component=kube-controller-manager -o jsonpath='{.items[0].spec.containers[0].command}' | grep -o 'cluster-cidr=[^,"]*' | cut -d= -f2)
|
|
echo "pod-cidr=$pc" > {WS}/q63/facts.txt
|
|
|
|
sc=$(k -n kube-system get pod -l component=kube-apiserver -o jsonpath='{.items[0].spec.containers[0].command}' | grep -o 'service-cluster-ip-range=[^,"]*' | cut -d= -f2)
|
|
echo "service-cidr=$sc" >> {WS}/q63/facts.txt
|
|
|
|
d=$(k -n kube-system get svc kube-dns -o jsonpath='{.spec.clusterIP}')
|
|
echo "dns-ip=$d" >> {WS}/q63/facts.txt
|
|
|
|
# Empty mode in the ConfigMap means kube-proxy defaults to iptables
|
|
m=$(k -n kube-system get cm kube-proxy -o jsonpath='{.data.config\.conf}' | grep '^mode:' | awk '{print $2}' | tr -d '"')
|
|
echo "proxy-mode=${m:-iptables}" >> {WS}/q63/facts.txt
|
|
|
|
echo "%{node}-pod-cidr=$(k get node %{node} -o jsonpath='{.spec.podCIDR}')" >> {WS}/q63/facts.txt
|
|
"""
|
|
|
|
S[66] = """# Step 1: Confirm nothing works through kubectl at all
|
|
kubectl get nodes # connection refused / timeout -- the API server itself is down
|
|
|
|
# Step 2: Access the control-plane node directly (kubectl can't help you here)
|
|
sudo -i # the task's host is the control-plane node: no hop needed
|
|
|
|
# Step 3: Check what containerd/kubelet is trying (and failing) to run
|
|
crictl ps -a | grep apiserver
|
|
journalctl -u kubelet -n 50 --no-pager | grep -i apiserver
|
|
|
|
# Step 4: Fix the static pod manifest
|
|
vi /etc/kubernetes/manifests/kube-apiserver.yaml
|
|
# kube-apiserverr -> kube-apiserver
|
|
|
|
# Step 5: Confirm recovery from your own terminal
|
|
exit
|
|
kubectl get nodes
|
|
kubectl -n kube-system get pods -l component=kube-apiserver
|
|
"""
|