974 lines
47 KiB
Python
974 lines
47 KiB
Python
"""Simulator Set B - 17 tasks, fixed set, killer.sh-grade difficulty.
|
|
|
|
Original scenarios. Weighted toward node-level and control-plane work: etcd flags, kube-proxy
|
|
rules, ServiceCIDR, bound tokens, PDB-blocked drains, PV reclaim, upgrade planning, CSR signing.
|
|
|
|
Run with: ./exam sim b
|
|
"""
|
|
from tracks.cka.common import C, NS
|
|
|
|
Q = []
|
|
|
|
|
|
def task(**kw):
|
|
kw.setdefault("difficulty", "hard")
|
|
kw["set"] = "b"
|
|
Q.append(kw)
|
|
|
|
|
|
# ------------------------------------------------------------------ 301 ----
|
|
task(
|
|
id=301, domain="ARCH", title="etcd configuration facts",
|
|
task="""
|
|
The security team wants the following about the etcd running on the control-plane node.
|
|
Write `{WS}/q301/etcd-info.txt` with exactly these five lines (values after the colon):
|
|
|
|
```
|
|
data-dir: <etcd data directory>
|
|
server-key: <path to the server private key>
|
|
trusted-ca: <path to the CA used for client certificate verification>
|
|
client-cert-auth: <yes|no>
|
|
server-cert-expiry: <notAfter value of the etcd server certificate>
|
|
```
|
|
|
|
Take every value from how etcd is actually started, not from memory.
|
|
""",
|
|
setup=r"""rm -rf "$WS/q301"; mkdir -p "$WS/q301" """,
|
|
checks=[
|
|
C("data-dir and server-key are correct", 3, r"""
|
|
f="$WS/q301/etcd-info.txt"; [ -f "$f" ] || exit 1
|
|
cmd=$(j -n kube-system pod -l component=etcd '{.items[0].spec.containers[0].command}')
|
|
dd=$(echo "$cmd" | tr ',' '\n' | grep -o 'data-dir=[^"]*' | cut -d= -f2)
|
|
sk=$(echo "$cmd" | tr ',' '\n' | grep -o '\-\-key-file=[^"]*' | cut -d= -f2)
|
|
grep -q "data-dir: *$dd" "$f" && grep -q "server-key: *$sk" "$f" """),
|
|
C("trusted-ca and client-cert-auth are correct", 3, r"""
|
|
f="$WS/q301/etcd-info.txt"
|
|
cmd=$(j -n kube-system pod -l component=etcd '{.items[0].spec.containers[0].command}')
|
|
ca=$(echo "$cmd" | tr ',' '\n' | grep -o '\-\-trusted-ca-file=[^"]*' | cut -d= -f2)
|
|
grep -q "trusted-ca: *$ca" "$f" && grep -Eqi 'client-cert-auth: *(yes|true)' "$f" """),
|
|
C("server-cert-expiry matches the certificate on disk", 3, r"""
|
|
exp=$(nx "$CP" 'openssl x509 -in /etc/kubernetes/pki/etcd/server.crt -noout -enddate' | sed 's/notAfter=//')
|
|
yr=$(echo "$exp" | awk '{print $4}'); mon=$(echo "$exp" | awk '{print $1}'); day=$(echo "$exp" | awk '{print $2}')
|
|
f="$WS/q301/etcd-info.txt"; grep -q "$yr" "$f" 2>/dev/null && grep -qi "$mon" "$f" && grep -q "$day" "$f" """),
|
|
],
|
|
solution="""
|
|
{SHELL_CP}
|
|
cat /etc/kubernetes/manifests/etcd.yaml # every flag etcd runs with
|
|
# --data-dir=/var/lib/etcd
|
|
# --key-file=/etc/kubernetes/pki/etcd/server.key
|
|
# --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
|
|
# --client-cert-auth=true
|
|
openssl x509 -in /etc/kubernetes/pki/etcd/server.crt -noout -enddate
|
|
# etcd runs as a static pod, so the manifest in /etc/kubernetes/manifests is the source of truth.
|
|
# k -n kube-system get pod -l component=etcd -o yaml would show the same command.
|
|
""",
|
|
)
|
|
|
|
# ------------------------------------------------------------------ 302 ----
|
|
task(
|
|
id=302, domain="NET", title="Service rules on the node", breaks="proxy-rules",
|
|
task="""
|
|
Confirm that kube-proxy is programming this cluster.
|
|
|
|
1. In Namespace `project-proxy` create Pod `proxy-pod` (image `nginx:1-alpine`) and expose it
|
|
internally as Service `proxy-svc` on port **4000** targeting container port 80.
|
|
2. From the node where the Pod runs, capture the proxy rules that mention `proxy-svc` into
|
|
`{WS}/q302/rules.txt` (the node uses iptables; `iptables-save` is available as root).
|
|
3. Write the Service's ClusterIP to `{WS}/q302/clusterip.txt`.
|
|
4. Write the proxy mode kube-proxy is running in to `{WS}/q302/mode.txt`
|
|
(an empty `mode` in its ConfigMap means `iptables`).
|
|
""",
|
|
setup=NS + r"""
|
|
mkns project-proxy
|
|
rm -rf "$WS/q302"; mkdir -p "$WS/q302"
|
|
$K -n project-proxy delete pod proxy-pod --ignore-not-found --wait=false >/dev/null
|
|
$K -n project-proxy delete svc proxy-svc --ignore-not-found >/dev/null
|
|
""",
|
|
checks=[
|
|
C("Pod and Service exist, 4000 -> 80, with endpoints", 3, r"""
|
|
eq "$(j -n project-proxy pod proxy-pod '{.status.phase}')" Running \
|
|
&& eq "$(j -n project-proxy svc proxy-svc '{.spec.ports[0].port}')" 4000 && eq "$(j -n project-proxy svc proxy-svc '{.spec.ports[0].targetPort}')" 80 \
|
|
&& [ -n "$(j -n project-proxy endpointslices -l kubernetes.io/service-name=proxy-svc '{.items[*].endpoints[*].addresses[*]}')" ]"""),
|
|
C("clusterip.txt matches the Service", 2, r"""eq "$(tr -d '[:space:]' < "$WS/q302/clusterip.txt" 2>/dev/null)" "$(j -n project-proxy svc proxy-svc '{.spec.clusterIP}')" """),
|
|
C("rules.txt contains real proxy rules naming the Service and its ClusterIP", 4, r"""
|
|
f="$WS/q302/rules.txt"; grep -q 'proxy-svc' "$f" 2>/dev/null && grep -q "$(j -n project-proxy svc proxy-svc '{.spec.clusterIP}')" "$f" && grep -Eq 'KUBE-(SVC|SEP|SERVICES)' "$f" """),
|
|
C("mode.txt says iptables", 2, r"""grep -qi 'iptables' "$WS/q302/mode.txt" 2>/dev/null"""),
|
|
],
|
|
solution="""
|
|
k -n project-proxy run proxy-pod --image=nginx:1-alpine
|
|
k -n project-proxy expose pod proxy-pod --name proxy-svc --port 4000 --target-port 80
|
|
k -n project-proxy get svc proxy-svc -o jsonpath='{.spec.clusterIP}' > {WS}/q302/clusterip.txt
|
|
k -n project-proxy get pod proxy-pod -o wide # note the node
|
|
{SHELL_W1} # the node it landed on
|
|
iptables-save | grep proxy-svc # -> copy into {WS}/q302/rules.txt
|
|
exit
|
|
k -n kube-system get cm kube-proxy -o yaml | grep -E '^ *mode:' # empty -> iptables
|
|
echo iptables > {WS}/q302/mode.txt
|
|
# Deleting the Service makes the rules disappear within a second - kube-proxy watches the API
|
|
# and rewrites the rules on every Service/EndpointSlice change.
|
|
""",
|
|
)
|
|
|
|
# ------------------------------------------------------------------ 303 ----
|
|
task(
|
|
id=303, domain="NET", title="A second Service CIDR",
|
|
task="""
|
|
The cluster is running out of Service IPs.
|
|
|
|
1. Write the CIDR the cluster currently allocates Service IPs from to `{WS}/q303/current-cidr.txt`.
|
|
2. Add a **second** range `10.180.0.0/16` to the cluster by creating a `ServiceCIDR` object named
|
|
`extra-range` (do not remove the existing one).
|
|
3. In Namespace `default` create Deployment `cidr-demo` (image `nginx:1-alpine`) and a ClusterIP
|
|
Service `cidr-demo` on port 80 whose ClusterIP is taken **from the new range**
|
|
(request the address explicitly if you need to).
|
|
4. Write that Service's ClusterIP to `{WS}/q303/new-ip.txt`.
|
|
""",
|
|
setup=r"""
|
|
rm -rf "$WS/q303"; mkdir -p "$WS/q303"
|
|
$K delete svc cidr-demo --ignore-not-found >/dev/null 2>&1
|
|
$K delete deploy cidr-demo --ignore-not-found >/dev/null 2>&1
|
|
$K delete servicecidr extra-range --ignore-not-found >/dev/null 2>&1
|
|
$K get servicecidr >/dev/null 2>&1 || echo "warn: this cluster has no ServiceCIDR API (needs MultiCIDRServiceAllocator)"
|
|
""",
|
|
checks=[
|
|
C("current-cidr.txt matches the apiserver's service range", 2, r"""
|
|
sc=$(j -n kube-system pod -l component=kube-apiserver '{.items[0].spec.containers[0].command}' | tr ',' '\n' | grep -o 'service-cluster-ip-range=[^"]*' | cut -d= -f2)
|
|
grep -qF "$sc" "$WS/q303/current-cidr.txt" 2>/dev/null"""),
|
|
C("ServiceCIDR extra-range covers 10.180.0.0/16 and is Ready", 4, r"""
|
|
j servicecidr extra-range '{.spec.cidrs[*]}' | grep -q '10.180.0.0/16' || exit 1
|
|
for i in $(seq 1 10); do [ "$(j servicecidr extra-range '{.status.conditions[?(@.type=="Ready")].status}')" = True ] && exit 0; sleep 2; done; exit 1"""),
|
|
C("Service cidr-demo has an IP from the new range and ready endpoints", 4, r"""
|
|
ip=$(j -n default svc cidr-demo '{.spec.clusterIP}'); case "$ip" in 10.180.*) ;; *) exit 1;; esac
|
|
[ -n "$(j -n default endpointslices -l kubernetes.io/service-name=cidr-demo '{.items[*].endpoints[?(@.conditions.ready==true)].addresses[*]}')" ]"""),
|
|
C("new-ip.txt records that ClusterIP", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q303/new-ip.txt" 2>/dev/null)" "$(j -n default svc cidr-demo '{.spec.clusterIP}')" """),
|
|
],
|
|
solution="""
|
|
k -n kube-system get pod -l component=kube-apiserver -o yaml | grep service-cluster-ip-range
|
|
# -> e.g. 10.96.0.0/16 ; write it into {WS}/q303/current-cidr.txt
|
|
k get servicecidr # the built-in "kubernetes" ServiceCIDR
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: ServiceCIDR
|
|
metadata: {name: extra-range}
|
|
spec: {cidrs: ["10.180.0.0/16"]}
|
|
EOF
|
|
k get servicecidr extra-range -o yaml # wait for Ready=True
|
|
k create deploy cidr-demo --image=nginx:1-alpine
|
|
k expose deploy cidr-demo --port=80 --cluster-ip=10.180.0.50
|
|
k get svc cidr-demo -o jsonpath='{.spec.clusterIP}' > {WS}/q303/new-ip.txt
|
|
# New ranges are additive: existing Services keep their IPs, and allocation from a new range
|
|
# happens once the ServiceCIDR is Ready. Requesting --cluster-ip makes it deterministic.
|
|
""",
|
|
)
|
|
|
|
# ------------------------------------------------------------------ 304 ----
|
|
task(
|
|
id=304, domain="ARCH", title="Bound tokens versus legacy tokens",
|
|
task="""
|
|
In Namespace `project-token` there is ServiceAccount `reporter`.
|
|
|
|
1. Issue a **bound token** for it that is valid for exactly 2 hours and save it to
|
|
`{WS}/q304/token.jwt`.
|
|
2. Decode the token's payload and write its `exp` and `iat` claims, as
|
|
`exp=<value>` and `iat=<value>` on two lines, to `{WS}/q304/claims.txt`.
|
|
3. Also create a **long-lived** token for the same ServiceAccount, as a Secret named
|
|
`reporter-legacy` of the correct type, and confirm the controller populated it.
|
|
""",
|
|
setup=NS + r"""
|
|
mkns project-token
|
|
rm -rf "$WS/q304"; mkdir -p "$WS/q304"
|
|
$K -n project-token create sa reporter --dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
$K -n project-token delete secret reporter-legacy --ignore-not-found >/dev/null
|
|
""",
|
|
checks=[
|
|
C("token.jwt is a JWT issued for the reporter ServiceAccount", 3, r"""
|
|
f="$WS/q304/token.jwt"; [ -s "$f" ] || exit 1
|
|
python3 - "$f" <<'PY'
|
|
import base64,json,sys
|
|
t=open(sys.argv[1]).read().strip().split(".")
|
|
if len(t)!=3: sys.exit(1)
|
|
p=json.loads(base64.urlsafe_b64decode(t[1]+"=="*3))
|
|
sub=p.get("sub",""); ko=p.get("kubernetes.io",{}).get("serviceaccount",{}).get("name","")
|
|
sys.exit(0 if "reporter" in sub or ko=="reporter" else 1)
|
|
PY"""),
|
|
C("the token's lifetime is 2 hours", 3, r"""
|
|
python3 - "$WS/q304/token.jwt" <<'PY'
|
|
import base64,json,sys
|
|
p=json.loads(base64.urlsafe_b64decode(open(sys.argv[1]).read().strip().split(".")[1]+"=="*3))
|
|
d=p["exp"]-p["iat"]
|
|
sys.exit(0 if 7000 <= d <= 7400 else 1)
|
|
PY"""),
|
|
C("claims.txt records exp and iat matching the token", 2, r"""
|
|
python3 - "$WS/q304/token.jwt" "$WS/q304/claims.txt" <<'PY'
|
|
import base64,json,sys,re
|
|
p=json.loads(base64.urlsafe_b64decode(open(sys.argv[1]).read().strip().split(".")[1]+"=="*3))
|
|
try: t=open(sys.argv[2]).read()
|
|
except OSError: sys.exit(1)
|
|
ok=re.search(r"exp\s*=\s*%d"%p["exp"],t) and re.search(r"iat\s*=\s*%d"%p["iat"],t)
|
|
sys.exit(0 if ok else 1)
|
|
PY"""),
|
|
C("Secret reporter-legacy is a populated service-account-token for reporter", 3, r"""
|
|
eq "$(j -n project-token secret reporter-legacy '{.type}')" kubernetes.io/service-account-token \
|
|
&& eq "$(j -n project-token secret reporter-legacy '{.metadata.annotations.kubernetes\.io/service-account\.name}')" reporter \
|
|
&& [ -n "$(j -n project-token secret reporter-legacy '{.data.token}')" ]"""),
|
|
],
|
|
solution="""
|
|
k -n project-token create token reporter --duration=2h > {WS}/q304/token.jwt
|
|
# decode the middle segment (JWTs are base64url, pad it out):
|
|
cut -d. -f2 {WS}/q304/token.jwt | base64 -d 2>/dev/null | python3 -m json.tool
|
|
# -> copy exp= and iat= into {WS}/q304/claims.txt
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: reporter-legacy
|
|
namespace: project-token
|
|
annotations: {kubernetes.io/service-account.name: reporter}
|
|
type: kubernetes.io/service-account-token
|
|
EOF
|
|
k -n project-token get secret reporter-legacy -o yaml # the controller fills in .data.token
|
|
# Bound tokens expire and are tied to an audience; the Secret form never expires - avoid it
|
|
# unless something really needs a static token.
|
|
""",
|
|
)
|
|
|
|
# ------------------------------------------------------------------ 305 ----
|
|
task(
|
|
id=305, domain="WORK", title="Schedule without the scheduler",
|
|
task="""
|
|
In Namespace `project-orion`:
|
|
|
|
1. Create Pod `direct-assign` (image `busybox:1.36`, command `sleep 86400`) that runs on node
|
|
`{W3}` **without the scheduler being involved at all**.
|
|
2. Create Pod `scheduled` (same image and command) that lands on `{W3}` **through** the scheduler.
|
|
3. Write to `{WS}/q305/proof.txt` the value of the field you used for Pod 1, in the form
|
|
`field=<name>`, and on a second line `scheduler=<the schedulerName of the second Pod>`.
|
|
""",
|
|
setup=NS + r"""
|
|
mkns project-orion
|
|
rm -rf "$WS/q305"; mkdir -p "$WS/q305"
|
|
$K -n project-orion delete pod direct-assign scheduled --ignore-not-found --wait=false >/dev/null
|
|
$K label node "$W3" tier- >/dev/null 2>&1 || true
|
|
""",
|
|
checks=[
|
|
C("direct-assign Running on the right node with spec.nodeName set", 3, r"""
|
|
eq "$(j -n project-orion pod direct-assign '{.status.phase}')" Running && eq "$(j -n project-orion pod direct-assign '{.spec.nodeName}')" "$W3" """),
|
|
C("direct-assign was never scheduled (no Scheduled event from default-scheduler)", 3, r"""
|
|
! $K -n project-orion get events --field-selector involvedObject.name=direct-assign 2>/dev/null | grep -q 'default-scheduler'"""),
|
|
C("scheduled Running on the same node via the default scheduler", 3, r"""
|
|
eq "$(j -n project-orion pod scheduled '{.status.phase}')" Running && eq "$(j -n project-orion pod scheduled '{.spec.nodeName}')" "$W3" \
|
|
&& $K -n project-orion get events --field-selector involvedObject.name=scheduled 2>/dev/null | grep -q 'default-scheduler'"""),
|
|
C("proof.txt names nodeName and default-scheduler", 2, r"""
|
|
grep -q 'field= *nodeName' "$WS/q305/proof.txt" 2>/dev/null && grep -q 'scheduler= *default-scheduler' "$WS/q305/proof.txt" """),
|
|
],
|
|
solution="""
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: direct-assign, namespace: project-orion}
|
|
spec:
|
|
nodeName: {W3} # bypasses the scheduler entirely - the kubelet just picks it up
|
|
containers: [{name: c, image: busybox:1.36, command: [sleep, "86400"]}]
|
|
EOF
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: scheduled, namespace: project-orion}
|
|
spec:
|
|
nodeSelector: {kubernetes.io/hostname: {W3}} # the scheduler still makes the decision
|
|
containers: [{name: c, image: busybox:1.36, command: [sleep, "86400"]}]
|
|
EOF
|
|
k -n project-orion get events --field-selector involvedObject.name=direct-assign # no Scheduled event
|
|
k -n project-orion get events --field-selector involvedObject.name=scheduled # Scheduled by default-scheduler
|
|
printf 'field=nodeName\\nscheduler=default-scheduler\\n' > {WS}/q305/proof.txt
|
|
""",
|
|
)
|
|
|
|
# ------------------------------------------------------------------ 306 ----
|
|
task(
|
|
id=306, domain="STOR", title="Grow a volume in place",
|
|
task="""
|
|
In Namespace `project-orbit`:
|
|
|
|
1. Create StorageClass `expandable` using provisioner `rancher.io/local-path`,
|
|
`volumeBindingMode: WaitForFirstConsumer`, and allowing volume expansion.
|
|
2. Create PVC `growing` (1Gi, RWO) using that class and Pod `grower` (busybox:1.36, `sleep 86400`)
|
|
mounting it at `/data`, so the claim binds.
|
|
3. Then grow the claim to **2Gi** without deleting it, and write the claim's
|
|
`.status.capacity.storage` after the resize to `{WS}/q306/capacity.txt`.
|
|
""",
|
|
setup=NS + r"""
|
|
mkns project-orbit
|
|
rm -rf "$WS/q306"; mkdir -p "$WS/q306"
|
|
$K -n project-orbit delete pod grower --ignore-not-found --wait=false >/dev/null
|
|
$K -n project-orbit delete pvc growing --ignore-not-found >/dev/null
|
|
$K delete sc expandable --ignore-not-found >/dev/null
|
|
""",
|
|
checks=[
|
|
C("StorageClass expandable: local-path, WaitForFirstConsumer, expansion allowed", 3, r"""
|
|
s="$K get sc expandable"
|
|
eq "$($s -o jsonpath='{.provisioner}' 2>/dev/null)" rancher.io/local-path && eq "$($s -o jsonpath='{.volumeBindingMode}')" WaitForFirstConsumer \
|
|
&& eq "$($s -o jsonpath='{.allowVolumeExpansion}')" true"""),
|
|
C("PVC growing Bound on that class with grower Running", 3, r"""
|
|
eq "$(j -n project-orbit pvc growing '{.spec.storageClassName}')" expandable && eq "$(j -n project-orbit pvc growing '{.status.phase}')" Bound \
|
|
&& eq "$(j -n project-orbit pod grower '{.status.phase}')" Running"""),
|
|
C("claim requests 2Gi and reports at least 2Gi capacity", 4, r"""
|
|
eq "$(j -n project-orbit pvc growing '{.spec.resources.requests.storage}')" 2Gi || exit 1
|
|
for i in $(seq 1 12); do c=$(j -n project-orbit pvc growing '{.status.capacity.storage}'); [ "$c" = 2Gi ] && exit 0; sleep 5; done; exit 1"""),
|
|
C("capacity.txt records the new capacity", 1, r"""grep -q '2Gi' "$WS/q306/capacity.txt" 2>/dev/null"""),
|
|
],
|
|
solution="""
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: storage.k8s.io/v1
|
|
kind: StorageClass
|
|
metadata: {name: expandable}
|
|
provisioner: rancher.io/local-path
|
|
volumeBindingMode: WaitForFirstConsumer
|
|
allowVolumeExpansion: true
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: growing, namespace: project-orbit}
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: expandable
|
|
resources: {requests: {storage: 1Gi}}
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: grower, namespace: project-orbit}
|
|
spec:
|
|
volumes: [{name: d, persistentVolumeClaim: {claimName: growing}}]
|
|
containers: [{name: c, image: busybox:1.36, command: [sleep, "86400"], volumeMounts: [{name: d, mountPath: /data}]}]
|
|
EOF
|
|
k -n project-orbit patch pvc growing -p '{"spec":{"resources":{"requests":{"storage":"2Gi"}}}}'
|
|
k -n project-orbit get pvc growing -w # spec 2Gi first, then status.capacity follows
|
|
k -n project-orbit get pvc growing -o jsonpath='{.status.capacity.storage}' > {WS}/q306/capacity.txt
|
|
# Shrinking is never allowed. Without allowVolumeExpansion the patch is rejected outright.
|
|
""",
|
|
)
|
|
|
|
# ------------------------------------------------------------------ 307 ----
|
|
task(
|
|
id=307, domain="TSHOOT", title="Node accepts nothing new", breaks="w1-block",
|
|
task="""
|
|
Node `{W1}` is `Ready` and its kubelet is healthy, but no new Pod will schedule onto it.
|
|
|
|
Find every reason and remove them, so that a Pod with a `nodeSelector` pinning it to `{W1}`
|
|
starts successfully. Do not delete or recreate the node, and do not use `nodeName` to work around it.
|
|
""",
|
|
setup=r"""
|
|
$K cordon "$W1" >/dev/null 2>&1 || true
|
|
$K taint node "$W1" maintenance=inprogress:NoSchedule --overwrite >/dev/null 2>&1 || true
|
|
$K label node "$W1" kubernetes.io/os- >/dev/null 2>&1 || true
|
|
$K label node "$W1" kubernetes.io/os=linux --overwrite >/dev/null 2>&1 || true
|
|
""",
|
|
checks=[
|
|
C("node is schedulable (uncordoned)", 3, r"""[ "$(j node "$W1" '{.spec.unschedulable}')" != true ]"""),
|
|
C("the maintenance taint is gone", 3, r"""! j node "$W1" '{.spec.taints}' | grep -q 'maintenance'"""),
|
|
C("a pinned Pod actually runs there", 4, r"""
|
|
$K -n default delete pod pin-probe --ignore-not-found --wait=true >/dev/null 2>&1
|
|
cat <<EOF | $K apply -f - >/dev/null 2>&1
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: pin-probe, namespace: default}
|
|
spec:
|
|
nodeSelector: {kubernetes.io/hostname: $W1}
|
|
containers: [{name: c, image: busybox:1.36, command: [sleep, "120"]}]
|
|
EOF
|
|
ok=1; for i in $(seq 1 20); do [ "$(j -n default pod pin-probe '{.status.phase}')" = Running ] && { ok=0; break; }; sleep 3; done
|
|
$K -n default delete pod pin-probe --wait=false >/dev/null 2>&1; exit $ok"""),
|
|
],
|
|
solution="""
|
|
k describe node {W1} | head -20
|
|
# Taints: maintenance=inprogress:NoSchedule
|
|
# Unschedulable: true
|
|
k uncordon {W1}
|
|
k taint node {W1} maintenance- # trailing dash removes the taint
|
|
k get node {W1} -o jsonpath='{.spec.taints}{"\\n"}{.spec.unschedulable}{"\\n"}'
|
|
# Cordon and taints are two independent mechanisms - `kubectl drain` sets the cordon,
|
|
# and a taint may have been left by an operator or added by hand. Both must go.
|
|
""",
|
|
)
|
|
|
|
# ------------------------------------------------------------------ 308 ----
|
|
task(
|
|
id=308, domain="ARCH", title="API resource reference card",
|
|
task="""
|
|
Write `{WS}/q308/resources.txt` with one line per resource below, in this exact format:
|
|
|
|
```
|
|
<resource>|<shortname or ->|<apiVersion as served>|<namespaced true|false>
|
|
```
|
|
|
|
for these four: `networkpolicies`, `persistentvolumes`, `storageclasses`, `certificatesigningrequests`.
|
|
|
|
Take the values from the cluster (`kubectl api-resources`), not from memory. Use `-` when a
|
|
resource has no shortname. Lines in the order given above.
|
|
""",
|
|
setup=r"""rm -rf "$WS/q308"; mkdir -p "$WS/q308" """,
|
|
checks=[
|
|
C("all four lines present and correctly formatted", 3, r"""
|
|
f="$WS/q308/resources.txt"; [ -f "$f" ] || exit 1
|
|
for r in networkpolicies persistentvolumes storageclasses certificatesigningrequests; do grep -q "^$r|" "$f" || exit 1; done
|
|
[ "$(grep -c '|' "$f")" -ge 4 ]"""),
|
|
C("shortnames and namespaced flags correct", 4, r"""
|
|
f="$WS/q308/resources.txt"
|
|
grep -Eq '^networkpolicies\|netpol\|networking.k8s.io/v1\|true$' "$f" || exit 1
|
|
grep -Eq '^persistentvolumes\|pv\|v1\|false$' "$f" || exit 1
|
|
grep -Eq '^storageclasses\|sc\|storage.k8s.io/v1\|false$' "$f" || exit 1
|
|
grep -Eq '^certificatesigningrequests\|csr\|certificates.k8s.io/v1\|false$' "$f" || exit 1"""),
|
|
C("order matches the task", 2, r"""
|
|
f="$WS/q308/resources.txt"
|
|
[ "$(grep -o '^[a-z]*' "$f" | grep . | head -4 | tr '\n' ' ')" = "networkpolicies persistentvolumes storageclasses certificatesigningrequests " ]"""),
|
|
],
|
|
solution="""
|
|
k api-resources | grep -E '^(networkpolicies|persistentvolumes|storageclasses|certificatesigningrequests) '
|
|
# NAME SHORTNAMES APIVERSION NAMESPACED KIND
|
|
cat > {WS}/q308/resources.txt <<'EOF'
|
|
networkpolicies|netpol|networking.k8s.io/v1|true
|
|
persistentvolumes|pv|v1|false
|
|
storageclasses|sc|storage.k8s.io/v1|false
|
|
certificatesigningrequests|csr|certificates.k8s.io/v1|false
|
|
EOF
|
|
# k api-resources --namespaced=false narrows to cluster-scoped resources;
|
|
# k explain <resource> shows the same apiVersion when you're unsure.
|
|
""",
|
|
)
|
|
|
|
# ------------------------------------------------------------------ 309 ----
|
|
task(
|
|
id=309, domain="WORK", title="A PodDisruptionBudget that blocks maintenance", breaks="w3-drain",
|
|
task="""
|
|
Namespace `project-atlas` runs Deployment `atlas-api` with 2 replicas, and a PodDisruptionBudget
|
|
`atlas-api` that currently makes voluntary eviction impossible.
|
|
|
|
Node `{W3}` must be drained for maintenance, and `atlas-api` must keep at least one Pod serving
|
|
at all times.
|
|
|
|
1. Adjust the PodDisruptionBudget so the drain can complete while still guaranteeing one
|
|
available Pod. Keep the PDB — do not delete it.
|
|
2. Drain `{W3}` (DaemonSet Pods aside), then make it schedulable again.
|
|
3. Write the value of the PDB's `.status.disruptionsAllowed` after your change to
|
|
`{WS}/q309/allowed.txt`.
|
|
""",
|
|
setup=NS + r"""
|
|
mkns project-atlas
|
|
rm -rf "$WS/q309"; mkdir -p "$WS/q309"
|
|
$K uncordon "$W3" >/dev/null 2>&1 || true
|
|
$K -n project-atlas delete deploy atlas-api --ignore-not-found >/dev/null
|
|
$K -n project-atlas delete pdb atlas-api --ignore-not-found >/dev/null
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: atlas-api, namespace: project-atlas}
|
|
spec:
|
|
replicas: 2
|
|
selector: {matchLabels: {app: atlas-api}}
|
|
template:
|
|
metadata: {labels: {app: atlas-api}}
|
|
spec:
|
|
containers: [{name: api, image: nginx:1.27, resources: {requests: {cpu: 10m, memory: 16Mi}}}]
|
|
---
|
|
apiVersion: policy/v1
|
|
kind: PodDisruptionBudget
|
|
metadata: {name: atlas-api, namespace: project-atlas}
|
|
spec:
|
|
minAvailable: 2
|
|
selector: {matchLabels: {app: atlas-api}}
|
|
EOF
|
|
$K -n project-atlas rollout status deploy atlas-api --timeout=120s >/dev/null 2>&1
|
|
""",
|
|
checks=[
|
|
C("PDB still exists and now permits exactly one disruption", 3, r"""
|
|
p="$K -n project-atlas get pdb atlas-api"
|
|
$p >/dev/null 2>&1 || exit 1
|
|
ma=$($p -o jsonpath='{.spec.minAvailable}'); mu=$($p -o jsonpath='{.spec.maxUnavailable}')
|
|
{ [ "$ma" = 1 ] || [ "$mu" = 1 ]; }"""),
|
|
C("no atlas-api Pod left on the drained node", 3, r"""
|
|
n=$($K -n project-atlas get pod -l app=atlas-api -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' 2>/dev/null | grep -c "^$W3$")
|
|
eq "$n" 0"""),
|
|
C("both replicas Available and node schedulable again", 3, r"""
|
|
eq "$(j -n project-atlas deploy atlas-api '{.status.availableReplicas}')" 2 && [ "$(j node "$W3" '{.spec.unschedulable}')" != true ]"""),
|
|
C("allowed.txt records disruptionsAllowed", 1, r"""grep -Eq '^[0-9]+$' <(tr -dc '0-9\n' < "$WS/q309/allowed.txt" 2>/dev/null | grep .) """),
|
|
],
|
|
solution="""
|
|
k -n project-atlas get pdb atlas-api
|
|
# ALLOWED DISRUPTIONS: 0 -> minAvailable 2 with 2 replicas means nothing may be evicted;
|
|
# a drain would hang forever waiting for the eviction API.
|
|
k -n project-atlas patch pdb atlas-api -p '{"spec":{"minAvailable":1}}'
|
|
k -n project-atlas get pdb atlas-api # ALLOWED DISRUPTIONS: 1
|
|
k drain {W3} --ignore-daemonsets --delete-emptydir-data
|
|
k -n project-atlas rollout status deploy atlas-api
|
|
k uncordon {W3}
|
|
k -n project-atlas get pdb atlas-api -o jsonpath='{.status.disruptionsAllowed}' > {WS}/q309/allowed.txt
|
|
""",
|
|
)
|
|
|
|
# ------------------------------------------------------------------ 310 ----
|
|
task(
|
|
id=310, domain="NET", title="Route to something outside the cluster",
|
|
task="""
|
|
An external database lives at `10.190.7.25` port 5432. Applications should reach it through a
|
|
normal in-cluster Service name.
|
|
|
|
In Namespace `project-orbit`:
|
|
1. Create a Service `external-db` (ClusterIP, port 5432) that has **no selector**.
|
|
2. Make it resolve to that external address by creating the matching `EndpointSlice`
|
|
named `external-db-1` (port 5432, TCP, addressType IPv4) with the right labels so the
|
|
Service adopts it.
|
|
3. Write the ClusterIP of the Service to `{WS}/q310/ip.txt`.
|
|
|
|
Do not use an ExternalName Service.
|
|
""",
|
|
setup=NS + r"""
|
|
mkns project-orbit
|
|
rm -rf "$WS/q310"; mkdir -p "$WS/q310"
|
|
$K -n project-orbit delete svc external-db --ignore-not-found >/dev/null
|
|
$K -n project-orbit delete endpointslice external-db-1 --ignore-not-found >/dev/null 2>&1
|
|
""",
|
|
checks=[
|
|
C("Service external-db is ClusterIP on 5432 with no selector", 3, r"""
|
|
s="$K -n project-orbit get svc external-db"
|
|
eq "$($s -o jsonpath='{.spec.type}' 2>/dev/null)" ClusterIP && eq "$($s -o jsonpath='{.spec.ports[0].port}')" 5432 && [ -z "$($s -o jsonpath='{.spec.selector}')" ]"""),
|
|
C("EndpointSlice external-db-1 points at 10.190.7.25:5432", 4, r"""
|
|
e="$K -n project-orbit get endpointslice external-db-1"
|
|
eq "$($e -o jsonpath='{.addressType}' 2>/dev/null)" IPv4 && $e -o jsonpath='{.endpoints[*].addresses[*]}' | grep -q '10.190.7.25' \
|
|
&& eq "$($e -o jsonpath='{.ports[0].port}')" 5432"""),
|
|
C("the slice is linked to the Service by label", 2, r"""eq "$(j -n project-orbit endpointslice external-db-1 '{.metadata.labels.kubernetes\.io/service-name}')" external-db"""),
|
|
C("ip.txt records the Service ClusterIP", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q310/ip.txt" 2>/dev/null)" "$(j -n project-orbit svc external-db '{.spec.clusterIP}')" """),
|
|
],
|
|
solution="""
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata: {name: external-db, namespace: project-orbit}
|
|
spec:
|
|
ports: [{name: pg, port: 5432, protocol: TCP, targetPort: 5432}]
|
|
# no selector: nothing populates the endpoints automatically, so we do it ourselves
|
|
---
|
|
apiVersion: discovery.k8s.io/v1
|
|
kind: EndpointSlice
|
|
metadata:
|
|
name: external-db-1
|
|
namespace: project-orbit
|
|
labels: {kubernetes.io/service-name: external-db} # this label is the link to the Service
|
|
addressType: IPv4
|
|
ports: [{name: pg, port: 5432, protocol: TCP}]
|
|
endpoints:
|
|
- addresses: ["10.190.7.25"]
|
|
conditions: {ready: true}
|
|
EOF
|
|
k -n project-orbit get svc external-db -o jsonpath='{.spec.clusterIP}' > {WS}/q310/ip.txt
|
|
k -n project-orbit get endpointslice
|
|
# ExternalName would return a CNAME instead and gives you no ClusterIP or port mapping.
|
|
""",
|
|
)
|
|
|
|
# ------------------------------------------------------------------ 311 ----
|
|
task(
|
|
id=311, domain="ARCH", title="Image inventory across the cluster",
|
|
task="""
|
|
Produce `{WS}/q311/images.txt`: for every Deployment in every Namespace, one line
|
|
|
|
```
|
|
<namespace>/<deployment> <image>[,<image>...]
|
|
```
|
|
|
|
with the images of its Pod template (init containers included), comma-separated in the order they
|
|
appear, and the file sorted by the first column.
|
|
|
|
Then write the number of **distinct** images in use across those Deployments to
|
|
`{WS}/q311/distinct.txt`.
|
|
""",
|
|
setup=r"""rm -rf "$WS/q311"; mkdir -p "$WS/q311" """,
|
|
checks=[
|
|
C("images.txt matches the cluster exactly", 5, r"""
|
|
want=$($K get deploy -A -o json 2>/dev/null | python3 -c '
|
|
import sys,json
|
|
out=[]
|
|
for d in json.load(sys.stdin)["items"]:
|
|
s=d["spec"]["template"]["spec"]
|
|
imgs=[c["image"] for c in s.get("initContainers",[])]+[c["image"] for c in s["containers"]]
|
|
out.append("%s/%s %s"%(d["metadata"]["namespace"],d["metadata"]["name"],",".join(imgs)))
|
|
print("\n".join(sorted(out)))')
|
|
[ "$(grep . "$WS/q311/images.txt" 2>/dev/null | sort)" = "$want" ]"""),
|
|
C("distinct.txt has the right count", 3, r"""
|
|
want=$($K get deploy -A -o json 2>/dev/null | python3 -c '
|
|
import sys,json
|
|
s=set()
|
|
for d in json.load(sys.stdin)["items"]:
|
|
sp=d["spec"]["template"]["spec"]
|
|
for c in sp.get("initContainers",[])+sp["containers"]: s.add(c["image"])
|
|
print(len(s))')
|
|
eq "$(tr -dc '0-9' < "$WS/q311/distinct.txt" 2>/dev/null)" "$want" """),
|
|
],
|
|
solution="""
|
|
k get deploy -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{" "}{range .spec.template.spec.initContainers[*]}{.image}{","}{end}{range .spec.template.spec.containers[*]}{.image}{","}{end}{"\\n"}{end}' \\
|
|
| sed 's/,$//' | sort > {WS}/q311/images.txt
|
|
k get deploy -A -o jsonpath='{range .items[*]}{range .spec.template.spec.initContainers[*]}{.image}{"\\n"}{end}{range .spec.template.spec.containers[*]}{.image}{"\\n"}{end}{end}' \\
|
|
| grep . | sort -u | wc -l > {WS}/q311/distinct.txt
|
|
# Two `range` blocks over the same pod spec keeps init and normal containers in order.
|
|
# A trailing comma is easiest to strip afterwards with sed.
|
|
""",
|
|
)
|
|
|
|
# ------------------------------------------------------------------ 312 ----
|
|
task(
|
|
id=312, domain="WORK", title="A Job that fails, then one that doesn't",
|
|
task="""
|
|
In Namespace `project-orion`:
|
|
|
|
1. Job `flaky` exists and keeps failing. Let it reach its terminal state, then write the name of
|
|
the condition that marks it failed and its reason, as `<Type>/<Reason>`, to `{WS}/q312/why.txt`.
|
|
2. Create a **new** Job `steady` that does the same work correctly: image `busybox:1.36`,
|
|
6 completions, 3 in parallel, `backoffLimit: 2`, restartPolicy Never, each Pod printing
|
|
`ok` and exiting 0.
|
|
3. `steady` must complete all 6.
|
|
""",
|
|
setup=NS + r"""
|
|
mkns project-orion
|
|
rm -rf "$WS/q312"; mkdir -p "$WS/q312"
|
|
$K -n project-orion delete job flaky steady --ignore-not-found >/dev/null
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata: {name: flaky, namespace: project-orion}
|
|
spec:
|
|
completions: 2
|
|
parallelism: 1
|
|
backoffLimit: 1
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
containers: [{name: c, image: busybox:1.36, command: [sh, -c, "echo starting; exit 7"]}]
|
|
EOF
|
|
""",
|
|
checks=[
|
|
C("why.txt names the Failed condition and BackoffLimitExceeded", 3, r"""
|
|
f="$WS/q312/why.txt"; grep -qi 'Failed' "$f" 2>/dev/null && grep -qi 'BackoffLimitExceeded' "$f" """),
|
|
C("Job steady: 6 completions, parallelism 3, backoffLimit 2, restartPolicy Never", 3, r"""
|
|
j1="$K -n project-orion get job steady"
|
|
eq "$($j1 -o jsonpath='{.spec.completions}' 2>/dev/null)" 6 && eq "$($j1 -o jsonpath='{.spec.parallelism}')" 3 \
|
|
&& eq "$($j1 -o jsonpath='{.spec.backoffLimit}')" 2 && eq "$($j1 -o jsonpath='{.spec.template.spec.restartPolicy}')" Never"""),
|
|
C("steady completed all 6", 4, r"""for i in $(seq 1 20); do [ "$(j -n project-orion job steady '{.status.succeeded}')" = 6 ] && exit 0; sleep 3; done; exit 1"""),
|
|
],
|
|
solution="""
|
|
k -n project-orion get job flaky -o yaml | grep -A6 conditions
|
|
# type: Failed / reason: BackoffLimitExceeded
|
|
k -n project-orion describe job flaky
|
|
k -n project-orion logs job/flaky # exit 7
|
|
echo Failed/BackoffLimitExceeded > {WS}/q312/why.txt
|
|
k -n project-orion create job steady --image=busybox:1.36 $do -- sh -c "echo ok" > steady.yaml
|
|
# set spec.completions: 6, spec.parallelism: 3, spec.backoffLimit: 2
|
|
k apply -f steady.yaml
|
|
k -n project-orion wait --for=condition=complete job/steady --timeout=120s
|
|
""",
|
|
)
|
|
|
|
# ------------------------------------------------------------------ 313 ----
|
|
task(
|
|
id=313, domain="NET", title="Two hosts, TLS, and a default backend",
|
|
task="""
|
|
Namespace `project-r700` has Services `web-classic` and `web-next` (port 80).
|
|
|
|
Create Ingress `multi` in that Namespace with class `nginx`:
|
|
|
|
- host `classic.r700.io`, path `/` (Prefix) → `web-classic:80`, TLS with secret `classic-tls`
|
|
- host `next.r700.io`, path `/` (Prefix) → `web-next:80`, TLS with the same secret
|
|
- a **default backend** (requests matching no rule) → `web-classic:80`
|
|
|
|
Also create the TLS secret `classic-tls` from a self-signed certificate whose CN is `classic.r700.io`.
|
|
""",
|
|
setup=NS + r"""
|
|
mkns project-r700
|
|
for s in web-classic web-next; do
|
|
$K -n project-r700 create deploy $s --image=nginx:1.27 --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
$K -n project-r700 expose deploy $s --port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null
|
|
done
|
|
$K -n project-r700 delete ingress multi --ignore-not-found >/dev/null
|
|
$K -n project-r700 delete secret classic-tls --ignore-not-found >/dev/null
|
|
""",
|
|
checks=[
|
|
C("Secret classic-tls is a TLS secret with CN classic.r700.io", 3, r"""
|
|
eq "$(j -n project-r700 secret classic-tls '{.type}')" kubernetes.io/tls \
|
|
&& j -n project-r700 secret classic-tls '{.data.tls\.crt}' | base64 -d | openssl x509 -noout -subject 2>/dev/null | grep -q 'CN *= *classic.r700.io'"""),
|
|
C("both host rules route to the right Services on /", 3, r"""
|
|
$K -n project-r700 get ingress multi -o json 2>/dev/null | python3 -c '
|
|
import sys,json
|
|
rs=json.load(sys.stdin)["spec"]["rules"]
|
|
m={r["host"]:r["http"]["paths"][0] for r in rs if r.get("host")}
|
|
ok=all(h in m and m[h]["pathType"]=="Prefix" and m[h]["path"]=="/" for h in ("classic.r700.io","next.r700.io"))
|
|
ok=ok and m["classic.r700.io"]["backend"]["service"]["name"]=="web-classic" and m["next.r700.io"]["backend"]["service"]["name"]=="web-next"
|
|
sys.exit(0 if ok else 1)'"""),
|
|
C("TLS covers both hosts with classic-tls", 2, r"""
|
|
$K -n project-r700 get ingress multi -o json 2>/dev/null | python3 -c '
|
|
import sys,json
|
|
t=json.load(sys.stdin)["spec"].get("tls",[])
|
|
hosts={h for e in t for h in e.get("hosts",[])}; secs={e.get("secretName") for e in t}
|
|
sys.exit(0 if {"classic.r700.io","next.r700.io"}<=hosts and secs=={"classic-tls"} else 1)'"""),
|
|
C("defaultBackend points at web-classic:80, class is nginx", 3, r"""
|
|
i="$K -n project-r700 get ingress multi"
|
|
eq "$($i -o jsonpath='{.spec.ingressClassName}' 2>/dev/null)" nginx \
|
|
&& eq "$($i -o jsonpath='{.spec.defaultBackend.service.name}')" web-classic && eq "$($i -o jsonpath='{.spec.defaultBackend.service.port.number}')" 80"""),
|
|
],
|
|
solution="""
|
|
openssl req -x509 -newkey rsa:2048 -nodes -days 365 -subj "/CN=classic.r700.io" -keyout t.key -out t.crt
|
|
k -n project-r700 create secret tls classic-tls --cert=t.crt --key=t.key
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata: {name: multi, namespace: project-r700}
|
|
spec:
|
|
ingressClassName: nginx
|
|
defaultBackend: {service: {name: web-classic, port: {number: 80}}}
|
|
tls:
|
|
- hosts: [classic.r700.io, next.r700.io]
|
|
secretName: classic-tls
|
|
rules:
|
|
- host: classic.r700.io
|
|
http: {paths: [{path: /, pathType: Prefix, backend: {service: {name: web-classic, port: {number: 80}}}}]}
|
|
- host: next.r700.io
|
|
http: {paths: [{path: /, pathType: Prefix, backend: {service: {name: web-next, port: {number: 80}}}}]}
|
|
EOF
|
|
# `kubectl create ingress` can do most of this:
|
|
# k -n project-r700 create ingress multi --class=nginx \\
|
|
# --rule="classic.r700.io/*=web-classic:80,tls=classic-tls" \\
|
|
# --rule="next.r700.io/*=web-next:80,tls=classic-tls" \\
|
|
# --default-backend=web-classic:80
|
|
""",
|
|
)
|
|
|
|
# ------------------------------------------------------------------ 314 ----
|
|
task(
|
|
id=314, domain="TSHOOT", title="A static pod that never starts", breaks="w3-kubelet",
|
|
task="""
|
|
Someone added a static Pod manifest on node `{W3}` for a Pod called `edge-cache`, but no such Pod
|
|
ever appears in the cluster and the node is otherwise healthy.
|
|
|
|
Find out why, fix it on the node, and get the mirror Pod `edge-cache-{W3}` Running in Namespace
|
|
`default` with image `nginx:1.27`.
|
|
|
|
Write the one-line reason it wasn't starting to `{WS}/q314/reason.txt`.
|
|
""",
|
|
setup=r"""
|
|
rm -rf "$WS/q314"; mkdir -p "$WS/q314"
|
|
nx "$W3" 'rm -f /etc/kubernetes/manifests/edge-cache.yaml
|
|
d=$(grep staticPodPath /var/lib/kubelet/config.yaml | awk "{print \$2}")
|
|
mkdir -p "$d"
|
|
cat > "$d/edge-cache.yaml" <<EOF
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: edge-cache
|
|
namespace: default
|
|
spec:
|
|
containers:
|
|
- name: cache
|
|
image: nginx:1.27
|
|
volumeMounts:
|
|
- name: conf
|
|
mountPath: /etc/nginx/conf.d
|
|
volumes:
|
|
- name: conf
|
|
hostPath:
|
|
path: /opt/edge-cache/conf
|
|
type: Directory
|
|
EOF
|
|
rm -rf /opt/edge-cache
|
|
systemctl restart kubelet'
|
|
""",
|
|
checks=[
|
|
C("mirror pod edge-cache-<node> is Running with nginx:1.27", 5, r"""
|
|
for i in $(seq 1 12); do
|
|
[ "$(j -n default pod "edge-cache-$W3" '{.status.phase}')" = Running ] && [ "$(j -n default pod "edge-cache-$W3" '{.spec.containers[0].image}')" = nginx:1.27 ] && exit 0
|
|
sleep 5
|
|
done; exit 1"""),
|
|
C("the hostPath the manifest references now exists on the node", 3, r"""
|
|
p=$(nx "$W3" 'grep -A2 hostPath /etc/kubernetes/manifests/edge-cache.yaml /etc/kubernetes/static/edge-cache.yaml 2>/dev/null | grep "path:" | head -1' | awk '{print $2}')
|
|
[ -n "$p" ] && nx "$W3" "test -d $p" """),
|
|
C("reason.txt mentions the missing host directory", 2, r"""grep -Eqi 'director|hostpath|/opt/edge-cache|not exist|missing' "$WS/q314/reason.txt" 2>/dev/null"""),
|
|
],
|
|
solution="""
|
|
k get pod -A | grep edge-cache # nothing
|
|
{SHELL_W3}
|
|
journalctl -u kubelet -n 40 --no-pager | grep -i edge-cache
|
|
# ... failed to start ... hostPath type check failed: /opt/edge-cache/conf is not a directory
|
|
grep staticPodPath /var/lib/kubelet/config.yaml
|
|
cat <staticPodPath>/edge-cache.yaml # hostPath /opt/edge-cache/conf with type: Directory
|
|
mkdir -p /opt/edge-cache/conf # type: Directory means the kubelet will NOT create it
|
|
exit
|
|
k get pod edge-cache-{W3}
|
|
echo 'hostPath /opt/edge-cache/conf did not exist and type was Directory' > {WS}/q314/reason.txt
|
|
# The alternative fix is type: DirectoryOrCreate, but the task asks for the pod to run as written.
|
|
""",
|
|
)
|
|
|
|
# ------------------------------------------------------------------ 315 ----
|
|
task(
|
|
id=315, domain="STOR", title="Keep the data when the claim goes",
|
|
task="""
|
|
In Namespace `project-orbit`, Pod `keeper` writes to PVC `keeper-claim`, which was provisioned
|
|
dynamically by the default StorageClass — whose reclaim policy is `Delete`.
|
|
|
|
The data must survive the claim being deleted:
|
|
|
|
1. Without recreating anything, make sure the bound PersistentVolume is **retained** when its claim
|
|
goes away.
|
|
2. Write the PV's name to `{WS}/q315/pv.txt`.
|
|
3. Delete Pod `keeper` and PVC `keeper-claim`.
|
|
4. The PV must still exist afterwards, in phase `Released`.
|
|
""",
|
|
setup=NS + r"""
|
|
mkns project-orbit
|
|
rm -rf "$WS/q315"; mkdir -p "$WS/q315"
|
|
$K -n project-orbit delete pod keeper --ignore-not-found --wait=true >/dev/null
|
|
$K -n project-orbit delete pvc keeper-claim --ignore-not-found >/dev/null
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: {name: keeper-claim, namespace: project-orbit}
|
|
spec: {accessModes: [ReadWriteOnce], resources: {requests: {storage: 200Mi}}}
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: keeper, namespace: project-orbit}
|
|
spec:
|
|
volumes: [{name: d, persistentVolumeClaim: {claimName: keeper-claim}}]
|
|
containers: [{name: c, image: busybox:1.36, command: [sh, -c, "echo important > /data/keep.txt; sleep 86400"], volumeMounts: [{name: d, mountPath: /data}]}]
|
|
EOF
|
|
for i in $(seq 1 30); do [ "$(j -n project-orbit pod keeper '{.status.phase}')" = Running ] && break; sleep 2; done
|
|
""",
|
|
checks=[
|
|
C("pv.txt names the PV that was bound to keeper-claim", 3, r"""
|
|
p=$(tr -d '[:space:]' < "$WS/q315/pv.txt" 2>/dev/null); [ -n "$p" ] || exit 1
|
|
eq "$(j pv "$p" '{.spec.claimRef.name}')" keeper-claim"""),
|
|
C("that PV has reclaim policy Retain", 4, r"""eq "$(j pv "$(tr -d '[:space:]' < "$WS/q315/pv.txt")" '{.spec.persistentVolumeReclaimPolicy}')" Retain"""),
|
|
C("Pod and PVC are gone", 2, r"""! $K -n project-orbit get pod keeper >/dev/null 2>&1 && ! $K -n project-orbit get pvc keeper-claim >/dev/null 2>&1"""),
|
|
C("the PV still exists and is Released", 3, r"""
|
|
p=$(tr -d '[:space:]' < "$WS/q315/pv.txt"); for i in $(seq 1 10); do [ "$(j pv "$p" '{.status.phase}')" = Released ] && exit 0; sleep 3; done; exit 1"""),
|
|
],
|
|
solution="""
|
|
k -n project-orbit get pvc keeper-claim -o jsonpath='{.spec.volumeName}' | tee {WS}/q315/pv.txt
|
|
PV=$(cat {WS}/q315/pv.txt)
|
|
k patch pv $PV -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}'
|
|
k get pv $PV # RECLAIM POLICY: Retain
|
|
k -n project-orbit delete pod keeper --now
|
|
k -n project-orbit delete pvc keeper-claim
|
|
k get pv $PV # STATUS: Released - data still on disk
|
|
# Patch the PV, not the StorageClass: the class only sets the policy for volumes created later.
|
|
# To reuse it, clear .spec.claimRef and it becomes Available again.
|
|
""",
|
|
)
|
|
|
|
# ------------------------------------------------------------------ 316 ----
|
|
task(
|
|
id=316, domain="ARCH", title="Plan a control-plane upgrade",
|
|
task="""
|
|
The control-plane node must be upgraded to the next patch release.
|
|
|
|
1. Write the cluster's current control-plane version to `{WS}/q316/current.txt` (as `kubectl` reports it).
|
|
2. Run the kubeadm command that shows what an upgrade would do, and save its output to
|
|
`{WS}/q316/plan.txt`.
|
|
3. Write to `{WS}/q316/steps.sh` the ordered commands you would run to upgrade the control-plane
|
|
node — package install of kubeadm, the kubeadm upgrade step, draining, upgrading kubelet and
|
|
kubectl, restarting the kubelet, and uncordoning — one command per line, in the order they must
|
|
be executed. Do **not** execute them.
|
|
""",
|
|
setup=r"""rm -rf "$WS/q316"; mkdir -p "$WS/q316" """,
|
|
checks=[
|
|
C("current.txt matches the server version", 2, r"""
|
|
v=$($K version -o json 2>/dev/null | python3 -c 'import sys,json;print(json.load(sys.stdin)["serverVersion"]["gitVersion"])')
|
|
grep -qF "$v" "$WS/q316/current.txt" 2>/dev/null"""),
|
|
C("plan.txt is real kubeadm upgrade plan output", 3, r"""grep -Eqi 'upgrade|component|version' "$WS/q316/plan.txt" 2>/dev/null && [ "$(wc -l < "$WS/q316/plan.txt")" -ge 5 ]"""),
|
|
C("steps.sh has the commands in the correct order", 5, r"""
|
|
f="$WS/q316/steps.sh"; [ -f "$f" ] || exit 1
|
|
grep -q 'kubeadm upgrade apply' "$f" || exit 1
|
|
grep -Eq '(apt|yum|dnf).*(install).*kubeadm' "$f" || exit 1
|
|
grep -q 'drain' "$f" && grep -q 'uncordon' "$f" || exit 1
|
|
grep -Eq '(apt|yum|dnf).*(install).*kubelet' "$f" || exit 1
|
|
grep -Eq 'systemctl (restart|daemon-reload)' "$f" || exit 1
|
|
n=$(grep -n 'kubeadm upgrade apply' "$f" | head -1 | cut -d: -f1)
|
|
ka=$(grep -nE '(apt|yum|dnf).*install.*kubeadm' "$f" | head -1 | cut -d: -f1)
|
|
kl=$(grep -nE '(apt|yum|dnf).*install.*kubelet' "$f" | head -1 | cut -d: -f1)
|
|
dr=$(grep -n 'drain' "$f" | head -1 | cut -d: -f1)
|
|
un=$(grep -n 'uncordon' "$f" | head -1 | cut -d: -f1)
|
|
[ "$ka" -lt "$n" ] && [ "$n" -lt "$kl" ] && [ "$dr" -lt "$kl" ] && [ "$kl" -lt "$un" ]"""),
|
|
],
|
|
solution="""
|
|
k version -o json | python3 -c 'import sys,json;print(json.load(sys.stdin)["serverVersion"]["gitVersion"])' > {WS}/q316/current.txt
|
|
{SHELL_CP}
|
|
kubeadm upgrade plan | tee /tmp/plan.txt # copy into {WS}/q316/plan.txt
|
|
cat > {WS}/q316/steps.sh <<'EOF'
|
|
apt-get update && apt-get install -y kubeadm=1.34.1-1.1
|
|
kubeadm upgrade plan
|
|
kubeadm upgrade apply v1.34.1
|
|
kubectl drain <control-plane> --ignore-daemonsets
|
|
apt-get install -y kubelet=1.34.1-1.1 kubectl=1.34.1-1.1
|
|
systemctl daemon-reload
|
|
systemctl restart kubelet
|
|
kubectl uncordon <control-plane>
|
|
EOF
|
|
# Order matters and is the point of this task: kubeadm binary first, then `kubeadm upgrade apply`
|
|
# (it upgrades the static pod control plane), then drain, then kubelet/kubectl packages, restart,
|
|
# uncordon. Workers use `kubeadm upgrade node` instead of `upgrade apply`.
|
|
""",
|
|
)
|
|
|
|
# ------------------------------------------------------------------ 317 ----
|
|
task(
|
|
id=317, domain="TSHOOT", title="Certificates stop being issued", breaks="controller-manager",
|
|
task="""
|
|
Certificate signing requests are approved but never get a certificate: `kubectl get csr` shows
|
|
them `Approved` with an empty `CONDITION`-issued certificate forever.
|
|
|
|
Find the responsible control-plane component, repair it on the control-plane node, and prove it
|
|
works: a CertificateSigningRequest named `issue-probe` (CN `probe`, signer
|
|
`kubernetes.io/kube-apiserver-client`, usage `client auth`) that you create and approve must
|
|
receive a certificate.
|
|
""",
|
|
setup=r"""
|
|
$K delete csr issue-probe --ignore-not-found >/dev/null 2>&1
|
|
nx "$CP" '
|
|
f=/etc/kubernetes/manifests/kube-controller-manager.yaml
|
|
grep -q "cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt" $f && sed -i "s|--cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt|--cluster-signing-cert-file=/etc/kubernetes/pki/ca-signing.crt|" $f
|
|
'
|
|
""",
|
|
checks=[
|
|
C("kube-controller-manager points at a signing certificate that exists", 4, r"""
|
|
p=$(nx "$CP" 'grep -o "\-\-cluster-signing-cert-file=[^ ]*" /etc/kubernetes/manifests/kube-controller-manager.yaml' | cut -d= -f2)
|
|
[ -n "$p" ] && nx "$CP" "test -f $p" """),
|
|
C("kube-controller-manager pod is Running", 3, r"""
|
|
for i in $(seq 1 12); do [ "$(j -n kube-system pod -l component=kube-controller-manager '{.items[0].status.phase}')" = Running ] && exit 0; sleep 5; done; exit 1"""),
|
|
C("an approved CSR now receives a certificate", 5, r"""
|
|
$K get csr issue-probe >/dev/null 2>&1 || exit 1
|
|
$K get csr issue-probe -o jsonpath='{.status.conditions[*].type}' | grep -q Approved || exit 1
|
|
for i in $(seq 1 15); do [ -n "$(j csr issue-probe '{.status.certificate}')" ] && exit 0; sleep 4; done; exit 1"""),
|
|
],
|
|
solution="""
|
|
k get csr # Approved but never Issued
|
|
k -n kube-system logs kube-controller-manager-{CP} | tail -20
|
|
# ... unable to load client CA file .../ca-signing.crt: no such file or directory
|
|
{SHELL_CP}
|
|
grep signing /etc/kubernetes/manifests/kube-controller-manager.yaml
|
|
ls /etc/kubernetes/pki/ca.crt # the real file
|
|
vi /etc/kubernetes/manifests/kube-controller-manager.yaml
|
|
# --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
|
|
exit
|
|
openssl genrsa -out probe.key 2048
|
|
openssl req -new -key probe.key -subj "/CN=probe" -out probe.csr
|
|
cat <<EOF | k apply -f -
|
|
apiVersion: certificates.k8s.io/v1
|
|
kind: CertificateSigningRequest
|
|
metadata: {name: issue-probe}
|
|
spec: {request: $(base64 -w0 probe.csr), signerName: kubernetes.io/kube-apiserver-client, usages: [client auth]}
|
|
EOF
|
|
k certificate approve issue-probe
|
|
k get csr issue-probe # certificate now populated
|
|
# The csrsigning controller lives in kube-controller-manager: approval is the API's job,
|
|
# but issuing the certificate is the controller's.
|
|
""",
|
|
)
|
|
|
|
QUESTIONS = Q
|