639 lines
35 KiB
Python
639 lines
35 KiB
Python
"""CKA tasks 103-113: pool tasks for the curriculum objectives the bank covered thinly or not at all.
|
|
|
|
curriculum.py maps every task to the objectives of the CKA curriculum and turns the bank into a
|
|
study path (./exam objectives). Mapping the first 102 tasks left eight objectives with fewer than
|
|
two pool tasks; these close them:
|
|
|
|
2.3 Monitor cluster and application resource usage 103 kubectl top
|
|
2.4 Manage and evaluate container output streams 104 logs by selector, of one replica, of an init container
|
|
3.3 Configure workload autoscaling 105 an HPA on CPU with a scale-down policy
|
|
3.4 Robust, self-healing deployments 106 probes that check the wrong thing
|
|
4.2 Prepare underlying infrastructure 107 kernel modules and sysctls 108 the runtime behind a Pod
|
|
4.4 Manage the lifecycle of clusters 109 drain the control-plane node and bring it back
|
|
4.5 Highly-available control plane 110 the join of a second control-plane node 111 leaders and etcd members
|
|
4.7 Extension interfaces (CNI, CSI, CRI) 112 what this cluster plugs in (and 108)
|
|
5.1 Connectivity between Pods 113 Pod to Pod across nodes, Pod CIDRs, hostNetwork
|
|
|
|
103, 104 and 106 are the CKAD tasks for the same skill (kubectl top, logs, probes) under CKA ids:
|
|
the objective is word for word the same, so is the scenario. Everything else is CKA ground - nodes,
|
|
kubeadm, etcd, the runtime. All normal difficulty, all in the random pool; nothing here breaks a
|
|
node or a control-plane component (109 cordons the control-plane node while you work on it and has
|
|
you uncordon it; the session baseline uncordons it if you do not).
|
|
"""
|
|
from tracks.cka.common import C, NS, ns_pick, name_pick
|
|
from tracks.ckad import bank_extra as _ckad_extra, bank_more as _ckad_more
|
|
|
|
|
|
def _port(bank, src_id, new_id, domain, **over):
|
|
"""A CKAD task that serves a CKA objective as it is: same scenario and graders, CKA id and paths."""
|
|
src = next(q for q in bank.QUESTIONS if q["id"] == src_id)
|
|
old, new = f"q{src_id}", f"q{new_id}"
|
|
q = dict(src, id=new_id, domain=domain, **over)
|
|
for k in ("task", "setup", "solution"):
|
|
q[k] = src[k].replace(old, new)
|
|
q["checks"] = [dict(c, script=c["script"].replace(old, new), name=c["name"].replace(old, new)) for c in src["checks"]]
|
|
return q
|
|
|
|
|
|
ETCDCTL = ("etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt "
|
|
"--cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key")
|
|
|
|
QUESTIONS = [
|
|
# ---------------------------------------------------------------- TSHOOT ----
|
|
# needs a working metrics API, which Q23 breaks on purpose: never in one session with it
|
|
_port(_ckad_extra, 23, 103, "TSHOOT", breaks="metrics-server"),
|
|
_port(_ckad_more, 64, 104, "TSHOOT"),
|
|
# ------------------------------------------------------------------ WORK ----
|
|
{
|
|
"id": 105, "domain": "WORK", "title": "Autoscale on CPU, scale down calmly",
|
|
"params": lambda r: {"ns": ns_pick(r, "scale"), "dep": name_pick(r, "render"), "min": r.choice([2, 3]),
|
|
"max": r.choice([6, 8, 10]), "cpu": r.choice([50, 60, 70]), "win": r.choice([120, 180, 300])},
|
|
"task": """
|
|
Deployment `%{dep}` in Namespace `%{ns}` has to scale with its load.
|
|
|
|
1. An autoscaler that works on CPU utilisation needs a CPU request to measure against. Give the
|
|
container `web` a CPU request of `100m` and leave everything else in the Deployment as it is.
|
|
2. Create HorizontalPodAutoscaler `%{dep}` for the Deployment: at least **%{min}** replicas, at
|
|
most **%{max}**, target average CPU utilisation **%{cpu}%**.
|
|
3. Scaling down must be calm: a stabilisation window of **%{win}** seconds, and at most **1 Pod
|
|
removed per 60 seconds**. Leave scaling up at its defaults.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}
|
|
$K -n %{ns} delete hpa --all >/dev/null 2>&1
|
|
$K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: %{dep}, namespace: %{ns}, labels: {app: %{dep}}}
|
|
spec:
|
|
replicas: 1
|
|
selector: {matchLabels: {app: %{dep}}}
|
|
template:
|
|
metadata: {labels: {app: %{dep}}}
|
|
spec:
|
|
containers:
|
|
- name: web
|
|
image: nginx:1.27
|
|
ports: [{containerPort: 80}]
|
|
resources: {requests: {memory: 32Mi}}
|
|
EOF
|
|
$K -n %{ns} rollout status deploy %{dep} --timeout=150s >/dev/null
|
|
""",
|
|
"checks": [
|
|
C("container web requests cpu 100m, the rest of it unchanged", 2, r"""
|
|
eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].name}/{.spec.template.spec.containers[0].image}/{.spec.template.spec.containers[0].resources.requests.cpu}/{.spec.template.spec.containers[0].resources.requests.memory}')" \
|
|
"web/nginx:1.27/100m/32Mi" """),
|
|
C("HPA %{dep}: Deployment %{dep}, %{min}-%{max} replicas, %{cpu}% average CPU utilisation", 2, r"""
|
|
$K -n %{ns} get hpa %{dep} -o json 2>/dev/null | python3 -c '
|
|
import sys, json
|
|
s = json.load(sys.stdin)["spec"]; t = s["scaleTargetRef"]; m = s.get("metrics") or []
|
|
cpu = [x["resource"]["target"] for x in m if x.get("type") == "Resource" and x["resource"]["name"] == "cpu"]
|
|
ok = (t["kind"], t["name"]) == ("Deployment", "%{dep}") and (s.get("minReplicas"), s["maxReplicas"]) == (%{min}, %{max}) \
|
|
and len(m) == 1 and len(cpu) == 1 and cpu[0].get("type") == "Utilization" and cpu[0].get("averageUtilization") == %{cpu}
|
|
sys.exit(0 if ok else 1)'"""),
|
|
C("scale-down: %{win} s stabilisation window, at most 1 Pod per 60 s", 2, r"""
|
|
$K -n %{ns} get hpa %{dep} -o json 2>/dev/null | python3 -c '
|
|
import sys, json
|
|
d = (json.load(sys.stdin)["spec"].get("behavior") or {}).get("scaleDown") or {}
|
|
p = d.get("policies") or []
|
|
ok = d.get("stabilizationWindowSeconds") == %{win} and len(p) == 1 \
|
|
and (p[0].get("type"), p[0].get("value"), p[0].get("periodSeconds")) == ("Pods", 1, 60)
|
|
sys.exit(0 if ok else 1)'"""),
|
|
C("the autoscaler has taken the Deployment to at least %{min} replicas", 1, r"""
|
|
$K -n %{ns} get hpa %{dep} >/dev/null 2>&1 || exit 1
|
|
for i in $(seq 1 20); do
|
|
[ "$(j -n %{ns} deploy %{dep} '{.spec.replicas}')" -ge %{min} ] 2>/dev/null && exit 0; sleep 3
|
|
done; exit 1"""),
|
|
],
|
|
"solution": """
|
|
k -n %{ns} set resources deploy %{dep} -c web --requests=cpu=100m
|
|
k -n %{ns} autoscale deploy %{dep} --min=%{min} --max=%{max} --cpu=%{cpu}% # kubectl before 1.34: --cpu-percent=%{cpu}
|
|
k -n %{ns} edit hpa %{dep} # `autoscale` cannot write behaviour - add under spec:
|
|
# behavior:
|
|
# scaleDown:
|
|
# stabilizationWindowSeconds: %{win}
|
|
# policies:
|
|
# - {type: Pods, value: 1, periodSeconds: 60}
|
|
k -n %{ns} get hpa %{dep} # MINPODS %{min}, MAXPODS %{max}; REPLICAS climbs to %{min}
|
|
k -n %{ns} describe hpa %{dep}
|
|
# The whole object, for when you write it instead:
|
|
# apiVersion: autoscaling/v2
|
|
# kind: HorizontalPodAutoscaler
|
|
# metadata: {name: %{dep}, namespace: %{ns}}
|
|
# spec:
|
|
# scaleTargetRef: {apiVersion: apps/v1, kind: Deployment, name: %{dep}}
|
|
# minReplicas: %{min}
|
|
# maxReplicas: %{max}
|
|
# metrics:
|
|
# - type: Resource
|
|
# resource: {name: cpu, target: {type: Utilization, averageUtilization: %{cpu}}}
|
|
# behavior: {scaleDown: {stabilizationWindowSeconds: %{win}, policies: [{type: Pods, value: 1, periodSeconds: 60}]}}
|
|
# Utilisation is usage divided by the REQUEST, so without a CPU request the HPA shows <unknown>
|
|
# and never scales on load. It does enforce minReplicas at once, metrics or not. TARGETS stays
|
|
# <unknown> here until metrics-server is installed and has a minute of data.
|
|
""",
|
|
},
|
|
_port(_ckad_more, 62, 106, "WORK"),
|
|
# ------------------------------------------------------------------ ARCH ----
|
|
{
|
|
"id": 107, "domain": "ARCH", "title": "Kernel prerequisites for a Kubernetes node",
|
|
"params": lambda r: {"name": r.choice(["k8s", "kubernetes", "99-kubernetes-cri"])},
|
|
"task": """
|
|
Node `{W3}` is to be prepared the way a machine is before `kubeadm` puts Kubernetes on it. Reach
|
|
it with `{SHELL_W3}`.
|
|
|
|
1. Make the kernel modules `overlay` and `br_netfilter` load at every boot: list them, one per
|
|
line, in `/etc/modules-load.d/%{name}.conf`.
|
|
2. Make these kernel parameters permanent in `/etc/sysctl.d/%{name}.conf`:
|
|
`net.bridge.bridge-nf-call-iptables = 1`, `net.bridge.bridge-nf-call-ip6tables = 1` and
|
|
`net.ipv4.ip_forward = 1`.
|
|
3. Write what the node reports right now to `{WS}/q107/node.txt`, in exactly this format:
|
|
|
|
```
|
|
ip_forward=<current value of net.ipv4.ip_forward>
|
|
swap=<on or off - is any swap device active on the node>
|
|
fail-swap-on=<the kubelet's failSwapOn setting>
|
|
cgroup-driver=<the kubelet's cgroupDriver>
|
|
```
|
|
|
|
(The nodes of this simulator are containers, so a module cannot really be loaded here: the two
|
|
files and the report are what is graded.)
|
|
""",
|
|
"setup": r"""
|
|
rm -rf "$WS/q107"; mkdir -p "$WS/q107"
|
|
nx "$W3" 'rm -f /etc/modules-load.d/%{name}.conf /etc/sysctl.d/%{name}.conf' </dev/null
|
|
""",
|
|
"checks": [
|
|
C("/etc/modules-load.d/%{name}.conf lists overlay and br_netfilter", 2, r"""
|
|
got=$(nx "$W3" "grep -Ev '^[[:space:]]*(#|\$)' /etc/modules-load.d/%{name}.conf 2>/dev/null" </dev/null | tr -d ' \t\r' | sort | tr '\n' ' ')
|
|
eq "$got" "br_netfilter overlay " """),
|
|
C("/etc/sysctl.d/%{name}.conf sets the three parameters to 1", 3, r"""
|
|
f=$(nx "$W3" "cat /etc/sysctl.d/%{name}.conf 2>/dev/null" </dev/null | tr -d ' \t\r')
|
|
[ -n "$f" ] || exit 1
|
|
for kv in net.bridge.bridge-nf-call-iptables=1 net.bridge.bridge-nf-call-ip6tables=1 net.ipv4.ip_forward=1; do
|
|
echo "$f" | grep -qx "$kv" || exit 1
|
|
done"""),
|
|
C("node.txt: ip_forward, swap, fail-swap-on and cgroup-driver as the node has them", 2, r"""
|
|
fwd=$(nx "$W3" "sysctl -n net.ipv4.ip_forward" </dev/null | tr -d '[:space:]')
|
|
swap=off; [ "$(nx "$W3" "sed 1d /proc/swaps | grep -c ." </dev/null | tr -d '[:space:]')" -gt 0 ] 2>/dev/null && swap=on
|
|
fso=$(nx "$W3" "awk '/^failSwapOn:/{print \$2}' /var/lib/kubelet/config.yaml" </dev/null | tr -d '[:space:]'); [ -n "$fso" ] || fso=true
|
|
cg=$(nx "$W3" "awk '/^cgroupDriver:/{print \$2}' /var/lib/kubelet/config.yaml" </dev/null | tr -d '[:space:]')
|
|
[ -n "$fwd" ] && [ -n "$cg" ] || exit 1
|
|
want=$(printf 'ip_forward=%s\nswap=%s\nfail-swap-on=%s\ncgroup-driver=%s' "$fwd" "$swap" "$fso" "$cg")
|
|
eq "$(tr -d ' \t\r' < "$WS/q107/node.txt" 2>/dev/null | grep .)" "$want" """),
|
|
],
|
|
"solution": """
|
|
{SHELL_W3}
|
|
printf 'overlay\\nbr_netfilter\\n' > /etc/modules-load.d/%{name}.conf
|
|
cat > /etc/sysctl.d/%{name}.conf <<EOF
|
|
net.bridge.bridge-nf-call-iptables = 1
|
|
net.bridge.bridge-nf-call-ip6tables = 1
|
|
net.ipv4.ip_forward = 1
|
|
EOF
|
|
# on a real machine you now load and apply them without a reboot:
|
|
# modprobe overlay && modprobe br_netfilter && sysctl --system
|
|
sysctl -n net.ipv4.ip_forward # 1
|
|
cat /proc/swaps # any line under the header = swap is on (swapon --show)
|
|
grep -E 'failSwapOn|cgroupDriver' /var/lib/kubelet/config.yaml
|
|
exit
|
|
cat > {WS}/q107/node.txt <<EOF
|
|
ip_forward=1
|
|
swap=<on|off>
|
|
fail-swap-on=<true|false>
|
|
cgroup-driver=<systemd|cgroupfs>
|
|
EOF
|
|
# `sysctl -w` and `modprobe` alone are gone after a reboot - the files are what makes them stick.
|
|
# kubeadm's preflight refuses a node with swap on unless the kubelet is told to tolerate it
|
|
# (failSwapOn: false), and the kubelet's cgroup driver has to match the container runtime's
|
|
# (SystemdCgroup = true in /etc/containerd/config.toml goes with cgroupDriver: systemd).
|
|
""",
|
|
},
|
|
{
|
|
"id": 108, "domain": "ARCH", "title": "The container runtime behind a Pod",
|
|
"params": lambda r: {"ns": ns_pick(r, "runtime"), "pod": name_pick(r, "probe")},
|
|
"task": """
|
|
Pod `%{pod}` in Namespace `%{ns}` runs on node `{W2}`. Look at it from the node's side, through
|
|
the container runtime (`{SHELL_W2}`, then `crictl`), and write `{WS}/q108/runtime.txt` in exactly
|
|
this format:
|
|
|
|
```
|
|
runtime=<runtime name> <runtime version>
|
|
socket=<the runtime endpoint the kubelet on that node is configured with>
|
|
container-id=<the full ID of the Pod's container `app`>
|
|
pid=<the process ID of that container on the node>
|
|
log=<the full path of that container's log file on the node>
|
|
```
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}; rm -rf "$WS/q108"; mkdir -p "$WS/q108"
|
|
$K -n %{ns} delete pod %{pod} --ignore-not-found --wait=true --grace-period=1 >/dev/null 2>&1
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: %{pod}, namespace: %{ns}}
|
|
spec:
|
|
terminationGracePeriodSeconds: 1
|
|
nodeSelector: {kubernetes.io/hostname: "{W2}"}
|
|
containers:
|
|
- name: app
|
|
image: busybox:1.36
|
|
command: [sh, -c, "echo 'runtime probe started'; sleep 86400"]
|
|
EOF
|
|
$K -n %{ns} wait --for=condition=Ready pod/%{pod} --timeout=150s >/dev/null
|
|
""",
|
|
"checks": [
|
|
C("runtime and socket lines", 2, r"""
|
|
ver=$(nx "$W2" "crictl version 2>/dev/null" </dev/null)
|
|
name=$(echo "$ver" | awk '/^RuntimeName:/{print $2}'); rv=$(echo "$ver" | awk '/^RuntimeVersion:/{print $2}')
|
|
sock=$(nx "$W2" "awk '/^containerRuntimeEndpoint:/{print \$2}' /var/lib/kubelet/config.yaml" </dev/null | tr -d '"[:space:]')
|
|
[ -n "$name" ] && [ -n "$rv" ] && [ -n "$sock" ] || exit 1
|
|
f="$WS/q108/runtime.txt"
|
|
tr -d '\r' < "$f" 2>/dev/null | sed 's/[[:space:]]*$//' | grep -qx "runtime=$name $rv" && tr -d ' \t\r' < "$f" | grep -qx "socket=$sock" """),
|
|
C("container-id is the full ID of container app", 2, r"""
|
|
cid=$($K -n %{ns} get pod %{pod} -o jsonpath='{.status.containerStatuses[0].containerID}' 2>/dev/null | sed 's#^[a-z-]*://##')
|
|
[ "${#cid}" = 64 ] && tr -d ' \t\r' < "$WS/q108/runtime.txt" 2>/dev/null | grep -qx "container-id=$cid" """),
|
|
C("pid is that container's process on the node", 2, r"""
|
|
cid=$($K -n %{ns} get pod %{pod} -o jsonpath='{.status.containerStatuses[0].containerID}' 2>/dev/null | sed 's#^[a-z-]*://##')
|
|
pid=$(nx "$W2" "crictl inspect -o go-template --template '{{.info.pid}}' $cid 2>/dev/null" </dev/null | tr -d '[:space:]')
|
|
[ -n "$cid" ] && [ -n "$pid" ] && tr -d ' \t\r' < "$WS/q108/runtime.txt" 2>/dev/null | grep -qx "pid=$pid" """),
|
|
C("log is the container's log file on the node", 1, r"""
|
|
cid=$($K -n %{ns} get pod %{pod} -o jsonpath='{.status.containerStatuses[0].containerID}' 2>/dev/null | sed 's#^[a-z-]*://##')
|
|
log=$(nx "$W2" "crictl inspect -o go-template --template '{{.status.logPath}}' $cid 2>/dev/null" </dev/null | tr -d '[:space:]')
|
|
[ -n "$cid" ] && [ -n "$log" ] && tr -d ' \t\r' < "$WS/q108/runtime.txt" 2>/dev/null | grep -qx "log=$log" """),
|
|
],
|
|
"solution": """
|
|
k -n %{ns} get pod %{pod} -o wide # on {W2}
|
|
{SHELL_W2}
|
|
crictl version # RuntimeName: containerd RuntimeVersion: v2.x.y
|
|
grep containerRuntimeEndpoint /var/lib/kubelet/config.yaml # unix:///run/containerd/containerd.sock
|
|
crictl pods --name %{pod} # the Pod sandbox
|
|
crictl ps --no-trunc --name app --pod $(crictl pods --name %{pod} -q) # full container ID
|
|
crictl inspect <id> | grep -E '"pid"|logPath' # .info.pid and .status.logPath
|
|
# or: crictl inspect -o go-template --template '{{.info.pid}} {{.status.logPath}}' <id>
|
|
exit
|
|
cat > {WS}/q108/runtime.txt <<EOF
|
|
runtime=containerd <version>
|
|
socket=unix:///run/containerd/containerd.sock
|
|
container-id=<64 hex characters>
|
|
pid=<number>
|
|
log=/var/log/pods/%{ns}_%{pod}_<pod uid>/app/0.log
|
|
EOF
|
|
# The kubelet drives the runtime over the CRI socket, and crictl speaks the same protocol - which
|
|
# is why it still works when the API server does not. `kubectl get pod -o
|
|
# jsonpath='{.status.containerStatuses[0].containerID}'` shows the same ID with a containerd://
|
|
# prefix; /var/log/pods/... is the file `kubectl logs` reads.
|
|
""",
|
|
},
|
|
{
|
|
# drains the control-plane node: CoreDNS moves off it, so never beside the tasks that break DNS
|
|
"id": 109, "domain": "ARCH", "title": "Drain the control-plane node and bring it back", "breaks": "coredns",
|
|
"params": lambda r: {"ns": ns_pick(r, "maint"), "dep": name_pick(r, "beacon")},
|
|
"task": """
|
|
The control-plane node `{CP}` is about to be upgraded and has to be emptied first.
|
|
|
|
1. Drain `{CP}`: it must be unschedulable, and the only Pods left on it are static Pods and Pods
|
|
of DaemonSets. Deployment `%{dep}` in Namespace `%{ns}` has a Pod with an `emptyDir` volume on
|
|
that node - losing that volume is accepted.
|
|
2. While the node is drained, save the output of `kubectl get nodes` to `{WS}/q109/drained.txt`,
|
|
and write the phase the Pod of `%{dep}` is in now to `{WS}/q109/phase.txt`.
|
|
3. The upgrade is over: make the node schedulable again and wait until `%{dep}` has its Pod back.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}; rm -rf "$WS/q109"; mkdir -p "$WS/q109"
|
|
$K uncordon "$CP" >/dev/null 2>&1
|
|
$K -n %{ns} delete deploy %{dep} --ignore-not-found --wait=true >/dev/null 2>&1
|
|
$K -n %{ns} delete pod -l app=%{dep} --wait=true --grace-period=1 >/dev/null 2>&1
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata: {name: %{dep}, namespace: %{ns}, labels: {app: %{dep}}}
|
|
spec:
|
|
replicas: 1
|
|
selector: {matchLabels: {app: %{dep}}}
|
|
template:
|
|
metadata: {labels: {app: %{dep}}}
|
|
spec:
|
|
terminationGracePeriodSeconds: 2
|
|
nodeSelector: {kubernetes.io/hostname: "{CP}"}
|
|
tolerations:
|
|
- {key: node-role.kubernetes.io/control-plane, operator: Exists, effect: NoSchedule}
|
|
volumes: [{name: scratch, emptyDir: {}}]
|
|
containers:
|
|
- name: beacon
|
|
image: busybox:1.36
|
|
command: [sh, -c, "date > /scratch/started; sleep 86400"]
|
|
volumeMounts: [{name: scratch, mountPath: /scratch}]
|
|
EOF
|
|
$K -n %{ns} rollout status deploy %{dep} --timeout=150s >/dev/null
|
|
$K -n %{ns} get pod -l app=%{dep} -o jsonpath='{.items[0].metadata.uid}' > "$WS/.q109-uid"
|
|
""",
|
|
"checks": [
|
|
C("drained.txt shows {CP} as SchedulingDisabled", 2, r"""
|
|
grep -Eq "^$CP[[:space:]]+Ready,SchedulingDisabled([[:space:]]|\$)" "$WS/q109/drained.txt" 2>/dev/null"""),
|
|
C("phase.txt holds Pending", 1, r"""eq "$(tr -d '[:space:]' < "$WS/q109/phase.txt" 2>/dev/null)" Pending"""),
|
|
C("the Pod of %{dep} was evicted, and its replacement runs on {CP} again", 2, r"""
|
|
old=$(cat "$WS/.q109-uid" 2>/dev/null); [ -n "$old" ] || exit 1
|
|
for i in $(seq 1 20); do
|
|
new=$($K -n %{ns} get pod -l app=%{dep} -o jsonpath='{range .items[?(@.status.phase=="Running")]}{.metadata.uid} {.spec.nodeName}{"\n"}{end}' 2>/dev/null | head -1)
|
|
[ -n "$new" ] && break; sleep 3
|
|
done
|
|
[ -n "$new" ] && [ "${new%% *}" != "$old" ] && [ "${new##* }" = "$CP" ]"""),
|
|
C("{CP} is schedulable again and CoreDNS is fully available", 2, r"""
|
|
old=$(cat "$WS/.q109-uid" 2>/dev/null); [ -n "$old" ] || exit 1
|
|
$K -n %{ns} get pod -l app=%{dep} -o jsonpath='{.items[*].metadata.uid}' 2>/dev/null | grep -q "$old" && exit 1
|
|
[ "$(j node "$CP" '{.spec.unschedulable}')" != true ] || exit 1
|
|
for i in $(seq 1 20); do
|
|
[ "$(j -n kube-system deploy coredns '{.status.availableReplicas}')" = "$(j -n kube-system deploy coredns '{.spec.replicas}')" ] && exit 0; sleep 3
|
|
done; exit 1"""),
|
|
],
|
|
"solution": """
|
|
k drain {CP} --ignore-daemonsets --delete-emptydir-data
|
|
# without the flags it stops at: cannot delete DaemonSet-managed Pods ... / cannot delete Pods with local storage ...
|
|
k get nodes > {WS}/q109/drained.txt # {CP} Ready,SchedulingDisabled control-plane
|
|
k -n %{ns} get pod -l app=%{dep} # Pending: it may only run on {CP}, and that node is cordoned
|
|
echo Pending > {WS}/q109/phase.txt
|
|
k uncordon {CP}
|
|
k -n %{ns} rollout status deploy %{dep}
|
|
k get nodes; k -n kube-system get deploy coredns # CoreDNS moved to the workers during the drain and stays there
|
|
# drain = cordon + evict. It leaves DaemonSet Pods (they would come straight back) and static Pods,
|
|
# refuses Pods with emptyDir data until you accept the loss, and refuses bare Pods until --force -
|
|
# which deletes them for good. The node stays cordoned until you uncordon it: a real upgrade
|
|
# (kubeadm upgrade apply, then the kubelet package) happens between those two commands.
|
|
""",
|
|
},
|
|
{
|
|
"id": 110, "domain": "ARCH", "title": "Prepare the join of a second control-plane node",
|
|
"task": """
|
|
The cluster is getting a second control-plane node. Prepare what the new machine needs - on the
|
|
control-plane node, `{SHELL_CP}` - without joining anything.
|
|
|
|
1. Write the address every node uses to reach the API server - the cluster's control-plane
|
|
endpoint, as kubeadm has it configured - to `{WS}/q110/endpoint.txt`.
|
|
2. Upload the control-plane certificates to the cluster, so that a joining control-plane node can
|
|
fetch them.
|
|
3. Save the complete `kubeadm join` command a new **control-plane** node has to run - with a new
|
|
token, the CA certificate hash and the certificate key from step 2 - to
|
|
`{WS}/q110/join-control-plane.sh`, on one line.
|
|
""",
|
|
"setup": r"""
|
|
rm -rf "$WS/q110"; mkdir -p "$WS/q110"
|
|
$K -n kube-system delete secret kubeadm-certs --ignore-not-found >/dev/null 2>&1
|
|
date +%s > "$WS/.q110-t0"
|
|
""",
|
|
"checks": [
|
|
C("endpoint.txt holds the control-plane endpoint", 1, r"""
|
|
want=$($K -n kube-system get cm kubeadm-config -o jsonpath='{.data.ClusterConfiguration}' 2>/dev/null | awk '/^controlPlaneEndpoint:/{print $2}' | tr -d '"')
|
|
[ -n "$want" ] && eq "$(tr -d '[:space:]' < "$WS/q110/endpoint.txt" 2>/dev/null)" "$want" """),
|
|
C("the control-plane certificates were uploaded (Secret kubeadm-certs)", 2, r"""
|
|
ct=$(j -n kube-system secret kubeadm-certs '{.metadata.creationTimestamp}'); [ -n "$ct" ] || exit 1
|
|
[ "$(date -d "$ct" +%s)" -ge "$(( $(cat "$WS/.q110-t0" 2>/dev/null || echo 9999999999) - 5 ))" ]"""),
|
|
C("join command for a control-plane node: endpoint, CA hash, --control-plane, a certificate key", 2, r"""
|
|
f="$WS/q110/join-control-plane.sh"
|
|
ep=$($K -n kube-system get cm kubeadm-config -o jsonpath='{.data.ClusterConfiguration}' 2>/dev/null | awk '/^controlPlaneEndpoint:/{print $2}' | tr -d '"')
|
|
hash=$(nx "$CP" "openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'" </dev/null | tr -d '[:space:]')
|
|
[ -n "$ep" ] && [ -n "$hash" ] || exit 1
|
|
l=$(grep 'kubeadm join' "$f" 2>/dev/null | head -1)
|
|
echo "$l" | grep -q "kubeadm join $ep " && echo "$l" | grep -q -- "--discovery-token-ca-cert-hash sha256:$hash" \
|
|
&& echo "$l" | grep -Eq -- '--control-plane( |$)' && echo "$l" | grep -Eq -- '--certificate-key [0-9a-f]{64}( |$)' """),
|
|
C("its token is a real, unexpired bootstrap token", 2, r"""
|
|
tok=$(grep -oE -- '--token [a-z0-9]{6}\.[a-z0-9]{16}' "$WS/q110/join-control-plane.sh" 2>/dev/null | head -1 | awk '{print $2}'); [ -n "$tok" ] || exit 1
|
|
exp=$(j -n kube-system secret bootstrap-token-${tok%%.*} '{.data.expiration}' | base64 -d 2>/dev/null); [ -n "$exp" ] || exit 1
|
|
[ "$(date -d "$exp" +%s)" -gt "$(date +%s)" ]"""),
|
|
],
|
|
"solution": """
|
|
k -n kube-system get cm kubeadm-config -o yaml | grep controlPlaneEndpoint # {CP}:6443
|
|
echo {CP}:6443 > {WS}/q110/endpoint.txt
|
|
{SHELL_CP}
|
|
kubeadm init phase upload-certs --upload-certs
|
|
# [upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
|
|
# [upload-certs] Using certificate key:
|
|
# <64 hex characters>
|
|
kubeadm token create --print-join-command --certificate-key <that key>
|
|
# kubeadm join {CP}:6443 --token ... --discovery-token-ca-cert-hash sha256:... --control-plane --certificate-key ...
|
|
exit
|
|
# paste that line into {WS}/q110/join-control-plane.sh
|
|
# A control-plane node needs the cluster's CA, front-proxy CA, etcd CA and service-account keys.
|
|
# upload-certs puts them, encrypted with the certificate key, into the Secret kubeadm-certs for
|
|
# two hours; `kubeadm join --control-plane --certificate-key` downloads and decrypts them. All of
|
|
# it only works because the cluster was initialised with a controlPlaneEndpoint - the one stable
|
|
# address (a load balancer or VIP in a real HA cluster) that every node and kubeconfig points at.
|
|
""",
|
|
},
|
|
{
|
|
"id": 111, "domain": "ARCH", "title": "Who leads, and who is in etcd",
|
|
"task": """
|
|
Before a second control-plane node is added, record how the control plane is laid out today.
|
|
Write `{WS}/q111/ha.txt` in exactly this format:
|
|
|
|
```
|
|
control-plane-nodes=<number of control-plane nodes>
|
|
etcd-topology=<stacked or external>
|
|
etcd-members=<number of members of the etcd cluster>
|
|
etcd-leader=<name of the etcd member that is the leader>
|
|
scheduler-leader=<name of the node whose kube-scheduler holds the leader lease>
|
|
controller-manager-leader=<name of the node whose kube-controller-manager holds the leader lease>
|
|
```
|
|
|
|
`stacked` means etcd runs on the control-plane nodes themselves, `external` that the API server
|
|
uses etcd hosts of their own. `etcdctl` is on the control-plane node (`{SHELL_CP}`); etcd's
|
|
certificates are under `/etc/kubernetes/pki/etcd`.
|
|
""",
|
|
"setup": r"""
|
|
rm -rf "$WS/q111"; mkdir -p "$WS/q111"
|
|
ensure_etcd_tools >/dev/null 2>&1 || { echo "etcdctl could not be installed on $CP - run ./etcd-tools.sh"; exit 1; }
|
|
""",
|
|
"checks": [
|
|
C("control-plane-nodes and etcd-topology", 2, r"""
|
|
n=$($K get nodes -l node-role.kubernetes.io/control-plane --no-headers 2>/dev/null | grep -c .)
|
|
topo=external; [ "$($K -n kube-system get pods -l component=etcd,tier=control-plane --no-headers 2>/dev/null | grep -c .)" -ge 1 ] && topo=stacked
|
|
f=$(tr -d ' \t\r' < "$WS/q111/ha.txt" 2>/dev/null)
|
|
[ "$n" -ge 1 ] && echo "$f" | grep -qx "control-plane-nodes=$n" && echo "$f" | grep -qx "etcd-topology=$topo" """),
|
|
C("etcd-members and etcd-leader", 3, r"""
|
|
E='""" + ETCDCTL + r"""'
|
|
out=$( { nx "$CP" "$E endpoint status -w json 2>/dev/null" </dev/null; echo; nx "$CP" "$E member list -w json 2>/dev/null" </dev/null; } | python3 -c '
|
|
import sys, json
|
|
docs = [json.loads(l) for l in sys.stdin.read().splitlines() if l.strip()[:1] in "[{" and l.strip()]
|
|
status, members = docs[0], docs[1]["members"]
|
|
leader = status[0]["Status"]["leader"]
|
|
print(len(members), next(m["name"] for m in members if m["ID"] == leader))' 2>/dev/null)
|
|
[ -n "$out" ] || exit 1
|
|
f=$(tr -d ' \t\r' < "$WS/q111/ha.txt" 2>/dev/null)
|
|
echo "$f" | grep -qx "etcd-members=${out%% *}" && echo "$f" | grep -qx "etcd-leader=${out##* }" """),
|
|
C("scheduler-leader and controller-manager-leader", 2, r"""
|
|
s=$(j -n kube-system lease kube-scheduler '{.spec.holderIdentity}' | cut -d_ -f1)
|
|
c=$(j -n kube-system lease kube-controller-manager '{.spec.holderIdentity}' | cut -d_ -f1)
|
|
f=$(tr -d ' \t\r' < "$WS/q111/ha.txt" 2>/dev/null)
|
|
[ -n "$s" ] && [ -n "$c" ] && echo "$f" | grep -qx "scheduler-leader=$s" && echo "$f" | grep -qx "controller-manager-leader=$c" """),
|
|
],
|
|
"solution": """
|
|
k get nodes -l node-role.kubernetes.io/control-plane # how many
|
|
k -n kube-system get pods -l component=etcd -o wide # an etcd static Pod on the control-plane node = stacked
|
|
k -n kube-system get lease kube-scheduler kube-controller-manager # HOLDER is <node>_<uuid>
|
|
{SHELL_CP}
|
|
export ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCDCTL_CACERT=/etc/kubernetes/pki/etcd/ca.crt \\
|
|
ETCDCTL_CERT=/etc/kubernetes/pki/etcd/server.crt ETCDCTL_KEY=/etc/kubernetes/pki/etcd/server.key
|
|
etcdctl member list -w table # one line per member: ID, NAME, peer and client URLs
|
|
etcdctl endpoint status -w table # IS LEADER (add --cluster to ask every member)
|
|
exit
|
|
cat > {WS}/q111/ha.txt <<EOF
|
|
control-plane-nodes=1
|
|
etcd-topology=stacked
|
|
etcd-members=1
|
|
etcd-leader={CP}
|
|
scheduler-leader={CP}
|
|
controller-manager-leader={CP}
|
|
EOF
|
|
# In an HA control plane every kube-apiserver serves at once, but only one kube-scheduler and one
|
|
# kube-controller-manager act: they elect a leader through a Lease in kube-system, and the others
|
|
# wait. etcd elects its own leader and needs a majority of members to stay writable - three
|
|
# members survive the loss of one, five of two, and an even number adds nothing.
|
|
""",
|
|
},
|
|
{
|
|
"id": 112, "domain": "ARCH", "title": "CNI, CSI and CRI: what this cluster plugs in",
|
|
"task": """
|
|
Kubernetes leaves networking, storage and running containers to plugins. Record which ones this
|
|
cluster uses in `{WS}/q112/interfaces.txt`, in exactly this format:
|
|
|
|
```
|
|
cni-config=<full path of the CNI configuration file on node {W1}>
|
|
cni-plugins=<the type of every plugin in that file's chain, in order, separated by commas>
|
|
cni-ipam=<the type of the IPAM plugin it uses>
|
|
cri-socket=<the runtime endpoint the kubelet on {W1} is configured with>
|
|
csi-drivers=<number of CSIDriver objects in the cluster>
|
|
default-provisioner=<provisioner of the default StorageClass>
|
|
```
|
|
|
|
Reach the node with `{SHELL_W1}`.
|
|
""",
|
|
"setup": r"""
|
|
rm -rf "$WS/q112"; mkdir -p "$WS/q112"
|
|
""",
|
|
"checks": [
|
|
C("cni-config, cni-plugins and cni-ipam", 3, r"""
|
|
conf=$(nx "$W1" "ls /etc/cni/net.d/*.conflist /etc/cni/net.d/*.conf 2>/dev/null | sort | head -1" </dev/null | tr -d '[:space:]'); [ -n "$conf" ] || exit 1
|
|
out=$(nx "$W1" "cat $conf" </dev/null | python3 -c '
|
|
import sys, json
|
|
d = json.load(sys.stdin); p = d.get("plugins", [d])
|
|
print(",".join(x["type"] for x in p), next((x["ipam"]["type"] for x in p if "ipam" in x), ""))' 2>/dev/null)
|
|
[ -n "$out" ] || exit 1
|
|
f=$(tr -d ' \t\r' < "$WS/q112/interfaces.txt" 2>/dev/null)
|
|
echo "$f" | grep -qx "cni-config=$conf" && echo "$f" | grep -qx "cni-plugins=${out%% *}" && echo "$f" | grep -qx "cni-ipam=${out##* }" """),
|
|
C("cri-socket", 2, r"""
|
|
sock=$(nx "$W1" "awk '/^containerRuntimeEndpoint:/{print \$2}' /var/lib/kubelet/config.yaml" </dev/null | tr -d '"[:space:]')
|
|
[ -n "$sock" ] && tr -d ' \t\r' < "$WS/q112/interfaces.txt" 2>/dev/null | grep -qx "cri-socket=$sock" """),
|
|
C("csi-drivers and default-provisioner", 2, r"""
|
|
n=$($K get csidrivers --no-headers 2>/dev/null | grep -c .)
|
|
prov=$($K get sc -o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.provisioner}{"\n"}{end}' 2>/dev/null | head -1)
|
|
[ -n "$prov" ] || exit 1
|
|
f=$(tr -d ' \t\r' < "$WS/q112/interfaces.txt" 2>/dev/null)
|
|
echo "$f" | grep -qx "csi-drivers=$n" && echo "$f" | grep -qx "default-provisioner=$prov" """),
|
|
],
|
|
"solution": """
|
|
{SHELL_W1}
|
|
ls /etc/cni/net.d/ # 10-kindnet.conflist - the kubelet's runtime takes the first file
|
|
cat /etc/cni/net.d/10-kindnet.conflist # "plugins": [{"type": "ptp", "ipam": {"type": "host-local"}}, {"type": "portmap"}]
|
|
ls /opt/cni/bin # the plugin binaries those types name
|
|
grep containerRuntimeEndpoint /var/lib/kubelet/config.yaml
|
|
exit
|
|
k get csidrivers # No resources found = 0
|
|
k get sc # the (default) class and its PROVISIONER
|
|
cat > {WS}/q112/interfaces.txt <<EOF
|
|
cni-config=/etc/cni/net.d/10-kindnet.conflist
|
|
cni-plugins=ptp,portmap
|
|
cni-ipam=host-local
|
|
cri-socket=unix:///run/containerd/containerd.sock
|
|
csi-drivers=0
|
|
default-provisioner=rancher.io/local-path
|
|
EOF
|
|
# CRI: the kubelet talks to the runtime over that socket (crictl uses the same one). CNI: the
|
|
# runtime calls the plugins of the chain in order for every Pod sandbox - here ptp wires the veth
|
|
# pair, host-local hands out an address from the node's Pod CIDR, portmap serves hostPorts.
|
|
# CSI: a storage driver registers a CSIDriver object and a StorageClass names it as provisioner;
|
|
# this cluster's local-path provisioner is not a CSI driver, hence none.
|
|
""",
|
|
},
|
|
# ------------------------------------------------------------------- NET ----
|
|
{
|
|
"id": 113, "domain": "NET", "title": "Pod to Pod, across nodes",
|
|
"params": lambda r: {"ns": ns_pick(r, "wire"), "web": name_pick(r, "web"), "client": name_pick(r, "client"), "edge": name_pick(r, "edge")},
|
|
"task": """
|
|
Namespace `%{ns}` runs a few Pods on different nodes. No Service is involved in this task.
|
|
|
|
1. Write the IP address of Pod `%{web}` and the node it runs on to `{WS}/q113/web.txt`, as
|
|
`<ip> <node>`.
|
|
2. From Pod `%{client}`, which runs on another node, request `http://<that IP>` directly and save
|
|
the response to `{WS}/q113/direct.html`.
|
|
3. Write the Pod CIDR of every node of the cluster to `{WS}/q113/pod-cidrs.txt`, one node per
|
|
line as `<node> <cidr>`.
|
|
4. One Pod in `%{ns}` has an address that lies in none of those ranges. Write its name to
|
|
`{WS}/q113/outside.txt` and the Pod spec field that causes it to `{WS}/q113/why.txt`.
|
|
""",
|
|
"setup": NS + r"""
|
|
mkns %{ns}; rm -rf "$WS/q113"; mkdir -p "$WS/q113"
|
|
$K -n %{ns} delete pod --all --wait=true --grace-period=1 >/dev/null 2>&1
|
|
$K apply -f - >/dev/null <<'EOF'
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: %{web}, namespace: %{ns}, labels: {app: web}}
|
|
spec:
|
|
nodeSelector: {kubernetes.io/hostname: "{W1}"}
|
|
containers: [{name: web, image: "nginx:1.27", ports: [{containerPort: 80}]}]
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: %{client}, namespace: %{ns}, labels: {app: client}}
|
|
spec:
|
|
terminationGracePeriodSeconds: 1
|
|
nodeSelector: {kubernetes.io/hostname: "{W2}"}
|
|
containers: [{name: client, image: "busybox:1.36", command: [sleep, "86400"]}]
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: %{edge}, namespace: %{ns}, labels: {app: edge}}
|
|
spec:
|
|
terminationGracePeriodSeconds: 1
|
|
hostNetwork: true
|
|
nodeSelector: {kubernetes.io/hostname: "{W3}"}
|
|
containers: [{name: edge, image: "busybox:1.36", command: [sleep, "86400"]}]
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: spare-%{edge}, namespace: %{ns}, labels: {app: spare}}
|
|
spec:
|
|
terminationGracePeriodSeconds: 1
|
|
nodeSelector: {kubernetes.io/hostname: "{W3}"}
|
|
containers: [{name: spare, image: "busybox:1.36", command: [sleep, "86400"]}]
|
|
EOF
|
|
$K -n %{ns} wait --for=condition=Ready pod --all --timeout=150s >/dev/null
|
|
""",
|
|
"checks": [
|
|
C("web.txt holds the Pod's IP and node", 1, r"""
|
|
want=$(j -n %{ns} pod %{web} '{.status.podIP} {.spec.nodeName}')
|
|
[ -n "$want" ] && eq "$(tr -s ' \t' ' ' < "$WS/q113/web.txt" 2>/dev/null | tr -d '\r' | sed 's/^ *//; s/ *$//' | grep .)" "$want" """),
|
|
C("direct.html holds the page fetched from the Pod's IP", 2, r"""grep -q 'Welcome to nginx' "$WS/q113/direct.html" 2>/dev/null"""),
|
|
C("pod-cidrs.txt lists every node with its Pod CIDR", 2, r"""
|
|
want=$($K get nodes -o jsonpath='{range .items[*]}{.metadata.name} {.spec.podCIDR}{"\n"}{end}' 2>/dev/null | sort)
|
|
[ "$(echo "$want" | grep -c .)" -ge 2 ] || exit 1
|
|
eq "$(tr -s ' \t' ' ' < "$WS/q113/pod-cidrs.txt" 2>/dev/null | tr -d '\r' | sed 's/^ *//; s/ *$//' | grep . | sort)" "$want" """),
|
|
C("outside.txt names %{edge}, why.txt names hostNetwork", 2, r"""
|
|
eq "$(tr -d '[:space:]' < "$WS/q113/outside.txt" 2>/dev/null | sed 's#^pod/##')" %{edge} && grep -qi 'hostNetwork' "$WS/q113/why.txt" 2>/dev/null"""),
|
|
],
|
|
"solution": """
|
|
k -n %{ns} get pods -o wide # IP and NODE of every Pod
|
|
k -n %{ns} get pod %{web} -o jsonpath='{.status.podIP} {.spec.nodeName}' > {WS}/q113/web.txt
|
|
k -n %{ns} exec %{client} -- wget -qO- -T3 http://$(cut -d' ' -f1 {WS}/q113/web.txt) > {WS}/q113/direct.html
|
|
k get nodes -o jsonpath='{range .items[*]}{.metadata.name} {.spec.podCIDR}{"\\n"}{end}' > {WS}/q113/pod-cidrs.txt
|
|
# %{edge}'s IP is its node's own address (compare with `k get nodes -o wide`), not one from 10.244.x.0/24
|
|
echo %{edge} > {WS}/q113/outside.txt
|
|
k -n %{ns} get pod %{edge} -o yaml | grep hostNetwork
|
|
echo hostNetwork > {WS}/q113/why.txt
|
|
# Every Pod gets its own address from its node's Pod CIDR, and any Pod reaches any other Pod on
|
|
# any node at that address without NAT - that is the whole network model, and the CNI plugin is
|
|
# what makes it true. A hostNetwork Pod opts out: it lives in the node's network namespace and
|
|
# shares its address and ports. Pod addresses change with every new Pod, which is why Services exist.
|
|
""",
|
|
},
|
|
]
|