"""CKA lessons of the Cluster Architecture, Installation and Configuration domain (4.1-4.8). See cka/lessons.py for what a lesson is.""" from cka.common import backup_operator from sim.bank import CAN, NS, PY, step _ETCDCTL = ("ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt " "--cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key") # ---- 4.1 RBAC and ServiceAccounts ------------------------------------------------------------- L41 = { "id": 9401, "objective": "4.1", "workers": 0, "title": "RBAC: Roles, bindings, ServiceAccounts and their tokens", "intro": r"""Every request to the API server is made by someone - a **user** (a client certificate, or a token from outside), or a **ServiceAccount** (a token the cluster issues) - and RBAC decides what they may do: ``` Role / ClusterRole verbs on resources (in apiGroups) what may be done RoleBinding subjects -> a Role or a ClusterRole in ONE Namespace ClusterRoleBinding subjects -> a ClusterRole everywhere kubectl auth can-i VERB RESOURCE --as=USER | --as=system:serviceaccount:NS:NAME [-n NS] [--list] ``` A RoleBinding may point at a ClusterRole: the ClusterRole's rules then apply only in the binding's Namespace - the usual way to reuse one set of rules in many Namespaces. Pods authenticate as their ServiceAccount, with a short-lived token the kubelet mounts at `/var/run/secrets/kubernetes.io/serviceaccount/`. `kubectl create token` issues one for you. Namespace `team` is empty.""", "setup": NS + r""" mkns team mkdir -p "$WS/lesson-4.1" """, "steps": [ step("A ServiceAccount that deploys", r"""In `team`, create ServiceAccount `deployer`, Role `deploy-edit` that allows `get`, `list`, `watch`, `create`, `update`, `patch` and `delete` on **deployments** (API group `apps`) and `get`, `list` on **pods**, and RoleBinding `deployer` that gives the Role to the ServiceAccount. Prove it with `kubectl auth can-i ... --as=system:serviceaccount:team:deployer`.""", check=CAN + r""" S=system:serviceaccount:team:deployer can --as=$S -- create deployments.apps -n team && can --as=$S -- list pods -n team && cannot --as=$S -- delete pods -n team \ && cannot --as=$S -- create deployments.apps -n default""", hint=r"""`kubectl create role deploy-edit --verb=... --resource=deployments.apps` - a Role with two resources and different verbs needs two rules, so write YAML or create it and `kubectl edit`. `kubectl create rolebinding NAME --role=ROLE --serviceaccount=team:deployer`.""", show=r"""kubectl -n team create serviceaccount deployer kubectl -n team apply -f - <<'EOF' apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: deploy-edit rules: - apiGroups: [apps] resources: [deployments] verbs: [get, list, watch, create, update, patch, delete] - apiGroups: [""] resources: [pods] verbs: [get, list] EOF kubectl -n team create rolebinding deployer --role=deploy-edit --serviceaccount=team:deployer kubectl auth can-i create deployments -n team --as=system:serviceaccount:team:deployer kubectl auth can-i delete pods -n team --as=system:serviceaccount:team:deployer"""), step("One ClusterRole, one Namespace", r"""Create ClusterRole `pod-reader` (`get`, `list`, `watch` on pods). Give it to the user **`jane`** in Namespace `team` only - with a **RoleBinding** `jane-reads-pods`, not a ClusterRoleBinding. `jane` can then list Pods in `team`, and nowhere else.""", check=CAN + r""" eq "$(j clusterrole pod-reader '{.rules[0].verbs}')" '["get","list","watch"]' || exit 1 eq "$(j -n team rolebinding jane-reads-pods '{.roleRef.kind}/{.roleRef.name}')" ClusterRole/pod-reader || exit 1 can --as=jane -- list pods -n team && cannot --as=jane -- list pods -n default && cannot --as=jane -- list pods -A""", hint=r"""`kubectl create clusterrole pod-reader --verb=get,list,watch --resource=pods`, then `kubectl -n team create rolebinding jane-reads-pods --clusterrole=pod-reader --user=jane`. Users exist only in certificates and tokens: nothing to create for `jane`.""", show=r"""kubectl create clusterrole pod-reader --verb=get,list,watch --resource=pods kubectl -n team create rolebinding jane-reads-pods --clusterrole=pod-reader --user=jane kubectl auth can-i list pods -n team --as=jane kubectl auth can-i list pods -n default --as=jane"""), step("A token for the ServiceAccount", r"""Issue a token for `deployer` that is valid for **one hour**, and save it in `{WS}/lesson-4.1/deployer.token`. Then use it - as the **only** credential: your kubeconfig carries the admin's client certificate, and the API server checks a certificate before a token, so `kubectl --token=...` alone would still be you. With no kubeconfig, just the server, the CA and the token, `kubectl auth whoami` says `system:serviceaccount:team:deployer`, `get deployments` works in `team` and `get secrets` is forbidden.""", check=r""" t=$(tr -d ' \r\n' < "$WS/lesson-4.1/deployer.token" 2>/dev/null); [ -n "$t" ] || exit 1 srv=$($K config view --minify -o jsonpath='{.clusters[0].cluster.server}') eq "$($K --kubeconfig=/dev/null --server="$srv" --insecure-skip-tls-verify --token="$t" auth whoami -o jsonpath='{.status.userInfo.username}' 2>/dev/null)" system:serviceaccount:team:deployer || exit 1 echo "$t" | cut -d. -f2 | tr '_-' '/+' | python3 -c ' import base64, json, sys s = sys.stdin.read().strip(); c = json.loads(base64.b64decode(s + "=" * (-len(s) % 4))) sys.exit(0 if c["exp"] - c["iat"] == 3600 else 1)'""", hint=r"""`kubectl -n team create token deployer --duration=1h`. Then `kubectl --kubeconfig=/dev/null --server=URL --certificate-authority=/etc/kubernetes/pki/ca.crt --token=TOKEN ...`; the URL is in your kubeconfig. The token is a JWT: its middle part, base64-decoded, holds `sub`, `iat` and `exp`.""", show=r"""kubectl -n team create token deployer --duration=1h > {WS}/lesson-4.1/deployer.token server=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}') as_deployer() { kubectl --kubeconfig=/dev/null --server=$server --certificate-authority=/etc/kubernetes/pki/ca.crt --token=$(cat {WS}/lesson-4.1/deployer.token) "$@"; } as_deployer auth whoami as_deployer -n team get deployments as_deployer -n team get secrets || true"""), step("A Pod without a token", r"""Most Pods never talk to the API server, and do not need a token in their filesystem. Create ServiceAccount `quiet` in `team` with `automountServiceAccountToken: false`, and run Pod `notoken` (`busybox:1.36`, `sleep 86400`) as that ServiceAccount. Inside it, `/var/run/secrets/kubernetes.io/serviceaccount` does not exist.""", check=r""" eq "$(j -n team sa quiet '{.automountServiceAccountToken}')" false && eq "$(j -n team pod notoken '{.spec.serviceAccountName} {.status.phase}')" "quiet Running" || exit 1 ! $K -n team exec notoken -- ls /var/run/secrets/kubernetes.io/serviceaccount >/dev/null 2>&1""", hint=r"""The field sits at the top level of the ServiceAccount (and can be set on a Pod's `spec` too, which wins). The Pod names its ServiceAccount with `spec.serviceAccountName`.""", show=r"""kubectl -n team apply -f - <<'EOF' apiVersion: v1 kind: ServiceAccount metadata: name: quiet automountServiceAccountToken: false --- apiVersion: v1 kind: Pod metadata: name: notoken spec: serviceAccountName: quiet containers: - name: app image: busybox:1.36 command: [sleep, '86400'] EOF kubectl -n team wait --for=condition=Ready pod/notoken --timeout=120s kubectl -n team exec notoken -- ls /var/run/secrets/kubernetes.io/serviceaccount || echo 'no token mounted'"""), step("Everything one subject may do", r"""Write the complete list of what `deployer` may do in `team` to `{WS}/lesson-4.1/deployer-can.txt` - `kubectl auth can-i` prints it with `--list`. It includes what every authenticated subject gets from the built-in ClusterRoles, such as creating `selfsubjectreviews`.""", check=r"""f="$WS/lesson-4.1/deployer-can.txt"; grep -q '^deployments.apps' "$f" 2>/dev/null && grep -q '^pods ' "$f" && grep -qi 'verbs' "$f" """, hint=r"""`kubectl auth can-i --list -n team --as=system:serviceaccount:team:deployer`.""", show=r"""kubectl auth can-i --list -n team --as=system:serviceaccount:team:deployer > {WS}/lesson-4.1/deployer-can.txt cat {WS}/lesson-4.1/deployer-can.txt"""), ], "outro": r"""You gave a ServiceAccount a Role, reused a ClusterRole in one Namespace, issued and used a token, ran a Pod without one, and listed a subject's permissions. **For the exam** - `kubectl create role|clusterrole NAME --verb=... --resource=...` and `kubectl create rolebinding|clusterrolebinding NAME --role|--clusterrole=... --user|--group|--serviceaccount=NS:NAME`. - Resources in a group are written `RESOURCE.GROUP` on the command line (`deployments.apps`) and as `apiGroups: [apps]` in YAML; the core group is `""`. Subresources: `pods/log`, `pods/exec`, `deployments/scale`. - A binding's `roleRef` cannot be changed: delete and re-create. - `--as=system:serviceaccount:NS:NAME` to test a ServiceAccount; `--as=USER --as-group=GROUP` for users. - Users are certificates: a CSR with `CN=user, O=group`, signed via a `CertificateSigningRequest` (`kubectl certificate approve`), then a kubeconfig with `kubectl config set-credentials --embed-certs`. Drill 4.1 has that end to end. Next, prove it against the clock: **drill 4.1**.""", } # ---- 4.2 underlying infrastructure ------------------------------------------------------------ L42 = { "id": 9402, "objective": "4.2", "workers": 1, "title": "Preparing a machine for kubeadm: modules, sysctls, swap, runtime", "intro": r"""Before `kubeadm init` or `join`, each machine needs: ``` kernel modules overlay, br_netfilter /etc/modules-load.d/*.conf (modprobe now) sysctls net.ipv4.ip_forward = 1, net.bridge.bridge-nf-call-iptables = 1 /etc/sysctl.d/*.conf (sysctl --system) swap off, or the kubelet told to tolerate it (failSwapOn: false) container runtime containerd, CRI enabled, its cgroup driver matching the kubelet's (systemd) packages kubeadm, kubelet, kubectl from pkgs.k8s.io for ONE minor, held ports 6443 (API server), 2379-2380 (etcd), 10250 (kubelet), 30000-32767 (NodePorts) ``` A setting made with `sysctl -w` or `modprobe` alone is gone after the next reboot; the files are what make it stick. Worker `{W1}` stands in for the new machine. (The nodes of this simulator are containers: a module cannot really be loaded there, but the files and what you read from the node are real.)""", "setup": r""" mkdir -p "$WS/lesson-4.2" nx "$W1" 'rm -f /etc/modules-load.d/k8s.conf /etc/sysctl.d/k8s.conf' /dev/null | tr -d " " | grep . | sort | tr "\n" " "' /etc/modules-load.d/k8s.conf cat /etc/modules-load.d/k8s.conf EOF"""), step("Kernel parameters, for good", r"""On `{W1}`, put the three parameters Kubernetes needs in `/etc/sysctl.d/k8s.conf` - `net.bridge.bridge-nf-call-iptables = 1`, `net.bridge.bridge-nf-call-ip6tables = 1`, `net.ipv4.ip_forward = 1` - and apply every sysctl file now with `sysctl --system`.""", check=r""" f=$(nx "$W1" 'cat /etc/sysctl.d/k8s.conf 2>/dev/null' /etc/sysctl.d/k8s.conf <<'CONF' net.bridge.bridge-nf-call-iptables = 1 net.bridge.bridge-nf-call-ip6tables = 1 net.ipv4.ip_forward = 1 CONF sysctl --system 2>/dev/null | grep -A3 k8s.conf sysctl net.ipv4.ip_forward EOF"""), step("Swap and the cgroup driver", r"""Two more things kubeadm's preflight checks. Write to `{WS}/lesson-4.2/node.txt`, in this format: ``` swap= failSwapOn= kubelet-cgroup= containerd-systemd-cgroup= ``` The last two must agree, or Pods fail to start.""", check=r""" swap=$(nx "$W1" '[ "$(grep -c . /proc/swaps)" -gt 1 ] && echo on || echo off' /dev/null | grep .)" "$want" """, hint=r"""`cat /proc/swaps` (or `swapon --show`): any line below the header is swap. `grep -E 'failSwapOn|cgroupDriver' /var/lib/kubelet/config.yaml` - a field that is not there has its default (`failSwapOn` is `true`). `grep SystemdCgroup /etc/containerd/config.toml`; no such line means `false`.""", show=r"""ssh {W1} sudo bash <<'EOF' cat /proc/swaps grep -E 'failSwapOn|cgroupDriver' /var/lib/kubelet/config.yaml grep -n SystemdCgroup /etc/containerd/config.toml || echo 'SystemdCgroup not set' EOF swap=$(ssh -n {W1} '[ "$(grep -c . /proc/swaps)" -gt 1 ] && echo on || echo off') fso=$(kubectl get --raw /api/v1/nodes/{W1}/proxy/configz | jq -r .kubeletconfig.failSwapOn) cg=$(kubectl get --raw /api/v1/nodes/{W1}/proxy/configz | jq -r .kubeletconfig.cgroupDriver) sc=$(ssh -n {W1} 'grep -o "SystemdCgroup *= *[a-z]*" /etc/containerd/config.toml | head -1 | tr -d " " | cut -d= -f2') printf 'swap=%s\nfailSwapOn=%s\nkubelet-cgroup=%s\ncontainerd-systemd-cgroup=%s\n' "$swap" "$fso" "$cg" "${sc:-false}" > {WS}/lesson-4.2/node.txt cat {WS}/lesson-4.2/node.txt"""), step("The packages, as kubeadm wants them", r"""On an exam node, kubeadm, kubelet and kubectl come from the repository of **one** minor at `pkgs.k8s.io`, and are **held** so that an `apt upgrade` never moves them by accident. Write to `{WS}/lesson-4.2/packages.txt`: the line of `{W1}`'s Kubernetes repository (from `/etc/apt/sources.list.d/`), then the held packages, one per line, as `apt-mark` prints them.""", check=r""" want=$(nx "$W1" 'cat /etc/apt/sources.list.d/kubernetes.list; apt-mark showhold' /dev/null | tr -d '\r')" "$want" """, hint=r"""`cat /etc/apt/sources.list.d/kubernetes.list` and `apt-mark showhold`. `apt-cache madison kubeadm` lists the versions that repository offers; `dpkg -l | grep kube` what is installed.""", show=r"""ssh -n {W1} 'cat /etc/apt/sources.list.d/kubernetes.list; apt-mark showhold' > {WS}/lesson-4.2/packages.txt cat {WS}/lesson-4.2/packages.txt ssh -n {W1} 'dpkg -l | grep -E "kube(adm|let|ctl)"'"""), ], "outro": r"""You set up modules and sysctls that survive a reboot, checked swap and the cgroup drivers, and read how Kubernetes is installed on a node. **For the exam** - The docs page "Installing kubeadm" and "Container Runtimes" have every file to copy - module list, sysctl file, the containerd `SystemdCgroup = true` line, the apt repository and `apt-mark hold`. - `sysctl --system` applies every file under `/etc/sysctl.d`; `sysctl -p FILE` just one. - containerd: `containerd config default > /etc/containerd/config.toml`, set `SystemdCgroup = true`, `systemctl restart containerd`. `crictl info` shows what it runs with. - A machine with a CNI already configured has a file in `/etc/cni/net.d/` - the CNI itself is installed after `kubeadm init`, as a manifest. Next, prove it against the clock: **drill 4.2**.""", } # ---- 4.3 kubeadm clusters --------------------------------------------------------------------- L43 = { "id": 9403, "objective": "4.3", "workers": 1, "title": "kubeadm: the cluster's configuration, joining nodes, certificates, static Pods", "intro": r"""`kubeadm init` builds a control plane from a `ClusterConfiguration` and leaves you everything needed to manage it: ``` /etc/kubernetes/manifests/ static Pods: etcd, kube-apiserver, kube-controller-manager, kube-scheduler /etc/kubernetes/pki/ the cluster CA, etcd CA, front-proxy CA, and the certificates they signed /etc/kubernetes/*.conf kubeconfigs: admin.conf, super-admin.conf, controller-manager.conf, scheduler.conf, kubelet.conf ConfigMap kubeadm-config the ClusterConfiguration, in kube-system ``` Nodes join with a **bootstrap token** and the hash of the CA's public key: `kubeadm token create --print-join-command` prints the whole command. The certificates kubeadm makes last a year - the CA ten years - and `kubeadm certs check-expiration` / `renew` look after them. This cluster was made by kubeadm; its worker is `{W1}`.""", "setup": r""" mkdir -p "$WS/lesson-4.3" nx "$CP" 'openssl x509 -in /etc/kubernetes/pki/apiserver.crt -noout -startdate' "$PRIV/lesson-4.3.before" nx "$W1" 'rm -f /etc/kubernetes/manifests/node-web.yaml' /dev/null | tr -d '\r\n')" "$want" """, hint=r"""`kubectl -n kube-system get cm kubeadm-config -o yaml` - look under `networking:`. `kubeadm config print init-defaults` shows every field such a file can have.""", show=r"""kubectl -n kube-system get cm kubeadm-config -o jsonpath='{.data.ClusterConfiguration}' | grep -A4 networking kubectl -n kube-system get cm kubeadm-config -o jsonpath='{.data.ClusterConfiguration}' | awk '/podSubnet:/ {p=$2} /serviceSubnet:/ {s=$2} END {print p, s}' > {WS}/lesson-4.3/networks.txt cat {WS}/lesson-4.3/networks.txt"""), step("A join command for a new worker", r"""Create a new bootstrap token that expires in **2 hours**, and save the complete `kubeadm join` command for a worker in `{WS}/lesson-4.3/join.sh`, on one line. Run it on the control-plane node.""", check=r""" f="$WS/lesson-4.3/join.sh"; grep -q '^kubeadm join .*--token .*--discovery-token-ca-cert-hash sha256:' "$f" 2>/dev/null || exit 1 id=$(grep -o -- '--token [a-z0-9]*\.[a-z0-9]*' "$f" | awk '{print $2}' | cut -d. -f1); [ -n "$id" ] || exit 1 exp=$(j -n kube-system secret "bootstrap-token-$id" '{.data.expiration}' | base64 -d); [ -n "$exp" ] || exit 1 left=$(( $(date -d "$exp" +%s) - $(date +%s) )); [ "$left" -gt 6600 ] && [ "$left" -le 7300 ]""", hint=r"""`sudo kubeadm token create --ttl 2h --print-join-command`. `kubeadm token list` shows the tokens and when they expire; each is a Secret `bootstrap-token-ID` in `kube-system`.""", show=r"""sudo kubeadm token create --ttl 2h --print-join-command > {WS}/lesson-4.3/join.sh cat {WS}/lesson-4.3/join.sh sudo kubeadm token list"""), step("Renew the API server's certificate", r"""Check the certificates' expiry dates, then **renew** the API server's serving certificate - only that one - and make the API server use it. A running process keeps the certificate it read at start, so the static Pod has to be restarted: move its manifest out of `/etc/kubernetes/manifests`, wait until the container is gone, and move it back. Afterwards `kubeadm certs check-expiration` shows the new date, and the API server presents the new certificate.""", check=r""" b=$(cat "$PRIV/lesson-4.3.before") nb=$(nx "$CP" 'openssl x509 -in /etc/kubernetes/pki/apiserver.crt -noout -startdate' /dev/null | openssl x509 -noout -serial' /dev/null 2>&1 && break; sleep 3; done sudo kubeadm certs check-expiration | grep '^apiserver '"""), step("A static Pod on the worker", r"""Every kubelet runs static Pods, not only the control plane's. On `{W1}`, find the kubelet's `staticPodPath` and put a manifest there for Pod `node-web` (`nginx:1.27`). The API server shows it as a **mirror Pod** named after the node: `node-web-{W1}`. Deleting the mirror Pod with kubectl does nothing lasting - the kubelet puts it back; only removing the file does.""", check=r"""eq "$(j -n default pod "node-web-$W1" '{.status.phase} {.metadata.annotations.kubernetes\.io/config\.source}')" "Running file" """, hint=r"""`grep staticPodPath /var/lib/kubelet/config.yaml` on the node. `kubectl run node-web --image=nginx:1.27 --dry-run=client -o yaml` gives you the manifest; write it with `sudo tee`.""", show=r"""kubectl run node-web --image=nginx:1.27 --dry-run=client -o yaml > {WS}/lesson-4.3/node-web.yaml ssh -n {W1} 'grep staticPodPath /var/lib/kubelet/config.yaml' ssh {W1} 'sudo tee /etc/kubernetes/manifests/node-web.yaml >/dev/null' < {WS}/lesson-4.3/node-web.yaml sleep 15 kubectl get pods -o wide"""), ], "outro": r"""You read the cluster's kubeadm configuration, made a join command, renewed a certificate and made the API server load it, and ran a static Pod on a worker. **For the exam** - `kubeadm init --pod-network-cidr=... [--control-plane-endpoint=...] [--upload-certs]`, then copy `admin.conf` to `~/.kube/config`, then install a CNI. `kubeadm join` on the others. `kubeadm reset` undoes either. - `kubeadm token create --print-join-command`; for a control-plane node add `--certificate-key` from `kubeadm init phase upload-certs --upload-certs`. - `kubeadm certs check-expiration | renew all|apiserver|...`; restart the static Pods afterwards. `kubeadm kubeconfig user --client-name=NAME` writes a kubeconfig for a user. - Static Pods: the kubelet's `staticPodPath` (`/etc/kubernetes/manifests`); mirror Pods are named `POD-NODE`. Next, prove it against the clock: **drill 4.3**.""", } # ---- 4.4 cluster lifecycle: etcd and a real upgrade ---------------------------------------------- _APT = r""" pkgs() { nx "$1" "dpkg-query -W -f='\${Version} ' kubelet kubeadm kubectl" v1.37, apt install kubeadm kubeadm upgrade plan / apply v1.37.x drain, apt install kubelet kubectl, systemctl daemon-reload, restart kubelet, uncordon each worker drain (from the control plane); on the node: repo line, kubeadm, kubeadm upgrade node, kubelet kubectl, restart kubelet; uncordon ``` pkgs.k8s.io has one repository per minor, so the first step on every node is the line in `/etc/apt/sources.list.d/kubernetes.list`. The packages are held: `apt-mark unhold` before, `hold` after. Namespace `life` has ConfigMap `marker`; etcd's certificates are under `/etc/kubernetes/pki/etcd/`.""", "setup": NS + r""" mkns life $K -n life create configmap marker --from-literal=state=before-backup >/dev/null nx "$CP" 'mkdir -p /opt/backup; rm -f /opt/backup/*.db' /dev/null 2>&1' /dev/null \ && grep -qi 'total key' "$WS/lesson-4.4/snapshot.txt" """, hint=r"""`etcdctl --endpoints=https://127.0.0.1:2379 --cacert=.../ca.crt --cert=.../server.crt --key=.../server.key snapshot save FILE`, as root. Since etcd 3.6, `status` and `restore` are `etcdutl` only.""", show=r"""sudo grep -E -- '--(listen-client-urls|cert-file|key-file|trusted-ca-file)' /etc/kubernetes/manifests/etcd.yaml sudo ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt \ --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key snapshot save /opt/backup/etcd.db sudo etcdutl snapshot status /opt/backup/etcd.db -w table | tee {WS}/lesson-4.4/snapshot.txt"""), step("Lose something", r"""Now the mistake a backup is for: delete ConfigMap `marker` in `life`.""", check=r"""[ -z "$(j -n life cm marker '{.metadata.name}')" ] && nx "$CP" 'test -s /opt/backup/etcd.db' /dev/null 2>&1 && break; sleep 3; done kubectl -n life get configmap marker -o yaml | grep state"""), step("Plan the upgrade", r"""On the control-plane node, point the Kubernetes repository at **v1.37**, install the **newest 1.37 kubeadm** the repository offers (unhold it first, hold it again after), and save the output of `kubeadm upgrade plan` in `{WS}/lesson-4.4/plan.txt`. The plan shows two choices: the newest 1.36 patch, and the 1.37 release your kubeadm can go to.""", check=_APT + r""" t=$(newest); [ -n "$t" ] || exit 1 eq "$(ver "$CP" kubeadm)" "$t" && held "$CP" && grep -q "kubeadm upgrade apply v${t%%-*}" "$WS/lesson-4.4/plan.txt" 2>/dev/null""", hint=r"""`sudo sed -i 's|/v1.36/|/v1.37/|' /etc/apt/sources.list.d/kubernetes.list`, `sudo apt-get update`, `apt-cache madison kubeadm` (newest first), `sudo apt-mark unhold kubeadm && sudo apt-get install -y kubeadm=1.37.X-1.1 && sudo apt-mark hold kubeadm`.""", show=r"""sudo sed -i 's|/v1.36/|/v1.37/|' /etc/apt/sources.list.d/kubernetes.list sudo apt-get update -qq apt-cache madison kubeadm | head -3 v=$(apt-cache madison kubeadm | awk '$3 ~ /^1\.37\./ {print $3; exit}') sudo apt-mark unhold kubeadm && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq kubeadm=$v && sudo apt-mark hold kubeadm kubeadm version -o short sudo kubeadm upgrade plan | tee {WS}/lesson-4.4/plan.txt | tail -12"""), step("Upgrade the control plane", r"""Apply the upgrade to that 1.37 release (`kubeadm upgrade apply`, a few minutes - it upgrades etcd, the static Pods, CoreDNS and kube-proxy). Then drain the node, upgrade `kubelet` and `kubectl` to the same version, reload systemd, restart the kubelet, and uncordon. `kubectl get nodes` shows the kubelet's version: the control-plane node says v1.37 only after its kubelet has been upgraded and restarted.""", check=_APT + r""" t=$(newest); v="v${t%%-*}" eq "$($K version -o json 2>/dev/null | python3 -c 'import json,sys; print(json.load(sys.stdin)["serverVersion"]["gitVersion"])')" "$v" || exit 1 eq "$(j node "$CP" '{.status.nodeInfo.kubeletVersion}')" "$v" && eq "$(pkgs "$CP")" "$t $t $t " && held "$CP" \ && [ "$(j node "$CP" '{.spec.unschedulable}')" != true ]""", hint=r"""`sudo kubeadm upgrade apply v1.37.X -y`; `kubectl drain {CP} --ignore-daemonsets`; `sudo apt-mark unhold kubelet kubectl && sudo apt-get install -y kubelet=... kubectl=... && sudo apt-mark hold kubelet kubectl`; `sudo systemctl daemon-reload && sudo systemctl restart kubelet`; `kubectl uncordon {CP}`.""", show=r"""v=$(apt-cache madison kubeadm | awk '$3 ~ /^1\.37\./ {print $3; exit}') sudo kubeadm upgrade apply v${v%%-*} -y | tail -3 kubectl drain {CP} --ignore-daemonsets --delete-emptydir-data sudo apt-mark unhold kubelet kubectl && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq kubelet=$v kubectl=$v && sudo apt-mark hold kubelet kubectl sudo systemctl daemon-reload && sudo systemctl restart kubelet for i in $(seq 1 30); do kubectl uncordon {CP} >/dev/null 2>&1 && break; sleep 3; done kubectl get nodes"""), step("Upgrade the worker", r"""Now `{W1}`: drain it from the control-plane node; on `{W1}`, the repository line, kubeadm, **`kubeadm upgrade node`** (a worker has no control-plane Pods to replace - it only refreshes the kubelet's configuration), kubelet and kubectl, daemon-reload, restart; then uncordon it. Both nodes then report the new version, and every package is held again.""", check=_APT + r""" t=$(newest); v="v${t%%-*}" eq "$(j node "$W1" '{.status.nodeInfo.kubeletVersion}')" "$v" && eq "$(pkgs "$W1")" "$t $t $t " && held "$W1" \ && [ "$(j node "$W1" '{.spec.unschedulable}')" != true ] && eq "$(j node "$W1" '{.status.conditions[?(@.type=="Ready")].status}')" True""", hint=r"""The same packages, from the same repository line, as on the control plane - only `kubeadm upgrade node` instead of `apply`. Drain and uncordon are kubectl commands: run them on the control-plane node.""", show=r"""v=$(apt-cache madison kubeadm | awk '$3 ~ /^1\.37\./ {print $3; exit}') kubectl drain {W1} --ignore-daemonsets --delete-emptydir-data ssh {W1} sudo bash -s $v <<'EOF' sed -i 's|/v1.36/|/v1.37/|' /etc/apt/sources.list.d/kubernetes.list apt-get update -qq apt-mark unhold kubeadm kubelet kubectl >/dev/null DEBIAN_FRONTEND=noninteractive apt-get install -y -qq kubeadm=$1 kubeadm upgrade node | tail -2 DEBIAN_FRONTEND=noninteractive apt-get install -y -qq kubelet=$1 kubectl=$1 apt-mark hold kubeadm kubelet kubectl systemctl daemon-reload && systemctl restart kubelet EOF kubectl uncordon {W1} kubectl get nodes"""), ], "outro": r"""You backed etcd up, restored it after a loss, and took a whole cluster from v1.36 to v1.37: control plane first, then the worker, with the packages held at every step. **For the exam** - etcd: `etcdctl snapshot save` (with `--endpoints --cacert --cert --key`), `etcdutl snapshot status`, `etcdutl snapshot restore --data-dir NEWDIR`, then the `etcd-data` `hostPath` in `etcd.yaml` - or restore in place, as here. - Upgrade one minor at a time; kubeadm before `upgrade apply`, kubelet after it. `kubeadm upgrade plan` names the versions; take the exact patch the task asks for, or the newest. - Workers: `kubeadm upgrade node`, never `apply`. Drain and uncordon from where kubectl works. - Version skew: kubelets may be up to three minors behind the API server, never ahead. - `apt-mark unhold` → `apt-get install PKG=VERSION` → `apt-mark hold`; `systemctl daemon-reload` before restarting the kubelet. Next, prove it against the clock: **drill 4.4**.""", } # ---- 4.5 highly available control plane -------------------------------------------------------- L45 = { "id": 9405, "objective": "4.5", "workers": 0, "title": "A highly available control plane: endpoint, etcd members, leaders, joining", "intro": r"""An HA control plane is three (or five) control-plane nodes behind one address: ``` controlPlaneEndpoint one stable address - a load balancer or a VIP - for every kubelet and kubectl etcd stacked: a member on each control-plane node external: hosts of its own kube-apiserver active on every node at once, behind the endpoint kube-scheduler, kube-controller-manager one leader at a time, chosen through a Lease in kube-system ``` etcd needs a majority (quorum): three members survive one failure, five survive two. A second control-plane node joins with the certificates the first uploads (`kubeadm init phase upload-certs --upload-certs`, valid two hours) and `kubeadm join ... --control-plane --certificate-key KEY`. This cluster has one control-plane node, `{CP}`, made with a `controlPlaneEndpoint` so that more can join.""", "setup": r""" mkdir -p "$WS/lesson-4.5" $K -n kube-system delete secret kubeadm-certs --ignore-not-found >/dev/null 2>&1 """, "steps": [ step("The one stable address", r"""Write the cluster's control-plane endpoint, as kubeadm has it, to `{WS}/lesson-4.5/endpoint.txt`. It is also the `server:` of every kubeconfig on the nodes.""", check=r""" want=$($K -n kube-system get cm kubeadm-config -o jsonpath='{.data.ClusterConfiguration}' | awk '/controlPlaneEndpoint:/ {print $2}') [ -n "$want" ] && eq "$(tr -d ' \r\n' < "$WS/lesson-4.5/endpoint.txt" 2>/dev/null)" "$want" """, hint=r"""`controlPlaneEndpoint:` in the `ClusterConfiguration` of ConfigMap `kubeadm-config`.""", show=r"""kubectl -n kube-system get cm kubeadm-config -o jsonpath='{.data.ClusterConfiguration}' | awk '/controlPlaneEndpoint:/ {print $2}' > {WS}/lesson-4.5/endpoint.txt cat {WS}/lesson-4.5/endpoint.txt sudo grep server: /etc/kubernetes/kubelet.conf"""), step("etcd's members and its leader", r"""Ask etcd itself. Write to `{WS}/lesson-4.5/etcd.txt`: ``` members= leader= ```""", check=r""" out=$(nx "$CP" '""" + _ETCDCTL + r""" endpoint status -w json' /dev/null | grep .)" "$(printf 'members=%s\nleader=%s' "$n" "$lead")" """, hint=r"""`etcdctl member list -w table` and `etcdctl endpoint status -w table` (the IS LEADER column), with the endpoint and certificates as for a snapshot. `export ETCDCTL_ENDPOINTS=... ETCDCTL_CACERT=...` saves typing.""", show=r"""sudo bash <<'EOF' export ETCDCTL_API=3 ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCDCTL_CACERT=/etc/kubernetes/pki/etcd/ca.crt \ ETCDCTL_CERT=/etc/kubernetes/pki/etcd/server.crt ETCDCTL_KEY=/etc/kubernetes/pki/etcd/server.key etcdctl member list -w table etcdctl endpoint status -w table EOF printf 'members=1\nleader={CP}\n' > {WS}/lesson-4.5/etcd.txt"""), step("Who leads the scheduler and the controller manager", r"""Only one kube-scheduler and one kube-controller-manager act at a time; the others wait for the **Lease** in `kube-system` to expire. Write the node of each holder to `{WS}/lesson-4.5/leaders.txt`: ``` scheduler= controller-manager= ```""", check=r""" s=$(j -n kube-system lease kube-scheduler '{.spec.holderIdentity}'); c=$(j -n kube-system lease kube-controller-manager '{.spec.holderIdentity}') eq "$(tr -d ' \r' < "$WS/lesson-4.5/leaders.txt" 2>/dev/null | grep .)" "$(printf 'scheduler=%s\ncontroller-manager=%s' "${s%%_*}" "${c%%_*}")" """, hint=r"""`kubectl -n kube-system get lease kube-scheduler kube-controller-manager` - HOLDER is `NODE_UUID`.""", show=r"""kubectl -n kube-system get lease kube-scheduler kube-controller-manager s=$(kubectl -n kube-system get lease kube-scheduler -o jsonpath='{.spec.holderIdentity}') c=$(kubectl -n kube-system get lease kube-controller-manager -o jsonpath='{.spec.holderIdentity}') printf 'scheduler=%s\ncontroller-manager=%s\n' "${s%%_*}" "${c%%_*}" > {WS}/lesson-4.5/leaders.txt"""), step("What a second control-plane node needs", r"""Upload the control-plane certificates for a joining node, and save the complete `kubeadm join` command a new **control-plane** node would run - new token, CA hash, `--control-plane` and the certificate key - to `{WS}/lesson-4.5/join-cp.sh`, on one line. Do not run it anywhere.""", check=r""" f="$WS/lesson-4.5/join-cp.sh"; grep -q '^kubeadm join .*--token .*--discovery-token-ca-cert-hash sha256:.*--control-plane' "$f" 2>/dev/null || exit 1 key=$(grep -o -- '--certificate-key [0-9a-f]\{64\}' "$f" | awk '{print $2}'); [ -n "$key" ] && [ -n "$(j -n kube-system secret kubeadm-certs '{.metadata.name}')" ]""", hint=r"""`sudo kubeadm init phase upload-certs --upload-certs` prints the key as its last line; `sudo kubeadm token create --print-join-command --certificate-key KEY` adds `--control-plane --certificate-key` itself.""", show=r"""key=$(sudo kubeadm init phase upload-certs --upload-certs 2>/dev/null | tail -1) sudo kubeadm token create --print-join-command --certificate-key $key > {WS}/lesson-4.5/join-cp.sh cat {WS}/lesson-4.5/join-cp.sh kubectl -n kube-system get secret kubeadm-certs"""), ], "outro": r"""You read the endpoint that makes HA possible, asked etcd for its members and leader, found the leaders of the scheduler and controller manager, and prepared a control-plane join. **For the exam** - HA starts at `kubeadm init --control-plane-endpoint LB:6443 --upload-certs`; without an endpoint, adding control-plane nodes later means re-issuing certificates. - Stacked etcd: an etcd static Pod on each control-plane node. External: `etcd.external.endpoints` in the ClusterConfiguration, and the API server's `--etcd-servers` names other hosts. - Quorum is a majority of members: an even number adds no tolerance. - The uploaded certificates expire after two hours; re-run `upload-certs` for a fresh key. Next, prove it against the clock: **drill 4.5**.""", } # ---- 4.6 Helm and Kustomize ------------------------------------------------------------------- _CHART = r""" c="$WS/lesson-4.6/chart"; rm -rf "$WS/lesson-4.6"; mkdir -p "$c/templates" "$WS/lesson-4.6/kustomize/base" cat > "$c/Chart.yaml" <<'EOF' apiVersion: v2 name: shop description: A small web shop type: application version: 0.1.0 appVersion: "1.27" EOF cat > "$c/values.yaml" <<'EOF' replicaCount: 1 image: repository: nginx tag: "1.27" service: type: ClusterIP port: 80 EOF cat > "$c/templates/deployment.yaml" <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: name: {{ .Release.Name }}-shop labels: {app: {{ .Release.Name }}-shop} spec: replicas: {{ .Values.replicaCount }} selector: matchLabels: {app: {{ .Release.Name }}-shop} template: metadata: labels: {app: {{ .Release.Name }}-shop} spec: containers: - name: shop image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" ports: [{containerPort: 80}] EOF cat > "$c/templates/service.yaml" <<'EOF' apiVersion: v1 kind: Service metadata: name: {{ .Release.Name }}-shop spec: type: {{ .Values.service.type }} selector: {app: {{ .Release.Name }}-shop} ports: [{port: {{ .Values.service.port }}, targetPort: 80}] EOF b="$WS/lesson-4.6/kustomize/base" cat > "$b/deployment.yaml" <<'EOF' apiVersion: apps/v1 kind: Deployment metadata: name: api spec: replicas: 1 selector: matchLabels: {app: api} template: metadata: labels: {app: api} spec: containers: - name: api image: nginx:1.26 EOF cat > "$b/kustomization.yaml" <<'EOF' resources: - deployment.yaml EOF helm -n apps uninstall web >/dev/null 2>&1 || true """ L46 = { "id": 9406, "objective": "4.6", "workers": 1, "title": "Helm releases and Kustomize overlays", "intro": r"""Two ways to install cluster components from more than a single manifest: ``` Helm a chart = templates + values.yaml helm install | upgrade | rollback | uninstall a release = one installed copy, with numbered revisions (stored as Secrets in its Namespace) Kustomize a base + overlays that change it, no templates kubectl apply -k DIR / kubectl kustomize DIR ``` With Helm you change **values** (`--set`, `-f values.yaml`), never the templates; `helm template` renders without installing. With Kustomize you never edit the base; an overlay sets the Namespace, prefixes, images, replicas, labels, and patches the rest. A chart is at `{WS}/lesson-4.6/chart` (values `replicaCount`, `image.tag`, `service.type`, `service.port`); a Kustomize base is at `{WS}/lesson-4.6/kustomize/base`.""", "setup": _CHART, "steps": [ step("Render before you install", r"""Render the chart as release `web` with `replicaCount=2`, without installing anything, and save the manifests in `{WS}/lesson-4.6/rendered.yaml`. That is how you check what a set of values does.""", check=r"""f="$WS/lesson-4.6/rendered.yaml"; grep -q 'name: web-shop' "$f" 2>/dev/null && grep -q 'replicas: 2' "$f" && [ -z "$(j -n apps deploy web-shop '{.metadata.name}')" ]""", hint=r"""`helm template RELEASE CHART --set KEY=VALUE`.""", show=r"""helm template web {WS}/lesson-4.6/chart --set replicaCount=2 > {WS}/lesson-4.6/rendered.yaml grep -E 'kind:|replicas:|image:' {WS}/lesson-4.6/rendered.yaml"""), step("Install a release", r"""Install the chart as release `web` in Namespace `apps` (create it), with `replicaCount=2`.""", check=r"""eq "$(helm -n apps status web -o json 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d["info"]["status"], d["version"])')" "deployed 1" && eq "$(j -n apps deploy web-shop '{.status.readyReplicas}')" 2""", hint=r"""`helm install web CHART -n apps --create-namespace --set replicaCount=2`; `helm list -n apps`.""", show=r"""helm install web {WS}/lesson-4.6/chart -n apps --create-namespace --set replicaCount=2 helm list -n apps kubectl -n apps rollout status deploy web-shop --timeout=120s"""), step("Upgrade it, keeping what you set", r"""Upgrade release `web` so that its Service becomes a `NodePort` - and keep `replicaCount=2`. A plain `helm upgrade --set service.type=NodePort` would quietly put the replica count back to the chart's default.""", check=r"""eq "$(j -n apps svc web-shop '{.spec.type}')" NodePort && eq "$(j -n apps deploy web-shop '{.spec.replicas}')" 2 \ && [ "$(helm -n apps history web -o json 2>/dev/null | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))')" -ge 2 ]""", hint=r"""`--reuse-values` keeps the values of the last release and adds yours; `helm get values web -n apps` shows what is set.""", show=r"""helm upgrade web {WS}/lesson-4.6/chart -n apps --reuse-values --set service.type=NodePort helm get values web -n apps kubectl -n apps get svc web-shop"""), step("Roll the release back", r"""The NodePort was a mistake: roll release `web` back to **revision 1**. A rollback is itself a new revision.""", check=r""" eq "$(j -n apps svc web-shop '{.spec.type}')" ClusterIP || exit 1 helm -n apps history web -o json 2>/dev/null | python3 -c ' import json, sys; h = json.load(sys.stdin) sys.exit(0 if len(h) >= 3 and h[-1]["status"] == "deployed" and "Rollback to 1" in h[-1]["description"] else 1)'""", hint=r"""`helm rollback web 1 -n apps`, then `helm history web -n apps`.""", show=r"""helm rollback web 1 -n apps helm history web -n apps kubectl -n apps get svc web-shop"""), step("An overlay for production", r"""Create a Kustomize overlay at `{WS}/lesson-4.6/kustomize/prod` on top of the base, which: puts everything in Namespace `prod`, prefixes names with `prod-`, runs **3** replicas, and uses image `nginx:1.27`. Apply it. Leave the base as it is.""", check=r"""eq "$(j -n prod deploy prod-api '{.spec.replicas} {.spec.template.spec.containers[0].image} {.status.readyReplicas}')" "3 nginx:1.27 3" \ && grep -q 'image: nginx:1.26' "$WS/lesson-4.6/kustomize/base/deployment.yaml" """, hint=r"""A `kustomization.yaml` with `resources: [../base]`, `namespace: prod`, `namePrefix: prod-`, `replicas: [{name: api, count: 3}]`, `images: [{name: nginx, newTag: "1.27"}]`. The Namespace itself must exist (or be one of the resources). `kubectl kustomize DIR` shows the result first.""", show=r"""mkdir -p {WS}/lesson-4.6/kustomize/prod cat > {WS}/lesson-4.6/kustomize/prod/kustomization.yaml <<'EOF' resources: - ../base namespace: prod namePrefix: prod- replicas: - name: api count: 3 images: - name: nginx newTag: "1.27" EOF kubectl kustomize {WS}/lesson-4.6/kustomize/prod kubectl create namespace prod kubectl apply -k {WS}/lesson-4.6/kustomize/prod kubectl -n prod rollout status deploy prod-api --timeout=120s"""), ], "outro": r"""You rendered a chart, installed, upgraded and rolled back a release, and built a production overlay with Kustomize without touching its base. **For the exam** - `helm repo add NAME URL && helm repo update`, `helm search repo KEYWORD`, `helm show values CHART`, `helm pull CHART --untar`. - `helm install|upgrade RELEASE CHART -n NS [--create-namespace] [--set k=v] [-f values.yaml] [--version X]`; `helm upgrade --install` for either. `--reuse-values` on an upgrade, or you lose earlier `--set`s. - `helm list -A`, `helm history`, `helm rollback RELEASE REVISION`, `helm get values|manifest`, `helm uninstall`. - Kustomize: `resources`, `namespace`, `namePrefix`/`nameSuffix`, `labels`, `images`, `replicas`, `configMapGenerator`/`secretGenerator`, `patches` (strategic merge or JSON 6902). `kubectl apply -k`, `kubectl delete -k`. Next, prove it against the clock: **drill 4.6**.""", } # ---- 4.7 extension interfaces ----------------------------------------------------------------- L47 = { "id": 9407, "objective": "4.7", "workers": 1, "title": "CNI, CRI and CSI: what plugs into this cluster", "intro": r"""Kubernetes leaves three jobs to plugins, each behind an interface: ``` CRI Container Runtime Interface the kubelet asks the runtime (containerd, CRI-O) over a socket crictl CNI Container Network Interface the runtime calls the plugins in /etc/cni/net.d for each Pod sandbox CSI Container Storage Interface a driver (Deployment + DaemonSet) makes and mounts volumes CSIDriver, CSINode ``` A **RuntimeClass** names a handler the runtime has configured - `runc`, or a sandbox such as `runsc` (gVisor) - and a Pod picks it with `runtimeClassName`. This cluster runs containerd, kindnet as its CNI, and the local-path provisioner for storage. Its worker is `{W1}`.""", "setup": NS + r""" mkns plug mkdir -p "$WS/lesson-4.7" $K -n plug run probe --image=busybox:1.36 -- sleep 86400 >/dev/null $K -n plug wait --for=condition=Ready pod/probe --timeout=180s >/dev/null """, "steps": [ step("The CNI chain on a node", r"""On `{W1}`, find the CNI configuration (the runtime uses the first file in `/etc/cni/net.d`, by name) and write to `{WS}/lesson-4.7/cni.txt`: ``` config= plugins= ```""", check=r""" f=$(nx "$W1" 'ls /etc/cni/net.d/*.conf* 2>/dev/null | sort | head -1' /dev/null | grep .)" "$(printf 'config=%s\nplugins=%s' "$f" "$p")" """, hint=r"""`ls /etc/cni/net.d/` and `cat` the file: a `.conflist` has a `"plugins"` list, each with a `"type"` - the binary under `/opt/cni/bin` that is run.""", show=r"""ssh -n {W1} 'ls /etc/cni/net.d/; cat /etc/cni/net.d/10-kindnet.conflist; ls /opt/cni/bin' f=$(ssh -n {W1} 'ls /etc/cni/net.d/*.conf* | sort | head -1') p=$(ssh -n {W1} "cat $f" | jq -r '[.plugins[].type] | join(",")') printf 'config=%s\nplugins=%s\n' "$f" "$p" > {WS}/lesson-4.7/cni.txt cat {WS}/lesson-4.7/cni.txt"""), step("The runtime, through CRI", r"""Write to `{WS}/lesson-4.7/cri.txt`: ``` endpoint= runtime= ```""", check=r""" e=$($K get --raw "/api/v1/nodes/$W1/proxy/configz" | python3 -c 'import json,sys; print(json.load(sys.stdin)["kubeletconfig"]["containerRuntimeEndpoint"])') r=$(j node "$W1" '{.status.nodeInfo.containerRuntimeVersion}') eq "$(tr -d ' \r' < "$WS/lesson-4.7/cri.txt" 2>/dev/null | grep .)" "$(printf 'endpoint=%s\nruntime=%s' "$e" "$r")" """, hint=r"""`containerRuntimeEndpoint` in the kubelet's configuration (`/var/lib/kubelet/config.yaml`, or `configz`); `kubectl get node -o wide` shows CONTAINER-RUNTIME.""", show=r"""e=$(kubectl get --raw /api/v1/nodes/{W1}/proxy/configz | jq -r .kubeletconfig.containerRuntimeEndpoint) r=$(kubectl get node {W1} -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}') printf 'endpoint=%s\nruntime=%s\n' "$e" "$r" > {WS}/lesson-4.7/cri.txt cat {WS}/lesson-4.7/cri.txt"""), step("A Pod's container, from the runtime's side", r"""Pod `probe` in `plug` runs on `{W1}`. Ask the runtime about it there, with `crictl`: find its container, and write the container's **process ID on the node** to `{WS}/lesson-4.7/pid.txt`.""", check=r""" want=$(nx "$W1" 'crictl inspect $(crictl ps --name probe -q | head -1) 2>/dev/null' /dev/null)" "$want" """, hint=r"""`sudo crictl pods --name probe`, `sudo crictl ps --name probe`, `sudo crictl inspect ID` - the PID is under `info.pid`. `crictl` talks to the same socket as the kubelet.""", show=r"""ssh -n {W1} 'sudo crictl pods --name probe; sudo crictl ps --name probe' ssh -n {W1} 'sudo crictl inspect $(sudo crictl ps --name probe -q | head -1)' | jq .info.pid > {WS}/lesson-4.7/pid.txt cat {WS}/lesson-4.7/pid.txt"""), step("A RuntimeClass", r"""Create RuntimeClass `plain` for containerd's handler `runc`, and run Pod `classy` in `plug` (`busybox:1.36`, `sleep 86400`) with that runtime class. A Pod that names a handler the node does not have stays in `ContainerCreating`.""", check=r"""eq "$(j runtimeclass plain '{.handler}')" runc && eq "$(j -n plug pod classy '{.spec.runtimeClassName} {.status.phase}')" "plain Running" """, hint=r"""A RuntimeClass is `node.k8s.io/v1` with a top-level `handler:` and no `spec`. The Pod: `spec.runtimeClassName: plain`.""", show=r"""kubectl apply -f - <<'EOF' apiVersion: node.k8s.io/v1 kind: RuntimeClass metadata: name: plain handler: runc EOF kubectl -n plug run classy --image=busybox:1.36 --overrides='{"spec":{"runtimeClassName":"plain"}}' -- sleep 86400 kubectl -n plug wait --for=condition=Ready pod/classy --timeout=120s kubectl get runtimeclass"""), step("Storage drivers", r"""Write to `{WS}/lesson-4.7/csi.txt`: ``` csidrivers= provisioner= ``` A CSI driver registers a `CSIDriver` object and appears in each node's `CSINode`. The local-path provisioner is not one - it is an external provisioner of its own.""", check=r""" n=$($K get csidrivers --no-headers 2>/dev/null | grep -c .) p=$($K get sc -o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.provisioner}{end}') eq "$(tr -d ' \r' < "$WS/lesson-4.7/csi.txt" 2>/dev/null | grep .)" "$(printf 'csidrivers=%s\nprovisioner=%s' "$n" "$p")" """, hint=r"""`kubectl get csidrivers`, `kubectl get csinodes`, `kubectl get sc`.""", show=r"""kubectl get csidrivers kubectl get csinodes kubectl get sc printf 'csidrivers=%s\nprovisioner=%s\n' "$(kubectl get csidrivers --no-headers 2>/dev/null | grep -c .)" "$(kubectl get sc standard -o jsonpath='{.provisioner}')" > {WS}/lesson-4.7/csi.txt cat {WS}/lesson-4.7/csi.txt"""), ], "outro": r"""You read the CNI chain on a node, the runtime the kubelet talks to, a container as the runtime sees it, ran a Pod with a RuntimeClass, and counted the storage drivers. **For the exam** - CNI: `/etc/cni/net.d/` (config), `/opt/cni/bin/` (plugins); the CNI itself (Calico, Cilium, Flannel, ...) is installed as a manifest or a Helm chart after `kubeadm init`, and its Pods run in `kube-system` or a Namespace of its own. - CRI: `crictl ps|pods|images|inspect|logs|exec`; the endpoint is in `/etc/crictl.yaml` and in the kubelet's `containerRuntimeEndpoint`. - RuntimeClass `handler` must match a runtime configured in containerd (`[plugins..."cri".containerd.runtimes.NAME]`). - CSI: `kubectl get csidrivers,csinodes`; a StorageClass names the driver as its `provisioner`. Next, prove it against the clock: **drill 4.7**.""", } # ---- 4.8 CRDs and operators -------------------------------------------------------------------- _OPERATOR = r""" o="$WS/lesson-4.8/operator"; rm -rf "$WS/lesson-4.8"; mkdir -p "$o" cat > "$o/operator.yaml" <<'EOF' """ + backup_operator() + r"""EOF $K delete crd backups.data.example.com --ignore-not-found >/dev/null 2>&1 """ L48 = { "id": 9408, "objective": "4.8", "workers": 1, "title": "CustomResourceDefinitions, and an operator that acts on them", "intro": r"""A **CustomResourceDefinition** adds a kind to the API: once it is `Established`, you create, list and watch its objects like any other, and its OpenAPI schema validates them and fills in defaults. ``` CRD name PLURAL.GROUP backups.data.example.com names kind, plural, singular, shortNames scope Namespaced | Cluster versions each served, exactly one storage: true, each with a schema (and subresources: status) ``` An **operator** is a CRD plus a controller - a Deployment with RBAC - that watches those objects and makes the world match them. Installing one is applying its manifests or its Helm chart; using it is creating custom resources. The manifests of a small backup operator are in `{WS}/lesson-4.8/operator/`. It expects a `Backup` kind that you will define.""", "setup": NS + _OPERATOR + r""" mkns data """, "steps": [ step("Define the kind", r"""Create CRD `backups.data.example.com`: group `data.example.com`, kind `Backup` (plural `backups`, singular `backup`, short name `bk`), **Namespaced**, one version `v1` (served, storage) with: - `spec.schedule`: string, **required** - `spec.retention`: integer, at least `1`, **default `7`** - a `status` subresource, with `status.phase` a string""", check=PY + r""" c=backups.data.example.com eq "$(j crd $c '{.spec.scope} {.spec.names.kind} {.spec.names.shortNames[0]}')" "Namespaced Backup bk" || exit 1 py crd $c 's = o["spec"]["versions"][0]; p = s["schema"]["openAPIV3Schema"]["properties"]["spec"] s["served"] and s["storage"] and "status" in s.get("subresources", {}) and "schedule" in p.get("required", []) and p["properties"]["retention"].get("default") == 7 and p["properties"]["retention"].get("minimum") == 1' \ && eq "$(j crd $c '{.status.conditions[?(@.type=="Established")].status}')" True""", hint=r"""The docs page "Extend the Kubernetes API with CustomResourceDefinitions" has the skeleton. `required:` sits beside `properties:`, at the level of `spec`'s schema; `default:` and `minimum:` inside `retention`. `subresources: {status: {}}` goes under the version.""", show=r"""kubectl apply -f - <<'EOF' apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: backups.data.example.com spec: group: data.example.com scope: Namespaced names: {kind: Backup, plural: backups, singular: backup, shortNames: [bk]} versions: - name: v1 served: true storage: true subresources: status: {} schema: openAPIV3Schema: type: object properties: spec: type: object required: [schedule] properties: schedule: {type: string} retention: {type: integer, minimum: 1, default: 7} status: type: object properties: phase: {type: string} EOF kubectl wait --for condition=established crd/backups.data.example.com kubectl api-resources --api-group=data.example.com"""), step("Defaults and validation", r"""In Namespace `data`, create Backup `nightly` with `schedule: "0 2 * * *"` and no retention: the API server fills in `7`. Then try a Backup `broken` with `retention: 0` - it is refused. Save the refusal (kubectl writes it to **stderr**) in `{WS}/lesson-4.8/refused.txt`.""", check=r""" eq "$(j -n data bk nightly '{.spec.schedule} {.spec.retention}')" "0 2 * * * 7" && [ -z "$(j -n data bk broken '{.metadata.name}')" ] \ && grep -q 'retention' "$WS/lesson-4.8/refused.txt" 2>/dev/null && grep -qi 'greater than or equal to 1' "$WS/lesson-4.8/refused.txt" """, hint=r"""`apiVersion: data.example.com/v1`, `kind: Backup`. Redirect with `2>` - with `>` the file stays empty.""", show=r"""kubectl -n data apply -f - <<'EOF' apiVersion: data.example.com/v1 kind: Backup metadata: name: nightly spec: schedule: "0 2 * * *" EOF kubectl -n data get bk nightly -o jsonpath='{.spec}'; echo printf 'apiVersion: data.example.com/v1\nkind: Backup\nmetadata: {name: broken}\nspec: {schedule: "@hourly", retention: 0}\n' | kubectl -n data apply -f - 2> {WS}/lesson-4.8/refused.txt cat {WS}/lesson-4.8/refused.txt"""), step("What the schema says", r"""`kubectl explain` reads the CRD's schema like any built-in kind's. Save the output of `kubectl explain backup.spec` in `{WS}/lesson-4.8/explain.txt`.""", check=r"""f="$WS/lesson-4.8/explain.txt"; grep -q 'retention' "$f" 2>/dev/null && grep -q 'schedule' "$f" && grep -qi 'required' "$f" """, hint=r"""`kubectl explain backup.spec` - or `--recursive` for the whole tree.""", show=r"""kubectl explain backup.spec | tee {WS}/lesson-4.8/explain.txt"""), step("Install the operator", r"""Install the operator from `{WS}/lesson-4.8/operator/` and wait until its controller runs. Read the manifests first: a Namespace, a ServiceAccount, a ClusterRole for `backups` and ConfigMaps, the binding, and the controller's Deployment.""", check=r"""eq "$(j -n backup-operator deploy controller '{.status.readyReplicas}')" 1 && eq "$(j clusterrolebinding backup-operator '{.subjects[0].name}')" controller""", hint=r"""`kubectl apply -f DIR` applies every file in it; `kubectl -n backup-operator rollout status deploy controller`.""", show=r"""grep -E '^kind:|^ name:' {WS}/lesson-4.8/operator/operator.yaml kubectl apply -f {WS}/lesson-4.8/operator/ kubectl -n backup-operator rollout status deploy controller --timeout=120s"""), step("Let it work", r"""Create Backup `weekly` in `data` with `schedule: "0 3 * * 0"`. Within a few seconds the controller has acted on it - and on `nightly`: each has `status.phase: Scheduled`, and a ConfigMap `backup-NAME` holds its schedule. `kubectl -n backup-operator logs deploy/controller` is where you would look if it did not.""", check=r""" eq "$(j -n data bk weekly '{.status.phase}')" Scheduled && eq "$(j -n data cm backup-weekly '{.data.schedule}')" "0 3 * * 0" && eq "$(j -n data bk nightly '{.status.phase}')" Scheduled""", hint=r"""A Backup like `nightly`. `kubectl -n data get bk` - add `additionalPrinterColumns` to the CRD and the phase would show there too.""", show=r"""kubectl -n data apply -f - <<'EOF' apiVersion: data.example.com/v1 kind: Backup metadata: name: weekly spec: schedule: "0 3 * * 0" EOF sleep 10 kubectl -n data get bk -o custom-columns=NAME:.metadata.name,SCHEDULE:.spec.schedule,PHASE:.status.phase kubectl -n data get cm"""), ], "outro": r"""You defined a kind with validation and defaults, saw the schema refuse bad objects, installed an operator with its RBAC, and watched it act on your custom resources. **For the exam** - `kubectl get crd`, `kubectl api-resources --api-group=GROUP`, `kubectl explain KIND.spec --recursive` - the fastest way to learn an unfamiliar custom resource. - Wait for `Established` before creating objects of a new kind. - Installing an operator: its manifests (`kubectl apply -f URL|DIR`) or chart (`helm install`); then check its CRDs, its controller's Pods and logs, and create the custom resource the task asks for. - Deleting a CRD deletes every object of its kind. Next, prove it against the clock: **drill 4.8**.""", } LESSONS = [L41, L42, L43, L44, L45, L46, L47, L48]