"""CKA pool tasks 125-128: Services & Networking - CoreDNS configuration, the move from Ingress to the Gateway API, IngressClasses, and the nodes' own network (hostPort, hostNetwork). Written for the per-task hosts: a task may rewrite the cluster's CoreDNS configuration. Name resolution and hostPorts are tested for real; Ingress and Gateway objects are graded as objects, as no controller for them runs on the hosts. """ from cka.common import C, DNS, NS, ONE, PODS, PY, ns_pick, name_pick QUESTIONS = [ # ------------------------------------------------------------------ 125 ---- { "id": 125, "domain": "NET", "difficulty": "hard", "title": "Names of your own in cluster DNS", "params": lambda r: {"ns": ns_pick(r, "billing"), "svc": name_pick(r, "invoices"), "host": r.choice(["legacy-db", "mainframe", "ldap", "sap"]), "ip": f"10.{r.randint(120, 200)}.{r.randint(1, 250)}.{r.randint(2, 250)}"}, "task": """ Applications in this cluster still use two names from the old data centre. Teach cluster DNS both of them by changing the CoreDNS configuration: 1. `%{host}.dc.internal` must resolve to `%{ip}`. 2. `%{svc}.dc.internal` must resolve to whatever Service `%{svc}` in Namespace `%{ns}` resolves to - an alias for the Service's name, not a copy of its address. 3. Before you change anything, save the CoreDNS ConfigMap as it is to `{WS}/q125/coredns-backup.yaml`. Names in `cluster.local` must keep resolving, and CoreDNS must be running with 2 replicas when you are done. Pod `probe` in `%{ns}` is there for testing; busybox's `nslookup` wants fully qualified names. """, "setup": PODS + r""" mkns %{ns}; rm -rf "$WS/q125"; mkdir -p "$WS/q125" $K -n %{ns} delete pod,svc --all --wait=true --grace-period=1 >/dev/null 2>&1 srv %{ns} %{svc}; cli %{ns} probe; ready %{ns} || exit 1 $K -n kube-system get cm coredns -o jsonpath='{.data.Corefile}' > "$PRIV/q125.corefile" grep -q kubernetes "$PRIV/q125.corefile" || { echo "could not read the CoreDNS configuration"; exit 1; } """, "checks": [ C("%{host}.dc.internal resolves to %{ip} from a Pod", 3, DNS + r"""resolves %{ns} probe %{host}.dc.internal %{ip}"""), C("%{svc}.dc.internal resolves to the Service, as an alias", 3, DNS + r""" cip=$(j -n %{ns} svc %{svc} '{.spec.clusterIP}'); [ -n "$cip" ] || exit 1 resolves %{ns} probe %{svc}.dc.internal "$cip" || exit 1 ! $K -n kube-system get cm coredns -o jsonpath='{.data.Corefile}' | grep -qF "$cip" """), C("cluster names still resolve, and CoreDNS runs with 2 replicas", 1, DNS + r""" resolves %{ns} probe %{host}.dc.internal %{ip} || exit 1 for i in $(seq 1 20); do eq "$(j -n kube-system deploy coredns '{.status.updatedReplicas}/{.status.availableReplicas}/{.status.replicas}')" 2/2/2 && break; sleep 3; done eq "$(j -n kube-system deploy coredns '{.status.updatedReplicas}/{.status.availableReplicas}/{.status.replicas}')" 2/2/2 || exit 1 resolves %{ns} probe kubernetes.default.svc.cluster.local "$(j -n default svc kubernetes '{.spec.clusterIP}')" """), C("coredns-backup.yaml holds the ConfigMap as it was", 1, r""" ORIG="$PRIV/q125.corefile" python3 - "$WS/q125/coredns-backup.yaml" <<'PY' import os, sys try: backup = open(sys.argv[1]).read() except OSError: sys.exit(1) lines = [l.strip() for l in open(os.environ["ORIG"]) if l.strip()] ok = "ConfigMap" in backup and "coredns" in backup and "dc.internal" not in backup and all(l in backup for l in lines) sys.exit(0 if ok else 1) PY """), ], "solution": """ k -n kube-system get cm coredns -o yaml > {WS}/q125/coredns-backup.yaml k -n kube-system edit cm coredns # inside the .:53 { ... } block, e.g. above "kubernetes ...": # hosts { # %{ip} %{host}.dc.internal # fallthrough # } # rewrite name %{svc}.dc.internal %{svc}.%{ns}.svc.cluster.local k -n kube-system rollout restart deploy coredns # the reload plugin would pick it up too, in a minute or two k -n kube-system rollout status deploy coredns k -n %{ns} exec probe -- nslookup %{host}.dc.internal # %{ip} k -n %{ns} exec probe -- nslookup %{svc}.dc.internal # the Service's ClusterIP k -n %{ns} exec probe -- nslookup kubernetes.default.svc.cluster.local k -n kube-system logs -l k8s-app=kube-dns --tail=5 # a syntax error in the Corefile shows up here # hosts answers from a static list; without `fallthrough` every other name would end there with # NXDOMAIN. rewrite renames the question before the kubernetes plugin sees it, so the alias follows # the Service even if its ClusterIP changes. Back the ConfigMap up first: a Corefile that does not # parse takes cluster DNS down with it - `k apply -f` the backup is the way back. """, }, # ------------------------------------------------------------------ 126 ---- { "id": 126, "domain": "NET", "difficulty": "hard", "title": "From Ingress to the Gateway API", "params": lambda r: (lambda host: {"ns": ns_pick(r, "storefront"), "host": host, "gc": name_pick(r, "edge"), "gw": f"{host}-gateway"})(r.choice(["shop", "portal", "fleet", "tickets"])), "task": """ Namespace `%{ns}` publishes `%{host}.example.com` through Ingress `%{host}`. The cluster is moving to the Gateway API: its CRDs are installed and GatewayClass `%{gc}` exists. 1. Create Gateway `%{gw}` in `%{ns}` with class `%{gc}` and a single listener named `https`: protocol HTTPS, port 443, hostname `%{host}.example.com`, terminating TLS with the Secret the Ingress uses. 2. Create HTTPRoute `%{host}` in `%{ns}`, attached to that Gateway, for the same hostname, routing exactly as the Ingress does. 3. Delete the Ingress. (No Gateway controller runs here; you are graded on the objects.) """, "setup": NS + r""" ensure_gateway_crds || { echo "could not install the Gateway API CRDs (no network?) - the task needs them"; exit 1; } mkns %{ns} $K -n %{ns} delete ingress %{host} --ignore-not-found >/dev/null 2>&1 $K -n %{ns} delete httproute.gateway.networking.k8s.io,gateway.gateway.networking.k8s.io --all >/dev/null 2>&1 $K -n %{ns} delete secret %{host}-tls --ignore-not-found >/dev/null 2>&1 for s in web api; do $K -n %{ns} create deploy $s --image=nginx:1.27 --dry-run=client -o yaml | $K apply -f - >/dev/null; done $K -n %{ns} expose deploy web --port=80 --target-port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null $K -n %{ns} expose deploy api --port=8080 --target-port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null nx "$CP" "openssl req -x509 -nodes -newkey rsa:2048 -days 30 -subj /CN=%{host}.example.com -keyout /tmp/q126.key -out /tmp/q126.crt >/dev/null 2>&1 \ && kubectl --kubeconfig /etc/kubernetes/admin.conf -n %{ns} create secret tls %{host}-tls --cert=/tmp/q126.crt --key=/tmp/q126.key >/dev/null; rm -f /tmp/q126.key /tmp/q126.crt" "$PRIV/q126.uid" [ -s "$PRIV/q126.uid" ] || { echo "could not create the TLS Secret"; exit 1; } $K apply -f - >/dev/null <<'EOF' apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: {name: %{gc}} spec: {controllerName: example.com/gateway-controller} --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: {name: %{host}, namespace: %{ns}} spec: ingressClassName: nginx tls: - hosts: [%{host}.example.com] secretName: %{host}-tls rules: - host: %{host}.example.com http: paths: - {path: /api, pathType: Prefix, backend: {service: {name: api, port: {number: 8080}}}} - {path: /, pathType: Prefix, backend: {service: {name: web, port: {number: 80}}}} EOF """, "checks": [ C("Gateway %{gw}: class %{gc}, one listener https - HTTPS, 443, %{host}.example.com, TLS from %{host}-tls", 3, PY + r""" py -n %{ns} gateway.gateway.networking.k8s.io %{gw} ' ls = o["spec"].get("listeners", []) l = ls[0] if len(ls) == 1 else {} t = l.get("tls") or {} refs = [(r.get("kind", "Secret"), r.get("name"), r.get("namespace", "%{ns}")) for r in t.get("certificateRefs", [])] o["spec"].get("gatewayClassName") == "%{gc}" and l.get("name") == "https" and l.get("protocol") == "HTTPS" and l.get("port") == 443 and l.get("hostname") == "%{host}.example.com" and t.get("mode", "Terminate") == "Terminate" and refs == [("Secret", "%{host}-tls", "%{ns}")]'"""), C("HTTPRoute %{host}: attached to %{gw}, same hostname, /api -> api:8080 and / -> web:80", 4, PY + r""" py -n %{ns} httproute.gateway.networking.k8s.io %{host} ' s = o["spec"] parents = [(p.get("name"), p.get("namespace", "%{ns}")) for p in s.get("parentRefs", [])] def backs(r): return sorted((b.get("name"), b.get("port")) for b in r.get("backendRefs", [])) def paths(r): return tuple(sorted((m.get("path", {}).get("type", "PathPrefix"), m.get("path", {}).get("value", "/")) for m in (r.get("matches") or [{}]))) rules = {paths(r): backs(r) for r in s.get("rules", [])} parents == [("%{gw}", "%{ns}")] and s.get("hostnames") == ["%{host}.example.com"] and rules == {(("PathPrefix", "/api"),): [("api", 8080)], (("PathPrefix", "/"),): [("web", 80)]}'"""), C("the Ingress is deleted, the Secret is the one it used", 1, r""" [ -n "$(j -n %{ns} httproute.gateway.networking.k8s.io %{host} '{.metadata.name}')" ] || exit 1 out=$($K -n %{ns} get ingress %{host} 2>&1); case "$out" in *NotFound*) ;; *) exit 1 ;; esac eq "$(j -n %{ns} secret %{host}-tls '{.metadata.uid}')" "$(cat "$PRIV/q126.uid")" """), ], "solution": """ k -n %{ns} get ingress %{host} -o yaml # host, the TLS Secret (%{host}-tls), /api -> api:8080, / -> web:80 cat </dev/null 2>&1 $K delete ingressclass %{old} %{cls} --ignore-not-found >/dev/null 2>&1 for s in web cart blog; do $K -n %{ns} create deploy $s --image=nginx:1.27 --dry-run=client -o yaml | $K apply -f - >/dev/null; done $K apply -f - >/dev/null <<'EOF' apiVersion: v1 kind: Service metadata: {name: web, namespace: %{ns}} spec: {selector: {app: web}, ports: [{name: http, port: 80, targetPort: 80}]} --- apiVersion: v1 kind: Service metadata: {name: cart, namespace: %{ns}} spec: {selector: {app: cart}, ports: [{name: http, port: 8080, targetPort: 80}]} --- apiVersion: v1 kind: Service metadata: {name: blog, namespace: %{ns}} spec: {selector: {app: blog}, ports: [{name: http, port: 80, targetPort: 80}]} --- # created while the cluster has no IngressClass at all: it gets no class apiVersion: networking.k8s.io/v1 kind: Ingress metadata: {name: %{ing}, namespace: %{ns}} spec: rules: - host: shop.example.com http: paths: - {path: /cart, pathType: Prefix, backend: {service: {name: cart, port: {name: web}}}} - {path: /, pathType: Exact, backend: {service: {name: web, port: {name: http}}}} EOF $K apply -f - >/dev/null <<'EOF' apiVersion: networking.k8s.io/v1 kind: IngressClass metadata: name: %{old} annotations: {ingressclass.kubernetes.io/is-default-class: "true"} spec: {controller: example.com/legacy-lb} --- apiVersion: networking.k8s.io/v1 kind: IngressClass metadata: {name: %{cls}} spec: {controller: k8s.io/ingress-nginx} EOF cat > "$WS/q127/blog-ingress.yaml" <<'EOF' apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: %{new} namespace: %{ns} spec: rules: - host: blog.example.com http: paths: - path: / pathType: Prefix backend: service: name: blog port: name: http EOF md5sum < "$WS/q127/blog-ingress.yaml" > "$PRIV/q127.md5" """, "checks": [ C("%{cls} is the default IngressClass, and the only one", 2, PY + r""" py ingressclass ' d = sorted(x["metadata"]["name"] for x in o["items"] if (x["metadata"].get("annotations") or {}).get("ingressclass.kubernetes.io/is-default-class") == "true") d == ["%{cls}"] and {"%{old}", "%{cls}"} <= {x["metadata"]["name"] for x in o["items"]}'"""), C("Ingress %{ing}: class %{cls}; /cart -> cart by its port name, / -> web for every path", 3, PY + r""" py -n %{ns} ingress %{ing} ' s = o["spec"] p = {x.get("path"): x for r in s.get("rules", []) if r.get("host") == "shop.example.com" for x in r.get("http", {}).get("paths", [])} def be(x): return (x.get("pathType"), x["backend"]["service"]["name"], x["backend"]["service"]["port"].get("name")) s.get("ingressClassName") == "%{cls}" and set(p) == {"/cart", "/"} and be(p["/cart"]) == ("Prefix", "cart", "http") and be(p["/"]) == ("Prefix", "web", "http")'"""), C("Ingress %{new} was created from the unchanged file and has class %{cls}", 3, r""" eq "$(md5sum < "$WS/q127/blog-ingress.yaml" 2>/dev/null)" "$(cat "$PRIV/q127.md5")" || exit 1 eq "$(j -n %{ns} ingress %{new} '{.spec.ingressClassName}/{.spec.rules[0].host}/{.spec.rules[0].http.paths[0].backend.service.name}')" %{cls}/blog.example.com/blog"""), ], "solution": """ k get ingressclass # no column shows the default: it is an annotation k get ingressclass -o custom-columns='NAME:.metadata.name,DEFAULT:.metadata.annotations.ingressclass\\.kubernetes\\.io/is-default-class' k annotate ingressclass %{old} ingressclass.kubernetes.io/is-default-class- k annotate ingressclass %{cls} ingressclass.kubernetes.io/is-default-class=true k -n %{ns} get ingress %{ing} # CLASS : the default is only filled in when an Ingress is CREATED k -n %{ns} get svc cart -o jsonpath='{.spec.ports}'; echo # its port is named http, not web k -n %{ns} edit ingress %{ing} # spec.ingressClassName: %{cls} # /cart: backend.service.port.name: http # /: pathType: Prefix (Exact matches the one path "/" and nothing below it) k apply -f {WS}/q127/blog-ingress.yaml k -n %{ns} get ingress # both CLASS %{cls} # With two default IngressClasses, creating an Ingress without a class is refused outright - so # remove the old annotation before you set the new one. An Ingress that already exists is never # touched by a change of default: its class has to be set by hand. """, }, # ------------------------------------------------------------------ 128 ---- { "id": 128, "domain": "NET", "difficulty": "hard", "workers": 2, "title": "On the node's own address", "params": lambda r: {"ns": ns_pick(r, "telemetry"), "ds": name_pick(r, "node-agent"), "hp": r.choice([8181, 9191, 18080, 19100]), "pod": name_pick(r, "collector"), "svc": name_pick(r, "inventory")}, "task": """ Two workloads in Namespace `%{ns}` deal with the nodes' own network: 1. DaemonSet `%{ds}` serves HTTP on container port 8080 on every worker node. Monitoring must reach each agent at **its node's IP address, port %{hp}** - without a Service, and without putting the Pods on the host network. 2. Pod `%{pod}` has to stay on the host network, and its log keeps saying `bad address`: it cannot resolve Service `%{svc}`. Fix that; the Pod may be recreated from `{WS}/q128/collector.yaml`. 3. Write the IP address of Pod `%{pod}` and the IP address of the node it runs on to `{WS}/q128/ips.txt`, as ` `. """, "setup": PODS + r""" mkns %{ns}; rm -rf "$WS/q128"; mkdir -p "$WS/q128" $K -n %{ns} delete ds %{ds} --ignore-not-found --wait=true >/dev/null 2>&1 $K -n %{ns} delete pod,svc --all --wait=true --grace-period=1 >/dev/null 2>&1 srv %{ns} %{svc} $K apply -f - >/dev/null <<'EOF' apiVersion: apps/v1 kind: DaemonSet metadata: {name: %{ds}, namespace: %{ns}} spec: selector: {matchLabels: {app: %{ds}}} template: metadata: {labels: {app: %{ds}}} spec: terminationGracePeriodSeconds: 2 containers: - name: agent image: busybox:1.36 env: [{name: NODE_NAME, valueFrom: {fieldRef: {fieldPath: spec.nodeName}}}] command: [sh, -c, 'mkdir -p /www && echo "agent on $NODE_NAME" > /www/index.html && exec httpd -f -p 8080 -h /www'] ports: [{containerPort: 8080}] EOF cat > "$WS/q128/collector.yaml" <<'EOF' apiVersion: v1 kind: Pod metadata: {name: %{pod}, namespace: %{ns}, labels: {app: %{pod}}} spec: hostNetwork: true terminationGracePeriodSeconds: 2 containers: - name: collector image: busybox:1.36 command: - sh - -c - | while true; do if nslookup %{svc}.%{ns}.svc.cluster.local >/dev/null 2>&1; then echo "resolved %{svc}"; else echo "bad address %{svc}"; fi sleep 5 done EOF $K apply -f "$WS/q128/collector.yaml" >/dev/null $K -n %{ns} rollout status ds %{ds} --timeout=240s >/dev/null 2>&1 || { echo "DaemonSet %{ds} did not come up"; exit 1; } """, "checks": [ C("DaemonSet %{ds}: hostPort %{hp} on container port 8080, Pods not on the host network", 2, PY + r""" py -n %{ns} ds %{ds} ' s = o["spec"]["template"]["spec"] ports = [(p.get("containerPort"), p.get("hostPort")) for c in s["containers"] for p in c.get("ports", [])] (8080, %{hp}) in ports and not s.get("hostNetwork")'"""), C("each worker's agent answers on its node's IP, port %{hp}", 3, r""" [ "$(j -n %{ns} ds %{ds} '{.spec.template.spec.hostNetwork}')" != true ] || exit 1 for i in $(seq 1 20); do eq "$(j -n %{ns} ds %{ds} '{.status.updatedNumberScheduled}/{.status.numberAvailable}')" 2/2 && break; sleep 3 done for n in "$W1" "$W2"; do ip=$(j node "$n" '{.status.addresses[?(@.type=="InternalIP")].address}'); [ -n "$ip" ] || exit 1 ok="" for i in 1 2 3 4 5; do [ "$(nx "$CP" "curl -s -m 3 http://$ip:%{hp}/" /dev/null | tr -d '\r')" = "agent on $n" ] && { ok=1; break; }; sleep 3 done [ -n "$ok" ] || exit 1 done"""), C("Pod %{pod} is on the host network and resolves the Service", 3, r""" eq "$(j -n %{ns} pod %{pod} '{.spec.hostNetwork}/{.spec.dnsPolicy}')" true/ClusterFirstWithHostNet || exit 1 for i in $(seq 1 20); do $K -n %{ns} logs %{pod} --tail=1 2>/dev/null | grep -q 'resolved %{svc}' && exit 0; sleep 3 done; exit 1"""), C("ips.txt: the Pod's address is its node's address", 1, r""" eq "$(j -n %{ns} pod %{pod} '{.spec.hostNetwork}')" true || exit 1 pip=$(j -n %{ns} pod %{pod} '{.status.podIP}'); hip=$(j -n %{ns} pod %{pod} '{.status.hostIP}') [ -n "$pip" ] && eq "$pip" "$hip" && eq "$(tr -s ' \t\r\n' ' ' < "$WS/q128/ips.txt" 2>/dev/null | sed 's/^ //; s/ $//')" "$pip $hip" """), ], "solution": """ k -n %{ns} edit ds %{ds} # under the container: # ports: [{containerPort: 8080, hostPort: %{hp}}] k -n %{ns} rollout status ds %{ds} k get nodes -o wide # INTERNAL-IP curl -s http://:%{hp}/ # agent on k -n %{ns} logs %{pod} --tail=2 # bad address %{svc} k -n %{ns} get pod %{pod} -o jsonpath='{.spec.dnsPolicy}'; echo # ClusterFirst - which a hostNetwork Pod does not get vi {WS}/q128/collector.yaml # spec.dnsPolicy: ClusterFirstWithHostNet k -n %{ns} delete pod %{pod}; k apply -f {WS}/q128/collector.yaml k -n %{ns} logs %{pod} --tail=2 # resolved %{svc} k -n %{ns} get pod %{pod} -o jsonpath='{.status.podIP} {.status.hostIP}' > {WS}/q128/ips.txt # hostPort maps one port of the node to one Pod on it (the CNI's portmap plugin writes the rule); # hostNetwork puts the whole Pod into the node's network namespace - its IP IS the node's IP, and # it uses the node's /etc/resolv.conf, which knows nothing of cluster.local, unless dnsPolicy is # ClusterFirstWithHostNet. dnsPolicy cannot be changed on a running Pod: recreate it. """, }, ]