kubestronaut-sim/cka/bank_net.py

"""CKA pool tasks 125-128: Services & Networking - CoreDNS configuration, the move from Ingress to
the Gateway API, IngressClasses, and the nodes' own network (hostPort, hostNetwork).

Written for the per-task hosts: a task may rewrite the cluster's CoreDNS configuration. Name
resolution and hostPorts are tested for real; Ingress and Gateway objects are graded as objects,
as no controller for them runs on the hosts.
"""

from sim.bank import cmd, hand
from cka.common import C, DNS, NS, ONE, PODS, PY, ns_pick, name_pick

QUESTIONS = [
# ------------------------------------------------------------------ 125 ----
{
 "id": 125, "domain": "NET", "difficulty": "hard", "title": "Names of your own in cluster DNS",
 "params": lambda r: {"ns": ns_pick(r, "billing"), "svc": name_pick(r, "invoices"),
                      "host": r.choice(["legacy-db", "mainframe", "ldap", "sap"]),
                      "ip": f"10.{r.randint(120, 200)}.{r.randint(1, 250)}.{r.randint(2, 250)}"},
 "task": """
Applications in this cluster still use two names from the old data centre. Teach cluster DNS both
of them by changing the CoreDNS configuration:

1. `%{host}.dc.internal` must resolve to `%{ip}`.
2. `%{svc}.dc.internal` must resolve to whatever Service `%{svc}` in Namespace `%{ns}` resolves to
   - an alias for the Service's name, not a copy of its address.
3. Before you change anything, save the CoreDNS ConfigMap as it is to
   `{WS}/q125/coredns-backup.yaml`.

Names in `cluster.local` must keep resolving, and CoreDNS must be running with 2 replicas when you
are done. Pod `probe` in `%{ns}` is there for testing; busybox's `nslookup` wants fully qualified
names.
""",
 "setup": PODS + r"""
mkns %{ns}; rm -rf "$WS/q125"; mkdir -p "$WS/q125"
$K -n %{ns} delete pod,svc --all --wait=true --grace-period=1 >/dev/null 2>&1
srv %{ns} %{svc}; cli %{ns} probe; ready %{ns} || exit 1
$K -n kube-system get cm coredns -o jsonpath='{.data.Corefile}' > "$PRIV/q125.corefile"
grep -q kubernetes "$PRIV/q125.corefile" || { echo "could not read the CoreDNS configuration"; exit 1; }
""",
 "checks": [
  C("%{host}.dc.internal resolves to %{ip} from a Pod", 3, DNS + r"""resolves %{ns} probe %{host}.dc.internal %{ip}"""),
  C("%{svc}.dc.internal resolves to the Service, as an alias", 3, DNS + r"""
cip=$(j -n %{ns} svc %{svc} '{.spec.clusterIP}'); [ -n "$cip" ] || exit 1
resolves %{ns} probe %{svc}.dc.internal "$cip" || exit 1
! $K -n kube-system get cm coredns -o jsonpath='{.data.Corefile}' | grep -qF "$cip" """),
  C("cluster names still resolve, and CoreDNS runs with 2 replicas", 1, DNS + r"""
resolves %{ns} probe %{host}.dc.internal %{ip} || exit 1
for i in $(seq 1 20); do eq "$(j -n kube-system deploy coredns '{.status.updatedReplicas}/{.status.availableReplicas}/{.status.replicas}')" 2/2/2 && break; sleep 3; done
eq "$(j -n kube-system deploy coredns '{.status.updatedReplicas}/{.status.availableReplicas}/{.status.replicas}')" 2/2/2 || exit 1
resolves %{ns} probe kubernetes.default.svc.cluster.local "$(j -n default svc kubernetes '{.spec.clusterIP}')" """),
  C("coredns-backup.yaml holds the ConfigMap as it was", 1, r"""
ORIG="$PRIV/q125.corefile" python3 - "$WS/q125/coredns-backup.yaml" <<'PY'
import os, sys
try:
    backup = open(sys.argv[1]).read()
except OSError:
    sys.exit(1)
lines = [l.strip() for l in open(os.environ["ORIG"]) if l.strip()]
ok = "ConfigMap" in backup and "coredns" in backup and "dc.internal" not in backup and all(l in backup for l in lines)
sys.exit(0 if ok else 1)
PY
"""),
 ],
 "solution": """
k -n kube-system get cm coredns -o yaml > {WS}/q125/coredns-backup.yaml
k -n kube-system edit cm coredns           # inside the  .:53 { ... }  block, e.g. above "kubernetes ...":
#       hosts {
#           %{ip} %{host}.dc.internal
#           fallthrough
#       }
#       rewrite name %{svc}.dc.internal %{svc}.%{ns}.svc.cluster.local
k -n kube-system rollout restart deploy coredns      # the reload plugin would pick it up too, in a minute or two
k -n kube-system rollout status deploy coredns
k -n %{ns} exec probe -- nslookup %{host}.dc.internal             # %{ip}
k -n %{ns} exec probe -- nslookup %{svc}.dc.internal              # the Service's ClusterIP
k -n %{ns} exec probe -- nslookup kubernetes.default.svc.cluster.local
k -n kube-system logs -l k8s-app=kube-dns --tail=5                # a syntax error in the Corefile shows up here
# hosts answers from a static list; without `fallthrough` every other name would end there with
# NXDOMAIN. rewrite renames the question before the kubernetes plugin sees it, so the alias follows
# the Service even if its ClusterIP changes. Back the ConfigMap up first: a Corefile that does not
# parse takes cluster DNS down with it - `k apply -f` the backup is the way back.
""",
},
# ------------------------------------------------------------------ 126 ----
{
 "id": 126, "domain": "NET", "difficulty": "hard", "title": "From Ingress to the Gateway API",
 "params": lambda r: (lambda host: {"ns": ns_pick(r, "storefront"), "host": host, "gc": name_pick(r, "edge"),
                                    "gw": f"{host}-gateway"})(r.choice(["shop", "portal", "fleet", "tickets"])),
 "task": """
Namespace `%{ns}` publishes `%{host}.example.com` through Ingress `%{host}`. The cluster is moving
to the Gateway API: its CRDs are installed and GatewayClass `%{gc}` exists.

1. Create Gateway `%{gw}` in `%{ns}` with class `%{gc}` and a single listener named `https`:
   protocol HTTPS, port 443, hostname `%{host}.example.com`, terminating TLS with the Secret the
   Ingress uses.
2. Create HTTPRoute `%{host}` in `%{ns}`, attached to that Gateway, for the same hostname, routing
   exactly as the Ingress does.
3. Delete the Ingress.

(No Gateway controller runs here; you are graded on the objects.)
""",
 "setup": NS + r"""
ensure_gateway_crds || { echo "could not install the Gateway API CRDs (no network?) - the task needs them"; exit 1; }
mkns %{ns}
$K -n %{ns} delete ingress %{host} --ignore-not-found >/dev/null 2>&1
$K -n %{ns} delete httproute.gateway.networking.k8s.io,gateway.gateway.networking.k8s.io --all >/dev/null 2>&1
$K -n %{ns} delete secret %{host}-tls --ignore-not-found >/dev/null 2>&1
for s in web api; do $K -n %{ns} create deploy $s --image=nginx:1.27 --dry-run=client -o yaml | $K apply -f - >/dev/null; done
$K -n %{ns} expose deploy web --port=80 --target-port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null
$K -n %{ns} expose deploy api --port=8080 --target-port=80 --dry-run=client -o yaml | $K apply -f - >/dev/null
nx "$CP" "openssl req -x509 -nodes -newkey rsa:2048 -days 30 -subj /CN=%{host}.example.com -keyout /tmp/q126.key -out /tmp/q126.crt >/dev/null 2>&1 \
  && kubectl --kubeconfig /etc/kubernetes/admin.conf -n %{ns} create secret tls %{host}-tls --cert=/tmp/q126.crt --key=/tmp/q126.key >/dev/null; rm -f /tmp/q126.key /tmp/q126.crt" </dev/null
$K -n %{ns} get secret %{host}-tls -o jsonpath='{.metadata.uid}' > "$PRIV/q126.uid"
[ -s "$PRIV/q126.uid" ] || { echo "could not create the TLS Secret"; exit 1; }
$K apply -f - >/dev/null <<'EOF'
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata: {name: %{gc}}
spec: {controllerName: example.com/gateway-controller}
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata: {name: %{host}, namespace: %{ns}}
spec:
  ingressClassName: nginx
  tls:
    - hosts: [%{host}.example.com]
      secretName: %{host}-tls
  rules:
    - host: %{host}.example.com
      http:
        paths:
          - {path: /api, pathType: Prefix, backend: {service: {name: api, port: {number: 8080}}}}
          - {path: /, pathType: Prefix, backend: {service: {name: web, port: {number: 80}}}}
EOF
""",
 "checks": [
  C("Gateway %{gw}: class %{gc}, one listener https - HTTPS, 443, %{host}.example.com, TLS from %{host}-tls", 3, PY + r"""
py -n %{ns} gateway.gateway.networking.k8s.io %{gw} '
ls = o["spec"].get("listeners", [])
l = ls[0] if len(ls) == 1 else {}
t = l.get("tls") or {}
refs = [(r.get("kind", "Secret"), r.get("name"), r.get("namespace", "%{ns}")) for r in t.get("certificateRefs", [])]
o["spec"].get("gatewayClassName") == "%{gc}" and l.get("name") == "https" and l.get("protocol") == "HTTPS" and l.get("port") == 443 and l.get("hostname") == "%{host}.example.com" and t.get("mode", "Terminate") == "Terminate" and refs == [("Secret", "%{host}-tls", "%{ns}")]'"""),
  C("HTTPRoute %{host}: attached to %{gw}, same hostname, /api -> api:8080 and / -> web:80", 4, PY + r"""
py -n %{ns} httproute.gateway.networking.k8s.io %{host} '
s = o["spec"]
parents = [(p.get("name"), p.get("namespace", "%{ns}")) for p in s.get("parentRefs", [])]
def backs(r): return sorted((b.get("name"), b.get("port")) for b in r.get("backendRefs", []))
def paths(r): return tuple(sorted((m.get("path", {}).get("type", "PathPrefix"), m.get("path", {}).get("value", "/")) for m in (r.get("matches") or [{}])))
rules = {paths(r): backs(r) for r in s.get("rules", [])}
parents == [("%{gw}", "%{ns}")] and s.get("hostnames") == ["%{host}.example.com"] and rules == {(("PathPrefix", "/api"),): [("api", 8080)], (("PathPrefix", "/"),): [("web", 80)]}'"""),
  C("the Ingress is deleted, the Secret is the one it used", 1, r"""
[ -n "$(j -n %{ns} httproute.gateway.networking.k8s.io %{host} '{.metadata.name}')" ] || exit 1
out=$($K -n %{ns} get ingress %{host} 2>&1); case "$out" in *NotFound*) ;; *) exit 1 ;; esac
eq "$(j -n %{ns} secret %{host}-tls '{.metadata.uid}')" "$(cat "$PRIV/q126.uid")" """),
 ],
 "solution": [cmd(r"""k -n %{ns} get ingress %{host} -o yaml""", r"""The host, the TLS Secret `%{host}-tls`, `/api` to `api:8080` and `/` to `web:80`."""),
              hand(r"""vim gw.yaml""",
                   r"""What the Ingress keeps in one object, the Gateway API splits: the listener - port, hostname, certificate - goes in the Gateway, the routing in the HTTPRoute. The docs page *Gateway API* has both; `k explain gateway.spec.listeners.tls` the TLS fields:
```
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: %{gw}
  namespace: %{ns}
spec:
  gatewayClassName: %{gc}
  listeners:
  - name: https
    protocol: HTTPS
    port: 443
    hostname: %{host}.example.com
    tls:
      mode: Terminate
      certificateRefs:
      - kind: Secret
        name: %{host}-tls
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: %{host}
  namespace: %{ns}
spec:
  parentRefs:
  - name: %{gw}
  hostnames:
  - %{host}.example.com
  rules:
  - matches:
    - path:
        type: PathPrefix
        value: /api
    backendRefs:
    - name: api
      port: 8080
  - matches:
    - path:
        type: PathPrefix
        value: /
    backendRefs:
    - name: web
      port: 80
```""",
                   run=r"""cat > gw.yaml <<'EOF'
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: %{gw}
  namespace: %{ns}
spec:
  gatewayClassName: %{gc}
  listeners:
  - name: https
    protocol: HTTPS
    port: 443
    hostname: %{host}.example.com
    tls:
      mode: Terminate
      certificateRefs:
      - kind: Secret
        name: %{host}-tls
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: %{host}
  namespace: %{ns}
spec:
  parentRefs:
  - name: %{gw}
  hostnames:
  - %{host}.example.com
  rules:
  - matches:
    - path:
        type: PathPrefix
        value: /api
    backendRefs:
    - name: api
      port: 8080
  - matches:
    - path:
        type: PathPrefix
        value: /
    backendRefs:
    - name: web
      port: 80
EOF"""),
              cmd(r"""k apply -f gw.yaml
k -n %{ns} get gateway,httproute
k -n %{ns} delete ingress %{host}""")],
},
# ------------------------------------------------------------------ 127 ----
{
 "id": 127, "domain": "NET", "title": "A default IngressClass, and what it does not fix",
 "params": lambda r: {"ns": ns_pick(r, "market"), "cls": r.choice(["edge-nginx", "public-nginx", "nginx-v2"]),
                      "old": r.choice(["legacy-lb", "old-haproxy"]), "ing": name_pick(r, "shop"), "new": name_pick(r, "blog")},
 "task": """
This cluster has two IngressClasses, `%{old}` and `%{cls}`. `%{old}` is marked as the default, and
its controller has been switched off for good.

1. Make `%{cls}` the default IngressClass - the only one.
2. Ingress `%{ing}` in Namespace `%{ns}` was created before any default existed. Give it class
   `%{cls}` and correct its two rules: `/cart` must reach Service `cart` on a port that Service
   really has (keep referring to the port by name), and `/` must match every path, not only `/`
   itself.
3. Create the Ingress in `{WS}/q127/blog-ingress.yaml` exactly as it is - do not edit the file. It
   names no class, and it must end up with class `%{cls}`.

(No Ingress controller runs here; you are graded on the objects.)
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q127"; mkdir -p "$WS/q127"
$K -n %{ns} delete ingress --all >/dev/null 2>&1
$K delete ingressclass %{old} %{cls} --ignore-not-found >/dev/null 2>&1
for s in web cart blog; do $K -n %{ns} create deploy $s --image=nginx:1.27 --dry-run=client -o yaml | $K apply -f - >/dev/null; done
$K apply -f - >/dev/null <<'EOF'
apiVersion: v1
kind: Service
metadata: {name: web, namespace: %{ns}}
spec: {selector: {app: web}, ports: [{name: http, port: 80, targetPort: 80}]}
---
apiVersion: v1
kind: Service
metadata: {name: cart, namespace: %{ns}}
spec: {selector: {app: cart}, ports: [{name: http, port: 8080, targetPort: 80}]}
---
apiVersion: v1
kind: Service
metadata: {name: blog, namespace: %{ns}}
spec: {selector: {app: blog}, ports: [{name: http, port: 80, targetPort: 80}]}
---
# created while the cluster has no IngressClass at all: it gets no class
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata: {name: %{ing}, namespace: %{ns}}
spec:
  rules:
    - host: shop.example.com
      http:
        paths:
          - {path: /cart, pathType: Prefix, backend: {service: {name: cart, port: {name: web}}}}
          - {path: /, pathType: Exact, backend: {service: {name: web, port: {name: http}}}}
EOF
$K apply -f - >/dev/null <<'EOF'
apiVersion: networking.k8s.io/v1
kind: IngressClass
metadata:
  name: %{old}
  annotations: {ingressclass.kubernetes.io/is-default-class: "true"}
spec: {controller: example.com/legacy-lb}
---
apiVersion: networking.k8s.io/v1
kind: IngressClass
metadata: {name: %{cls}}
spec: {controller: k8s.io/ingress-nginx}
EOF
cat > "$WS/q127/blog-ingress.yaml" <<'EOF'
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: %{new}
  namespace: %{ns}
spec:
  rules:
    - host: blog.example.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: blog
                port:
                  name: http
EOF
md5sum < "$WS/q127/blog-ingress.yaml" > "$PRIV/q127.md5"
""",
 "checks": [
  C("%{cls} is the default IngressClass, and the only one", 2, PY + r"""
py ingressclass '
d = sorted(x["metadata"]["name"] for x in o["items"] if (x["metadata"].get("annotations") or {}).get("ingressclass.kubernetes.io/is-default-class") == "true")
d == ["%{cls}"] and {"%{old}", "%{cls}"} <= {x["metadata"]["name"] for x in o["items"]}'"""),
  C("Ingress %{ing}: class %{cls}; /cart -> cart by its port name, / -> web for every path", 3, PY + r"""
py -n %{ns} ingress %{ing} '
s = o["spec"]
p = {x.get("path"): x for r in s.get("rules", []) if r.get("host") == "shop.example.com" for x in r.get("http", {}).get("paths", [])}
def be(x): return (x.get("pathType"), x["backend"]["service"]["name"], x["backend"]["service"]["port"].get("name"))
s.get("ingressClassName") == "%{cls}" and set(p) == {"/cart", "/"} and be(p["/cart"]) == ("Prefix", "cart", "http") and be(p["/"]) == ("Prefix", "web", "http")'"""),
  C("Ingress %{new} was created from the unchanged file and has class %{cls}", 3, r"""
eq "$(md5sum < "$WS/q127/blog-ingress.yaml" 2>/dev/null)" "$(cat "$PRIV/q127.md5")" || exit 1
eq "$(j -n %{ns} ingress %{new} '{.spec.ingressClassName}/{.spec.rules[0].host}/{.spec.rules[0].http.paths[0].backend.service.name}')" %{cls}/blog.example.com/blog"""),
 ],
 "solution": """
k get ingressclass                         # no column shows the default: it is an annotation
k get ingressclass -o custom-columns='NAME:.metadata.name,DEFAULT:.metadata.annotations.ingressclass\\.kubernetes\\.io/is-default-class'
k annotate ingressclass %{old} ingressclass.kubernetes.io/is-default-class-
k annotate ingressclass %{cls} ingressclass.kubernetes.io/is-default-class=true
k -n %{ns} get ingress %{ing}              # CLASS <none>: the default is only filled in when an Ingress is CREATED
k -n %{ns} get svc cart -o jsonpath='{.spec.ports}'; echo       # its port is named http, not web
k -n %{ns} edit ingress %{ing}
#   spec.ingressClassName: %{cls}
#   /cart: backend.service.port.name: http
#   /:     pathType: Prefix                 (Exact matches the one path "/" and nothing below it)
k apply -f {WS}/q127/blog-ingress.yaml
k -n %{ns} get ingress                     # both CLASS %{cls}
# With two default IngressClasses, creating an Ingress without a class is refused outright - so
# remove the old annotation before you set the new one. An Ingress that already exists is never
# touched by a change of default: its class has to be set by hand.
""",
},
# ------------------------------------------------------------------ 128 ----
{
 "id": 128, "domain": "NET", "difficulty": "hard", "workers": 2, "title": "On the node's own address",
 "params": lambda r: {"ns": ns_pick(r, "telemetry"), "ds": name_pick(r, "node-agent"), "hp": r.choice([8181, 9191, 18080, 19100]),
                      "pod": name_pick(r, "collector"), "svc": name_pick(r, "inventory")},
 "task": """
Two workloads in Namespace `%{ns}` deal with the nodes' own network:

1. DaemonSet `%{ds}` serves HTTP on container port 8080 on every worker node. Monitoring must
   reach each agent at **its node's IP address, port %{hp}** - without a Service, and without
   putting the Pods on the host network.
2. Pod `%{pod}` has to stay on the host network, and its log keeps saying `bad address`: it cannot
   resolve Service `%{svc}`. Fix that; the Pod may be recreated from `{WS}/q128/collector.yaml`.
3. Write the IP address of Pod `%{pod}` and the IP address of the node it runs on to
   `{WS}/q128/ips.txt`, as `<pod ip> <node ip>`.
""",
 "setup": PODS + r"""
mkns %{ns}; rm -rf "$WS/q128"; mkdir -p "$WS/q128"
$K -n %{ns} delete ds %{ds} --ignore-not-found --wait=true >/dev/null 2>&1
$K -n %{ns} delete pod,svc --all --wait=true --grace-period=1 >/dev/null 2>&1
srv %{ns} %{svc}
$K apply -f - >/dev/null <<'EOF'
apiVersion: apps/v1
kind: DaemonSet
metadata: {name: %{ds}, namespace: %{ns}}
spec:
  selector: {matchLabels: {app: %{ds}}}
  template:
    metadata: {labels: {app: %{ds}}}
    spec:
      terminationGracePeriodSeconds: 2
      containers:
        - name: agent
          image: busybox:1.36
          env: [{name: NODE_NAME, valueFrom: {fieldRef: {fieldPath: spec.nodeName}}}]
          command: [sh, -c, 'mkdir -p /www && echo "agent on $NODE_NAME" > /www/index.html && exec httpd -f -p 8080 -h /www']
          ports: [{containerPort: 8080}]
EOF
cat > "$WS/q128/collector.yaml" <<'EOF'
apiVersion: v1
kind: Pod
metadata: {name: %{pod}, namespace: %{ns}, labels: {app: %{pod}}}
spec:
  hostNetwork: true
  terminationGracePeriodSeconds: 2
  containers:
    - name: collector
      image: busybox:1.36
      command:
        - sh
        - -c
        - |
          while true; do
            if nslookup %{svc}.%{ns}.svc.cluster.local >/dev/null 2>&1; then echo "resolved %{svc}"; else echo "bad address %{svc}"; fi
            sleep 5
          done
EOF
$K apply -f "$WS/q128/collector.yaml" >/dev/null
$K -n %{ns} rollout status ds %{ds} --timeout=240s >/dev/null 2>&1 || { echo "DaemonSet %{ds} did not come up"; exit 1; }
""",
 "checks": [
  C("DaemonSet %{ds}: hostPort %{hp} on container port 8080, Pods not on the host network", 2, PY + r"""
py -n %{ns} ds %{ds} '
s = o["spec"]["template"]["spec"]
ports = [(p.get("containerPort"), p.get("hostPort")) for c in s["containers"] for p in c.get("ports", [])]
(8080, %{hp}) in ports and not s.get("hostNetwork")'"""),
  C("each worker's agent answers on its node's IP, port %{hp}", 3, r"""
[ "$(j -n %{ns} ds %{ds} '{.spec.template.spec.hostNetwork}')" != true ] || exit 1
for i in $(seq 1 20); do
  eq "$(j -n %{ns} ds %{ds} '{.status.updatedNumberScheduled}/{.status.numberAvailable}')" 2/2 && break; sleep 3
done
for n in "$W1" "$W2"; do
  ip=$(j node "$n" '{.status.addresses[?(@.type=="InternalIP")].address}'); [ -n "$ip" ] || exit 1
  ok=""
  for i in 1 2 3 4 5; do
    [ "$(nx "$CP" "curl -s -m 3 http://$ip:%{hp}/" </dev/null 2>/dev/null | tr -d '\r')" = "agent on $n" ] && { ok=1; break; }; sleep 3
  done
  [ -n "$ok" ] || exit 1
done"""),
  C("Pod %{pod} is on the host network and resolves the Service", 3, r"""
eq "$(j -n %{ns} pod %{pod} '{.spec.hostNetwork}/{.spec.dnsPolicy}')" true/ClusterFirstWithHostNet || exit 1
for i in $(seq 1 20); do
  $K -n %{ns} logs %{pod} --tail=1 2>/dev/null | grep -q 'resolved %{svc}' && exit 0; sleep 3
done; exit 1"""),
  C("ips.txt: the Pod's address is its node's address", 1, r"""
eq "$(j -n %{ns} pod %{pod} '{.spec.hostNetwork}')" true || exit 1
pip=$(j -n %{ns} pod %{pod} '{.status.podIP}'); hip=$(j -n %{ns} pod %{pod} '{.status.hostIP}')
[ -n "$pip" ] && eq "$pip" "$hip" && eq "$(tr -s ' \t\r\n' ' ' < "$WS/q128/ips.txt" 2>/dev/null | sed 's/^ //; s/ $//')" "$pip $hip" """),
 ],
 "solution": [hand(r"""k -n %{ns} edit ds %{ds}""",
                   r"""Give the container's port a `hostPort`, then `:wq`:
```
        ports:
        - containerPort: 8080
          hostPort: %{hp}
          protocol: TCP
```
`hostPort` maps one port of the node to the Pod on it (the CNI's `portmap` plugin writes the rule); `hostNetwork` would put the whole Pod in the node's network.""",
                   run=r"""kubectl -n %{ns} patch ds %{ds} --type=json -p '[{"op": "add", "path": "/spec/template/spec/containers/0/ports/0/hostPort", "value": %{hp}}]'"""),
              cmd(r"""k -n %{ns} rollout status ds %{ds}
k get nodes -o wide""", r"""Note a worker's `INTERNAL-IP`."""),
              hand(r"""curl -s http://172.18.0.3:%{hp}/""",
                   r"""With that IP: `agent on <that node>`.""",
                   run=r""""""),
              cmd(r"""k -n %{ns} logs %{pod} --tail=2
k -n %{ns} get pod %{pod} -o yaml | grep -e hostNetwork -e dnsPolicy""", r"""`bad address`: a `hostNetwork` Pod uses the node's `/etc/resolv.conf`, which knows nothing of `cluster.local` - unless its `dnsPolicy` is `ClusterFirstWithHostNet`."""),
              hand(r"""vim {WS}/q128/collector.yaml""",
                   r"""In the Pod's `spec:`, below `hostNetwork: true`:
```
  dnsPolicy: ClusterFirstWithHostNet
```""",
                   run=r"""yq -i '.spec.dnsPolicy = "ClusterFirstWithHostNet"' {WS}/q128/collector.yaml"""),
              cmd(r"""k replace --force -f {WS}/q128/collector.yaml""", r"""A running Pod's `dnsPolicy` cannot change: `--force` deletes it and creates it again."""),
              hand(r"""k -n %{ns} logs %{pod} --tail=2
k -n %{ns} get pod %{pod} -o wide""",
                   r"""Again until `resolved %{svc}`. Its IP is its node's: note it.""",
                   run=r"""kubectl -n %{ns} wait --for=condition=Ready pod/%{pod} --timeout=180s
sleep 8"""),
              hand(r"""echo "172.18.0.3 172.18.0.3" > {WS}/q128/ips.txt""",
                   r"""The Pod's IP and its node's - the same address.""",
                   run=r"""kubectl -n %{ns} get pod %{pod} -o jsonpath='{.status.podIP} {.status.hostIP}' > {WS}/q128/ips.txt""")],
},
]