kubestronaut-sim/cks/bank_harden.py

"""CKS tasks 25-39: Cluster Hardening - the pool tasks that take the domain from 2 to 17.

  2.1 RBAC              25 one Secret instead of all of them       26 a group that is cluster-admin
                        27 bind, escalate, impersonate, exec       28 who can read this Secret?
  2.2 ServiceAccounts   29 a token for something else than the API 30 tokens that never expire
                        31 a ServiceAccount of its own
  2.3 API access        32 the API answers without credentials     33 anonymous for health checks only
                        34 AlwaysAllow and no NodeRestriction      35 the API on a NodePort
                        36 a client certificate for an auditor
  2.4 Upgrade           37 a worker on a vulnerable kubelet        38 upgrade the control plane
                        39 prepare an upgrade

Permissions are tested with `kubectl auth can-i` as the subject, tokens and certificates by calling
the API with them, and the API server's settings by what an unauthenticated or a kubelet's request
gets. 37 and 38 are real upgrades: CKS hosts carry Kubernetes as apt packages (images/node-cks),
so a node can be held back on an older patch release and upgraded with apt and kubeadm.
Every task has a scripted solution in solve.py.
"""
from cks.common import API, C, PY, NS, ns_pick, name_pick
from sim.bank import CAN  # noqa: F401  (bank_runtime imports it from here)


# code URL-PATH [CURL-ARGS..]: the HTTP status the API server gives a request made on the control
# plane node - without credentials unless the arguments supply some.
CURL = r"""
code() { local p="$1"; shift; nx "$CP" "curl -sk -m 8 -o /dev/null -w '%{http_code}' $* https://localhost:6443$p" </dev/null; }
"""

# jwt FIELD < token: a claim of a ServiceAccount token (dotted path into its payload)
JWT = r"""
jwt() { python3 -c '
import base64, json, sys
t = sys.stdin.read().strip().split(".")
p = json.loads(base64.urlsafe_b64decode(t[1] + "=" * (-len(t[1]) % 4)))
for k in sys.argv[1].split("."):
    p = p[k]
print(json.dumps(p) if isinstance(p, (list, dict)) else p)' "$1" 2>/dev/null; }
"""

# The apt repository of Kubernetes only (the node image has no package lists), then a patch release
# older than the node's for the three packages, held as kubeadm's documentation asks.
APT = r"""
K8S_LIST='-o Dir::Etc::sourcelist=/etc/apt/sources.list.d/kubernetes.list -o Dir::Etc::sourceparts=- -o APT::Get::List-Cleanup=0'
# hold_back NODE VERSION: put NODE's kubelet, kubeadm and kubectl on VERSION and restart its kubelet.
# pkgs.k8s.io has one repository per minor, so the node's repository line moves to VERSION's minor
# first, and stays there - as on a node installed with that minor.
hold_back() {
  nx "$1" "sed -i 's#/core:/stable:/v[0-9.]*/deb/#/core:/stable:/v${2%.*}/deb/#' /etc/apt/sources.list.d/kubernetes.list
apt-get update -qq $K8S_LIST >/dev/null 2>&1
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --allow-downgrades --allow-change-held-packages -o Dpkg::Options::=--force-confold kubelet=$2-1.1 kubeadm=$2-1.1 kubectl=$2-1.1 >/dev/null 2>&1 || exit 1
apt-mark hold kubelet kubeadm kubectl >/dev/null; systemctl daemon-reload; systemctl restart kubelet" </dev/null || return 1
  local i; for i in $(seq 1 40); do [ "$(j node "$1" '{.status.nodeInfo.kubeletVersion}')" = "v$2" ] && return 0; sleep 3; done; return 1
}
# pkgs NODE: "<kubelet version> <kubeadm version> <kubectl version>" as dpkg knows them
pkgs() { nx "$1" "dpkg-query -W -f='\${Version} ' kubelet kubeadm kubectl" </dev/null; }
held() { [ "$(nx "$1" 'apt-mark showhold | grep -c -x -e kubelet -e kubeadm -e kubectl' </dev/null)" = 3 ]; }
"""

USERS = ["dana", "omar", "priya", "lucas", "mei", "tariq"]

QUESTIONS = [
# ==================================================================== 2.1 RBAC ====
{
 "id": 25, "domain": "HARDEN", "title": "One Secret instead of all of them",
 "params": lambda r: {"ns": ns_pick(r, "billing"), "sa": name_pick(r, "invoicer")},
 "task": """
ServiceAccount `%{sa}` in Namespace `%{ns}` reads its API key from Secret `api-key`. Role
`app-reader`, bound to it by RoleBinding `app-reader`, lets it read **every** Secret of the
Namespace — and list them, which shows their contents too.

Change the Role (keep its name, and the RoleBinding) so that `%{sa}`:

- may `get` Secret `api-key`, and nothing else about Secrets: no `list`, no `watch`, no other Secret;
- keeps `get`, `list` and `watch` on ConfigMaps.
""",
 "setup": NS + r"""
mkns %{ns}
$K -n %{ns} create sa %{sa} >/dev/null
$K -n %{ns} create secret generic api-key --from-literal=key=k-%{sa} >/dev/null
$K -n %{ns} create secret generic db-root --from-literal=password=root-%{sa} >/dev/null
$K -n %{ns} create role app-reader --verb=get,list,watch --resource=secrets,configmaps >/dev/null
$K -n %{ns} create rolebinding app-reader --role=app-reader --serviceaccount=%{ns}:%{sa} >/dev/null
""",
 "checks": [
  C("%{sa} can still get Secret api-key", 3, CAN + r"""
S=system:serviceaccount:%{ns}:%{sa}
can --as=$S -- get secret/api-key -n %{ns} && cannot --as=$S -- list secrets -n %{ns}"""),
  C("it cannot get another Secret, nor list or watch Secrets", 4, CAN + r"""
S=system:serviceaccount:%{ns}:%{sa}
can --as=$S -- get secret/api-key -n %{ns} && cannot --as=$S -- get secret/db-root -n %{ns} \
 && cannot --as=$S -- list secrets -n %{ns} && cannot --as=$S -- watch secrets -n %{ns}"""),
  C("ConfigMaps are still readable, through Role and RoleBinding app-reader", 3, CAN + r"""
S=system:serviceaccount:%{ns}:%{sa}
cannot --as=$S -- list secrets -n %{ns} || exit 1
for v in get list watch; do can --as=$S -- $v configmaps -n %{ns} || exit 1; done
eq "$(j -n %{ns} rolebinding app-reader '{.roleRef.name}')" app-reader && [ -n "$(j -n %{ns} role app-reader '{.metadata.name}')" ]"""),
 ],
 "solution": """
k -n %{ns} get role app-reader -o yaml
cat <<EOF | k apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata: {name: app-reader, namespace: %{ns}}
rules:
  - apiGroups: [""]
    resources: [configmaps]
    verbs: [get, list, watch]
  - apiGroups: [""]
    resources: [secrets]
    resourceNames: [api-key]
    verbs: [get]
EOF
k -n %{ns} auth can-i get secret/api-key --as=system:serviceaccount:%{ns}:%{sa}     # yes
k -n %{ns} auth can-i get secret/db-root --as=system:serviceaccount:%{ns}:%{sa}     # no
k -n %{ns} auth can-i list secrets --as=system:serviceaccount:%{ns}:%{sa}           # no
# resourceNames narrows get, update, patch and delete to named objects. It cannot narrow list or
# watch, which return whole collections: a rule with resourceNames and `list` grants nothing
# useful, and `list` on secrets without it returns every value. Hence `get` only.
# The generator does it too:  k create role X --verb=get --resource=secrets --resource-name=api-key
""",
},
{
 "id": 26, "domain": "HARDEN", "title": "A group that is cluster-admin",
 "params": lambda r: {"ns": ns_pick(r, "team"), "user": r.choice(USERS)},
 "task": """
Everyone in group `dev` is cluster-admin, through ClusterRoleBinding `dev-team`. Replace that with
what the group needs, using the built-in ClusterRoles (create no Role of your own):

- in Namespace `%{ns}-dev`: `edit`, with RoleBinding `dev-edit`;
- in Namespace `%{ns}-prod`: `view`, with RoleBinding `dev-view`;
- nothing anywhere else.

Test as a member of the group: `--as=%{user} --as-group=dev`.
""",
 "setup": NS + r"""
mkns %{ns}-dev; mkns %{ns}-prod
$K create clusterrolebinding dev-team --clusterrole=cluster-admin --group=dev >/dev/null
""",
 "checks": [
  C("group dev is no longer cluster-admin", 3, CAN + r"""
A="--as=%{user} --as-group=dev"
cannot $A -- '*' '*' && cannot $A -- list nodes && cannot $A -- get secrets -n kube-system"""),
  C("in %{ns}-dev the group has edit, through RoleBinding dev-edit", 3, CAN + r"""
A="--as=%{user} --as-group=dev"
eq "$(j -n %{ns}-dev rolebinding dev-edit '{.roleRef.kind}/{.roleRef.name}')" ClusterRole/edit || exit 1
can $A -- create deployments -n %{ns}-dev && can $A -- delete pods -n %{ns}-dev && cannot $A -- create rolebindings -n %{ns}-dev"""),
  C("in %{ns}-prod the group has view, through RoleBinding dev-view", 3, CAN + r"""
A="--as=%{user} --as-group=dev"
eq "$(j -n %{ns}-prod rolebinding dev-view '{.roleRef.kind}/{.roleRef.name}')" ClusterRole/view || exit 1
can $A -- list pods -n %{ns}-prod && cannot $A -- delete pods -n %{ns}-prod && cannot $A -- get secrets -n %{ns}-prod"""),
  C("and nothing in any other Namespace", 1, CAN + r"""
A="--as=%{user} --as-group=dev"
can $A -- list pods -n %{ns}-prod && cannot $A -- list pods -n default && cannot $A -- list pods -n kube-system"""),
 ],
 "solution": """
k get clusterrolebinding dev-team -o yaml          # subjects: Group dev -> ClusterRole cluster-admin
k delete clusterrolebinding dev-team
k -n %{ns}-dev create rolebinding dev-edit --clusterrole=edit --group=dev
k -n %{ns}-prod create rolebinding dev-view --clusterrole=view --group=dev
k auth can-i '*' '*' --as=%{user} --as-group=dev                        # no
k -n %{ns}-dev auth can-i create deploy --as=%{user} --as-group=dev     # yes
k -n %{ns}-prod auth can-i delete pods --as=%{user} --as-group=dev      # no
# A RoleBinding may refer to a ClusterRole: the rules then hold in the RoleBinding's Namespace only.
# That is how the built-in roles (view, edit, admin) are meant to be handed out.
# `view` leaves out Secrets on purpose; `edit` leaves out Roles and RoleBindings.
""",
},
{
 "id": 27, "domain": "HARDEN", "title": "bind, escalate, impersonate, exec",
 "params": lambda r: {"ns": ns_pick(r, "release"), "user": r.choice(USERS)},
 "task": """
Role `release-manager` in Namespace `%{ns}` is bound to user `%{user}`, who rolls out Deployments.
The Role carries permissions with which its holder can become anything in the Namespace.

Edit the Role in place (do not delete it, and leave the RoleBinding alone) and remove:

- every permission on Roles and RoleBindings — with `bind` and `escalate`, a user can grant
  themselves whatever they like;
- `impersonate`;
- `pods/exec`;
- every permission on Secrets.

Keep: `get`, `list`, `watch`, `create`, `update`, `patch` on Deployments, and `get`, `list`,
`watch` on Pods.
""",
 "setup": NS + r"""
mkns %{ns}
$K apply -f - >/dev/null <<EOF
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata: {name: release-manager, namespace: %{ns}}
rules:
  - {apiGroups: [apps], resources: [deployments], verbs: [get, list, watch, create, update, patch]}
  - {apiGroups: [""], resources: [pods], verbs: [get, list, watch]}
  - {apiGroups: [""], resources: [pods/exec], verbs: [create]}
  - {apiGroups: [rbac.authorization.k8s.io], resources: [roles, rolebindings], verbs: [get, list, create, update, bind, escalate]}
  - {apiGroups: [""], resources: [serviceaccounts], verbs: [impersonate]}
  - {apiGroups: [""], resources: [secrets], verbs: [get, list]}
EOF
$K -n %{ns} create rolebinding release-manager --role=release-manager --user=%{user} >/dev/null
j -n %{ns} role release-manager '{.metadata.uid}' > "$PRIV/q27.uid"; [ -s "$PRIV/q27.uid" ]
""",
 "checks": [
  C("no permission on Roles and RoleBindings: not bind, not escalate, not create", 3, CAN + r"""
A="--as=%{user}"
for v in get create update bind escalate; do cannot $A -- $v roles.rbac.authorization.k8s.io -n %{ns} || exit 1; done
cannot $A -- create rolebindings.rbac.authorization.k8s.io -n %{ns}"""),
  C("no impersonate, no pods/exec", 2, CAN + r"""
cannot --as=%{user} -- impersonate serviceaccounts -n %{ns} && cannot --as=%{user} -- create pods/exec -n %{ns}"""),
  C("no access to Secrets", 2, CAN + r"""cannot --as=%{user} -- get secrets -n %{ns} && cannot --as=%{user} -- list secrets -n %{ns}"""),
  C("the same Role still manages Deployments and reads Pods", 3, CAN + r"""
A="--as=%{user}"
eq "$(j -n %{ns} role release-manager '{.metadata.uid}')" "$(cat "$PRIV/q27.uid")" && cannot $A -- escalate roles.rbac.authorization.k8s.io -n %{ns} || exit 1
for v in get list watch create update patch; do can $A -- $v deployments.apps -n %{ns} || exit 1; done
for v in get list watch; do can $A -- $v pods -n %{ns} || exit 1; done"""),
 ],
 "solution": """
k -n %{ns} edit role release-manager         # delete the four rules; what stays:
#   rules:
#     - {apiGroups: [apps], resources: [deployments], verbs: [get, list, watch, create, update, patch]}
#     - {apiGroups: [""], resources: [pods], verbs: [get, list, watch]}
k -n %{ns} auth can-i --list --as=%{user}
# Why each one is dangerous:
#   bind       create a RoleBinding to a role you do not hold yourself (to admin, say)
#   escalate   write rules into a Role that you do not hold yourself
#   impersonate  act as another user or ServiceAccount, with its permissions
#   pods/exec  a shell in any Pod - and with it that Pod's ServiceAccount token and Secrets
# Without bind and escalate the API server refuses to let anyone hand out more than they have.
""",
},
{
 "id": 28, "domain": "HARDEN", "title": "Who can read this Secret?",
 "params": lambda r: {"ns": ns_pick(r, "hr")},
 "task": """
Secret `payroll` in Namespace `%{ns}` must be readable by as few identities as possible. The
Namespace has six ServiceAccounts: `builder`, `deployer`, `viewer`, `auditor`, `ingest` and
`reporter`.

1. Find out which of them can `get` Secret `payroll`, and write their names to
   `{WS}/q28/can-read.txt`, one per line.
2. `auditor` has left the project: remove what gives **that** ServiceAccount its access to
   Secrets — in every Namespace — without changing what the others can do.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q28"; mkdir -p "$WS/q28"
$K -n %{ns} create secret generic payroll --from-literal=iban=XX00-%{ns} >/dev/null
for s in builder deployer viewer auditor ingest reporter; do $K -n %{ns} create sa $s >/dev/null; done
$K -n %{ns} create role cfg-reader --verb=get,list --resource=configmaps >/dev/null
$K -n %{ns} create rolebinding builder --role=cfg-reader --serviceaccount=%{ns}:builder >/dev/null
$K -n %{ns} create rolebinding deployer --clusterrole=edit --serviceaccount=%{ns}:deployer >/dev/null
$K -n %{ns} create rolebinding viewer --clusterrole=view --serviceaccount=%{ns}:viewer >/dev/null
$K create clusterrole %{ns}-secret-audit --verb=get,list --resource=secrets >/dev/null
$K create clusterrolebinding %{ns}-secret-audit --clusterrole=%{ns}-secret-audit --serviceaccount=%{ns}:auditor >/dev/null
$K -n %{ns} create role ingest --verb=get --resource=secrets --resource-name=ingest-key >/dev/null
$K -n %{ns} create rolebinding ingest --role=ingest --serviceaccount=%{ns}:ingest >/dev/null
$K -n %{ns} create role payroll-reader --verb=get --resource=secrets --resource-name=payroll >/dev/null
$K -n %{ns} create rolebinding reporter --role=payroll-reader --serviceaccount=%{ns}:reporter >/dev/null
printf 'auditor\ndeployer\nreporter\n' > "$PRIV/q28.can"
""",
 "checks": [
  C("can-read.txt names exactly the ServiceAccounts that could read payroll", 5, r"""
[ -s "$WS/q28/can-read.txt" ] && diff <(tr -d ' \r' < "$WS/q28/can-read.txt" | sed 's#.*:##' | grep . | sort -u) "$PRIV/q28.can" >/dev/null"""),
  C("auditor can no longer read Secrets, here or in any other Namespace", 3, CAN + r"""
S=system:serviceaccount:%{ns}:auditor
cannot --as=$S -- get secret/payroll -n %{ns} && cannot --as=$S -- list secrets -n %{ns} && cannot --as=$S -- get secrets -n kube-system"""),
  C("the others are as they were: deployer and reporter can, the rest cannot", 2, CAN + r"""
P=system:serviceaccount:%{ns}
cannot --as=$P:auditor -- get secret/payroll -n %{ns} || exit 1
can --as=$P:deployer -- get secret/payroll -n %{ns} && can --as=$P:reporter -- get secret/payroll -n %{ns} \
 && can --as=$P:deployer -- create deployments -n %{ns} && can --as=$P:viewer -- list pods -n %{ns} \
 && cannot --as=$P:viewer -- get secret/payroll -n %{ns} && cannot --as=$P:ingest -- get secret/payroll -n %{ns} && cannot --as=$P:builder -- get secret/payroll -n %{ns}"""),
 ],
 "solution": """
for sa in builder deployer viewer auditor ingest reporter; do
  echo "$sa $(k -n %{ns} auth can-i get secret/payroll --as=system:serviceaccount:%{ns}:$sa)"
done                                              # yes: deployer, auditor, reporter -> {WS}/q28/can-read.txt
k get rolebinding,clusterrolebinding -A -o wide | grep -w auditor      # where its access comes from
k delete clusterrolebinding %{ns}-secret-audit
# Why each answer: deployer has `edit` (which includes Secrets), auditor a ClusterRoleBinding (all
# Namespaces), reporter a Role naming payroll. viewer has `view`, which leaves Secrets out; ingest
# may only get the Secret named ingest-key.
# Bindings are found by subject, not by name - `-o wide` prints the subjects.
""",
},
# ========================================================== 2.2 ServiceAccounts ====
{
 "id": 29, "domain": "HARDEN", "title": "A token for something else than the API",
 "params": lambda r: {"ns": ns_pick(r, "pay"), "pod": name_pick(r, "signer")},
 "task": """
Pod `%{pod}` in Namespace `%{ns}` authenticates to an external vault with a ServiceAccount token.
That token must be short-lived and useless against the Kubernetes API.

Create ServiceAccount `vault-client` and Pod `%{pod}` (image `busybox:1.36`, command
`sleep 86400`) in `%{ns}`:

- the Pod runs as `vault-client`;
- the usual API token is **not** mounted;
- a projected ServiceAccount token is mounted at `/var/run/secrets/vault/token`, with audience
  `vault` and a lifetime of 600 seconds.

The Pod must be Running.
""",
 "setup": NS + r"""mkns %{ns}""",
 "checks": [
  C("the Pod runs as vault-client and mounts no API token", 3, r"""
eq "$(j -n %{ns} pod %{pod} '{.spec.serviceAccountName}')" vault-client && eq "$(j -n %{ns} pod %{pod} '{.status.phase}')" Running || exit 1
! $K -n %{ns} exec %{pod} -- test -e /var/run/secrets/kubernetes.io/serviceaccount/token 2>/dev/null"""),
  C("a projected token volume: audience vault, 600 seconds, path token", 3, PY + r"""
py -n %{ns} pod %{pod} '
src = [s["serviceAccountToken"] for v in o["spec"]["volumes"] for s in v.get("projected", {}).get("sources", []) if "serviceAccountToken" in s]
any(s.get("audience") == "vault" and s.get("expirationSeconds") == 600 and s.get("path") == "token" for s in src)'"""),
  C("the token in the Pod is for audience vault only and lives 10 minutes", 4, JWT + r"""
t=$($K -n %{ns} exec %{pod} -- cat /var/run/secrets/vault/token 2>/dev/null); [ -n "$t" ] || exit 1
eq "$(echo "$t" | jwt aud)" '["vault"]' && eq "$(echo "$t" | jwt sub)" system:serviceaccount:%{ns}:vault-client || exit 1
[ "$(( $(echo "$t" | jwt exp) - $(echo "$t" | jwt iat) ))" -le 660 ]"""),
 ],
 "solution": """
k -n %{ns} create sa vault-client
cat <<EOF | k apply -f -
apiVersion: v1
kind: Pod
metadata: {name: %{pod}, namespace: %{ns}}
spec:
  serviceAccountName: vault-client
  automountServiceAccountToken: false
  containers:
    - name: c
      image: busybox:1.36
      command: [sleep, "86400"]
      volumeMounts:
        - {name: vault-token, mountPath: /var/run/secrets/vault, readOnly: true}
  volumes:
    - name: vault-token
      projected:
        sources:
          - serviceAccountToken: {path: token, audience: vault, expirationSeconds: 600}
EOF
k -n %{ns} exec %{pod} -- cat /var/run/secrets/vault/token | cut -d. -f2 | base64 -d 2>/dev/null; echo
# The API server only accepts tokens whose audience is its own, so a token for "vault" cannot be
# replayed against it. 600 seconds is the minimum; the kubelet replaces the file before it expires.
# docs: Configure Service Accounts for Pods -> "ServiceAccount token volume projection".
""",
},
{
 "id": 30, "domain": "HARDEN", "title": "Tokens that never expire",
 "params": lambda r: {"ns": ns_pick(r, "build")},
 "task": """
Secret-based ServiceAccount tokens never expire: whoever copies one keeps their access for as long
as the Secret exists. Namespace `%{ns}` still has some.

1. Write the names of all Secrets of type `kubernetes.io/service-account-token` in `%{ns}` to
   `{WS}/q30/legacy.txt`, one per line.
2. Delete them — and only them.
3. ServiceAccount `ci` still needs a token for tonight's job: issue one that is valid for
   30 minutes and save it to `{WS}/q30/ci.token`.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q30"; mkdir -p "$WS/q30"
$K -n %{ns} create sa ci >/dev/null
$K -n %{ns} create role pod-reader --verb=get,list --resource=pods >/dev/null
$K -n %{ns} create rolebinding ci --role=pod-reader --serviceaccount=%{ns}:ci >/dev/null
$K -n %{ns} create secret generic ci-settings --from-literal=registry=registry.internal >/dev/null
$K apply -f - >/dev/null <<EOF
apiVersion: v1
kind: Secret
metadata: {name: ci-token-2023, namespace: %{ns}, annotations: {kubernetes.io/service-account.name: ci}}
type: kubernetes.io/service-account-token
---
apiVersion: v1
kind: Secret
metadata: {name: default-token-legacy, namespace: %{ns}, annotations: {kubernetes.io/service-account.name: default}}
type: kubernetes.io/service-account-token
EOF
for i in $(seq 1 30); do t=$(j -n %{ns} secret ci-token-2023 '{.data.token}' | base64 -d); [ -n "$t" ] && break; sleep 1; done
[ -n "$t" ] || exit 1
echo "$t" > "$PRIV/q30.old"; printf 'ci-token-2023\ndefault-token-legacy\n' > "$PRIV/q30.legacy"
""",
 "checks": [
  C("legacy.txt names the two token Secrets", 3, r"""
[ -s "$WS/q30/legacy.txt" ] && diff <(tr -d ' \r' < "$WS/q30/legacy.txt" | sed 's#.*/##' | grep . | sort -u) "$PRIV/q30.legacy" >/dev/null"""),
  C("both are deleted, Secret ci-settings is not, and the old token is refused", 3, CURL + r"""
[ -z "$(j -n %{ns} secret ci-token-2023 '{.metadata.name}')" ] && [ -z "$(j -n %{ns} secret default-token-legacy '{.metadata.name}')" ] \
 && [ -n "$(j -n %{ns} secret ci-settings '{.metadata.name}')" ] || exit 1
eq "$(code /api/v1/namespaces/%{ns}/pods -H "'Authorization: Bearer $(cat "$PRIV/q30.old")'")" 401"""),
  C("ci.token is a token for ServiceAccount ci, valid for 30 minutes, that the API accepts", 4, JWT + CURL + r"""
t=$(tr -d ' \n\r' < "$WS/q30/ci.token" 2>/dev/null); [ -n "$t" ] || exit 1
eq "$(echo "$t" | jwt sub)" system:serviceaccount:%{ns}:ci && eq "$(( $(echo "$t" | jwt exp) - $(echo "$t" | jwt iat) ))" 1800 || exit 1
eq "$(code /api/v1/namespaces/%{ns}/pods -H "'Authorization: Bearer $t'")" 200"""),
 ],
 "solution": """
k -n %{ns} get secret --field-selector type=kubernetes.io/service-account-token
k -n %{ns} get secret --field-selector type=kubernetes.io/service-account-token -o name | cut -d/ -f2 > {WS}/q30/legacy.txt
k -n %{ns} delete secret $(cat {WS}/q30/legacy.txt)
k -n %{ns} create token ci --duration=30m > {WS}/q30/ci.token
cut -d. -f2 {WS}/q30/ci.token | base64 -d 2>/dev/null; echo          # exp - iat = 1800
# Deleting the Secret revokes the token at once: the API server checks that the Secret still exists.
# `create token` asks the TokenRequest API for a signed, time-limited token that is stored nowhere.
# Since v1.24 no token Secret is created for a ServiceAccount automatically - any you find was made
# by hand or is left over.
""",
},
{
 "id": 31, "domain": "HARDEN", "title": "A ServiceAccount of its own",
 "params": lambda r: {"ns": ns_pick(r, "ops")},
 "task": """
Deployment `reporter` in Namespace `%{ns}` runs as the Namespace's `default` ServiceAccount, which
somebody bound to ClusterRole `edit` (RoleBinding `default-edit`). All `reporter` does with the
API is read ConfigMaps and create Events.

1. Create ServiceAccount `reporter`, Role `reporter` and RoleBinding `reporter` in `%{ns}` with
   exactly that: `get`, `list`, `watch` on ConfigMaps and `create` on Events.
2. Make the Deployment run as `reporter`.
3. Take the `default` ServiceAccount's rights away, and stop it from mounting its token into Pods
   automatically.
""",
 "setup": NS + r"""
mkns %{ns}
$K -n %{ns} create rolebinding default-edit --clusterrole=edit --serviceaccount=%{ns}:default >/dev/null
$K -n %{ns} create deploy reporter --image=busybox:1.36 -- sleep 86400 >/dev/null
$K -n %{ns} rollout status deploy reporter --timeout=180s >/dev/null
""",
 "checks": [
  C("the Deployment's Pods run as ServiceAccount reporter", 3, PY + r"""
eq "$(j -n %{ns} deploy reporter '{.spec.template.spec.serviceAccountName}')" reporter || exit 1
for i in $(seq 1 20); do
  py -n %{ns} pod -l app=reporter '
p = [x for x in o["items"] if "deletionTimestamp" not in x["metadata"]]
len(p) == 1 and p[0]["spec"]["serviceAccountName"] == "reporter" and p[0]["status"].get("phase") == "Running"' && exit 0
  sleep 3
done; exit 1"""),
  C("reporter may read ConfigMaps and create Events, and nothing more", 3, CAN + r"""
S=system:serviceaccount:%{ns}:reporter
for v in get list watch; do can --as=$S -- $v configmaps -n %{ns} || exit 1; done
can --as=$S -- create events -n %{ns} && cannot --as=$S -- get secrets -n %{ns} && cannot --as=$S -- delete configmaps -n %{ns} \
 && cannot --as=$S -- create pods -n %{ns} && cannot --as=$S -- list configmaps -n default"""),
  C("the default ServiceAccount has no rights left", 2, CAN + r"""
S=system:serviceaccount:%{ns}:default
[ -n "$(j -n %{ns} sa reporter '{.metadata.name}')" ] && cannot --as=$S -- list configmaps -n %{ns} && cannot --as=$S -- get pods -n %{ns} && cannot --as=$S -- get secrets -n %{ns}"""),
  C("and no longer mounts its token automatically", 2, r"""eq "$(j -n %{ns} sa default '{.automountServiceAccountToken}')" false"""),
 ],
 "solution": """
k -n %{ns} create sa reporter
k -n %{ns} create role reporter --verb=get,list,watch --resource=configmaps
k -n %{ns} edit role reporter                  # add:  - {apiGroups: [""], resources: [events], verbs: [create]}
k -n %{ns} create rolebinding reporter --role=reporter --serviceaccount=%{ns}:reporter
k -n %{ns} set serviceaccount deploy reporter reporter
k -n %{ns} delete rolebinding default-edit
k -n %{ns} patch sa default -p '{"automountServiceAccountToken": false}'
k -n %{ns} auth can-i --list --as=system:serviceaccount:%{ns}:reporter
# `create role` takes one set of verbs for all its resources, so two rules with different verbs
# need an edit (or a YAML file). The default ServiceAccount is what every Pod gets when nobody
# says otherwise: it should be able to do nothing, and need not hand out a token.
""",
},
# ======================================================= 2.3 Restrict API access ====
{
 "id": 32, "domain": "HARDEN", "title": "The API answers without credentials",
 "params": lambda r: {"ns": ns_pick(r, "payroll")},
 "task": """
Anyone who can reach this cluster's API server can read it without logging in. On `{CP}`:

    curl -k https://localhost:6443/api/v1/namespaces/%{ns}/secrets

1. Save the response of that request to `{WS}/q32/before.json`.
2. Find every ClusterRoleBinding and RoleBinding that grants something to user
   `system:anonymous` or to group `system:unauthenticated`. Write the names of those that are
   **not** part of Kubernetes itself to `{WS}/q32/bindings.txt`, one per line.
3. Delete them. Kubernetes' own bindings (their names start with `system:`) stay: `/version` and
   the health endpoints must keep answering anonymous requests.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q32"; mkdir -p "$WS/q32"
$K -n %{ns} create secret generic salaries --from-literal=total=%{ns}-4711 >/dev/null
$K create clusterrole all-secrets-reader --verb=get,list --resource=secrets >/dev/null
$K create clusterrolebinding monitoring-anon --clusterrole=all-secrets-reader --user=system:anonymous >/dev/null
$K create clusterrolebinding legacy-discovery --clusterrole=view --group=system:unauthenticated >/dev/null
$K -n %{ns} create rolebinding guest-access --clusterrole=edit --group=system:unauthenticated >/dev/null
printf 'guest-access\nlegacy-discovery\nmonitoring-anon\n' > "$PRIV/q32.bindings"
""",
 "checks": [
  C("before.json shows the Secrets an anonymous request could list", 2, r"""grep -q '"salaries"' "$WS/q32/before.json" 2>/dev/null && grep -q SecretList "$WS/q32/before.json" """),
  C("bindings.txt names the three bindings", 3, r"""
[ -s "$WS/q32/bindings.txt" ] && diff <(tr -d ' \r' < "$WS/q32/bindings.txt" | sed 's#.*/##' | grep . | sort -u) "$PRIV/q32.bindings" >/dev/null"""),
  C("anonymous requests for Secrets and Pods now get 403", 3, CURL + r"""
eq "$(code /api/v1/namespaces/%{ns}/secrets)" 403 && eq "$(code /api/v1/namespaces/%{ns}/pods)" 403 && eq "$(code /api/v1/namespaces/default/configmaps)" 403"""),
  C("while /version and /readyz still answer them", 2, CURL + r"""
eq "$(code /api/v1/namespaces/%{ns}/secrets)" 403 && eq "$(code /version)" 200 && eq "$(code /readyz)" 200"""),
 ],
 "solution": """
curl -sk https://localhost:6443/api/v1/namespaces/%{ns}/secrets > {WS}/q32/before.json
k get clusterrolebinding,rolebinding -A -o wide | grep -E 'system:anonymous|system:unauthenticated'
#   monitoring-anon, legacy-discovery (cluster-wide), guest-access (in %{ns})  -> {WS}/q32/bindings.txt
#   system:public-info-viewer is Kubernetes' own: it lets anyone read /version, /healthz, /livez, /readyz
k delete clusterrolebinding monitoring-anon legacy-discovery
k -n %{ns} delete rolebinding guest-access
curl -sk https://localhost:6443/api/v1/namespaces/%{ns}/secrets | grep -E 'code|message'      # 403 Forbidden
curl -sk https://localhost:6443/version | head -3                                             # still answers
# A request without credentials is not rejected: it becomes user system:anonymous in group
# system:unauthenticated, and RBAC decides. 403 means "I know who you are (nobody), and you may
# not"; 401 would mean anonymous requests are switched off altogether.
""",
},
{
 "id": 33, "domain": "HARDEN", "title": "Anonymous access for health checks only", "difficulty": "hard",
 "task": """
Anonymous requests to this cluster's API server are to be accepted for the health endpoints only.
Switching them off altogether (`--anonymous-auth=false`) is not an option: the kubelet probes the
API server's static Pod at `/livez` and `/readyz` without credentials.

On `{CP}`:

1. Write an `AuthenticationConfiguration` (API group `apiserver.config.k8s.io`) to
   `/etc/kubernetes/auth/authn.yaml` that enables anonymous requests for the paths `/livez`,
   `/readyz` and `/healthz`, and for nothing else.
2. Make the API server use it (`--authentication-config`); the file has to be visible inside
   the static Pod.
3. When the API server is back, save the HTTP status code an anonymous request to
   `https://localhost:6443/version` gets to `{WS}/q33/version.txt`.
""",
 "setup": r"""
rm -rf "$WS/q33"; mkdir -p "$WS/q33"
nx "$CP" 'rm -rf /etc/kubernetes/auth' </dev/null
""",
 "checks": [
  C("authn.yaml allows anonymous requests for the three health paths only", 2, r"""
f=/etc/kubernetes/auth/authn.yaml
eq "$(nx "$CP" "yq .kind $f" </dev/null)" AuthenticationConfiguration && eq "$(nx "$CP" "yq .anonymous.enabled $f" </dev/null)" true \
 && eq "$(nx "$CP" "yq '[.anonymous.conditions[].path] | sort | join(\",\")' $f" </dev/null)" /healthz,/livez,/readyz"""),
  C("the API server runs with --authentication-config and is healthy", 3, API + r"""
runs kube-apiserver '--authentication-config=/etc/kubernetes/auth/authn.yaml' && wait_api 90 && pod_ready kube-system "kube-apiserver-$CP" 90"""),
  C("anonymous: /livez and /readyz answer 200, /version and /api get 401", 4, API + CURL + r"""
runs kube-apiserver '--authentication-config=' && wait_api 60 || exit 1
eq "$(code /livez)" 200 && eq "$(code /readyz)" 200 && eq "$(code /version)" 401 && eq "$(code /api/v1/namespaces)" 401"""),
  C("version.txt says 401", 1, r"""eq "$(tr -d ' \n\r' < "$WS/q33/version.txt" 2>/dev/null)" 401"""),
 ],
 "solution": """
sudo -i
mkdir -p /etc/kubernetes/auth
cat > /etc/kubernetes/auth/authn.yaml <<EOF
apiVersion: apiserver.config.k8s.io/v1
kind: AuthenticationConfiguration
anonymous:
  enabled: true
  conditions:
    - path: /livez
    - path: /readyz
    - path: /healthz
EOF
cp /etc/kubernetes/manifests/kube-apiserver.yaml /root/                  # a backup outside manifests/
vi /etc/kubernetes/manifests/kube-apiserver.yaml
#   command:       - --authentication-config=/etc/kubernetes/auth/authn.yaml
#   volumeMounts:  - {name: authn, mountPath: /etc/kubernetes/auth, readOnly: true}
#   volumes:       - {name: authn, hostPath: {path: /etc/kubernetes/auth, type: DirectoryOrCreate}}
watch crictl ps                                                          # until kube-apiserver is back
curl -sk -o /dev/null -w '%{http_code}\\n' https://localhost:6443/livez      # 200
curl -sk -o /dev/null -w '%{http_code}\\n' https://localhost:6443/version | tee {WS}/q33/version.txt     # 401
# docs: Authenticating -> "Anonymous Authenticator Configuration". The file and --anonymous-auth
# exclude each other: with both, the API server refuses to start. If it does not come back, the
# reason is in `crictl logs $(crictl ps -a --name kube-apiserver -q | head -1)`.
""",
},
{
 "id": 34, "domain": "HARDEN", "title": "AlwaysAllow, and kubelets that may change anything", "workers": 1, "difficulty": "hard",
 "task": """
The API server of this cluster authorises every request (`--authorization-mode=AlwaysAllow`), and
nothing restricts what a kubelet may change.

1. Set the authorisation mode back to `Node,RBAC`.
2. Enable the `NodeRestriction` admission plugin.
3. When the API server is back, prove it: with the kubelet's own credentials on `{W1}`
   (`/etc/kubernetes/kubelet.conf` there), try to label the **control-plane** node:

       kubectl --kubeconfig /etc/kubernetes/kubelet.conf label node {CP} audit=failed

   Save the error message to `{WS}/q34/denied.txt`.
""",
 "setup": API + r"""
rm -rf "$WS/q34"; mkdir -p "$WS/q34"
medit kube-apiserver.yaml '(.spec.containers[0].command[] | select(. == "--authorization-mode=Node,RBAC")) = "--authorization-mode=AlwaysAllow" | .spec.containers[0].command |= map(select(test("^--enable-admission-plugins=") | not))' || exit 1
wait_proc kube-apiserver '--authorization-mode=AlwaysAllow' && wait_api
""",
 "checks": [
  C("the API server runs with --authorization-mode=Node,RBAC", 3, API + r"""
eq "$(flag kube-apiserver authorization-mode)" Node,RBAC && runs kube-apiserver '--authorization-mode=Node,RBAC' && wait_api 90"""),
  C("and with the NodeRestriction admission plugin", 3, API + r"""
flag kube-apiserver enable-admission-plugins | tr ',' '\n' | grep -qx NodeRestriction \
 && runs kube-apiserver '--enable-admission-plugins=[^ ]*NodeRestriction' && wait_api 90"""),
  C("the kubelet of {W1} can update its own Node but not label another one", 3, API + r"""
runs kube-apiserver '--authorization-mode=Node,RBAC' && wait_api 60 || exit 1
KC='kubectl --kubeconfig /etc/kubernetes/kubelet.conf'
nx "$W1" "$KC get node $W1 -o name" </dev/null >/dev/null 2>&1 || exit 1          # its credentials work at all
out=$(nx "$W1" "$KC label node $CP ksim-probe=1 --overwrite" </dev/null 2>&1); rc=$?
$K label node "$CP" ksim-probe- >/dev/null 2>&1
[ $rc -ne 0 ] && echo "$out" | grep -qi forbidden"""),
  C("denied.txt holds the refusal", 1, r"""grep -qi 'forbidden' "$WS/q34/denied.txt" 2>/dev/null"""),
 ],
 "solution": """
sudo cp /etc/kubernetes/manifests/kube-apiserver.yaml /root/
sudo vi /etc/kubernetes/manifests/kube-apiserver.yaml
#     - --authorization-mode=Node,RBAC
#     - --enable-admission-plugins=NodeRestriction
watch sudo crictl ps                                         # until kube-apiserver is back
ssh {W1} sudo kubectl --kubeconfig /etc/kubernetes/kubelet.conf label node {CP} audit=failed 2>&1 | tee {WS}/q34/denied.txt
# Node authorisation lets a kubelet read what the Pods on its node need; NodeRestriction (an
# admission plugin - it goes in --enable-admission-plugins, not in the authorisation modes) lets
# it modify only its own Node object and the Pods bound to it. Without it, a compromised node can
# relabel other nodes and attract their workloads. Order matters in Node,RBAC only for speed.
""",
},
{
 "id": 35, "domain": "HARDEN", "title": "The API server on a NodePort",
 "task": """
The API server of this cluster is published on port **31000** of every node: Service `kubernetes`
in Namespace `default` is of type `NodePort`.

1. Find what makes it so and remove it from the API server's configuration.
2. Service `kubernetes` must be of type `ClusterIP` again. (The API server maintains that Service
   itself — if you delete it, it is created anew.)
""",
 "setup": API + r"""
medit kube-apiserver.yaml '.spec.containers[0].command += ["--kubernetes-service-node-port=31000"]' || exit 1
wait_proc kube-apiserver '--kubernetes-service-node-port=31000' && wait_api || exit 1
# the flag only takes effect when the API server creates its Service: let it create it again
$K -n default delete svc kubernetes >/dev/null 2>&1
for i in $(seq 1 30); do [ "$(j -n default svc kubernetes '{.spec.type}')" = NodePort ] && exit 0; sleep 2; done
echo "Service kubernetes did not become a NodePort" >&2; exit 1
""",
 "checks": [
  C("the API server no longer runs with --kubernetes-service-node-port, and is healthy", 4, API + r"""
[ -z "$(flag kube-apiserver kubernetes-service-node-port)" ] && ! runs kube-apiserver '--kubernetes-service-node-port' && wait_api 90 \
 && pod_ready kube-system "kube-apiserver-$CP" 90"""),
  C("Service kubernetes is of type ClusterIP, without a node port", 4, API + r"""
! runs kube-apiserver '--kubernetes-service-node-port' || exit 1
for i in $(seq 1 10); do [ "$(j -n default svc kubernetes '{.spec.type}')" = ClusterIP ] && break; sleep 3; done
eq "$(j -n default svc kubernetes '{.spec.type}')" ClusterIP && [ -z "$(j -n default svc kubernetes '{.spec.ports[0].nodePort}')" ]"""),
  C("port 31000 of the node no longer answers, port 6443 does", 2, r"""
eq "$(j -n default svc kubernetes '{.spec.type}')" ClusterIP && eq "$(nx "$CP" "curl -sk -m 5 -o /dev/null -w '%{http_code}' https://$CP:6443/version" </dev/null)" 200 || exit 1
# kube-proxy may need most of a minute to notice, after the API server it watches was restarted
for i in $(seq 1 25); do nx "$CP" "curl -sk -m 3 -o /dev/null https://$CP:31000/version" </dev/null || exit 0; sleep 3; done; exit 1"""),
 ],
 "solution": """
k get svc kubernetes                                          # NodePort 443:31000
sudo grep -n node-port /etc/kubernetes/manifests/kube-apiserver.yaml
sudo vi /etc/kubernetes/manifests/kube-apiserver.yaml         # delete: - --kubernetes-service-node-port=31000
watch sudo crictl ps                                          # until kube-apiserver is back
k delete svc kubernetes                                       # the API server re-creates it, as ClusterIP
k get svc kubernetes
# The flag makes the API server create its own Service as a NodePort. Removing the flag does not
# change the existing Service - deleting it does, and is safe: the reconciler puts it back within
# seconds. Editing the Service's type by hand works too.
""",
},
{
 "id": 36, "domain": "HARDEN", "title": "A client certificate for an auditor",
 "params": lambda r: {"ns": ns_pick(r, "ledger"), "user": r.choice(USERS)},
 "task": """
Auditor `%{user}` needs read-only access to Namespace `%{ns}` for one day, with a client certificate.

1. Create a private key `{WS}/q36/%{user}.key` (RSA, 2048 bits) and a certificate request for
   common name `%{user}` and organisation `auditors`.
2. Submit it as CertificateSigningRequest `%{user}`: signer `kubernetes.io/kube-apiserver-client`,
   usage `client auth`, valid for 86400 seconds. Approve it and save the issued certificate to
   `{WS}/q36/%{user}.crt`.
3. Give group `auditors` the built-in ClusterRole `view` in Namespace `%{ns}` only, with
   RoleBinding `auditors-view`.
""",
 "setup": NS + r"""
mkns %{ns}; rm -rf "$WS/q36"; mkdir -p "$WS/q36"
$K -n %{ns} create secret generic books --from-literal=k=v >/dev/null
$K delete csr %{user} --ignore-not-found >/dev/null 2>&1
""",
 "checks": [
  C("CertificateSigningRequest %{user}: client signer, client auth, 86400 seconds, approved", 2, PY + r"""
py csr %{user} '
s = o["spec"]
s["signerName"] == "kubernetes.io/kube-apiserver-client" and "client auth" in s.get("usages", []) and s.get("expirationSeconds") == 86400 and any(c["type"] == "Approved" for c in o.get("status", {}).get("conditions", []))'"""),
  C("%{user}.crt: CN %{user}, O auditors, signed by the cluster's CA, for one day, matching the key", 3, r"""
d=/opt/course/q36
nx "$CP" "openssl verify -CAfile /etc/kubernetes/pki/ca.crt $d/%{user}.crt" </dev/null >/dev/null 2>&1 || exit 1
s=$(nx "$CP" "openssl x509 -in $d/%{user}.crt -noout -subject -nameopt RFC2253" </dev/null)
echo "$s" | grep -q 'CN=%{user}\(,\|$\)' && echo "$s" | grep -q 'O=auditors\(,\|$\)' || exit 1
eq "$(nx "$CP" "openssl x509 -in $d/%{user}.crt -noout -pubkey | sha256sum" </dev/null)" "$(nx "$CP" "openssl pkey -in $d/%{user}.key -pubout | sha256sum" </dev/null)" || exit 1
a=$(date -d "$(nx "$CP" "openssl x509 -in $d/%{user}.crt -noout -startdate" </dev/null | cut -d= -f2)" +%s)
b=$(date -d "$(nx "$CP" "openssl x509 -in $d/%{user}.crt -noout -enddate" </dev/null | cut -d= -f2)" +%s)
[ $(( b - a )) -le 87000 ]"""),
  C("with it, %{user} lists Pods in %{ns} - and gets no Secrets and no other Namespace", 3, CURL + r"""
A="--cert /opt/course/q36/%{user}.crt --key /opt/course/q36/%{user}.key"
eq "$(code /api/v1/namespaces/%{ns}/pods $A)" 200 && eq "$(code /api/v1/namespaces/%{ns}/secrets $A)" 403 && eq "$(code /api/v1/namespaces/default/pods $A)" 403"""),
  C("RoleBinding auditors-view gives group auditors the ClusterRole view", 2, PY + r"""
py -n %{ns} rolebinding auditors-view '
o["roleRef"]["kind"] == "ClusterRole" and o["roleRef"]["name"] == "view" and [(s["kind"], s["name"]) for s in o.get("subjects", [])] == [("Group", "auditors")]'"""),
 ],
 "solution": """
cd {WS}/q36
openssl genrsa -out %{user}.key 2048
openssl req -new -key %{user}.key -subj "/CN=%{user}/O=auditors" -out %{user}.csr
cat <<EOF | k apply -f -
apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata: {name: %{user}}
spec:
  request: $(base64 -w0 < %{user}.csr)
  signerName: kubernetes.io/kube-apiserver-client
  expirationSeconds: 86400
  usages: [client auth]
EOF
k certificate approve %{user}
# the signer issues the certificate a moment AFTER the approval - read it too early and the file is empty
until [ -n "$(k get csr %{user} -o jsonpath='{.status.certificate}')" ]; do sleep 1; done
k get csr %{user} -o jsonpath='{.status.certificate}' | base64 -d > %{user}.crt
k -n %{ns} create rolebinding auditors-view --clusterrole=view --group=auditors
openssl x509 -in %{user}.crt -noout -subject -dates
# CN becomes the user name, O the group. The request field is the base64 of the whole PEM file on
# one line (-w0). There is no revoking a client certificate: its short life and the RoleBinding
# of the group are what you control. docs: "Certificates and Certificate Signing Requests".
""",
},
# ================================================================= 2.4 Upgrade ====
{
 "id": 37, "domain": "HARDEN", "title": "A worker on a vulnerable kubelet", "workers": 1, "difficulty": "hard",
 "params": lambda r: {"ns": ns_pick(r, "web"), "old": r.choice(["1.36.3", "1.36.4", "1.36.5"])},
 "task": """
Node `{W1}` still runs Kubernetes **v%{old}**, a release with known vulnerabilities. The control
plane is newer (`k get nodes`).

Upgrade `{W1}` to the control plane's version, the documented way:

1. Drain `{W1}`.
2. On `{W1}`: upgrade the `kubeadm` package, run `kubeadm upgrade node`, upgrade `kubelet` and
   `kubectl`, and restart the kubelet.
3. Make the node schedulable again.

The three packages are held (`apt-mark`) and must be held again when you are done. Deployment
`web` in `%{ns}` must be available afterwards.
""",
 "setup": NS + API + APT + r"""
mkns %{ns}
hold_back "$W1" %{old} || { echo "could not put $W1 on v%{old}" >&2; exit 1; }
$K -n %{ns} create deploy web --image=nginx:1.27 --replicas=2 >/dev/null
$K -n %{ns} rollout status deploy web --timeout=240s >/dev/null && node_ready "$W1" 60
""",
 "checks": [
  C("{W1} runs the control plane's Kubernetes version and is Ready", 4, API + r"""
v=$(j node "$CP" '{.status.nodeInfo.kubeletVersion}'); [ -n "$v" ] || exit 1
for i in $(seq 1 10); do [ "$(j node "$W1" '{.status.nodeInfo.kubeletVersion}')" = "$v" ] && break; sleep 3; done
eq "$(j node "$W1" '{.status.nodeInfo.kubeletVersion}')" "$v" && node_ready "$W1" 30"""),
  C("kubelet, kubeadm and kubectl packages on {W1} are at that version", 2, APT + r"""
v=$(j node "$CP" '{.status.nodeInfo.kubeletVersion}'); v=${v#v}; [ -n "$v" ] || exit 1
eq "$(pkgs "$W1")" "$v-1.1 $v-1.1 $v-1.1 " """),
  C("and are held again", 2, APT + r"""
v=$(j node "$CP" '{.status.nodeInfo.kubeletVersion}'); eq "$(pkgs "$W1" | cut -d' ' -f1)" "${v#v}-1.1" && held "$W1" """),
  C("{W1} is schedulable and Deployment web is available", 2, r"""
eq "$(j node "$W1" '{.status.nodeInfo.kubeletVersion}')" "$(j node "$CP" '{.status.nodeInfo.kubeletVersion}')" || exit 1
[ "$(j node "$W1" '{.spec.unschedulable}')" != true ] || exit 1
for i in $(seq 1 20); do [ "$(j -n %{ns} deploy web '{.status.availableReplicas}')" = 2 ] && exit 0; sleep 3; done; exit 1"""),
 ],
 "solution": """
k get nodes                                        # {W1}: v%{old}; the control plane's version is the target, say 1.37.X
k drain {W1} --ignore-daemonsets --delete-emptydir-data
{SHELL_W1}
sudo -i
cat /etc/apt/sources.list.d/kubernetes.list                   # .../core:/stable:/v1.36/deb/ - the node's minor
sed -i 's|/v1.36/|/v1.37/|' /etc/apt/sources.list.d/kubernetes.list
apt-get update && apt-cache madison kubeadm | head -5         # the package versions on offer
apt-mark unhold kubeadm && apt-get install -y kubeadm=1.37.X-1.1 && apt-mark hold kubeadm
kubeadm upgrade node                                          # a worker: `node`, not `apply`
apt-mark unhold kubelet kubectl && apt-get install -y kubelet=1.37.X-1.1 kubectl=1.37.X-1.1 && apt-mark hold kubelet kubectl
systemctl daemon-reload && systemctl restart kubelet
exit; exit
k uncordon {W1}
k get nodes                                                   # both on the same version, Ready
# docs: "Upgrading kubeadm clusters" -> "Changing the package repository", then "Upgrade worker
# nodes". {W1} is a minor behind, and pkgs.k8s.io serves one repository per minor: until the line
# names v1.37, apt offers nothing newer than 1.36. kubeadm first, because it rewrites the kubelet's
# configuration for the new version; the kubelet package last, then the restart - `k get nodes`
# shows the version of the running kubelet, not of the package.
""",
},
{
 "id": 38, "domain": "HARDEN", "title": "Upgrade the control plane", "workers": 0, "difficulty": "hard",
 "task": """
A vulnerability in the API server was fixed in a newer patch release of Kubernetes v1.37.

Upgrade this cluster — a single control-plane node, `{CP}` — to the **newest v1.37 patch release**
that the node's package repository offers:

1. Upgrade the `kubeadm` package, look at `kubeadm upgrade plan`, and apply the upgrade.
2. Drain the node, upgrade `kubelet` and `kubectl`, restart the kubelet, make the node
   schedulable again.

The three packages are held (`apt-mark`) and must be held again when you are done.
""",
 "setup": API + APT + r"""
nx "$CP" "apt-get update -qq $K8S_LIST >/dev/null 2>&1; apt-cache madison kubeadm | head -1" </dev/null | awk '{print $3}' > "$PRIV/q38.target"
t=$(cat "$PRIV/q38.target"); cur=$(pkgs "$CP" | cut -d' ' -f1)
[ -n "$t" ] && [ -n "$cur" ] || { echo "no package versions on $CP" >&2; exit 1; }
[ "$t" != "$cur" ] || { echo "the repository has nothing newer than $cur: there is no upgrade to do" >&2; exit 1; }
""",
 "checks": [
  C("the API server runs the newest v1.37 patch release", 4, API + r"""
t=$(cat "$PRIV/q38.target"); wait_api 60 || exit 1
eq "$($K version -o json 2>/dev/null | python3 -c 'import json,sys; print(json.load(sys.stdin)["serverVersion"]["gitVersion"])')" "v${t%%-*}" """),
  C("the node's kubelet does too, and the node is Ready and schedulable", 3, API + r"""
t=$(cat "$PRIV/q38.target")
for i in $(seq 1 10); do [ "$(j node "$CP" '{.status.nodeInfo.kubeletVersion}')" = "v${t%%-*}" ] && break; sleep 3; done
eq "$(j node "$CP" '{.status.nodeInfo.kubeletVersion}')" "v${t%%-*}" && node_ready "$CP" 30 && [ "$(j node "$CP" '{.spec.unschedulable}')" != true ]"""),
  C("kubelet, kubeadm and kubectl packages are at that version and held", 2, APT + r"""
t=$(cat "$PRIV/q38.target"); eq "$(pkgs "$CP")" "$t $t $t " && held "$CP" """),
  C("controller manager and scheduler run the new images", 1, r"""
t=$(cat "$PRIV/q38.target"); v="v${t%%-*}"
j -n kube-system pod "kube-controller-manager-$CP" '{.spec.containers[0].image}' | grep -q ":$v$" \
 && j -n kube-system pod "kube-scheduler-$CP" '{.spec.containers[0].image}' | grep -q ":$v$" """),
 ],
 "solution": """
sudo -i
apt-get update && apt-cache madison kubeadm | head -3         # newest first, say 1.37.X-1.1
apt-mark unhold kubeadm && apt-get install -y kubeadm=1.37.X-1.1 && apt-mark hold kubeadm
kubeadm version -o short
kubeadm upgrade plan                                          # what would change, and to which version
kubeadm upgrade apply v1.37.X                                 # a few minutes; answers y
kubectl drain {CP} --ignore-daemonsets --delete-emptydir-data
apt-mark unhold kubelet kubectl && apt-get install -y kubelet=1.37.X-1.1 kubectl=1.37.X-1.1 && apt-mark hold kubelet kubectl
systemctl daemon-reload && systemctl restart kubelet
kubectl uncordon {CP}
kubectl get nodes; kubectl version
# docs: "Upgrading kubeadm clusters". `upgrade apply` replaces the static Pod manifests of the
# API server, controller manager and scheduler, and upgrades CoreDNS and kube-proxy; the kubelet
# is a package and stays old until you install and restart it. On a cluster with more nodes: the
# other control planes with `kubeadm upgrade node`, then the workers one at a time.
""",
},
{
 "id": 39, "domain": "HARDEN", "title": "Prepare an upgrade", "workers": 2,
 "params": lambda r: {"old": r.choice(["1.36.3", "1.36.4", "1.36.5"]), "n": r.choice([1, 2])},
 "task": """
A CVE affects every kubelet older than the version this cluster's control plane runs. Prepare the
upgrade — do **not** upgrade anything yet.

1. Write the names of the nodes whose kubelet is older than the control plane's to
   `{WS}/q39/outdated.txt`, one per line.
2. Cordon those nodes, so that no new Pod lands on a vulnerable kubelet. Do not drain them, and
   leave the other nodes schedulable.
3. Write the newest v1.37 package version of `kubeadm` that the repository on `{CP}` offers
   (as apt prints it, for example `1.37.2-1.1`) to `{WS}/q39/target.txt`.
4. Save the output of `kubeadm upgrade plan` on `{CP}` to `{WS}/q39/plan.txt`.
""",
 "setup": API + APT + r"""
rm -rf "$WS/q39"; mkdir -p "$WS/q39"
old=$([ %{n} = 1 ] && echo "$W1" || echo "$W2")
hold_back "$old" %{old} || { echo "could not put $old on v%{old}" >&2; exit 1; }
echo "$old" > "$PRIV/q39.outdated"
nx "$CP" "apt-get update -qq $K8S_LIST >/dev/null 2>&1; apt-cache madison kubeadm | head -1" </dev/null | awk '{print $3}' > "$PRIV/q39.target"
[ -s "$PRIV/q39.target" ] && node_ready "$old" 60
""",
 "checks": [
  C("outdated.txt names the node with the old kubelet", 3, r"""
[ -s "$WS/q39/outdated.txt" ] && diff <(tr -d ' \r' < "$WS/q39/outdated.txt" | sed 's#.*/##' | grep . | sort -u) "$PRIV/q39.outdated" >/dev/null"""),
  C("that node is cordoned and not drained; the others are schedulable", 3, r"""
old=$(cat "$PRIV/q39.outdated")
eq "$(j node "$old" '{.spec.unschedulable}')" true || exit 1
for n in "$CP" "$W1" "$W2"; do [ "$n" = "$old" ] || [ "$(j node "$n" '{.spec.unschedulable}')" != true ] || exit 1; done
eq "$(j node "$old" '{.status.nodeInfo.kubeletVersion}')" v%{old}"""),
  C("target.txt is the newest kubeadm package version on offer", 2, r"""eq "$(tr -d ' \n\r' < "$WS/q39/target.txt" 2>/dev/null)" "$(cat "$PRIV/q39.target")" """),
  C("plan.txt is the output of kubeadm upgrade plan", 2, r"""
grep -q 'kube-apiserver' "$WS/q39/plan.txt" 2>/dev/null && grep -qi 'upgrade' "$WS/q39/plan.txt" && grep -q 'kubelet' "$WS/q39/plan.txt" """),
 ],
 "solution": """
k get nodes                                              # the VERSION column is each node's kubelet
k get nodes -o custom-columns=NAME:.metadata.name,KUBELET:.status.nodeInfo.kubeletVersion
#   -> the node on v%{old} goes into {WS}/q39/outdated.txt
k cordon <that node>
sudo apt-get update && apt-cache madison kubeadm | head -1       # third column -> {WS}/q39/target.txt
sudo kubeadm upgrade plan | tee {WS}/q39/plan.txt
# cordon only marks the node unschedulable: running Pods stay. drain = cordon + evict.
# `upgrade plan` reads the cluster and changes nothing: it lists the current and the target version
# of each component and which kubelets it found. The kubelet may be up to three minor versions
# behind the API server, never ahead - but an old release still carries its bugs.
""",
},
]