kubestronaut-sim/ckad/bank_deploy.py

"""CKAD tasks 85-93: Application Deployment - the pool tasks that take the domain from 8 to 17.

  2.1 Strategies        85 promote a canary                       86 never two versions at once
  2.2 Rolling updates   87 three changes, one rollout             88 which revision ran that image?
                        89 a rotated Secret the Pods have not seen
  2.3 Helm              90 a release you did not install          91 a chart from a repository
                        92 a release that failed
  2.4 Kustomize         93 a generated ConfigMap that rolls the Deployment

Rollouts are judged by what runs afterwards (images, environment, revisions), Helm by the release's
own history, Kustomize by building the directory and by the objects in the cluster. The Helm tasks
are seeded with the helm on this machine and solved with the one on the host. Every task has a
scripted solution in solve.py.
"""
from ckad.common import C, ONE, PY, ROLL, NS, ns_pick, name_pick

# A chart "webapp" in two versions, as task 21 has it: a Deployment <release>-webapp and a Service.
WEBAPP = r"""
# webapp_chart DIR VERSION TAG: write the chart's source into DIR/webapp
webapp_chart() {
  local d="$1/webapp"; mkdir -p "$d/templates"
  printf 'apiVersion: v2\nname: webapp\nversion: %s\nappVersion: "%s"\n' "$2" "$3" > "$d/Chart.yaml"
  printf 'replicaCount: 1\nimage:\n  repository: nginx\n  tag: "%s"\nservice:\n  type: ClusterIP\n  port: 80\n' "$3" > "$d/values.yaml"
  cat > "$d/templates/deployment.yaml" <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata:
  name: {{ .Release.Name }}-webapp
  labels: {app.kubernetes.io/instance: {{ .Release.Name }}, chart: {{ .Chart.Name }}-{{ .Chart.Version }}}
spec:
  replicas: {{ .Values.replicaCount }}
  selector: {matchLabels: {app.kubernetes.io/instance: {{ .Release.Name }}}}
  template:
    metadata: {labels: {app.kubernetes.io/instance: {{ .Release.Name }}}}
    spec:
      containers:
        - name: web
          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
          ports: [{containerPort: 80}]
EOF
  cat > "$d/templates/service.yaml" <<'EOF'
apiVersion: v1
kind: Service
metadata: {name: {{ .Release.Name }}-webapp}
spec:
  type: {{ .Values.service.type }}
  selector: {app.kubernetes.io/instance: {{ .Release.Name }}}
  ports: [{port: {{ .Values.service.port }}, targetPort: 80}]
EOF
}
# hist NS RELEASE 'PYTHON EXPRESSION over h': the release's history (a list of revisions, oldest first) satisfies it
hist() { helm -n "$1" history "$2" -o json 2>/dev/null | CODE="$3" python3 -c '
import json, os, sys
try:
    h = json.load(sys.stdin)
except Exception:
    sys.exit(1)
sys.exit(0 if eval(os.environ["CODE"], {"h": h}) else 1)'; }
# vals NS RELEASE 'PYTHON EXPRESSION over v': the values the release was given satisfy it
vals() { helm -n "$1" get values "$2" -o json 2>/dev/null | CODE="$3" python3 -c '
import json, os, sys
try:
    v = json.load(sys.stdin) or {}
except Exception:
    sys.exit(1)
sys.exit(0 if eval(os.environ["CODE"], {"v": v}) else 1)'; }
"""

QUESTIONS = [
# ===================================================================== 2.1 Strategies ====
{
 "id": 85, "domain": "DEPLOY", "title": "Promote a canary",
 "params": lambda r: {"ns": ns_pick(r, "shop"), "app": name_pick(r, "cart")},
 "task": """
Namespace `%{ns}` runs a canary release behind Service `%{app}`: Deployment `%{app}-stable`
(4 replicas) and Deployment `%{app}-canary` (1 replica) with a newer image. The canary has done
well for a week. Promote it:

1. Write the image the canary runs to `{WS}/q85/image.txt`.
2. Bring `%{app}-stable` to that image, on all 4 replicas, with a normal rolling update.
3. When that rollout is finished, delete `%{app}-canary`.

Do not touch the Service. It must end up with exactly 4 endpoints, all on the new image.
""",
 "setup": NS + ROLL + r"""
mkns %{ns}; rm -rf "$WS/q85"; mkdir -p "$WS/q85"
for t in stable:4:nginx:1.25 canary:1:nginx:1.27; do
  IFS=: read -r track n repo tag <<< "$t"
  $K apply -f - >/dev/null <<EOF
apiVersion: apps/v1
kind: Deployment
metadata: {name: %{app}-$track, namespace: %{ns}}
spec:
  replicas: $n
  selector: {matchLabels: {app: %{app}, track: $track}}
  template:
    metadata: {labels: {app: %{app}, track: $track}}
    spec:
      containers:
        - {name: web, image: "$repo:$tag", ports: [{containerPort: 80}]}
EOF
done
$K -n %{ns} create service clusterip %{app} --tcp=80:80 >/dev/null
$K -n %{ns} patch svc %{app} -p '{"spec":{"selector":{"app":"%{app}"}}}' >/dev/null
rolled %{ns} %{app}-stable 4 240 && rolled %{ns} %{app}-canary 1 240 || exit 1
j -n %{ns} svc %{app} '{.metadata.uid}' > "$PRIV/q85.uid"
""",
 "checks": [
  C("image.txt names the canary's image", 2, ONE + r"""eq "$(one "$WS/q85/image.txt")" nginx:1.27"""),
  C("%{app}-stable runs that image on all 4 replicas, the rollout finished", 4, ROLL + r"""
eq "$(j -n %{ns} deploy %{app}-stable '{.spec.template.spec.containers[0].image}')" nginx:1.27 && rolled %{ns} %{app}-stable 4 60"""),
  C("%{app}-canary is deleted", 2, r"""
eq "$(j -n %{ns} deploy %{app}-stable '{.spec.template.spec.containers[0].image}')" nginx:1.27 && [ -z "$(j -n %{ns} deploy %{app}-canary '{.metadata.name}')" ]"""),
  C("the same Service has 4 endpoints, all on the new image", 2, PY + r"""
eq "$(j -n %{ns} svc %{app} '{.metadata.uid}')" "$(cat "$PRIV/q85.uid")" && eq "$(j -n %{ns} svc %{app} '{.spec.selector}')" '{"app":"%{app}"}' || exit 1
py -n %{ns} pod -l app=%{app} '
p = [x for x in o["items"] if "deletionTimestamp" not in x["metadata"]]
len(p) == 4 and all(x["spec"]["containers"][0]["image"] == "nginx:1.27" and any(c["type"] == "Ready" and c["status"] == "True" for c in x["status"]["conditions"]) for x in p)'"""),
 ],
 "solution": """
k -n %{ns} get deploy -o wide                                  # the IMAGES column: stable 1.25, canary 1.27
echo nginx:1.27 > {WS}/q85/image.txt
k -n %{ns} set image deploy/%{app}-stable web=nginx:1.27
k -n %{ns} rollout status deploy/%{app}-stable                 # wait: only then is the canary redundant
k -n %{ns} delete deploy %{app}-canary
k -n %{ns} get endpointslices -l kubernetes.io/service-name=%{app}
# A canary by replica count is two Deployments under one Service selector: the share of traffic is
# the share of Pods. Promoting it is an ordinary rolling update of the stable Deployment - the
# Service keeps answering throughout - and the canary goes last, not first.
""",
},
{
 "id": 86, "domain": "DEPLOY", "title": "Never two versions at once",
 "params": lambda r: {"ns": ns_pick(r, "ledger"), "dep": name_pick(r, "posting")},
 "task": """
The application of Deployment `%{dep}` in Namespace `%{ns}` (3 replicas, `nginx:1.25`) migrates a
shared database when it starts: an old and a new version must **never run side by side**, not even
for a moment. A short outage during an update is accepted.

1. Change the Deployment's update strategy accordingly.
2. Then update its image to `nginx:1.27`.
3. Write the name of the strategy type you chose to `{WS}/q86/strategy.txt`.
""",
 "setup": NS + ROLL + r"""
mkns %{ns}; rm -rf "$WS/q86"; mkdir -p "$WS/q86"
$K -n %{ns} create deploy %{dep} --image=nginx:1.25 --replicas=3 >/dev/null
rolled %{ns} %{dep} 3 240
""",
 "checks": [
  C("the Deployment's strategy is Recreate", 4, r"""
eq "$(j -n %{ns} deploy %{dep} '{.spec.strategy.type}')" Recreate && [ -z "$(j -n %{ns} deploy %{dep} '{.spec.strategy.rollingUpdate}')" ]"""),
  C("it runs nginx:1.27 on all 3 replicas", 4, ROLL + r"""
eq "$(j -n %{ns} deploy %{dep} '{.spec.strategy.type}')" Recreate && eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].image}')" nginx:1.27 && rolled %{ns} %{dep} 3 90"""),
  C("strategy.txt says Recreate", 2, ONE + r"""eq "$(one "$WS/q86/strategy.txt")" Recreate"""),
 ],
 "solution": """
k -n %{ns} patch deploy %{dep} --type=json -p '[{"op":"replace","path":"/spec/strategy","value":{"type":"Recreate"}}]'
#   with `kubectl edit`: set type: Recreate AND delete the rollingUpdate block, or the change is refused
k -n %{ns} set image deploy/%{dep} nginx=nginx:1.27
k -n %{ns} get pod -w                              # all three terminate, then three new ones start
echo Recreate > {WS}/q86/strategy.txt
# RollingUpdate (the default) keeps the application up by overlapping old and new Pods; Recreate
# scales the old ReplicaSet to zero first and trades availability for never mixing versions.
# maxSurge 0 would still leave old and new Pods running together during the update.
""",
},
# ================================================================ 2.2 Rolling updates ====
{
 "id": 87, "domain": "DEPLOY", "title": "Three changes, one rollout",
 "params": lambda r: {"ns": ns_pick(r, "api"), "dep": name_pick(r, "orders")},
 "task": """
Deployment `%{dep}` in Namespace `%{ns}` (3 replicas, `nginx:1.25`) is due for three changes:

- image `nginx:1.27`,
- environment variable `LOG_LEVEL=debug`,
- resource requests of `50m` CPU and `64Mi` memory.

Each change on its own would restart every Pod. Make all three reach the Pods in **one** rollout:
afterwards the Deployment has exactly one more revision than it has now.
""",
 "setup": NS + ROLL + r"""
mkns %{ns}
$K -n %{ns} create deploy %{dep} --image=nginx:1.25 --replicas=3 >/dev/null
rolled %{ns} %{dep} 3 240 && eq "$(j -n %{ns} deploy %{dep} '{.metadata.annotations.deployment\.kubernetes\.io/revision}')" 1
""",
 "checks": [
  C("the Pod template has all three changes and the Deployment is not paused", 3, PY + r"""
py -n %{ns} deploy %{dep} '
c = o["spec"]["template"]["spec"]["containers"][0]; r = c.get("resources", {}).get("requests", {})
c["image"] == "nginx:1.27" and any(e["name"] == "LOG_LEVEL" and e.get("value") == "debug" for e in c.get("env", [])) and r.get("cpu") == "50m" and r.get("memory") == "64Mi" and not o["spec"].get("paused")'"""),
  C("all 3 replicas run the new template", 3, ROLL + r"""
eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].image}')" nginx:1.27 && rolled %{ns} %{dep} 3 90"""),
  C("it took one rollout: the Deployment is at revision 2, with two ReplicaSets", 4, r"""
eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].image}')" nginx:1.27 || exit 1
eq "$(j -n %{ns} deploy %{dep} '{.metadata.annotations.deployment\.kubernetes\.io/revision}')" 2 && eq "$($K -n %{ns} get rs -l app=%{dep} -o name | wc -l)" 2"""),
 ],
 "solution": """
k -n %{ns} rollout pause deploy/%{dep}
k -n %{ns} set image deploy/%{dep} nginx=nginx:1.27
k -n %{ns} set env deploy/%{dep} LOG_LEVEL=debug
k -n %{ns} set resources deploy/%{dep} --requests=cpu=50m,memory=64Mi
k -n %{ns} rollout resume deploy/%{dep}
k -n %{ns} rollout status deploy/%{dep}; k -n %{ns} rollout history deploy/%{dep}      # revisions 1 and 2
# A paused Deployment records changes to its template without acting on them; resume starts one
# rollout for the lot. A single `kubectl edit` or one `kubectl apply` of the three changes does the
# same - the point is that every separate change of the template is a revision and a restart.
""",
},
{
 "id": 88, "domain": "DEPLOY", "title": "Which revision ran that image?",
 "params": lambda r: (lambda rev: {"ns": ns_pick(r, "media"), "dep": name_pick(r, "encoder"), "rev": rev, "tag": "1.3%d" % (3 + rev)})(r.choice([1, 2, 3])),
 "task": """
Deployment `%{dep}` in Namespace `%{ns}` has been through four revisions. Since the last one it
leaks memory, and the release notes say the last version without the leak used image
`busybox:%{tag}`.

1. Find the revision that ran that image and write its number to `{WS}/q88/revision.txt`.
2. Roll the Deployment back to exactly that revision.
3. Set the change cause of the revision this produces to `rollback to busybox:%{tag}`.
""",
 "setup": NS + ROLL + r"""
mkns %{ns}; rm -rf "$WS/q88"; mkdir -p "$WS/q88"
$K -n %{ns} create deploy %{dep} --image=busybox:1.34 --replicas=2 -- sleep 86400 >/dev/null
$K -n %{ns} annotate deploy %{dep} kubernetes.io/change-cause="initial release" >/dev/null
rolled %{ns} %{dep} 2 240 || exit 1
for t in 1.35 1.36 1.37; do
  $K -n %{ns} set image deploy/%{dep} busybox=busybox:$t >/dev/null
  $K -n %{ns} annotate deploy %{dep} kubernetes.io/change-cause="update to $t" --overwrite >/dev/null
  rolled %{ns} %{dep} 2 240 || exit 1
done
eq "$(j -n %{ns} deploy %{dep} '{.metadata.annotations.deployment\.kubernetes\.io/revision}')" 4
""",
 "checks": [
  C("revision.txt holds the number of the revision that ran busybox:%{tag}", 3, ONE + r"""eq "$(one "$WS/q88/revision.txt")" %{rev}"""),
  C("the Deployment is rolled back to it: busybox:%{tag} on both replicas, as revision 5", 4, ROLL + r"""
eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].image}')" busybox:%{tag} && rolled %{ns} %{dep} 2 90 \
 && eq "$(j -n %{ns} deploy %{dep} '{.metadata.annotations.deployment\.kubernetes\.io/revision}')" 5"""),
  C("its change cause is the one asked for", 3, r"""
eq "$(j -n %{ns} deploy %{dep} '{.spec.template.spec.containers[0].image}')" busybox:%{tag} \
 && eq "$(j -n %{ns} deploy %{dep} '{.metadata.annotations.kubernetes\.io/change-cause}')" "rollback to busybox:%{tag}" """),
 ],
 "solution": """
k -n %{ns} rollout history deploy/%{dep}                       # REVISION and CHANGE-CAUSE, no images
for i in 1 2 3 4; do echo "== $i"; k -n %{ns} rollout history deploy/%{dep} --revision=$i | grep -i image; done
#   revision %{rev} is busybox:%{tag}                               -> {WS}/q88/revision.txt
k -n %{ns} rollout undo deploy/%{dep} --to-revision=%{rev}
k -n %{ns} rollout status deploy/%{dep}
k -n %{ns} annotate deploy %{dep} kubernetes.io/change-cause="rollback to busybox:%{tag}" --overwrite
k -n %{ns} rollout history deploy/%{dep}                       # revision %{rev} is gone: it has become revision 5
# `rollout undo` without --to-revision goes back one step only. A rollback does not rewind the
# counter: the old ReplicaSet is reused under a new revision number. The CHANGE-CAUSE column is the
# annotation kubernetes.io/change-cause of the Deployment at the time of the rollout.
""",
},
{
 "id": 89, "domain": "DEPLOY", "title": "A rotated Secret the Pods have not seen",
 "params": lambda r: {"ns": ns_pick(r, "billing"), "app": name_pick(r, "invoicer"),
                      "pw": "pw-" + "".join(r.choice("abcdefghjkmnpqrstuvwxyz23456789") for _ in range(10))},
 "task": """
Secret `%{app}-db` in Namespace `%{ns}` was rotated an hour ago, but the Pods of Deployment `%{app}`
still run with the old password: they read it into the environment variable `DB_PASSWORD` when
they started.

1. Make every Pod of the Deployment pick up the current value **without** changing the
   Deployment's image, command or replica count, and without deleting Pods by hand: restart the
   rollout.
2. Record the reason: set the Deployment's change cause to `secret rotation`.
3. Save the first line one of the new Pods logged to `{WS}/q89/started.txt`.
""",
 "setup": NS + ROLL + r"""
mkns %{ns}; rm -rf "$WS/q89"; mkdir -p "$WS/q89"
$K -n %{ns} create secret generic %{app}-db --from-literal=password=old-%{pw} >/dev/null
$K apply -f - >/dev/null <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata: {name: %{app}, namespace: %{ns}}
spec:
  replicas: 2
  selector: {matchLabels: {app: %{app}}}
  template:
    metadata: {labels: {app: %{app}}}
    spec:
      containers:
        - name: app
          image: busybox:1.36
          command: ["sh", "-c", "echo \"started with $DB_PASSWORD\"; exec sleep 2147483647"]
          env:
            - {name: DB_PASSWORD, valueFrom: {secretKeyRef: {name: %{app}-db, key: password}}}
EOF
rolled %{ns} %{app} 2 240 || exit 1
$K -n %{ns} create secret generic %{app}-db --from-literal=password=%{pw} --dry-run=client -o yaml | $K apply -f - >/dev/null 2>&1
""",
 "checks": [
  C("every Pod of the Deployment now has the current password", 4, ROLL + r"""
rolled %{ns} %{app} 2 60 || exit 1
n=0; for p in $($K -n %{ns} get pod -l app=%{app} -o json | python3 -c '
import json, sys
[print(x["metadata"]["name"]) for x in json.load(sys.stdin)["items"] if "deletionTimestamp" not in x["metadata"]]'); do
  eq "$($K -n %{ns} exec "$p" -- printenv DB_PASSWORD 2>/dev/null)" %{pw} || exit 1; n=$((n+1))
done; [ $n -ge 2 ]"""),
  C("through a restart of the rollout: revision 2, image, command and replicas unchanged", 3, PY + r"""
py -n %{ns} deploy %{app} '
t = o["spec"]["template"]; c = t["spec"]["containers"][0]
"kubectl.kubernetes.io/restartedAt" in t["metadata"].get("annotations", {}) and o["metadata"]["annotations"].get("deployment.kubernetes.io/revision") == "2" and c["image"] == "busybox:1.36" and "started with" in c["command"][-1] and o["spec"]["replicas"] == 2'"""),
  C("the change cause is recorded", 2, r"""eq "$(j -n %{ns} deploy %{app} '{.metadata.annotations.kubernetes\.io/change-cause}')" "secret rotation" """),
  C("started.txt is a new Pod's first line", 1, r"""grep -qx 'started with %{pw}' <(tr -d '\r' < "$WS/q89/started.txt" 2>/dev/null)"""),
 ],
 "solution": """
k -n %{ns} exec deploy/%{app} -- printenv DB_PASSWORD           # old-...: the value from when the Pod started
k -n %{ns} rollout restart deploy/%{app}
k -n %{ns} rollout status deploy/%{app}
k -n %{ns} annotate deploy %{app} kubernetes.io/change-cause="secret rotation" --overwrite
k -n %{ns} logs deploy/%{app} | head -1 | tee {WS}/q89/started.txt
# Environment variables from a Secret or ConfigMap are copied in when the container starts; nothing
# updates them afterwards (mounted files are updated, env never is). `rollout restart` stamps an
# annotation into the Pod template - which makes it a new revision and a normal rolling update,
# with no other change to the Deployment.
""",
},
# =========================================================================== 2.3 Helm ====
{
 "id": 90, "domain": "DEPLOY", "title": "A Helm release you did not install",
 "params": lambda r: {"ns": ns_pick(r, "tracking"), "rel": name_pick(r, "fleet"), "mode": r.choice(["strict", "relaxed", "audit"]), "region": r.choice(["eu", "us", "ap"])},
 "task": """
A Helm release named `%{rel}` runs somewhere in this cluster. Nobody on the team installed it.
Find out what it is, and write to `{WS}/q90/`:

1. `namespace.txt` — the Namespace the release is in;
2. `chart.txt` — its chart and chart version, as `helm list` shows them (for example `nginx-1.4.2`);
3. `values.yaml` — the values that were **supplied** when it was installed or upgraded (not the
   chart's defaults), as YAML;
4. `manifest.yaml` — the manifest Helm rendered and applied for it.

Change nothing.
""",
 "setup": NS + r"""
command -v helm >/dev/null || { echo "helm is not installed on this machine - the task needs it"; exit 1; }
mkns %{ns}; mkns %{ns}-old; rm -rf "$WS/q90"; mkdir -p "$WS/q90"
d=$(mktemp -d)/tracker; mkdir -p "$d/templates"
printf 'apiVersion: v2\nname: tracker\nversion: 0.4.2\nappVersion: "2.1"\n' > "$d/Chart.yaml"
printf 'replicaCount: 1\nimage:\n  tag: "1.27"\nconfig:\n  mode: standard\n  region: eu\n  retention: 30\n' > "$d/values.yaml"
cat > "$d/templates/all.yaml" <<'EOF'
apiVersion: v1
kind: ConfigMap
metadata: {name: {{ .Release.Name }}-tracker}
data:
  MODE: {{ .Values.config.mode | quote }}
  REGION: {{ .Values.config.region | quote }}
  RETENTION: {{ .Values.config.retention | quote }}
---
apiVersion: apps/v1
kind: Deployment
metadata: {name: {{ .Release.Name }}-tracker}
spec:
  replicas: {{ .Values.replicaCount }}
  selector: {matchLabels: {app: {{ .Release.Name }}-tracker}}
  template:
    metadata: {labels: {app: {{ .Release.Name }}-tracker}}
    spec:
      containers:
        - name: tracker
          image: "nginx:{{ .Values.image.tag }}"
          envFrom: [{configMapRef: {name: {{ .Release.Name }}-tracker}}]
EOF
helm -n %{ns} install %{rel} "$d" --set replicaCount=3 --set config.mode=%{mode} --set config.region=%{region} >/dev/null || exit 1
helm -n %{ns}-old install %{rel}-legacy "$d" >/dev/null
rm -rf "$(dirname "$d")"
""",
 "checks": [
  C("namespace.txt and chart.txt", 4, ONE + r"""eq "$(one "$WS/q90/namespace.txt")" %{ns} && eq "$(one "$WS/q90/chart.txt")" tracker-0.4.2"""),
  C("values.yaml holds the supplied values, and only those", 3, r"""
nx "$CP" 'yq -o=json /opt/course/q90/values.yaml' </dev/null 2>/dev/null | python3 -c '
import json, sys
v = json.load(sys.stdin); v.pop("USER-SUPPLIED VALUES", None)
sys.exit(0 if v == {"replicaCount": 3, "config": {"mode": "%{mode}", "region": "%{region}"}} else 1)'"""),
  C("manifest.yaml is the release's rendered manifest", 3, r"""
f="$WS/q90/manifest.yaml"
grep -q '^# Source: tracker/templates/all.yaml' "$f" 2>/dev/null && grep -q 'name: %{rel}-tracker' "$f" && grep -q 'kind: Deployment' "$f" && grep -q 'MODE: "%{mode}"' "$f" """),
 ],
 "solution": """
helm list -A | grep %{rel}                        # NAMESPACE %{ns} ... CHART tracker-0.4.2
cd {WS}/q90
echo %{ns} > namespace.txt; echo tracker-0.4.2 > chart.txt
helm -n %{ns} get values %{rel} -o yaml > values.yaml          # supplied values only; --all adds the defaults
helm -n %{ns} get manifest %{rel} > manifest.yaml
helm -n %{ns} status %{rel}; helm -n %{ns} history %{rel}
# Helm commands are per Namespace: without -n (or -A for list) a release elsewhere does not show.
# `helm get` reads back what was stored with the release: values, manifest, notes, hooks, all.
# `helm show values CHART` is the other direction - a chart's defaults, before anything is installed.
""",
},
{
 "id": 91, "domain": "DEPLOY", "title": "A chart from a repository",
 "params": lambda r: {"ns": ns_pick(r, "portal"), "rel": name_pick(r, "front")},
 "task": """
The team publishes its charts in a Helm repository that is reachable from this host at
`http://localhost:30080`.

1. Add it to helm on this host under the name `internal`.
2. Write the versions of chart `webapp` it offers to `{WS}/q91/versions.txt`, one per line.
3. Install release `%{rel}` into Namespace `%{ns}` (create it) from that repository, at chart
   version **1.2.0** — not the newest — with `replicaCount=2`.
4. Then upgrade the release to the newest version of the chart, keeping its values.
""",
 "setup": NS + WEBAPP + r"""
command -v helm >/dev/null || { echo "helm is not installed on this machine - the task needs it"; exit 1; }
rm -rf "$WS/q91"; mkdir -p "$WS/q91"; mkns chart-repo
src=$(mktemp -d); mkdir -p "$src/repo"
webapp_chart "$src/1.2.0" 1.2.0 1.25; webapp_chart "$src/1.3.0" 1.3.0 1.27
helm package "$src/1.2.0/webapp" -d "$src/repo" >/dev/null && helm package "$src/1.3.0/webapp" -d "$src/repo" >/dev/null && helm repo index "$src/repo" >/dev/null || exit 1
$K -n chart-repo create configmap charts --from-file="$src/repo" >/dev/null || exit 1
rm -rf "$src"
$K apply -f - >/dev/null <<'EOF'
apiVersion: v1
kind: Pod
metadata: {name: chart-repo, namespace: chart-repo, labels: {app: chart-repo}}
spec:
  volumes: [{name: charts, configMap: {name: charts}}]
  containers:
    - name: nginx
      image: nginx:1.27
      volumeMounts: [{name: charts, mountPath: /usr/share/nginx/html}]
---
apiVersion: v1
kind: Service
metadata: {name: chart-repo, namespace: chart-repo}
spec:
  type: NodePort
  selector: {app: chart-repo}
  ports: [{port: 80, targetPort: 80, nodePort: 30080}]
EOF
$K -n chart-repo wait --for=condition=Ready pod/chart-repo --timeout=240s >/dev/null || exit 1
for i in $(seq 1 20); do nx "$CP" 'curl -s -m 3 http://localhost:30080/index.yaml' </dev/null 2>/dev/null | grep -q 'webapp-1.3.0.tgz' && exit 0; sleep 2; done
echo "the chart repository does not answer on the host" >&2; exit 1
""",
 "checks": [
  C("versions.txt lists the two versions of webapp", 2, r"""
[ -s "$WS/q91/versions.txt" ] && diff <(tr -d ' \rv' < "$WS/q91/versions.txt" | grep . | sort -u) <(printf '1.2.0\n1.3.0\n') >/dev/null"""),
  C("release %{rel} was installed at chart 1.2.0 and upgraded to 1.3.0, and is deployed", 4, WEBAPP + r"""
hist %{ns} %{rel} 'len(h) >= 2 and h[0]["chart"] == "webapp-1.2.0" and h[-1]["chart"] == "webapp-1.3.0" and h[-1]["status"] == "deployed"'"""),
  C("it kept replicaCount=2 through the upgrade", 2, WEBAPP + r"""
hist %{ns} %{rel} 'h[-1]["chart"] == "webapp-1.3.0"' && vals %{ns} %{rel} 'v.get("replicaCount") == 2' && eq "$(j -n %{ns} deploy %{rel}-webapp '{.spec.replicas}')" 2"""),
  C("the repository is configured on the host as internal", 2, r"""
for u in candidate root; do
  nx "$CP" "su - $u -c 'helm repo list -o json'" </dev/null 2>/dev/null | python3 -c '
import json, sys
try:
    r = json.load(sys.stdin)
except Exception:
    sys.exit(1)
sys.exit(0 if any(x["name"] == "internal" and x["url"].rstrip("/") == "http://localhost:30080" for x in r) else 1)' && exit 0
done; exit 1"""),
 ],
 "solution": """
helm repo add internal http://localhost:30080
helm repo update
helm search repo internal/webapp --versions          # 1.3.0 and 1.2.0     -> {WS}/q91/versions.txt
helm install %{rel} internal/webapp --version 1.2.0 -n %{ns} --create-namespace --set replicaCount=2
helm upgrade %{rel} internal/webapp -n %{ns} --reuse-values          # no --version: the newest
helm -n %{ns} history %{rel}; helm -n %{ns} get values %{rel}
# `search repo` shows only the newest version of each chart unless you ask for --versions, and it
# searches the local copy of the index: `repo update` first. An upgrade without --reuse-values (or
# the same --set again) puts every value back to the new chart's default.
""",
},
{
 "id": 92, "domain": "DEPLOY", "title": "A release that failed",
 "params": lambda r: {"ns": ns_pick(r, "search"), "rel": name_pick(r, "indexer")},
 "task": """
Helm release `%{rel}` in Namespace `%{ns}` is in status `failed` since its last upgrade. Its chart
is in `{WS}/q92/chart`.

1. Find the value that broke it, and write its name — as `--set` would spell it, for example
   `service.type` — to `{WS}/q92/key.txt`.
2. Bring the release back to `deployed` with another **upgrade** that corrects that one value to
   `1.27` and keeps every other value the release has. Do not roll back and do not reinstall.
3. Release `%{rel}-old` in Namespace `%{ns}-archive` is no longer used: uninstall it.
""",
 "setup": NS + WEBAPP + ROLL + r"""
command -v helm >/dev/null || { echo "helm is not installed on this machine - the task needs it"; exit 1; }
mkns %{ns}; mkns %{ns}-archive; rm -rf "$WS/q92"; mkdir -p "$WS/q92"
webapp_chart "$WS/q92" 1.3.0 1.27 && mv "$WS/q92/webapp" "$WS/q92/chart"
helm -n %{ns} install %{rel} "$WS/q92/chart" --set replicaCount=2 --set service.port=8080 >/dev/null || exit 1
rolled %{ns} %{rel}-webapp 2 240 || exit 1
helm -n %{ns}-archive install %{rel}-old "$WS/q92/chart" >/dev/null
helm -n %{ns} upgrade %{rel} "$WS/q92/chart" --reuse-values --set image.tag=1.99-nope --wait --timeout 25s >/dev/null 2>&1
hist %{ns} %{rel} 'len(h) == 2 and h[-1]["status"] == "failed"' || { echo "the upgrade did not leave the release failed" >&2; exit 1; }
""",
 "checks": [
  C("key.txt names the value", 2, ONE + r"""eq "$(one "$WS/q92/key.txt")" image.tag"""),
  C("the release is deployed again, on nginx:1.27, with its other values kept", 4, WEBAPP + ROLL + r"""
hist %{ns} %{rel} 'h[-1]["status"] == "deployed"' && vals %{ns} %{rel} 'v.get("replicaCount") == 2 and v.get("service", {}).get("port") == 8080 and str(v.get("image", {}).get("tag")) == "1.27"' || exit 1
eq "$(j -n %{ns} deploy %{rel}-webapp '{.spec.template.spec.containers[0].image}')" nginx:1.27 && rolled %{ns} %{rel}-webapp 2 60"""),
  C("by a further upgrade: the failed revision is still in its history", 2, WEBAPP + r"""
hist %{ns} %{rel} 'len(h) >= 3 and h[-1]["status"] == "deployed" and any(x["status"] == "failed" for x in h) and "ollback" not in h[-1]["description"]'"""),
  C("release %{rel}-old is uninstalled", 2, r"""
! helm -n %{ns}-archive status %{rel}-old >/dev/null 2>&1 && [ -z "$(j -n %{ns}-archive deploy %{rel}-old-webapp '{.metadata.name}')" ] && helm -n %{ns} status %{rel} >/dev/null 2>&1"""),
 ],
 "solution": """
helm -n %{ns} list -a; helm -n %{ns} history %{rel}             # revision 2: failed
helm -n %{ns} get values %{rel}                                # image.tag: 1.99-nope  <- no such image
k -n %{ns} get pod                                             # ImagePullBackOff says the same
echo image.tag > {WS}/q92/key.txt
helm -n %{ns} upgrade %{rel} {WS}/q92/chart --reuse-values --set image.tag=1.27 --wait
helm -n %{ns} history %{rel}                                   # 1 superseded, 2 failed, 3 deployed
helm -n %{ns}-archive uninstall %{rel}-old
# A failed upgrade leaves the release at the failed revision with the old Pods still running (the
# rolling update never finished). `helm rollback` would have gone back to revision 1; an upgrade
# with --reuse-values takes the failed revision's values and changes only what you --set.
""",
},
# ====================================================================== 2.4 Kustomize ====
{
 "id": 93, "domain": "DEPLOY", "title": "A generated ConfigMap that rolls the Deployment",
 "params": lambda r: {"ns": ns_pick(r, "render"), "app": name_pick(r, "painter")},
 "task": """
`{WS}/q93/app/` is a Kustomize directory with a Deployment `%{app}` that reads its settings from a
ConfigMap named `%{app}-config` — which nothing creates yet. `settings.env` beside it holds the
settings, one `KEY=value` per line.

1. Extend `kustomization.yaml`: everything goes into Namespace `%{ns}`, and a ConfigMap
   `%{app}-config` is **generated** from `settings.env`, each line becoming a key. Keep the hash
   that Kustomize appends to generated names.
2. Apply the directory with kubectl's built-in Kustomize.
3. Change `MODE` in `settings.env` from `blue` to `green` and apply again. The Deployment must
   roll to the new settings by itself — that is what the hash is for.
""",
 "setup": NS + r"""
mkns %{ns}; d="$WS/q93/app"; rm -rf "$WS/q93"; mkdir -p "$d"
cat > "$d/deployment.yaml" <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata:
  name: %{app}
spec:
  replicas: 2
  selector:
    matchLabels:
      app: %{app}
  template:
    metadata:
      labels:
        app: %{app}
    spec:
      containers:
        - name: app
          image: busybox:1.36
          command: ["sleep", "86400"]
          envFrom:
            - configMapRef:
                name: %{app}-config
EOF
printf 'apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nresources:\n  - deployment.yaml\n' > "$d/kustomization.yaml"
printf 'MODE=blue\nREGION=eu\n' > "$d/settings.env"
""",
 "checks": [
  C("the directory builds: a ConfigMap generated from settings.env with a hashed name, in %{ns}", 3, r"""
kubectl kustomize "$WS/q93/app" 2>/dev/null | python3 -c '
import re, sys
t = sys.stdin.read(); docs = t.split("\n---\n")
cm = [d for d in docs if re.search(r"^kind: ConfigMap$", d, re.M)]
dep = [d for d in docs if re.search(r"^kind: Deployment$", d, re.M)]
ok = len(cm) == 1 and len(dep) == 1
m = ok and re.search(r"^  name: (%{app}-config-[a-z0-9]{6,})$", cm[0], re.M)
ok = bool(m) and "MODE: green" in cm[0] and "REGION: eu" in cm[0] and "namespace: %{ns}" in cm[0] and "namespace: %{ns}" in dep[0] and ("name: " + m.group(1)) in dep[0]
sys.exit(0 if ok else 1)'"""),
  C("in %{ns} the Deployment runs with MODE=green from a generated ConfigMap", 4, ROLL + r"""
n=$(j -n %{ns} deploy %{app} '{.spec.template.spec.containers[0].envFrom[0].configMapRef.name}')
echo "$n" | grep -Eq '^%{app}-config-[a-z0-9]{6,}$' && eq "$(j -n %{ns} cm "$n" '{.data.MODE}')" green && rolled %{ns} %{app} 2 90 || exit 1
eq "$($K -n %{ns} exec "$(newest %{ns} app=%{app})" -- printenv MODE 2>/dev/null)" green"""),
  C("it rolled once, from the ConfigMap that said blue", 3, r"""
eq "$(j -n %{ns} deploy %{app} '{.metadata.annotations.deployment\.kubernetes\.io/revision}')" 2 || exit 1
[ "$($K -n %{ns} get cm -o name | grep -c '/%{app}-config-')" -ge 2 ] && $K -n %{ns} get cm -o jsonpath='{range .items[*]}{.metadata.name}={.data.MODE}{"\n"}{end}' | grep -q '^%{app}-config-.*=blue$'"""),
 ],
 "solution": """
cd {WS}/q93/app
cat > kustomization.yaml <<EOF
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: %{ns}
resources:
  - deployment.yaml
configMapGenerator:
  - name: %{app}-config
    envs:
      - settings.env
EOF
k kustomize .                       # the ConfigMap is %{app}-config-<hash>, and the Deployment refers to that name
k apply -k . && k -n %{ns} rollout status deploy/%{app}
sed -i 's/^MODE=blue/MODE=green/' settings.env
k apply -k . && k -n %{ns} rollout status deploy/%{app}
k -n %{ns} exec deploy/%{app} -- printenv MODE          # green
# The hash is of the content: new content, new name - and Kustomize rewrites every reference to it,
# so the Pod template changes and the Deployment rolls. `envs:` makes each line a key; `files:`
# would make the whole file one key. The old ConfigMap stays behind until somebody prunes it.
""",
},
]