Workflow: Only What It Needs

#!/usr/bin/env bash
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "${repo_root}"

for tool in kubectl yq jq flux; do
  command -v "${tool}" >/dev/null || { echo "check-app-security: ${tool} is required" >&2; exit 1; }
done

failures=()
MIN_PRESTOP=5   # seconds; on kind 10 s did no better than 5 s
MIN_SHUTDOWN=10 # seconds the app needs after the sleep: the backend's graceful shutdown timeout (cmd/api/main.go)

for overlay in "${overlays[@]}"; do
  # Render the overlay exactly as Flux would build it; every rule below reads this output.
  if ! rendered="$(kubectl kustomize "${overlay}" 2>&1)"; then
    failures+=("${overlay}: kustomize build failed: ${rendered}")
    continue
  fi

  # Rule: automountServiceAccountToken: false on every Deployment - and count the Deployments,
  # so an overlay that renders none fails instead of passing with nothing checked.
  deployments=0
  while IFS=$'\t' read -r name automount; do
    [[ -z "${name}" ]] && continue
    deployments=$((deployments + 1))
    if [[ "${automount}" != "false" ]]; then
      failures+=("${overlay}: Deployment ${name} must set automountServiceAccountToken: false (got: ${automount})")
    fi
  done < <(yq 'select(.kind == "Deployment") | [.metadata.name, (.spec.template.spec.automountServiceAccountToken | tostring)] | @tsv' <<<"${rendered}")
  if ((deployments == 0)); then
    failures+=("${overlay}: renders no Deployment - nothing was checked")
  fi

  # One line per container: deployment, container, runAsNonRoot (container, pod), readOnlyRootFilesystem,
  # allowPrivilegeEscalation, capabilities.drop, seccomp type (container, pod). Both levels are printed
  # and decided here: yq's // would treat an explicit false on the container as unset.
  # shellcheck disable=SC2016 # $pod and $d below are yq variables, not shell ones
  while IFS=$'\t' read -r dep ctr nonroot_c nonroot_p rofs ape drop seccomp_c seccomp_p; do
    [[ -z "${ctr}" ]] && continue
    where="${overlay}: Deployment ${dep} container ${ctr}"
    nonroot="${nonroot_c}"; [[ "${nonroot}" == "null" ]] && nonroot="${nonroot_p}"
    seccomp="${seccomp_c}"; [[ "${seccomp}" == "null" ]] && seccomp="${seccomp_p}"
    [[ "${nonroot}" == "true" ]] || failures+=("${where}: runAsNonRoot must be true (got: ${nonroot})")
    [[ "${rofs}" == "true" ]] || failures+=("${where}: readOnlyRootFilesystem must be true (got: ${rofs})")
    [[ "${ape}" == "false" ]] || failures+=("${where}: allowPrivilegeEscalation must be false (got: ${ape})")
    [[ ",${drop}," == *",ALL,"* ]] || failures+=("${where}: capabilities.drop must include ALL (got: '${drop}')")
    [[ "${seccomp}" == "RuntimeDefault" || "${seccomp}" == "Localhost" ]] ||
      failures+=("${where}: seccompProfile must be RuntimeDefault (got: ${seccomp})")
  done < <(yq 'select(.kind == "Deployment") | .spec.template.spec as $pod | .metadata.name as $d
      | ($pod.containers + ($pod.initContainers // []))[]
      | [$d, .name,
         (.securityContext.runAsNonRoot | tostring), ($pod.securityContext.runAsNonRoot | tostring),
         (.securityContext.readOnlyRootFilesystem | tostring),
         (.securityContext.allowPrivilegeEscalation | tostring),
         ((.securityContext.capabilities.drop // []) | join(",")),
         (.securityContext.seccompProfile.type | tostring), ($pod.securityContext.seccompProfile.type | tostring)]
      | @tsv' <<<"${rendered}")

  # Rule: every app container sleeps MIN_PRESTOP seconds or more before it is stopped, and the
  # grace period (default 30 s) leaves room for that sleep plus the app's own shutdown.
  while IFS=$'\t' read -r dep container sleep grace; do
    [[ -n "${dep}" && "${dep}" != "---" ]] || continue   # yq separates the documents with ---
    where="${overlay}: Deployment ${dep} container ${container}"
    if [[ ! "${grace}" =~ ^[0-9]+$ ]]; then
      failures+=("${where}: terminationGracePeriodSeconds must be a whole number of seconds (got: ${grace})")
      continue
    fi
    if [[ ! "${sleep}" =~ ^[0-9]+$ ]] || ((sleep < MIN_PRESTOP)); then
      failures+=("${where}: lifecycle.preStop.sleep.seconds must be at least ${MIN_PRESTOP} (got: ${sleep})")
    elif ((grace < sleep + MIN_SHUTDOWN)); then
      failures+=("${where}: terminationGracePeriodSeconds (${grace}s) must cover the preStop sleep (${sleep}s) plus ${MIN_SHUTDOWN}s of shutdown")
    fi
  done < <(yq 'select(.kind == "Deployment") | .metadata.name as $d
      | (.spec.template.spec.terminationGracePeriodSeconds // 30) as $g
      | .spec.template.spec.containers[]
      | [$d, .name, (.lifecycle.preStop.sleep.seconds // "none" | tostring), ($g | tostring)] | @tsv' <<<"${rendered}")

  # Rule: every emptyDir has a sizeLimit - a writable path is the price of a read-only root
  # filesystem, and it must not be able to fill the node's disk.
  # shellcheck disable=SC2016 # $d below is a yq variable, not a shell one
  while IFS=$'\t' read -r dep vol limit; do
    [[ -z "${vol}" ]] && continue
    if [[ "${limit}" == "null" || -z "${limit}" ]]; then
      failures+=("${overlay}: Deployment ${dep} emptyDir volume ${vol} must set a sizeLimit")
    fi
  done < <(yq 'select(.kind == "Deployment") | .metadata.name as $d
      | (.spec.template.spec.volumes // [])[] | select(has("emptyDir"))
      | [$d, .name, (.emptyDir.sizeLimit | tostring)] | @tsv' <<<"${rendered}")

  # Rule: for every Deployment an HPA scales, no spec.replicas in Git.
  while IFS= read -r target; do
    [[ -z "${target}" ]] && continue
    replicas="$(yq "select(.kind == \"Deployment\" and .metadata.name == \"${target}\") | .spec.replicas" <<<"${rendered}")"
    if [[ -n "${replicas}" && "${replicas}" != "null" ]]; then
      failures+=("${overlay}: Deployment ${target} has an HPA and must not set spec.replicas (got: ${replicas})")
    fi
  done < <(yq 'select(.kind == "HorizontalPodAutoscaler" and .spec.scaleTargetRef.kind == "Deployment") | .spec.scaleTargetRef.name' <<<"${rendered}")

  # Rule: no injected errors in production, in any container of either app.
  if [[ "${overlay}" == */production ]]; then
    # shellcheck disable=SC2016 # $d below is a yq variable, not a shell one
    while IFS=$'\t' read -r dep container; do
      [[ -z "${dep}" ]] && continue
      failures+=("${overlay}: Deployment ${dep} container ${container} must not set RANDOM_ERROR_RATE in production")
    done < <(yq -N 'select(.kind == "Deployment") | .metadata.name as $d
        | ((.spec.template.spec.containers // []) + (.spec.template.spec.initContainers // []))[]
        | select((.env // [])[] | .name == "RANDOM_ERROR_RATE")
        | [$d, .name] | @tsv' <<<"${rendered}")
  fi

  # Rules for the backend only: PPROF_ENABLED off, CHAOS_ENABLED as expected for the environment.
  if [[ "${overlay}" == flux/apps/backend/* ]]; then
    pprof="$(backend_env "${rendered}" PPROF_ENABLED)"
    chaos="$(backend_env "${rendered}" CHAOS_ENABLED)"
    if [[ "${pprof}" != "false" ]]; then
      failures+=("${overlay}: backend PPROF_ENABLED must be \"false\" in Git (got: '${pprof}')")
    fi
    expected="$(expected_chaos_for "${overlay}")"
    if [[ -z "${expected}" ]]; then
      failures+=("${overlay}: no expected CHAOS_ENABLED value - add the overlay to expected_chaos_for")
    elif [[ "${chaos}" != "${expected}" ]]; then
      failures+=("${overlay}: backend CHAOS_ENABLED must be \"${expected}\" (got: '${chaos}')")
    fi
  fi
done

for profile in flux/bootstrap/profiles/*/; do
  if ! kubectl kustomize --load-restrictor=LoadRestrictionsNone "${profile}" >"${render_tmp}/profile.yaml" 2>"${render_tmp}/err"; then
    failures+=("${profile}: kustomize build failed: $(cat "${render_tmp}/err")")
    continue
  fi
  enqueue "${render_tmp}/profile.yaml" "${profile}"
done
builds=0
next=0
while ((next < ${#pending[@]})); do
  file="${pending[next]}"
  next=$((next + 1))
  # Name, path (Flux's default: the source root) and source as kind/namespace/name - the source's
  # namespace defaults to the Kustomization's own.
  if ! IFS=$'\t' read -r name source path < <(yq -N '[.metadata.name,
      .spec.sourceRef.kind + "/" + (.spec.sourceRef.namespace // .metadata.namespace // "flux-system") + "/" + .spec.sourceRef.name,
      .spec.path // "./"] | @tsv' "${file}"); then
    failures+=("${file##*/}: a Flux Kustomization yq cannot read")
    continue
  fi
  if [[ "${source}" != "GitRepository/flux-system/flux-system" ]]; then
    if [[ " ${FOREIGN_SOURCES[*]} " != *" ${name} "* ]]; then
      failures+=("Flux Kustomization ${name}: its source ${source} is not this repository, so it cannot be built here - if it deploys no workload of ours, add it to FOREIGN_SOURCES")
    fi
    continue
  fi
  builds=$((builds + 1))
  if ! flux build kustomization "${name}" --path "${path}" --kustomization-file "${file}" \
    --dry-run >"${render_tmp}/built.yaml" 2>"${render_tmp}/err"; then
    failures+=("Flux Kustomization ${name} (${path}): flux build failed: $(cat "${render_tmp}/err")")
    continue
  fi
  collect_links "Flux Kustomization ${name} (${path})" "${render_tmp}/built.yaml"
  enqueue "${render_tmp}/built.yaml" "Flux Kustomization ${name} (${path})"
done

workloads="$(cut -f1,2 "${links_rows}" | sort -u | grep -c . || true)"

if ((${#failures[@]} > 0)); then
  echo "check-app-security: ${#failures[@]} problem(s):" >&2
  printf '  - %s\n' "${failures[@]}" >&2
  exit 1
fi
echo "check-app-security: OK (${#overlays[@]} overlays; ${builds} Flux Kustomizations built, ${workloads} workloads; ${dirs} kustomize directories rendered; ${documents} workload documents checked as written; ${ip_rules} NetworkPolicy ipBlock rules, both families)"

---
apiVersion: v1
kind: Namespace
metadata:
  name: auth
  labels:
    name: auth
    managed-by: flux
---
apiVersion: v1
kind: Namespace
metadata:
  name: cert-manager
  labels:
    name: cert-manager
    managed-by: flux
---
# Traefik, the ingress (flux/infrastructure/ingress/traefik). The application network policies admit
# traffic from this namespace by its name.
apiVersion: v1
kind: Namespace
metadata:
  name: traefik
  labels:
    name: traefik
    managed-by: flux
---
# external-dns is its own tool (DNS records for Ingresses), not part of cert-manager (TLS).
apiVersion: v1
kind: Namespace
metadata:
  name: external-dns
  labels:
    name: external-dns
    managed-by: flux
---
apiVersion: v1
kind: Namespace
metadata:
  annotations:
    # A Namespace must never be garbage-collected by Flux, even if another
    # Kustomization once applied it (see the duplicate namespace.yaml files).
    kustomize.toolkit.fluxcd.io/prune: Disabled
  name: develop
  labels:
    name: develop
    environment: development
    managed-by: flux
    pod-security.kubernetes.io/enforce: restricted
    pod-security.kubernetes.io/enforce-version: latest
    pod-security.kubernetes.io/warn: restricted
    pod-security.kubernetes.io/warn-version: latest
---
apiVersion: v1
kind: Namespace
metadata:
  annotations:
    # A Namespace must never be garbage-collected by Flux, even if another
    # Kustomization once applied it (see the duplicate namespace.yaml files).
    kustomize.toolkit.fluxcd.io/prune: Disabled
  name: staging
  labels:
    name: staging
    environment: staging
    managed-by: flux
    pod-security.kubernetes.io/enforce: restricted
    pod-security.kubernetes.io/enforce-version: latest
    pod-security.kubernetes.io/warn: restricted
    pod-security.kubernetes.io/warn-version: latest
---
apiVersion: v1
kind: Namespace
metadata:
  annotations:
    # A Namespace must never be garbage-collected by Flux, even if another
    # Kustomization once applied it (see the duplicate namespace.yaml files).
    kustomize.toolkit.fluxcd.io/prune: Disabled
  name: production
  labels:
    name: production
    environment: production
    managed-by: flux
    pod-security.kubernetes.io/enforce: restricted
    pod-security.kubernetes.io/enforce-version: latest
    pod-security.kubernetes.io/warn: restricted
    pod-security.kubernetes.io/warn-version: latest

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: backend
  labels:
    app: backend
    app.kubernetes.io/name: backend
    app.kubernetes.io/component: api
spec:
  # No spec.replicas: the HorizontalPodAutoscaler owns the replica count (hpa.yaml per environment,
  # minReplicas is the floor). A replicas value in Git makes Flux reset what the HPA scaled on every
  # reconcile. Existing clusters: run scripts/hpa-replicas-handover.sh once before this change.
  revisionHistoryLimit: 3
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 0
      maxSurge: 1
  selector:
    matchLabels:
      app: backend
  template:
    metadata:
      labels:
        app: backend
        app.kubernetes.io/name: backend
        app.kubernetes.io/component: api
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8080"
        prometheus.io/path: "/metrics"
    spec:
      # No <SERVICE>_PORT=tcp://... variables for the namespace's Services; KUBERNETES_SERVICE_* stay
      # (scripts/check-app-security.sh says why).
      enableServiceLinks: false
      # The backend never calls the Kubernetes API: no ServiceAccount token in the pod.
      automountServiceAccountToken: false
      securityContext:
        runAsNonRoot: true
        runAsUser: 10001
        runAsGroup: 10001
        fsGroup: 10001
        seccompProfile:
          type: RuntimeDefault
      # Schema migrations before the app (Chapter 18): the same image reference as the app (the
      # overlay's images: transformer sets both tags; Kyverno verifies the digest each tag resolves to
      # at admission - a tag is not pinned, see Chapter 17) runs `backend migrate`, applies the migrations
      # embedded in it that the database lacks (golang-migrate, under a Postgres advisory lock) and exits.
      # It runs whenever a backend pod is created - rollout, reschedule, scale-out; with nothing pending
      # it only checks. The recovery point is the continuous WAL archive (Chapter 12), not a backup taken
      # here; a migration's pull request is merged only while that archive is healthy. The app then only
      # checks the schema version and refuses to start on one it cannot use. Migrations only expand:
      # the previous release keeps working on the new schema, so rolling back the image never needs
      # rolling back the data.
      initContainers:
      - name: migrate
        image: backend:latest  # name/tag set by the environment overlay (images: transformer)
        imagePullPolicy: IfNotPresent
        args: ["migrate"]
        securityContext:
          allowPrivilegeEscalation: false
          readOnlyRootFilesystem: true
          runAsNonRoot: true
          runAsUser: 10001
          runAsGroup: 10001
          capabilities:
            drop:
              - ALL
        env:
        - name: POSTGRES_USER
          valueFrom:
            secretKeyRef:
              name: app-postgres-app
              key: username
        - name: POSTGRES_PASSWORD
          valueFrom:
            secretKeyRef:
              name: app-postgres-app
              key: password
        - name: POSTGRES_HOST
          value: app-postgres-rw
        - name: POSTGRES_DB
          value: app
        resources:
          requests:
            cpu: 10m
            memory: 32Mi
          limits:
            cpu: 100m
            memory: 64Mi
      containers:
      - name: backend
        image: backend:latest  # name/tag set by the environment overlay (images: transformer)
        imagePullPolicy: IfNotPresent
        # Stopping, the pod is removed from the Service's endpoints and from Traefik - but not at once,
        # while the container already gets SIGTERM. The sleep keeps it serving until they have caught
        # up; then the app shuts down. Measured on kind (a rollout, ~115 requests/s): without it 14-25
        # failed requests (refused, or 502 from Traefik), with it 0-3 (Chapter 20). The kubelet's own
        # sleep - no shell in the image needed. 5 s + the app's shutdown fit in the 30 s grace period.
        lifecycle:
          preStop:
            sleep:
              seconds: 5
        securityContext:
          allowPrivilegeEscalation: false
          readOnlyRootFilesystem: true
          runAsNonRoot: true
          runAsUser: 10001
          runAsGroup: 10001
          capabilities:
            drop:
              - ALL
        ports:
        - containerPort: 8080
          name: http
          protocol: TCP
        env:
        - name: PORT
          value: "8080"
        - name: NAMESPACE
          value: "${NAMESPACE}"
        - name: ENVIRONMENT
          value: "${ENVIRONMENT}"
        - name: LOG_LEVEL
          value: "${LOG_LEVEL}"
        - name: SERVICE_NAME
          value: "backend"
        # No SERVICE_VERSION: the backend reports the version baked into the image (the same as
        # /version), so every span says which release produced it.
        - name: DEPLOYMENT_ENVIRONMENT
          value: "${ENVIRONMENT}"
        # Where a span came from: the cluster, the namespace (= the environment) and the pod - set from
        # the downward API, so traces can be filtered per environment and per pod.
        - name: K8S_NAMESPACE_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: K8S_POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: OTEL_RESOURCE_ATTRIBUTES
          value: "k8s.cluster.name=${cluster_name},k8s.namespace.name=$(K8S_NAMESPACE_NAME),k8s.pod.name=$(K8S_POD_NAME)"
        - name: UPTRACE_DSN
          valueFrom:
            secretKeyRef:
              name: backend-secrets
              key: uptrace-dsn
        - name: JWT_SECRET
          valueFrom:
            secretKeyRef:
              name: backend-secrets
              key: jwt-secret
        - name: POSTGRES_USER
          valueFrom:
            secretKeyRef:
              name: app-postgres-app
              key: username
        - name: POSTGRES_PASSWORD
          valueFrom:
            secretKeyRef:
              name: app-postgres-app
              key: password
        - name: POSTGRES_HOST
          value: app-postgres-rw
        - name: POSTGRES_DB
          value: app
        # Security switches (backend >= the 2026-09-27 hardening). Safe values here; an environment
        # overlay may only turn chaos on (develop, staging). scripts/check-app-security.sh enforces it.
        - name: CHAOS_ENABLED       # /panic + readiness/liveness toggles (still need a token)
          value: "false"
        - name: PPROF_ENABLED       # profiling listener on loopback; never on in Git
          value: "false"
        - name: AUTH_REGISTRATION_ENABLED
          value: "true"
        livenessProbe:
          httpGet:
            path: /livez
            port: http
          initialDelaySeconds: 15
          periodSeconds: 20
          timeoutSeconds: 5
          failureThreshold: 3
        readinessProbe:
          httpGet:
            path: /readyz
            port: http
          initialDelaySeconds: 5
          periodSeconds: 10
          timeoutSeconds: 3
          failureThreshold: 3
        startupProbe:
          httpGet:
            path: /healthz
            port: http
          initialDelaySeconds: 0
          periodSeconds: 5
          timeoutSeconds: 3
          failureThreshold: 30
        # The production values (staging gets the same; develop patches its own). Requests are what
        # the scheduler reserves and the quota counts; limits are the ceiling: memory over the limit
        # is OOMKilled, CPU over it is throttled. requests < limits makes the pod Burstable.
        resources:
          requests:
            cpu: 100m
            memory: 128Mi
            ephemeral-storage: 64Mi
          limits:
            cpu: 250m
            memory: 256Mi
            ephemeral-storage: 128Mi
        volumeMounts:
        - name: tmp
          mountPath: /tmp
        - name: cache
          mountPath: /home/app/.cache
      volumes:
      # Every writable path is an emptyDir with a size limit. It is not a quota that stops the write:
      # the kubelet measures the volume on a schedule and evicts the pod once it is over the limit -
      # a very fast writer can still fill the node's disk before that. It bounds a slow leak and makes
      # the pod's budget explicit. /tmp holds request bodies Go buffers to disk - measured at 0 KB at
      # rest, so 64Mi leaves room for uploads.
      - name: tmp
        emptyDir:
          sizeLimit: 64Mi
      - name: cache
        emptyDir:
          sizeLimit: 10Mi