Containers Should Define Liveness And Readiness Probes

kubectl get pods --all-namespaces -o json | jq -r '
  [ .items[]
  | select(.metadata.namespace as $n | ["kube-system","kube-public","kube-node-lease"] | index($n) | not)
  | .metadata as $m
  | ([ ($m.ownerReferences // [])[] | select(.controller) ] | first) as $own
  | (.spec.containers // [])[]
  | (.livenessProbe != null) as $live
  | (.readinessProbe != null) as $ready
  | select(( $live and $ready ) | not)
  | "\($own.kind) \($m.namespace) \($own.name)"
  ] | unique[]' | sort -u

kubectl -n prod get deploy my-app -o yaml > /tmp/deploy-my-app.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-app
  namespace: prod
spec:
  template:
    spec:
      containers:
      - name: app
        image: my-registry.example.com/app:1.0.0
        ports:
        - containerPort: 8080
        livenessProbe:
          httpGet:
            path: /healthz
            port: 8080
          initialDelaySeconds: 30
          periodSeconds: 10
          timeoutSeconds: 5
          failureThreshold: 3
        readinessProbe:
          httpGet:
            path: /readyz
            port: 8080
          initialDelaySeconds: 10
          periodSeconds: 5
          timeoutSeconds: 3
          failureThreshold: 3

kubectl apply -f /tmp/deploy-my-app.yaml

kubectl -n my-namespace edit pod my-standalone-pod

kubectl get pods --all-namespaces -o json | jq -r '
  [ .items[]
  | select(.metadata.namespace as $n | ["kube-system","kube-public","kube-node-lease"] | index($n) | not)
  | .metadata as $m
  | (.spec.containers // [])[]
  | (.livenessProbe != null) as $live
  | (.readinessProbe != null) as $ready
  | select(( $live and $ready ) | not)
  ] | if (length == 0) then "is_compliant=true" else . end'

#!/usr/bin/env bash
# Purpose:
#   Report long-running containers (non-job pods outside AKS system namespaces)
#   that are missing livenessProbe and/or readinessProbe.
#
# NOTE:
#   Probe configuration is application-specific; this script CANNOT safely
#   auto-add probes without knowing container ports, paths, or health semantics.
#   Use this as an automation aid to identify and review/fix manifests.
#
# Requirements:
#   - Run on any machine with kubectl access to the AKS cluster
#   - Tools: kubectl, jq, awk
#
# Usage:
#   ./check_probes.sh
set -euo pipefail

echo "Scanning for containers missing liveness/readiness probes..."

# Reuse the authoritative audit logic to gather non-compliant containers
NON_COMPLIANT_JSON="$(kubectl get pods --all-namespaces -o json \
  | jq -c '
    [ .items[]
    | select(.metadata.namespace as $n
             | ["kube-system","kube-public","kube-node-lease","gatekeeper-system","azure-arc"]  # extend ignore list as needed
             | index($n)
             | not)
    | .metadata as $m
    | (.spec.containers // [])[]
    | select((.livenessProbe == null) or (.readinessProbe == null))
    | {
        namespace: $m.namespace,
        pod: $m.name,
        container: .name,
        liveness_defined: (.livenessProbe != null),
        readiness_defined: (.readinessProbe != null),
        owner_kind: ( ($m.ownerReferences // []) | map(select(.controller)) | first | .kind // "Pod"),
        owner_name: ( ($m.ownerReferences // []) | map(select(.controller)) | first | .name // $m.name)
      }
    ]')"

if [[ "${NON_COMPLIANT_JSON}" == "[]" ]]; then
  echo "All scanned containers already define both livenessProbe and readinessProbe."
  exit 0
fi

echo
echo "The following containers are missing one or both probes:"
echo

echo "${NON_COMPLIANT_JSON}" | jq -r '
  .[]
  | "namespace=\(.namespace)\tpod=\(.pod)\tcontainer=\(.container)\towner=\(.owner_kind)/\(.owner_name)\tliveness=\(.liveness_defined)\treadiness=\(.readiness_defined)"
' | column -t -s $'\t'

cat <<'EOF'

NEXT STEPS (manual, per workload):

1. Identify the owning object and manifest source:
   - For native Kubernetes objects:
       kubectl -n <namespace> get deployment/<name> -o yaml
       kubectl -n <namespace> get statefulset/<name> -o yaml
       kubectl -n <namespace> get daemonset/<name> -o yaml
   - For Helm-managed workloads:
       helm list -A | grep <release>
       helm get values -n <namespace> <release> > values.yaml

2. For EACH listed container, edit its manifest/values to add both probes, for example:
   # Example ONLY – adjust path/port/thresholds for your app
   livenessProbe:
     httpGet:
       path: /healthz
       port: 8080
     initialDelaySeconds: 30
     periodSeconds: 10
     timeoutSeconds: 5
     failureThreshold: 3
   readinessProbe:
     httpGet:
       path: /ready
       port: 8080
     initialDelaySeconds: 5
     periodSeconds: 5
     timeoutSeconds: 3
     failureThreshold: 3

3. Apply the change using the same mechanism used to deploy:
   - kubectl apply -f <manifest>.yaml
   - or: helm upgrade -n <namespace> <release> <chart> -f values.yaml
   - or: update CI/CD/IaC (e.g., GitOps) and let it reconcile.

4. Verify the fix (idempotent verification):

   kubectl get pods --all-namespaces -o json | jq -r '
     [ .items[]
     | select(.metadata.namespace as $n | ["kube-system","kube-public","kube-node-lease","gatekeeper-system","azure-arc"] | index($n) | not)
     | .metadata as $m
     | (.spec.containers // [])[]
     | (.livenessProbe != null) as $live
     | (.readinessProbe != null) as $ready
     | "ns=\($m.namespace) pod=\($m.name) container=\(.name) livenessProbe=\($live) readinessProbe=\($ready)"
       + " is_compliant=\(if ($live and $ready) then "true" else "false" end)"
     ] as $rows
     | if ($rows | length) == 0 then "is_compliant=true" else $rows[] end
   ' | grep -v 'is_compliant=true' || echo "All remaining scanned containers are compliant."
EOF