#!/usr/bin/env bash
# Purpose: Report and optionally patch Kubernetes workloads on EKS whose containers
# lack livenessProbe and/or readinessProbe, by generating patched manifests.
# Scope: Run on any machine with kubectl access to the cluster.
#
# NOTES:
# - This script does NOT directly modify running objects server-side.
# It generates patch files you can review and apply with kubectl.
# - You must decide appropriate probe types/paths/ports per application.
# This script only scaffolds probes where missing.
#
# Requirements: kubectl, jq, yq (https://mikefarah.gitbook.io/yq/)
set -euo pipefail
# -------- Configurable defaults for newly added probes --------
# Adjust these defaults before using in production.
DEFAULT_HTTP_PATH="/healthz"
DEFAULT_HTTP_PORT=8080
DEFAULT_INITIAL_DELAY=10
DEFAULT_PERIOD_SECONDS=10
DEFAULT_TIMEOUT_SECONDS=1
DEFAULT_FAILURE_THRESHOLD=3
DEFAULT_SUCCESS_THRESHOLD=1
OUTDIR="probe-patches-$(date +%Y%m%d-%H%M%S)"
mkdir -p "${OUTDIR}"
echo "Discovering pods with containers missing liveness/readiness probes ..."
echo "Output directory for generated manifests: ${OUTDIR}"
echo
# Reuse the audit logic to identify non-compliant containers (excluding system namespaces)
NON_COMPLIANT_JSON="$(kubectl get pods --all-namespaces -o json | jq '
.items[]
| select(.metadata.namespace as $n | ["kube-system","kube-public","kube-node-lease"] | index($n) | not)
| . as $pod
| .spec.containers[]
| select((.livenessProbe == null) or (.readinessProbe == null))
| {
namespace: $pod.metadata.namespace,
pod: $pod.metadata.name,
uid: $pod.metadata.uid,
owner: ($pod.metadata.ownerReferences // [] | map(select(.controller)) | first),
container: .name
}'
)"
if [ -z "${NON_COMPLIANT_JSON}" ]; then
echo "No pods with missing liveness/readiness probes found (outside system namespaces)."
exit 0
fi
echo "Non-compliant containers detected:"
echo "${NON_COMPLIANT_JSON}" | jq -r '. | "ns=\(.namespace) pod=\(.pod) owner_kind=\(.owner.kind // "Pod") owner_name=\(.owner.name // .pod) container=\(.container)"' | sort
echo
# Build a list of unique owning workload objects (kind/ns/name) to export manifests
# If pod has no controller owner, treat the Pod itself as the target object.
TARGETS_JSON="$(echo "${NON_COMPLIANT_JSON}" | jq -r '
{
ns: .namespace,
kind: (.owner.kind // "Pod"),
name: (.owner.name // .pod)
}' | jq -s 'unique')"
TARGET_COUNT="$(echo "${TARGETS_JSON}" | jq 'length')"
echo "Unique owning workloads to export and patch: ${TARGET_COUNT}"
echo
# Export and patch each target workload
for i in $(seq 0 $((TARGET_COUNT-1))); do
NS="$(echo "${TARGETS_JSON}" | jq -r ".[$i].ns")"
KIND="$(echo "${TARGETS_JSON}" | jq -r ".[$i].kind")"
NAME="$(echo "${TARGETS_JSON}" | jq -r ".[$i].name")"
# Map controller kinds to kubectl resource types where needed
case "${KIND}" in
ReplicaSet) RES="replicaset" ;;
Deployment) RES="deployment" ;;
DaemonSet) RES="daemonset" ;;
StatefulSet) RES="statefulset" ;;
Job) RES="job" ;;
CronJob) RES="cronjob" ;;
Pod) RES="pod" ;;
*) RES=$(echo "${KIND}" | tr '[:upper:]' '[:lower:]') ;;
esac
echo "Exporting ${KIND}/${NS}/${NAME} ..."
MANIFEST="${OUTDIR}/${NS}_${RES}_${NAME}.yaml"
# Export the current manifest (without cluster-specific status)
kubectl get "${RES}" "${NAME}" -n "${NS}" -o yaml \
| yq 'del(.metadata.uid, .metadata.resourceVersion, .metadata.creationTimestamp, .metadata.generation, .metadata.managedFields, .status)' \
> "${MANIFEST}"
echo "Patching containers in ${MANIFEST} to add missing probes (non-destructive) ..."
# Patch spec.containers (for Pod or controller templates)
yq -i "
(.. | select(has(\"containers\")) | .containers[] ) |= (
. as \$c |
(if \$c.livenessProbe == null then
.livenessProbe = {
httpGet: { path: \"${DEFAULT_HTTP_PATH}\", port: ${DEFAULT_HTTP_PORT} },
initialDelaySeconds: ${DEFAULT_INITIAL_DELAY},
periodSeconds: ${DEFAULT_PERIOD_SECONDS},
timeoutSeconds: ${DEFAULT_TIMEOUT_SECONDS},
failureThreshold: ${DEFAULT_FAILURE_THRESHOLD},
successThreshold: ${DEFAULT_SUCCESS_THRESHOLD}
}
else . end
)
|
(if \$c.readinessProbe == null then
.readinessProbe = {
httpGet: { path: \"${DEFAULT_HTTP_PATH}\", port: ${DEFAULT_HTTP_PORT} },
initialDelaySeconds: ${DEFAULT_INITIAL_DELAY},
periodSeconds: ${DEFAULT_PERIOD_SECONDS},
timeoutSeconds: ${DEFAULT_TIMEOUT_SECONDS},
failureThreshold: ${DEFAULT_FAILURE_THRESHOLD},
successThreshold: ${DEFAULT_SUCCESS_THRESHOLD}
}
else . end
)
)
" "${MANIFEST}"
echo "Patched manifest created: ${MANIFEST}"
echo
done
cat <<EOF
REVIEW & APPLY MANUALLY
1) Review each generated manifest under:
${OUTDIR}
Ensure the livenessProbe and readinessProbe definitions are correct for each container
(HTTP path, port, initialDelaySeconds, etc.). Adjust as needed.
2) Apply the updated manifests back to the cluster (this is idempotent):
# Example for all files in the directory:
kubectl apply -f ${OUTDIR}
This will trigger rolling updates for Deployments/DaemonSets/StatefulSets/Jobs
and direct updates for standalone Pods.
3) Verify all non-system pods now have both probes defined:
kubectl get pods --all-namespaces -o json | jq -r '
[ .items[]
| select(.metadata.namespace as \$n | [\"kube-system\",\"kube-public\",\"kube-node-lease\"] | index(\$n) | not)
| .metadata as \$m
| (.spec.containers // [])[]
| (.livenessProbe != null) as \$live
| (.readinessProbe != null) as \$ready
| \"ns=\(\$m.namespace) pod=\(\$m.name) container=\(.name) is_compliant=\(if (\$live and \$ready) then \"true\" else \"false\" end)\"
] as \$rows
| if (\$rows | map(select(contains(\"is_compliant=false\"))) | length) == 0
then \"All checked containers have livenessProbe and readinessProbe defined.\"
else \$rows[]
end
'
EOF
