Controller Manager Terminated Pod GC Threshold Should Be Set

#!/usr/bin/env bash
#
# Automation: Ensure --terminated-pod-gc-threshold is set on kube-controller-manager
# Scope: Run on every control plane node with direct access to /etc/kubernetes/manifests
# Impact: Editing the static pod manifest will cause the kube-controller-manager pod to restart.

set -euo pipefail

MANIFEST="/etc/kubernetes/manifests/kube-controller-manager.yaml"
BACKUP_DIR="/etc/kubernetes/manifests/backup-terminated-pod-gc-threshold"
THRESHOLD_VALUE="10"   # Adjust if your organization requires a different threshold

echo "==> Ensuring kube-controller-manager --terminated-pod-gc-threshold is set to ${THRESHOLD_VALUE}"

if [[ $EUID -ne 0 ]]; then
  echo "This script must be run as root on each control plane node."
  exit 1
fi

if [[ ! -f "${MANIFEST}" ]]; then
  echo "Manifest not found at ${MANIFEST}. This node might not be a control plane node."
  exit 0
fi

mkdir -p "${BACKUP_DIR}"
TS="$(date +%Y%m%d%H%M%S)"
cp "${MANIFEST}" "${BACKUP_DIR}/kube-controller-manager.yaml.${TS}"

echo "Current --terminated-pod-gc-threshold flags in manifest (if any):"
grep -n -- "--terminated-pod-gc-threshold" "${MANIFEST}" || echo "  (none found)"

# If flag already exists, normalize its value to THRESHOLD_VALUE
if grep -q -- "--terminated-pod-gc-threshold" "${MANIFEST}"; then
  echo "Updating existing --terminated-pod-gc-threshold to ${THRESHOLD_VALUE}"
  # Replace any existing value after '=' with the desired threshold, preserving YAML formatting
  perl -pi -e '
    s/(--terminated-pod-gc-threshold=)\d+/$1'"${THRESHOLD_VALUE}"'/g
  ' "${MANIFEST}"
else
  echo "Adding --terminated-pod-gc-threshold=${THRESHOLD_VALUE} to kube-controller-manager args"

  # Insert the flag into the args list under the kube-controller-manager container.
  # This assumes a standard kubeadm-style manifest with a container named kube-controller-manager.
  # The insertion is idempotent because we already checked that the flag does not exist.
  perl -0pi -e '
    s/(name:\s*kube-controller-manager\s*\n\s*image:.*?\n(\s*command:\s*\n(?:\s*-\s*\S+\s*\n)*)(\s*args:\s*\n))/\1\3\2/g
  ' "${MANIFEST}" 2>/dev/null || true

  # If args: section already exists, append the flag; otherwise create args: with the flag.
  if grep -qE '^\s*args:\s*$' "${MANIFEST}"; then
    # Append as a new line under args:
    perl -pi -e '
      if (/^\s*args:\s*$/ && !$x) {
        $x=1;
        my $indent = ($ARGV =~ /^(.*)$/) ? $1 : "";
      }
    ' "${MANIFEST}" 2>/dev/null || true

    # Use a simple awk-based append to avoid complex YAML parsing
    awk -v val="--terminated-pod-gc-threshold=${THRESHOLD_VALUE}" '
      /^\s*args:\s*$/ && !added {
        print;
        indent = match($0,/^(\s*)args:/,m) ? m[1] "  " : "    ";
        print indent "- " val;
        added=1;
        next
      }
      {print}
    ' "${MANIFEST}" > "${MANIFEST}.tmp"

    mv "${MANIFEST}.tmp" "${MANIFEST}"
  else
    # No args: section; create one under the kube-controller-manager container
    awk -v val="--terminated-pod-gc-threshold=${THRESHOLD_VALUE}" '
      $0 ~ /name:\s*kube-controller-manager/ && !done {
        print;
        getline;
        print; # likely image: line
        print "    args:";
        print "      - " val;
        done=1;
        next
      }
      {print}
    ' "${MANIFEST}" > "${MANIFEST}.tmp"

    mv "${MANIFEST}.tmp" "${MANIFEST}"
  fi
fi

echo "Manifest updated. kubelet will restart the kube-controller-manager static pod automatically."

echo "Waiting up to 120 seconds for kube-controller-manager process to reflect new flag..."
end=$((SECONDS+120))
success=0
while (( SECONDS < end )); do
  if /bin/ps -ef | grep kube-controller-manager | grep -v grep | grep -q -- "--terminated-pod-gc-threshold=${THRESHOLD_VALUE}"; then
    success=1
    break
  fi
  sleep 5
done

echo "Verification output:"
/bin/ps -ef | grep kube-controller-manager | grep -v grep || true

if (( success == 1 )); then
  echo "SUCCESS: kube-controller-manager is running with --terminated-pod-gc-threshold=${THRESHOLD_VALUE}"
  exit 0
else
  echo "WARNING: kube-controller-manager process does not yet show --terminated-pod-gc-threshold=${THRESHOLD_VALUE}."
  echo "The component may still be restarting. Re-run the verification in a few minutes:"
  echo "  /bin/ps -ef | grep kube-controller-manager | grep -v grep"
  exit 1
fi