How to fix Eviction Hard Threshold in Kubernetes

kubectl top nodes
kubectl describe node <node-name>  # Shows Allocatable and current pressure
kubectl get nodes -o wide

kubectl describe node <node-name> | grep -A 10 "Conditions:"

ssh <node-ip>
df -h  # Filesystem disk space usage
du -sh /*  # Directory sizes

# Check kubelet root directory:
du -sh /var/lib/kubelet
du -sh /var/log/pods

# Find large files:
find / -type f -size +500M 2>/dev/null

kubectl describe node <node-name> | grep -E "Allocated|Requested|Limits"
kubectl top nodes --no-headers | awk '{print $1, $5}'  # Memory usage %

# On the node itself:
free -h
cat /proc/meminfo
ps aux --sort=-%mem | head -10  # Top memory consumers

kubectl top pods -A --sort-by=memory

kubectl get node <node-name> -o yaml | grep -i eviction

# Check kubelet config:
kubectl get -n kube-system cm kubelet-config-1.xx -o yaml  # kubeadm clusters

# SSH to node and check:
ssh <node-ip>
ps aux | grep kubelet  # Show kubelet command line args
catcat /var/lib/kubelet/kubelet.conf  # Kubelet config file

ssh <node-ip>

# Remove old container images (careful!):
docker image prune -a --filter "until=168h"  # Remove images older than 7 days
kubectl debug node/<node-name> -it --image=ubuntu  # Debug pod if SSH unavailable

# Clear old pod logs:
find /var/log/pods -type f -name "*.log" -mtime +7 -delete  # Logs older than 7 days

# Clear kubelet temporary files:
rm -rf /var/lib/kubelet/pods/*/volume-subpaths/*

# Clear old evicted pod data:
kubectl get pods -A --field-selector=status.phase=Failed -o json | \
  jq -r '.items[] | select(.status.reason=="Evicted") | "\(.metadata.namespace) \(.metadata.name)" ' | \
  xargs -I {} sh -c 'kubectl delete pod -n {} --grace-period=0 --force'

apiVersion: apps/v1
kind: Deployment
metadata:
  name: bounded-app
spec:
  template:
    spec:
      containers:
      - name: app
        image: myapp:latest
        resources:
          requests:
            memory: 256Mi
            cpu: 100m
          limits:
            memory: 512Mi  # Pod cannot exceed 512Mi
            cpu: 500m

kubectl apply -f deployment.yaml

ssh <node-ip>

# Check current kubelet log rotation:
kubectl get -n kube-system pod <kubelet-pod> -o yaml | grep -i "max-size"

# Or edit kubelet config:
sudo nano /etc/kubernetes/kubelet/kubelet.conf

serializationFormat: json
serializationFormatVersion: v1

# Add logging config:
logging:
  flushFrequency: 5s
  format: json
  level: info

{
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "10m",
    "max-file": "3"
  }
}

kubectl edit node <node-name>

metadata:
  annotations:
    kubelet.kubernetes.io/eviction-hard: "memory.available<50Mi,nodefs.available<1%"

ssh <node-ip>
sudo nano /etc/kubernetes/manifests/kubelet-config.yaml
# OR
sudo nano /etc/sysconfig/kubelet

KUBELET_EXTRA_ARGS="--eviction-hard=memory.available<50Mi,nodefs.available<1%"

sudo systemctl restart kubelet