Kubernetes Troubleshooting: A Practical Guide

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# Get pod status with more details
kubectl get pods -o wide

# Describe pod for events and conditions
kubectl describe pod <pod-name>

# Get pods with specific status
kubectl get pods --field-selector=status.phase=Pending

# Watch pod status changes
kubectl get pods -w

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

# Get logs from a pod
kubectl logs <pod-name>

# Get logs from a specific container in multi-container pod
kubectl logs <pod-name> -c <container-name>

# Follow logs in real-time
kubectl logs -f <pod-name>

# Get logs from previous container instance (after restart)
kubectl logs <pod-name> --previous

# Get last N lines
kubectl logs --tail=100 <pod-name>

# Get logs with timestamps
kubectl logs --timestamps <pod-name>

1
2
3
4
5
6
7
8

# Open a shell in a running container
kubectl exec -it <pod-name> -- /bin/bash

# Run a specific command
kubectl exec <pod-name> -- cat /etc/config/app.yaml

# Exec into a specific container
kubectl exec -it <pod-name> -c <container-name> -- /bin/sh

1
2
3
4
5
6

kubectl describe pod <pod-name>

# Check for events like:
# - "0/3 nodes are available: insufficient cpu"
# - "no nodes available to schedule pods"
# - "0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims"

1
2
3
4
5
6
7
8

# Reduce resource requests
resources:
  requests:
    cpu: "100m"      # Reduced from 500m
    memory: "128Mi"  # Reduced from 512Mi
  limits:
    cpu: "500m"
    memory: "512Mi"

1
2
3
4
5

# Check logs from the crashed container
kubectl logs <pod-name> --previous

# Check container exit code
kubectl get pod <pod-name> -o jsonpath='{.status.containerStatuses[0].lastState.terminated.exitCode}'

1
2
3
4

# Increase memory limits
resources:
  limits:
    memory: "1Gi"  # Increased from 512Mi

1
2
3
4
5
6

kubectl describe pod <pod-name> | grep -A5 "Events:"

# Common error messages:
# - "unauthorized: authentication required"
# - "manifest unknown"
# - "ImagePullBackOff"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# Verify image exists and tag is correct
docker pull <image-name>:<tag>

# Check imagePullSecrets
kubectl get secrets
kubectl get pod <pod-name> -o jsonpath='{.spec.imagePullSecrets}'

# Create docker registry secret
kubectl create secret docker-registry regcred \
  --docker-server=<registry> \
  --docker-username=<user> \
  --docker-password=<password>

1
2
3
4
5
6
7
8
9

# Check service endpoints
kubectl get endpoints <service-name>

# If endpoints are empty, check selector matching
kubectl get svc <service-name> -o wide
kubectl get pods --show-labels

# Test from within the cluster
kubectl run test-pod --rm -it --image=busybox -- wget -qO- http://<service-name>:<port>

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# 1. Verify pod labels match service selector
kubectl get pods --show-labels | grep <app-label>

# 2. Check if pods are Ready
kubectl get pods | grep <app-name>

# 3. Verify service port configuration
kubectl get svc <service-name> -o yaml

# 4. Test connectivity from within pod
kubectl exec -it <pod-name> -- curl localhost:<container-port>

1
2
3
4
5
6
7
8

# Check CoreDNS pods
kubectl get pods -n kube-system -l k8s-app=kube-dns

# Test DNS resolution
kubectl run test-dns --rm -it --image=busybox -- nslookup kubernetes.default

# Check CoreDNS logs
kubectl logs -n kube-system -l k8s-app=kube-dns

1
2
3
4
5

# Restart CoreDNS
kubectl rollout restart deployment/coredns -n kube-system

# Check DNS config in pod
kubectl exec -it <pod-name> -- cat /etc/resolv.conf

1
2
3
4
5

# Add debug container to running pod (K8s 1.23+)
kubectl debug -it <pod-name> --image=busybox --target=<container-name>

# Create a debug copy of the pod
kubectl debug <pod-name> -it --copy-to=debug-pod --container=debug

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

# Deploy network debug pod
kubectl run netshoot --rm -it --image=nicolaka/netshoot -- /bin/bash

# Inside the pod:
# Test TCP connectivity
nc -zv <service-name> <port>

# DNS lookup
dig <service-name>.<namespace>.svc.cluster.local

# Trace route
traceroute <pod-ip>

# Check network policies
curl -v http://<service-name>:<port>

1
2
3
4
5
6
7
8
9

# Get resource usage
kubectl top pods
kubectl top nodes

# Check resource quotas
kubectl describe resourcequota -n <namespace>

# Check limit ranges
kubectl describe limitrange -n <namespace>

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

apiVersion: v1
kind: Pod
metadata:
  name: debug-toolkit
  namespace: default
spec:
  containers:
  - name: debug
    image: nicolaka/netshoot
    command: ["sleep", "infinity"]
    securityContext:
      capabilities:
        add: ["NET_ADMIN", "SYS_TIME"]
  hostNetwork: false
  dnsPolicy: ClusterFirst

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

# Pod debugging
kubectl describe pod <name>         # Events and conditions
kubectl logs <name> --previous      # Previous container logs
kubectl exec -it <name> -- sh       # Shell access

# Service debugging
kubectl get endpoints <svc>         # Check endpoints
kubectl port-forward svc/<name> 8080:80  # Local access

# Cluster debugging
kubectl get events --sort-by='.lastTimestamp'  # Recent events
kubectl get nodes -o wide           # Node status
kubectl cluster-info dump           # Full cluster dump

# Resource debugging
kubectl top pods --containers       # Container resource usage
kubectl describe node <name> | grep -A5 "Allocated resources"