쿠버네티스 MTU 문제 진단법 — ping은 되는데 큰 응답만 멈출 때

# 1) ping 완전 정상
$ kubectl exec -it client-pod -- ping -c 4 10.244.2.15
PING 10.244.2.15 (10.244.2.15) 56(84) bytes of data.
64 bytes from 10.244.2.15: icmp_seq=1 ttl=62 time=0.412 ms
64 bytes from 10.244.2.15: icmp_seq=2 ttl=62 time=0.388 ms
--- 10.244.2.15 ping statistics ---
4 packets transmitted, 4 received, 0% packet loss

# 2) 작은 응답 정상 (약 200B)
$ kubectl exec -it client-pod -- curl -s -o /dev/null -w '%{http_code} %{size_download}\n' \
    http://api-svc/health
200 187

# 3) 큰 응답 무한 대기 (약 80KB)
$ kubectl exec -it client-pod -- curl -v --max-time 10 http://api-svc/api/list
* Connected to api-svc (10.96.31.7) port 80 (#0)
> GET /api/list HTTP/1.1
> Host: api-svc
>
< HTTP/1.1 200 OK
< Content-Type: application/json
< Transfer-Encoding: chunked
<
* Operation timed out after 10001 milliseconds with 8192 bytes received

$ kubectl exec -it client-pod -- openssl s_client -connect internal-api:443 -servername internal-api
CONNECTED(00000003)
write to 0x... [0x...] (318 bytes => 318 (0x13E))
# ... 여기서 아무것도 오지 않고 정지

$ dmesg -T | grep -i conntrack
[Mon Aug 24 09:12:31 2026] nf_conntrack: nf_conntrack: table full, dropping packet
$ sysctl net.netfilter.nf_conntrack_count net.netfilter.nf_conntrack_max
net.netfilter.nf_conntrack_count = 262144
net.netfilter.nf_conntrack_max = 262144

# Pod 안에서 실행. -M do = Don't Fragment 고정
$ kubectl exec -it client-pod -- ping -M do -s 1472 -c 1 10.244.2.15
PING 10.244.2.15 (10.244.2.15) 1472(1500) bytes of data.
ping: local error: message too long, mtu=1450

$ kubectl exec -it client-pod -- ping -M do -s 1422 -c 1 10.244.2.15
1430 bytes from 10.244.2.15: icmp_seq=1 ttl=62 time=0.51 ms

From 10.0.1.1 icmp_seq=1 Frag needed and DF set (mtu = 1450)

# 노드에서
$ ip link show | grep -E '^[0-9]+:|mtu' | grep -E 'eth0|cni0|flannel|vxlan|cilium|tunl'
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP
4: flannel.1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN
5: cni0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP
7: vethb31a4f2@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 ...

2: eth0: ... mtu 1500
4: flannel.1: ... mtu 1450
5: cni0: ... mtu 1450
7: vethb31a4f2@if3: ... mtu 1450

$ kubectl exec -it client-pod -- ip link show eth0
3: eth0@if7: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP

$ kubectl exec -it client-pod -- tracepath 10.244.2.15
 1?: [LOCALHOST]      pmtu 1500
 1:  10.244.1.1       0.132ms
 2:  10.0.1.1         0.418ms pmtu 1450
 3:  10.244.2.15      0.522ms reached
     Resume: pmtu 1450 hops 3 back 3

$ for n in 500 1000 1400 1500 2000; do
    echo -n "size=$n -> "
    kubectl exec -it client-pod -- curl -s --max-time 5 \
      -o /dev/null -w '%{http_code}\n' "http://api-svc/echo?bytes=$n" || echo TIMEOUT
  done
size=500  -> 200
size=1000 -> 200
size=1400 -> 200
size=1500 -> TIMEOUT
size=2000 -> TIMEOUT

Pod MTU = 기저 링크 MTU − 캡슐화 오버헤드 (중첩 시 오버헤드 합산)

$ kubectl edit configmap kube-flannel-cfg -n kube-flannel

{
  "Network": "10.244.0.0/16",
  "Backend": {
    "Type": "vxlan",
    "MTU": 1410
  }
}

$ kubectl rollout restart daemonset kube-flannel-ds -n kube-flannel
$ kubectl rollout restart deployment -n <앱 네임스페이스> --all

$ kubectl patch configmap calico-config -n kube-system \
  --type merge -p '{"data":{"veth_mtu":"1410"}}'

# IPIP 모드 터널 MTU
$ kubectl set env daemonset/calico-node -n kube-system FELIX_IPINIPMTU=1440
# VXLAN 모드
$ kubectl set env daemonset/calico-node -n kube-system FELIX_VXLANMTU=1410
$ kubectl rollout restart daemonset calico-node -n kube-system

$ helm upgrade cilium cilium/cilium \
  --namespace kube-system --reuse-values \
  --set MTU=1410

$ kubectl rollout restart daemonset cilium -n kube-system
$ kubectl rollout restart deployment -n <앱 네임스페이스> --all

# 새로 뜬 Pod에서 확인
$ kubectl exec -it <신규-pod> -- ip link show eth0
3: eth0@if11: ... mtu 1410 ...

$ kubectl exec -it <신규-pod> -- ping -M do -s 1382 -c 1 <상대 Pod IP>
1390 bytes from ...: icmp_seq=1 ttl=62 time=0.44 ms

# 모든 노드에서 실행 (SYN 패킷의 MSS를 경로 MTU에 맞춤)
sudo iptables -t mangle -A FORWARD -p tcp --tcp-flags SYN,RST SYN \
  -j TCPMSS --clamp-mss-to-pmtu

# PMTUD가 아예 안 되는 환경이면 고정값으로
sudo iptables -t mangle -A FORWARD -p tcp --tcp-flags SYN,RST SYN \
  -j TCPMSS --set-mss 1370

# 확인
sudo iptables -t mangle -L FORWARD -n -v | grep TCPMSS

# 송신 노드에서 ICMP 수신 여부 관찰
$ sudo tcpdump -ni any 'icmp[icmptype] == 3 and icmp[icmpcode] == 4' -c 5
listening on any, link-type LINUX_SLL (Linux cooked v1)
0 packets captured

10:14:22.331 IP 10.0.1.1 > 10.0.2.31: ICMP 10.244.2.15 unreachable -
  need to frag (mtu 1450), length 556

$ ip route get 10.244.2.15
10.244.2.15 via 10.0.1.1 dev eth0 src 10.0.2.31 mtu 1450