[K8S] Troubleshoot Network

certified-kubernetes-administrator-with-practice-tests 304강

 

실습 1] CNI

root@controlplane ~ ➜  kubectl get pods
NAME                           READY   STATUS              RESTARTS   AGE
mysql                          0/1     ContainerCreating   0          39s
webapp-mysql-d89894b4b-l86gz   0/1     ContainerCreating   0          39s

root@controlplane ~ ➜  kubectl describe pod mysql 
Name:             mysql
Namespace:        triton
Priority:         0
Service Account:  default
Node:             controlplane/192.168.121.127
Start Time:       Wed, 12 Feb 2025 02:06:14 +0000
Labels:           name=mysql
Annotations:      <none>
Status:           Pending
IP:               
IPs:              <none>
...
Events:
  Type     Reason                  Age               From               Message
  ----     ------                  ----              ----               -------
  Normal   Scheduled               81s               default-scheduler  Successfully assigned triton/mysql to controlplane
  Warning  FailedCreatePodSandBox  81s               kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "44a62527ae3fa9632695a3218f443632bdabe13e7d9a15c9de4e1ed58d5e4579": plugin type="weave-net" name="weave" failed (add): unable to allocate IP address: Post "http://127.0.0.1:6784/ip/44a62527ae3fa9632695a3218f443632bdabe13e7d9a15c9de4e1ed58d5e4579": dial tcp 127.0.0.1:6784: connect: connection refused
  Normal   SandboxChanged          3s (x7 over 80s)  kubelet            Pod sandbox changed, it will be killed and re-created.
# Weave Net이 새로운 파드에 IP 주소를 할당하는 데 실패했음

root@controlplane ~ ➜  kubectl get pods -n kube-system
NAME                                   READY   STATUS    RESTARTS   AGE
coredns-6f6b679f8f-hcqwr               1/1     Running   0          9m48s
coredns-6f6b679f8f-qxf7r               1/1     Running   0          9m48s
etcd-controlplane                      1/1     Running   0          9m54s
kube-apiserver-controlplane            1/1     Running   0          9m55s
kube-controller-manager-controlplane   1/1     Running   0          9m54s
kube-proxy-wx249                       1/1     Running   0          9m48s
kube-scheduler-controlplane            1/1     Running   0          9m54s
# calico, flannel, weave-net, cilium 등의 이름을 가진 파드가 존재하지 않는다.

root@controlplane ~ ➜  ls /etc/cni/net.d/
10-weave.conflist $ weave가 깔려 있어야 함.

 

• Weave 설치

root@controlplane /etc/cni/net.d ➜  kubectl apply -f https://reweave.azurewebsites.net/k8s/v1.29/net.yaml
serviceaccount/weave-net created
clusterrole.rbac.authorization.k8s.io/weave-net created
clusterrolebinding.rbac.authorization.k8s.io/weave-net created
role.rbac.authorization.k8s.io/weave-net created
rolebinding.rbac.authorization.k8s.io/weave-net created
daemonset.apps/weave-net created

# Weave 생성됨
root@controlplane /etc/cni/net.d ➜  kubectl get pods -n kube-system
NAME                                   READY   STATUS    RESTARTS   AGE
coredns-6f6b679f8f-hcqwr               1/1     Running   0          16m
coredns-6f6b679f8f-qxf7r               1/1     Running   0          16m
etcd-controlplane                      1/1     Running   0          17m
kube-apiserver-controlplane            1/1     Running   0          17m
kube-controller-manager-controlplane   1/1     Running   0          17m
kube-proxy-wx249                       1/1     Running   0          16m
kube-scheduler-controlplane            1/1     Running   0          17m
weave-net-89vln                        2/2     Running   0          19s

# Pod 제대로 Running
root@controlplane /etc/cni/net.d ➜  kubectl get pods
NAME                           READY   STATUS    RESTARTS   AGE
mysql                          1/1     Running   0          7m59s
webapp-mysql-d89894b4b-l86gz   1/1     Running   0          7m59s

 

실습2 ] kube-proxy

 

kube-proxy는 서비스(Service)의 네트워크 프록시 역할로 클러스터 내에서 서비스 디스커버리와 로드 밸런싱을 제공하며, 클러스터의 네트워크 트래픽이 효율적으로 관리되도록 돕는다.

kube-proxy는 Deployment 또는 DeploySet으로 배포된다.

 ConfigMap은 설정 데이터를 저장하는 리소스이므로, kube-proxy의 실행 방식과 직접적인 관련은 없지만, 설정 관리에 사용한다.

root@controlplane ~ ➜  kubectl get pods -n kube-system
NAME                                   READY   STATUS             RESTARTS      AGE
coredns-6f6b679f8f-682q8               1/1     Running            0             8m46s
coredns-6f6b679f8f-tdwlr               1/1     Running            0             8m46s
etcd-controlplane                      1/1     Running            0             8m53s
kube-apiserver-controlplane            1/1     Running            0             8m53s
kube-controller-manager-controlplane   1/1     Running            0             8m53s
kube-proxy-t6m2l                       0/1     CrashLoopBackOff   4 (65s ago)   2m43s
kube-scheduler-controlplane            1/1     Running            0             8m53s
weave-net-fbzlb                        2/2     Running            0             27s

root@controlplane ~ ➜  kubectl logs kube-proxy-t6m2l -n kube-system
E0212 04:27:31.556699       1 run.go:74] "command failed" err="failed complete: open /var/lib/kube-proxy/configuration.conf: no such file or directory"

root@controlplane /etc/cni/net.d ➜  ls /var/lib/kube-proxy/configuration.conf
ls: cannot access '/var/lib/kube-proxy/configuration.conf': No such file or directory

root@controlplane ~ ✖ kubectl describe ds kube-proxy -n kube-system | grep -i
 config
      --config=/var/lib/kube-proxy/configuration.conf
    Type:      ConfigMap (a volume populated by a ConfigMap)

root@controlplane ~ ➜  kubectl describe cm kube-proxy -n kube-system | grep -
i config
  kubeconfig: /var/lib/kube-proxy/kubeconfig.conf

#  ds와 cm 둘이 configuration path 다르게 설정되어 있는 것 확인 가능

kubeconfig: /var/lib/kube-proxy/configuration.conf

 

• config 설정 변경

kubectl describe cm kube-proxy -n kube-system
Name:         kube-proxy
Namespace:    kube-system
Labels:       app=kube-proxy
Annotations:  kubeadm.kubernetes.io/component-config.hash: sha256:906b8697200819e8263843f43965bb3614545800b82206dcee8ef93a08bc4f4b

Data
====
config.conf:  # 이 이름으로 사용해야 한다.

root@controlplane ~ ➜  kubectl describe ds kube-proxy -n kube-system | grep config
      --config=/var/lib/kube-proxy/config.conf # 이렇게 경로 변경해주기
      
root@controlplane ~ ➜  kubectl get pods -n kube-system
NAME                                   READY   STATUS    RESTARTS   AGE
coredns-6f6b679f8f-fkqc6               1/1     Running   0          21m
coredns-6f6b679f8f-vnl4b               1/1     Running   0          21m
etcd-controlplane                      1/1     Running   0          22m
kube-apiserver-controlplane            1/1     Running   0          22m
kube-controller-manager-controlplane   1/1     Running   0          22m
kube-proxy-qwppj                       1/1     Running   0          16s # 정상 작동 확인
kube-scheduler-controlplane            1/1     Running   0          22m
weave-net-hm9ww                        2/2     Running   0          3m18s