K8S 故障排错新手段：kubectl debug 实战

## 控制面开启 EphemeralContainers featureGate. ### 进入 master 节点，编辑 /etc/kubernetes/manifests/ 下的 kube-apiserver.yaml，kube-controller-manager.yaml 及 kube-scheduler.yaml，在 command 部分添加 - --feature-gates=EphemeralContainers=true； ## Kubelet 服务开启该功能 ### 在节点上编辑 /var/lib/kubelet/kubeadm-flags.env，添加 --feature-gates=EphemeralContainers=true；或者设置KUBELET_EXTRA_ARGS="--feature-gates=EphemeralContainers=true" ## 重启 Kubelet： ### systemctl restart kubelet

$ kubectl debug -it ${pod_name} --image=busybox:1.28 --target=${container_name}

[centos@ml-k8s-1 test1]$ kubectl apply -f pod1.yaml secret/pod1-secret created clusterrole.rbac.authorization.k8s.io/pod1-get created clusterrolebinding.rbac.authorization.k8s.io/pod1-get-rbac created serviceaccount/pod1-sa created pod/pod1 created [centos@ml-k8s-1 test1]$ kubectl get pod NAME READY STATUS RESTARTS AGE pod1 1/1 Running 0 7s

[centos@ml-k8s-1 deploy]$ kubectl describe pod pod1 ... Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Scheduled 5m44s default-scheduler Successfully assigned default/pod1 to ml-k8s-2.novalocal Normal Pulling 5m42s kubelet Pulling image "beyond.io:5000/debug-test:0.1.1" Normal Pulled 5m42s kubelet Successfully pulled image "beyond.io:5000/debug-test:0.1.1" in 59.162221ms Normal Created 5m42s kubelet Created container pod1 Normal Started 5m42s kubelet Started container pod1

[centos@ml-k8s-1 test1]$ kubectl logs pod1 I0429 10:24:34.913853 1 main.go:18] Test start! I0429 10:24:34.914013 1 main.go:19] Pulling data from mybucket 1 and storing it in mybucket 2. Wrong in getting object from mybucket1. 

[centos@ml-k8s-1 test1]$ kubectl exec -it pod1 -- sh OCI runtime exec failed: exec failed: container_linux.go:380: starting container process caused: exec: "sh": executable file not found in $PATH: unknown command terminated with exit code 126 

[centos@ml-k8s-1 test1]$ kubectl debug -it pod1 --image=busybox:1.28 --target=pod1 Defaulting debug container name to debugger-h59bb. If you don't see a command prompt, try pressing enter. / # ls bin dev etc home proc root sys tmp usr var (unreachable)/data # cat test.txt 

[centos@ml-k8s-1 test3]$ kubectl apply -f pod3.yaml pod/pod3 created 

[centos@ml-k8s-1 test3]$ kubectl get pod NAME READY STATUS RESTARTS AGE pod3 0/1 Completed 0 7s 

[centos@ml-k8s-1 test3]$ kubectl logs pod3 Hello ldsdsy Today is 

[centos@ml-k8s-1 test3]$ kubectl exec -it pod3 -- sh error: cannot exec into a container in a completed pod; current phase is Succeeded 

[centos@ml-k8s-1 test3]$ kubectl debug pod3 -it --copy-to=pod3-debug --container=pod3 -- sh If you don't see a command prompt, try pressing enter. / # ls app bin dev etc home proc root sys tmp usr var / # cd app /app # ls test.sh /app # cat test.sh #! /bin/sh echo "Hello ldsdsy" time=$(date +"%Y-%m-%d %H:%M:%S") echo "Today is $ttime" /app # 

[centos@ml-k8s-1 test3]$ kubectl debug pod3 --copy-to=pod3-debug --set-image=pod3=beyond.io:5000/debug-test:0.1.4 // --set-image=*=xxx 表示把 Pod 的所有容器镜像全换成 xxx 

[centos@ml-k8s-1 test3]$ kubectl get pod NAME READY STATUS RESTARTS AGE pod3 0/1 Completed 0 13m pod3-debug 0/1 Completed 0 8s [centos@ml-k8s-1 test3]$ kubectl logs pod3-debug Hello ldsdsy Today is 2022-05-02 09:50:25 [centos@ml-k8s-1 test3]$ 

package main import ( "context" "io" "os" "time" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/klog/v2" ) func main() { klog.Info("Test start!") klog.Info("Pulling data from mybucket 1 and storing it in mybucket 2.") // creates the in-cluster config config, err := rest.InClusterConfig() if err != nil { klog.Errorln("Wrong in creating config: ", err) } // create the clientset clientset, err := kubernetes.NewForConfig(config) if err != nil { klog.Errorln("Wrong in creating clientset: ", err) } // Get info of minio from s3-secret secret, err := clientset.CoreV1().Secrets("default").Get(context.TODO(), "minio-secret", metav1.GetOptions{}) if err != nil { klog.Errorln("Wrong in getting secret: ", err) time.Sleep(1 * time.Hour) } id := string(secret.Data["id"]) key := string(secret.Data["key"]) endpoint := string(secret.Data["endpoint"]) useSSL := false //true 会走 https // Initialize minio client object. minioClient, err := minio.New(endpoint, &minio.Options{ Creds: credentials.NewStaticV4(id, key, ""), Secure: useSSL, }) if err != nil { klog.Errorln("Wrong in getting minioClient : ", err) time.Sleep(1 * time.Hour) } object, err := minioClient.GetObject(context.Background(), "mybucket1", "test.txt", minio.GetObjectOptions{}) if err != nil { klog.Errorln("Wrong in getting object from mybucket1: ", err) time.Sleep(1 * time.Hour) } //以读写方式打开文件，如果不存在，则创建(只创建文件，不能创建文件夹) localFile, err := os.OpenFile("/data/test.txt", os.O_RDWR|os.O_CREATE, 0766) if err != nil { klog.Errorln("Wrong in creating /data/test.txt: ", err) time.Sleep(1 * time.Hour) } klog.Info(localFile) defer localFile.Close() if _, err = io.Copy(localFile, object); err != nil { klog.Errorln("Wrong in coping object from mybucket1 to localFile: ", err) time.Sleep(1 * time.Hour) } file, err := os.Open("/data/test.txt") if err != nil { klog.Errorln("Wrong in getting object from /data/test.txt: ", err) time.Sleep(1 * time.Hour) } defer file.Close() fileStat, err := file.Stat() if err != nil { klog.Errorln("Wrong in getting fileStat: ", err) time.Sleep(1 * time.Hour) } // Create a bucket at region 'us-east-1' with object locking enabled. err = minioClient.MakeBucket(context.Background(), "mybucket2", minio.MakeBucketOptions{Region: "cn-north-1", ObjectLocking: false}) if err != nil { klog.Errorln("Wrong in creating mybucket2: ", err) time.Sleep(1 * time.Hour) } uploadInfo, err := minioClient.PutObject(context.Background(), "mybucket2", "test.txt", file, fileStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { klog.Errorln("Wrong in putting myobject to mybucket2: ", err) time.Sleep(1 * time.Hour) } klog.Infoln("Successfully uploaded bytes: ", uploadInfo) }

FROM scratch ADD ./app / CMD ["/app"]

apiVersion: v1 stringData: id: AKIAIOSFODNN7EXAMPLE key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY endpoint: 10.20.9.60:30009 kind: Secret metadata: name: pod1-secret type: Opaque --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: pod1-get rules: - apiGroups: [""] resources: ["secrets"] verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: pod1-get-rbac subjects: - kind: ServiceAccount namespace: default name: pod1-sa roleRef: kind: ClusterRole name: pod1-get apiGroup: rbac.authorization.k8s.io --- apiVersion: v1 kind: ServiceAccount metadata: name: pod1-sa namespace: default --- apiVersion: v1 kind: Pod metadata: name: pod1 labels: k8s-app: pod1 spec: serviceAccountName: pod1-sa restartPolicy: Never containers: - name: pod1 image: beyond.io:5000/debug-test:0.1.1 imagePullPolicy: Always volumeMounts: - name: volume mountPath: /data readOnly: False volumes: - name: volume emptyDir: {} 

#! /bin/sh echo "Hello ldsdsy" time=$(date +"%Y-%m-%d %H:%M:%S") echo "Today is $ttime"

FROM busybox:1.28 RUN mkdir /app ADD ./test.sh /app RUN chmod +x /app/test.sh CMD ["sh","-c","/app/test.sh"]

apiVersion: v1 kind: Pod metadata: name: pod3 labels: k8s-app: pod3 spec: restartPolicy: Never containers: - name: pod3 image: beyond.io:5000/debug-test:0.1.3 imagePullPolicy: Always