深入探索Kubernetes中的Mirror Pod删除过程及其对Static Pod 的影响

apiVersion: v1
kind: Pod
metadata:
  name: nginx-static-pod
  namespace: default
spec:
  containers:
  - name: nginx-container
    image: nginx:latest
    ports:
    - containerPort: 80

apiVersion: v1
kind: Pod
metadata:
  annotations:
    kubernetes.io/config.hash: a8712c005851ee6b29cff91b9ab4b9c6
    kubernetes.io/config.mirror: a8712c005851ee6b29cff91b9ab4b9c6
    kubernetes.io/config.seen: "2023-11-23T14:05:21.400551188+08:00"
    kubernetes.io/config.source: file
  creationTimestamp: "2023-11-23T06:05:21Z"
  name: nginx-static-pod-10.11.251.2
  namespace: default
  ownerReferences:
  - apiVersion: v1
    controller: true
    kind: Node
    name: 10.11.251.2
    uid: d2e3ebf6-2a70-434f-9142-0f74a1dd16bc
  resourceVersion: "292107092"
  uid: 1facfed2-ba67-40f8-9982-ab86a5e7fd33
spec:
  containers:
  - image: nginx:latest
    imagePullPolicy: Always
    name: nginx-container
    ports:
    - containerPort: 80
      protocol: TCP
    resources: {}
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
  dnsPolicy: ClusterFirst
  enableServiceLinks: true
  nodeName: 10.11.251.2
  preemptionPolicy: PreemptLowerPriority
  priority: 0
  restartPolicy: Always
  schedulerName: default-scheduler
  securityContext: {}
  terminationGracePeriodSeconds: 30
  tolerations:
  - effect: NoExecute
    operator: Exists
status:
  conditions:
  - lastProbeTime: null
    lastTransitionTime: "2023-11-23T06:05:21Z"
    status: "True"
    type: Initialized
  - lastProbeTime: null
    lastTransitionTime: "2023-11-23T06:05:29Z"
    status: "True"
    type: Ready
  - lastProbeTime: null
    lastTransitionTime: "2023-11-23T06:05:29Z"
    status: "True"
    type: ContainersReady
  - lastProbeTime: null
    lastTransitionTime: "2023-11-23T06:05:21Z"
    status: "True"
    type: PodScheduled
  containerStatuses:
  - containerID: docker://b6ca55d329230c8f5776eb1160fe161d6fefa01f2b31e55dbb820add90aadccc
    image: nginx:latest
    imageID: docker-pullable://nginx@sha256:10d1f5b58f74683ad34eb29287e07dab1e90f10af243f151bb50aa5dbb4d62ee
    lastState: {}
    name: nginx-container
    ready: true
    restartCount: 0
    started: true
    state:
      running:
        startedAt: "2023-11-23T06:05:29Z"
  hostIP: 10.11.251.2
  phase: Running
  podIP: 10.26.124.96
  podIPs:
  - ip: 10.26.124.96
  qosClass: BestEffort
  startTime: "2023-11-23T06:05:21Z"

# kubectl get pod nginx-static-pod-10.11.251.2 -o yaml -w
apiVersion: v1
kind: Pod
metadata:
  annotations:
    kubernetes.io/config.hash: a8712c005851ee6b29cff91b9ab4b9c6
    kubernetes.io/config.mirror: a8712c005851ee6b29cff91b9ab4b9c6
    kubernetes.io/config.seen: "2023-11-23T14:05:21.400551188+08:00"
    kubernetes.io/config.source: file
  creationTimestamp: "2023-11-23T06:05:21Z"
  name: nginx-static-pod-10.11.251.2
  namespace: default
  ownerReferences:
  - apiVersion: v1
    controller: true
    kind: Node
    name: 10.11.251.2
    uid: d2e3ebf6-2a70-434f-9142-0f74a1dd16bc
  resourceVersion: "292107092"
  uid: 1facfed2-ba67-40f8-9982-ab86a5e7fd33

apiVersion: v1
kind: Pod
metadata:
  annotations:
    kubernetes.io/config.hash: a8712c005851ee6b29cff91b9ab4b9c6
    kubernetes.io/config.mirror: a8712c005851ee6b29cff91b9ab4b9c6
    kubernetes.io/config.seen: "2023-11-23T14:05:21.400551188+08:00"
    kubernetes.io/config.source: file
  creationTimestamp: "2023-11-23T06:05:21Z"
  deletionGracePeriodSeconds: 30
  deletionTimestamp: "2023-11-23T06:11:45Z"
  name: nginx-static-pod-10.11.251.2
  namespace: default
  ownerReferences:
  - apiVersion: v1
    controller: true
    kind: Node
    name: 10.11.251.2
    uid: d2e3ebf6-2a70-434f-9142-0f74a1dd16bc
  resourceVersion: "292110805"
  uid: 1facfed2-ba67-40f8-9982-ab86a5e7fd33

apiVersion: v1
kind: Pod
metadata:
  annotations:
    kubernetes.io/config.hash: a8712c005851ee6b29cff91b9ab4b9c6
    kubernetes.io/config.mirror: a8712c005851ee6b29cff91b9ab4b9c6
    kubernetes.io/config.seen: "2023-11-23T14:05:21.400551188+08:00"
    kubernetes.io/config.source: file
  creationTimestamp: "2023-11-23T06:05:21Z"
  deletionGracePeriodSeconds: 0
  deletionTimestamp: "2023-11-23T06:11:15Z"
  name: nginx-static-pod-10.11.251.2
  namespace: default
  ownerReferences:
  - apiVersion: v1
    controller: true
    kind: Node
    name: 10.11.251.2
    uid: d2e3ebf6-2a70-434f-9142-0f74a1dd16bc
  resourceVersion: "292110806"
  uid: 1facfed2-ba67-40f8-9982-ab86a5e7fd33

apiVersion: v1
kind: Pod
metadata:
  annotations:
    kubernetes.io/config.hash: a8712c005851ee6b29cff91b9ab4b9c6
    kubernetes.io/config.mirror: a8712c005851ee6b29cff91b9ab4b9c6
    kubernetes.io/config.seen: "2023-11-23T14:05:21.400551188+08:00"
    kubernetes.io/config.source: file
  creationTimestamp: "2023-11-23T06:05:21Z"
  deletionGracePeriodSeconds: 0
  deletionTimestamp: "2023-11-23T06:11:15Z"
  name: nginx-static-pod-10.11.251.2
  namespace: default
  ownerReferences:
  - apiVersion: v1
    controller: true
    kind: Node
    name: 10.11.251.2
    uid: d2e3ebf6-2a70-434f-9142-0f74a1dd16bc
  resourceVersion: "292110807"
  uid: 1facfed2-ba67-40f8-9982-ab86a5e7fd33

apiVersion: v1
kind: Pod
metadata:
  annotations:
    kubernetes.io/config.hash: a8712c005851ee6b29cff91b9ab4b9c6
    kubernetes.io/config.mirror: a8712c005851ee6b29cff91b9ab4b9c6
    kubernetes.io/config.seen: "2023-11-23T14:05:21.400551188+08:00"
    kubernetes.io/config.source: file
  creationTimestamp: "2023-11-23T06:11:15Z"
  name: nginx-static-pod-10.11.251.2
  namespace: default
  ownerReferences:
  - apiVersion: v1
    controller: true
    kind: Node
    name: 10.11.251.2
    uid: d2e3ebf6-2a70-434f-9142-0f74a1dd16bc
  resourceVersion: "292110808"
  uid: 7bdace8a-8cc5-4cb4-886b-f3961f07d3b8
  ...
status:
  phase: Pending
  qosClass: BestEffort

status:
  conditions:
  - lastProbeTime: null
    lastTransitionTime: "2023-11-23T06:05:21Z"
    status: "True"
    type: Initialized
  - lastProbeTime: null
    lastTransitionTime: "2023-11-23T06:05:29Z"
    status: "True"
    type: Ready
  - lastProbeTime: null
    lastTransitionTime: "2023-11-23T06:05:29Z"
    status: "True"
    type: ContainersReady
  - lastProbeTime: null
    lastTransitionTime: "2023-11-23T06:05:21Z"
    status: "True"
    type: PodScheduled
  containerStatuses:
  - containerID: docker://b6ca55d329230c8f5776eb1160fe161d6fefa01f2b31e55dbb820add90aadccc
    image: nginx:latest
    imageID: docker-pullable://nginx@sha256:10d1f5b58f74683ad34eb29287e07dab1e90f10af243f151bb50aa5dbb4d62ee
    lastState: {}
    name: nginx-container
    ready: true
    restartCount: 0
    started: true
    state:
      running:
        startedAt: "2023-11-23T06:05:29Z"
  hostIP: 10.11.251.2
  phase: Running
  podIP: 10.26.124.96
  podIPs:
  - ip: 10.26.124.96
  qosClass: BestEffort
  startTime: "2023-11-23T06:05:21Z"

	// Create Mirror Pod for Static Pod if it doesn't already exist
	// 如果是static pod，则检查是否存在mirror pod，当mirror pod不存在时候进行创建
	if kubetypes.IsStaticPod(pod) {
		deleted := false
		// 存在mirror pod
		if mirrorPod != nil {
			// mirrorPod被删除或mirrorPod不是pod的mirror pod
			if mirrorPod.DeletionTimestamp != nil || !kl.podManager.IsMirrorPodOf(mirrorPod, pod) {
				// The mirror pod is semantically different from the static pod. Remove
				// it. The mirror pod will get recreated later.
				klog.InfoS("Trying to delete pod", "pod", klog.KObj(pod), "podUID", mirrorPod.ObjectMeta.UID)
				// pod.Name + "_" + pod.Namespace
				podFullName := kubecontainer.GetPodFullName(pod)
				var err error
				// 调用api删除mirror pod（从podFullName解析出name和namespace）
				deleted, err = kl.podManager.DeleteMirrorPod(podFullName, &mirrorPod.ObjectMeta.UID)
				if deleted {
					klog.InfoS("Deleted mirror pod because it is outdated", "pod", klog.KObj(mirrorPod))
				} else if err != nil {
					klog.ErrorS(err, "Failed deleting mirror pod", "pod", klog.KObj(mirrorPod))
				}
			}
		}
		// static pod没有mirror pod或mirror pod被删除，则在获取node未发生错误且node未被删除条件下，创建mirror pod
		if mirrorPod == nil || deleted {
			node, err := kl.GetNode()
			// 当获取node发生错误或node被删除，则无需创建mirror pod
			if err != nil || node.DeletionTimestamp != nil {
				klog.V(4).InfoS("No need to create a mirror pod, since node has been removed from the cluster", "node", klog.KRef("", string(kl.nodeName)))
			} else {
				klog.V(4).InfoS("Creating a mirror pod for static pod", "pod", klog.KObj(pod))
				// node没有被删除，则创建static pod的mirror pod
				if err := kl.podManager.CreateMirrorPod(pod); err != nil {
					klog.ErrorS(err, "Failed creating a mirror pod for", "pod", klog.KObj(pod))
				}
			}
		}
	}