【kubernetes/k8s源码分析】 kubelet container gc 垃圾回收源码分析

叁歲伎倆 2022-10-02 10:49 184阅读 0赞

# WHY #

容器停止文件系统并不会自动清除，通过 `docker ps -a` 也能够看到这些资源。kubelet有**container gc**的方案用于清理宿主机上的非所需容器

容器镜像耗存储资源，会导致宿主机上会留下不再需要的容器镜像，kubelet设计了一套i**mage gc**的方案

# WHAT #

>        Kubernetes 垃圾回收（Garbage Collection）机制由kubelet完成，kubelet定期清理不再使用的容器和镜像，每分钟进行一次容器的GC，每五分钟进行一次镜像的GC

# HOW #

**NewMainKubelet**

-->  kubecontainer.ContainerGCPolicy

-->  kubecontainer.NewContainerGC

*  MinAge： minimum-container-ttl-duration 默认为 0s，容器生命周期结束，间隔时间被回收
 *  MaxPerPodContainer：maximum-dead-containers-per-container，默认为1，指每个pods最大可以保存多少个已结束的容器，默认是1，负数表示不限制，这些容器会浪费磁盘空间
 *  MaxContainers：maximum-dead-containers，默认为 -1表示不做限制，节点上最多允许保留多少个已结束的容器

containerGCPolicy := kubecontainer.ContainerGCPolicy{
    	MinAge:             minimumGCAge.Duration,
    	MaxPerPodContainer: int(maxPerPodContainerCount),
    	MaxContainers:      int(maxContainerCount),
    }

# 1. 实例化 container gc #

路径： pkg/kubelet/kubelet.go

// setup containerGC
    containerGC, err := kubecontainer.NewContainerGC(klet.containerRuntime, containerGCPolicy, klet.sourcesReady)
    if err != nil {
    	return nil, err
    }
    klet.containerGC = containerGC
    klet.containerDeletor = newPodContainerDeletor(klet.containerRuntime, integer.IntMax(containerGCPolicy.MaxPerPodContainer, minDeadContainerInPod))

###     1.1 ContainerGC 接口 ###

// Manages garbage collection of dead containers.
    //
    // Implementation is thread-compatible.
    type ContainerGC interface {
    	// Garbage collect containers.
    	GarbageCollect() error
    	// Deletes all unused containers, including containers belonging to pods that are terminated but not deleted
    	DeleteAllUnusedContainers() error
    }

###     1.2 realContainerGC 实现了 ContainerGC 接口 ###

// New ContainerGC instance with the specified policy.
    func NewContainerGC(runtime Runtime, policy ContainerGCPolicy, sourcesReadyProvider SourcesReadyProvider) (ContainerGC, error) {
    	if policy.MinAge < 0 {
    		return nil, fmt.Errorf("invalid minimum garbage collection age: %v", policy.MinAge)
    	}
    
    	return &realContainerGC{
    		runtime:              runtime,
    		policy:               policy,
    		sourcesReadyProvider: sourcesReadyProvider,
    	}, nil
    }

> createAndInitKubelet
> 
>        -->  StartGarbageCollection
> 
>                       -->  kl.containerGC.GarbageCollect
> 
>                                    -->  GarbageCollect
> 
>                                                -->  evictContainers
> 
>                                                -->  evictSandboxes
> 
>                       -->  kl.imageManager.GarbageCollect
> 
>    本文只分析 container gc部分

# 2. StartGarbageCollection #

主要是定期异步执行 container gc 流程

// StartGarbageCollection starts garbage collection threads.
    func (kl *Kubelet) StartGarbageCollection() {
    	loggedContainerGCFailure := false
    	go wait.Until(func() {
    		if err := kl.containerGC.GarbageCollect(); err != nil {
    			klog.Errorf("Container garbage collection failed: %v", err)
    			kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, events.ContainerGCFailed, err.Error())
    			loggedContainerGCFailure = true
    		} else {
    			var vLevel klog.Level = 4
    			if loggedContainerGCFailure {
    				vLevel = 1
    				loggedContainerGCFailure = false
    			}
    
    			klog.V(vLevel).Infof("Container garbage collection succeeded")
    		}
    	}, ContainerGCPeriod, wait.NeverStop)

###     2.1 GarbageCollect ###

看看GarbageCollect 究竟做了什么，继续

func (cgc *realContainerGC) GarbageCollect() error {
    	return cgc.runtime.GarbageCollect(cgc.policy, cgc.sourcesReadyProvider.AllReady(), false)
    }

// GarbageCollect removes dead containers using the specified container gc policy.
    func (m *kubeGenericRuntimeManager) GarbageCollect(gcPolicy kubecontainer.ContainerGCPolicy, allSourcesReady bool, evictNonDeletedPods bool) error {
    	return m.containerGC.GarbageCollect(gcPolicy, allSourcesReady, evictNonDeletedPods)
    }

###     2.2 GarbageCollect ###

路径：pkg/kubelet/kuberuntime/kuberuntime\_gc.go

删除挂掉的容器

// GarbageCollect removes dead containers using the specified container gc policy.
    // Note that gc policy is not applied to sandboxes. Sandboxes are only removed when they are
    // not ready and containing no containers.
    //
    // GarbageCollect consists of the following steps:
    // * gets evictable containers which are not active and created more than gcPolicy.MinAge ago.
    // * removes oldest dead containers for each pod by enforcing gcPolicy.MaxPerPodContainer.
    // * removes oldest dead containers by enforcing gcPolicy.MaxContainers.
    // * gets evictable sandboxes which are not ready and contains no containers.
    // * removes evictable sandboxes.
    func (cgc *containerGC) GarbageCollect(gcPolicy kubecontainer.ContainerGCPolicy, allSourcesReady bool, evictTerminatedPods bool) error {
    	errors := []error{}
    	// Remove evictable containers
    	if err := cgc.evictContainers(gcPolicy, allSourcesReady, evictTerminatedPods); err != nil {
    		errors = append(errors, err)
    	}
    
    	// Remove sandboxes with zero containers
    	if err := cgc.evictSandboxes(evictTerminatedPods); err != nil {
    		errors = append(errors, err)
    	}
    
    	// Remove pod sandbox log directory
    	if err := cgc.evictPodLogsDirectories(allSourcesReady); err != nil {
    		errors = append(errors, err)
    	}
    	return utilerrors.NewAggregate(errors)
    }

# 3. evictContainers 驱逐容器 #

###     3.1 evictableContainers 函数获得可驱逐的容器 ###

不是running的，并且创建时间  >  minAge

// evict all containers that are evictable
    func (cgc *containerGC) evictContainers(gcPolicy kubecontainer.ContainerGCPolicy, allSourcesReady bool, evictTerminatedPods bool) error {
    	// Separate containers by evict units.
    	evictUnits, err := cgc.evictableContainers(gcPolicy.MinAge)
    	if err != nil {
    		return err
    	}

###     3.2 如果 pod 是已经删除的，将所有这个pod的容器删除 ###

// Remove deleted pod containers if all sources are ready.
    if allSourcesReady {
    	for key, unit := range evictUnits {
    		if cgc.podStateProvider.IsPodDeleted(key.uid) || (cgc.podStateProvider.IsPodTerminated(key.uid) && evictTerminatedPods) {
    			cgc.removeOldestN(unit, len(unit)) // Remove all.
    			delete(evictUnits, key)
    		}
    	}
    }

###     3.3 如果 gcPolicy.MaxPerPodContainer 大于 0 ，将留下gcPolicy.MaxPerPodContainer 个数的容器，其他都删除 ###

// enforceMaxContainersPerEvictUnit enforces MaxPerPodContainer for each evictUnit.
    func (cgc *containerGC) enforceMaxContainersPerEvictUnit(evictUnits containersByEvictUnit, MaxContainers int) {
    	for key := range evictUnits {
    		toRemove := len(evictUnits[key]) - MaxContainers
    
    		if toRemove > 0 {
    			evictUnits[key] = cgc.removeOldestN(evictUnits[key], toRemove)
    		}
    	}
    }

###     3.4 如果 MaxContainers 大于 0 情况 （ 默认值为 -1 ，将不进行清理） ###

流程也比较简单，就是需不需要在清理容器

// Enforce max total number of containers.
    if gcPolicy.MaxContainers >= 0 && evictUnits.NumContainers() > gcPolicy.MaxContainers {
    	// Leave an equal number of containers per evict unit (min: 1).
    	numContainersPerEvictUnit := gcPolicy.MaxContainers / evictUnits.NumEvictUnits()
    	if numContainersPerEvictUnit < 1 {
    		numContainersPerEvictUnit = 1
    	}
    	cgc.enforceMaxContainersPerEvictUnit(evictUnits, numContainersPerEvictUnit)
    
    	// If we still need to evict, evict oldest first.
    	numContainers := evictUnits.NumContainers()
    	if numContainers > gcPolicy.MaxContainers {
    		flattened := make([]containerGCInfo, 0, numContainers)
    		for key := range evictUnits {
    			flattened = append(flattened, evictUnits[key]...)
    		}
    		sort.Sort(byCreated(flattened))
    
    		cgc.removeOldestN(flattened, numContainers-gcPolicy.MaxContainers)
    	}
    }

# 4. evictSandboxes 删除所有可驱逐的 sandbox #

可驱逐的sanbox满足如下条件：

*  不是 ready 状态的
 *  不包含容器
 *  不应该存在的（已经被删除的），或者不是为pod创建最新的

// evictSandboxes remove all evictable sandboxes. An evictable sandbox must
    // meet the following requirements:
    //   1. not in ready state
    //   2. contains no containers.
    //   3. belong to a non-existent (i.e., already removed) pod, or is not the
    //      most recently created sandbox for the pod.
    func (cgc *containerGC) evictSandboxes(evictTerminatedPods bool) error {

###     4.1 getKubeletContainers ###

获得 kubelet 所管理的所有容器

// getKubeletContainers lists containers managed by kubelet.
    // The boolean parameter specifies whether returns all containers including
    // those already exited and dead containers (used for garbage collection).
    func (m *kubeGenericRuntimeManager) getKubeletContainers(allContainers bool) ([]*runtimeapi.Container, error) {
    	filter := &runtimeapi.ContainerFilter{}
    	if !allContainers {
    		filter.State = &runtimeapi.ContainerStateValue{
    			State: runtimeapi.ContainerState_CONTAINER_RUNNING,
    		}
    	}
    
    	containers, err := m.runtimeService.ListContainers(filter)
    	if err != nil {
    		klog.Errorf("getKubeletContainers failed: %v", err)
    		return nil, err
    	}
    
    	return containers, nil
    }

###     4.2 获得所有 sanbox 以及 ID ###

// collect all the PodSandboxId of container
    sandboxIDs := sets.NewString()
    for _, container := range containers {
    	sandboxIDs.Insert(container.PodSandboxId)
    }
    
    sandboxes, err := cgc.manager.getKubeletSandboxes(true)
    if err != nil {
    	return err
    }

###     4.3 sanbox 为 ready 以及还有容器的则设置为 active ###

// Set ready sandboxes to be active.
    if sandbox.State == runtimeapi.PodSandboxState_SANDBOX_READY {
    	sandboxInfo.active = true
    }
    
    // Set sandboxes that still have containers to be active.
    if sandboxIDs.Has(sandbox.Id) {
    	sandboxInfo.active = true
    }

###     4.4 如果 sandbox 所属的 pod 被删除，那么删除所有 sandbox ###

否则删除旧的sandbox，留下最新的 sandbox

for podUID, sandboxes := range sandboxesByPod {
    	if cgc.podStateProvider.IsPodDeleted(podUID) || (cgc.podStateProvider.IsPodTerminated(podUID) && evictTerminatedPods) {
    		// Remove all evictable sandboxes if the pod has been removed.
    		// Note that the latest dead sandbox is also removed if there is
    		// already an active one.
    		cgc.removeOldestNSandboxes(sandboxes, len(sandboxes))
    	} else {
    		// Keep latest one if the pod still exists.
    		cgc.removeOldestNSandboxes(sandboxes, len(sandboxes)-1)
    	}
    }

# 5. evictPodLogsDirectories #

驱逐可驱逐的日志，也就是删除，没有pod就删除日志

###     5.1 删除 /var/log/pods目录下那些pods是deleted的目录 ###

// evictPodLogsDirectories evicts all evictable pod logs directories. Pod logs directories
    // are evictable if there are no corresponding pods.
    func (cgc *containerGC) evictPodLogsDirectories(allSourcesReady bool) error {
    	osInterface := cgc.manager.osInterface
    	if allSourcesReady {
    		// Only remove pod logs directories when all sources are ready.
    		dirs, err := osInterface.ReadDir(podLogsRootDirectory)
    		if err != nil {
    			return fmt.Errorf("failed to read podLogsRootDirectory %q: %v", podLogsRootDirectory, err)
    		}
    		for _, dir := range dirs {
    			name := dir.Name()
    			podUID := parsePodUIDFromLogsDirectory(name)
    			if !cgc.podStateProvider.IsPodDeleted(podUID) {
    				continue
    			}
    			err := osInterface.RemoveAll(filepath.Join(podLogsRootDirectory, name))
    			if err != nil {
    				klog.Errorf("Failed to remove pod logs directory %q: %v", name, err)
    			}
    		}
    	}

###     5.2 删除 /var/logs/containers目录下的链接 ###

// Remove dead container log symlinks.
    // TODO(random-liu): Remove this after cluster logging supports CRI container log path.
    logSymlinks, _ := osInterface.Glob(filepath.Join(legacyContainerLogsDir, fmt.Sprintf("*.%s", legacyLogSuffix)))
    for _, logSymlink := range logSymlinks {
    	if _, err := osInterface.Stat(logSymlink); os.IsNotExist(err) {
    		err := osInterface.Remove(logSymlink)
    		if err != nil {
    			klog.Errorf("Failed to remove container log dead symlink %q: %v", logSymlink, err)
    		}
    	}
    }

## 总结： ##

**      evictContainers 驱逐容器**

*  获取所有可驱逐的容器，这种容器不是running并且创建时间大于minAge
 *  如果 pod 是已经删除的，将所有这个pod的容器删除
 *  根据 MaxPerPodContainer 和 MaxContainers 清理容器

evictSandboxes 删除所有可驱逐的 sandbox

*  如果 sandbox 所属的 pod 被删除，那么删除所有 sandbox，否则删除旧的sandbox，留下最新的 sandbox

evictPodLogsDirectories

*    删除日志目录以及链接