【kubernetes/k8s源码分析】 kubernetes csi node driver registrar 源码分析

电玩女神 2022-02-03 11:57 186阅读 0赞

**Git Repository:** [https://github.com/kubernetes-csi/node-driver-registrar][https_github.com_kubernetes-csi_node-driver-registrar]

**Status:** GA/Stable

<table> 
 <thead> 
  <tr> 
   <th>Latest stable release</th> 
   <th>Branch</th> 
   <th>Min CSI Version</th> 
   <th>Max CSI Version</th> 
   <th>Container Image</th> 
   <th>Min k8s Version</th> 
   <th>Max k8s version</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td><a href="https://github.com/kubernetes-csi/node-driver-registrar/releases/tag/v1.1.0" rel="nofollow">node-driver-registrar v1.1.0</a></td> 
   <td><a href="https://github.com/kubernetes-csi/node-driver-registrar/tree/release-1.1" rel="nofollow">release-1.1</a></td> 
   <td><a href="https://github.com/container-storage-interface/spec/releases/tag/v1.0.0" rel="nofollow">v1.0.0</a></td> 
   <td>-</td> 
   <td>quay.io/k8scsi/csi-node-driver-registrar:v1.1.0</td> 
   <td>v1.13</td> 
   <td>-</td> 
  </tr> 
  <tr> 
   <td><a href="https://github.com/kubernetes-csi/node-driver-registrar/releases/tag/v1.0.2" rel="nofollow">node-driver-registrar v1.0.2</a></td> 
   <td><a href="https://github.com/kubernetes-csi/node-driver-registrar/tree/release-1.0" rel="nofollow">release-1.0</a></td> 
   <td><a href="https://github.com/container-storage-interface/spec/releases/tag/v1.0.0" rel="nofollow">v1.0.0</a></td> 
   <td>-</td> 
   <td>quay.io/k8scsi/csi-node-driver-registrar:v1.0.2</td> 
   <td>v1.13</td> 
   <td>-</td> 
  </tr> 
  <tr> 
   <td><a href="https://github.com/kubernetes-csi/driver-registrar/releases/tag/v0.4.2" rel="nofollow">driver-registrar v0.4.2</a></td> 
   <td><a href="https://github.com/kubernetes-csi/driver-registrar/tree/release-0.4" rel="nofollow">release-0.4</a></td> 
   <td><a href="https://github.com/container-storage-interface/spec/releases/tag/v0.3.0" rel="nofollow">v0.3.0</a></td> 
   <td><a href="https://github.com/container-storage-interface/spec/releases/tag/v0.3.0" rel="nofollow">v0.3.0</a></td> 
   <td>quay.io/k8scsi/driver-registrar:v0.4.2</td> 
   <td>v1.10</td> 
   <td>-</td> 
  </tr> 
 </tbody> 
</table>

# 概述 #

负责请求 Identity Service 来获取插件信息并且注册到 kubelet，初始化的时候通过 csi-sock 的 RPC 获取 driver name 和 node id。主要功能给 node 打上 annotations，device-driver-registar 主要是注册 Node annotation

> metadata:  
>   annotations:  
>     **csi.volume.kubernetes.io/nodeid: '\{"cephfs.csi.ceph.com":"node1","rbd.csi.ceph.com":"node1"\}'**  
>     node.alpha.kubernetes.io/ttl: "0"  
>     volumes.kubernetes.io/controller-managed-attach-detach: "true"

### Registration socket: ###

>  *  Registers the driver with kubelet.
>  *  Created by the `node-driver-registrar`.
>  *  Exposed on a Kubernetes node via hostpath in the Kubelet plugin registry. (typically `/var/lib/kubelet/plugins_registry/<drivername.example.com>-reg.sock`). The hostpath volume must be mounted at `/registration`

### CSI driver socket: ###

>  *  Used by kubelet to interact with the CSI driver.
>  *  Created by the CSI driver.
>  *  Exposed on a Kubernetes node via hostpath somewhere other than the Kubelet. plugin registry (typically `/var/lib/kubelet/plugins/<drivername.example.com>/csi.sock`).
>  *  This is the socket referenced by the `--csi-address` and `--kubelet-registration-path` arguments.

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3pob25nbGluemhhbmc_size_16_color_FFFFFF_t_70][]

>     ① node-driver 调用 CSI-plugin 插件的 GetPluginInfo 获得 driver 名字
> 
>     ② node-driver 创建 GRPC-Server，socket 地址在宿主机 /var/lib/kubelet/plugins\_registry/$\{driver-name\}-reg.sock，容器里为 /registration/$\{driver-name\}-reg.sock
> 
>     ③ kubelet 与 node-driver  建立 GRPC 连接，调用 GetInfo 获得插件信息，包括 plugin 名字，GRPC-Server socket 地址，这个是 Watcher 目录，为 socket 
> 
>     ④ kubelet 向 CSI-Plugin 发送 NodeGetInfo GRPC 请求，获得节点信息，将 CSI-Plugin 加入插件列表
> 
>     ⑤ kubelet 更新 node 资源，主要是更新注解 Annotations，创建 CSINode 资源
> 
>     ⑥ kubelet 调用 NotifyRegistrationStatus 通知 node-driver 是否成功

# 启动命令 #

si-address：CSI driver socket，node-driver-registrar将连到这socket上

kubelet-registration-path：在kubelet节点的CSI driver socket路径

>      node-driver-registrar --v=5
> 
> \--csi-address=/csi/csi.sock
> 
> \--kubelet-registration-path=/var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock

# 1. main函数 #

###     1.1 与CSI driver的socket建立GRPC ###

klog.V(1).Infof("Attempting to open a gRPC connection with: %q", *csiAddress)
    csiConn, err := connection.Connect(*csiAddress)
    if err != nil {
    	klog.Errorf("error connecting to CSI driver: %v", err)
    	os.Exit(1)
    }

###      1.2 GetDriverName函数返回CSI driver ###

发送 GetPluginInfoRequest GRPC 请求 GetPluginInfo，例如 rbd.csi.ceph.com

// GetDriverName returns name of CSI driver.
    func GetDriverName(ctx context.Context, conn *grpc.ClientConn) (string, error) {
    	client := csi.NewIdentityClient(conn)
    
    	req := csi.GetPluginInfoRequest{}
    	rsp, err := client.GetPluginInfo(ctx, &req)
    	if err != nil {
    		return "", err
    	}
    	name := rsp.GetName()
    	if name == "" {
    		return "", fmt.Errorf("driver name is empty")
    	}
    	return name, nil
    }

# 2. nodeRegister函数 #

监听 /registration/rbd.csi.ceph.com-reg.sock

registrar := newRegistrationServer(csiDriverName, *kubeletRegistrationPath, supportedVersions)
    socketPath := fmt.Sprintf("/registration/%s-reg.sock", csiDriverName)
    fi, err := os.Stat(socketPath)
    if err == nil && (fi.Mode()&os.ModeSocket) != 0 {
    	// Remove any socket, stale or not, but fall through for other files
    	if err := os.Remove(socketPath); err != nil {
    		klog.Errorf("failed to remove stale socket %s with error: %+v", socketPath, err)
    		os.Exit(1)
    	}
    }
    if err != nil && !os.IsNotExist(err) {
    	klog.Errorf("failed to stat the socket %s with error: %+v", socketPath, err)
    	os.Exit(1)
    }
    // Default to only user accessible socket, caller can open up later if desired
    oldmask := unix.Umask(0077)
    
    klog.Infof("Starting Registration Server at: %s\n", socketPath)
    lis, err := net.Listen("unix", socketPath)
    if err != nil {
    	klog.Errorf("failed to listen on socket: %s with error: %+v", socketPath, err)
    	os.Exit(1)
    }

###     2.1 Registers kubelet plugin watcher api ###

建立GRPC server服务

grpcServer := grpc.NewServer()
    // Registers kubelet plugin watcher api.
    registerapi.RegisterRegistrationServer(grpcServer, registrar)
    
    // Starts service
    if err := grpcServer.Serve(lis); err != nil {
    	klog.Errorf("Registration Server stopped serving: %v", err)
    	os.Exit(1)
    }

# 3. newRegistrationServer实例化registrationServer #

// NewregistrationServer returns an initialized registrationServer instance
    func newRegistrationServer(driverName string, endpoint string, versions []string) registerapi.RegistrationServer {
    	return &registrationServer{
    		driverName: driverName,
    		endpoint:   endpoint,
    		version:    versions,
    	}
    }

实现了两个接口，在 kubelet 发送 GRPC 请求时使用

// GetInfo is the RPC invoked by plugin watcher
    func (e registrationServer) GetInfo(ctx context.Context, req *registerapi.InfoRequest) (*registerapi.PluginInfo, error) {
    	klog.Infof("Received GetInfo call: %+v", req)
    	return &registerapi.PluginInfo{
    		Type:              registerapi.CSIPlugin,
    		Name:              e.driverName,
    		Endpoint:          e.endpoint,
    		SupportedVersions: e.version,
    	}, nil
    }
    
    func (e registrationServer) NotifyRegistrationStatus(ctx context.Context, status *registerapi.RegistrationStatus) (*registerapi.RegistrationStatusResponse, error) {
    	klog.Infof("Received NotifyRegistrationStatus call: %+v", status)
    	if !status.PluginRegistered {
    		klog.Errorf("Registration process failed with error: %+v, restarting registration container.", status.Error)
    		os.Exit(1)
    	}
    
    	return &registerapi.RegistrationStatusResponse{}, nil
    }

### 总结： ###

node driver registrar 作为 GRPC 客户端，与 CSI socket 建立 GRPC 连接，发送 GetPluginInfoRequest 请求获得 CSI driver 名字，与 CSI plugin 进行交互

在 /registration/%s-reg.sock 建立 GRPC server 监听，这个与 kubelet 进行交互

### Dockerfile ###

> FROM gcr.io/distroless/static:latest  
> LABEL maintainers="Kubernetes Authors"  
> LABEL description="CSI Node driver registrar"
> 
> COPY ./bin/csi-node-driver-registrar csi-node-driver-registrar  
> ENTRYPOINT \["/csi-node-driver-registrar"\]

### 参考： ###

[https://github.com/kubernetes-csi/node-driver-registrar][https_github.com_kubernetes-csi_node-driver-registrar]

[https_github.com_kubernetes-csi_node-driver-registrar]: https://github.com/kubernetes-csi/node-driver-registrar
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3pob25nbGluemhhbmc_size_16_color_FFFFFF_t_70]: /images/20220203/537df5d1908b478988e981e93d2fb1e6.png