【docker 17 源码分析】 Docker Daemon启动

客官°小女子只卖身不卖艺 2022-09-30 15:57 220阅读 0赞

#      基础知识       Daemon通过三种方式监听请求，unix，tcp，fd，默认使用unix domain socket(/var/run/[Docker][].sock)。对于远程请求，可以开启tcp socket(-H tcp:0.0.0.0:2375)，或者固定IP(-H tcp://192.168.0.1:2375)。可以使用多种配置如下：         $ sudo dockerd -H unix:///var/run/[docker][Docker].sock -H tcp://192.168.59.106 -H tcp://10.10.10.2       docker为docker client 和 docker daemon，client端发送命令，daemon端负责执行client发送过来的命令（获取和存储镜像、管理容器等）。两者可以通过TCP，HTTP和UNIX SOCKET来进行通信  创建 Docker 运行环境  httpserver 服务于Docker Client，接收并处理相应请求 #

# **一. Docker Daemon 启动源码分析** #

入口代码为cmd/dockerd/docker.go。

func main() {
           if reexec.Init() {
                  return }
     _, stdout, stderr := term.StdStreams()
           logrus.SetOutput(stderr)
    
           cmd := newDaemonCommand()
           cmd.SetOutput(stdout)
           ......
    }

1.1 reexec 是 docker 自己实现的一个 package，https://groups.google.com/forum/\#!topic/docker-dev/ePLDji\_qBvE 给的解释可能是个过期的代码

1.2 logrus 第三方的 log 模块，initLogging 就是将输出定向到 stderr。

1.3 newDaemonCommand

func newDaemonCommand() *cobra.Command {
           opts := daemonOptions{
                  daemonConfig: config.New(),
                  common:       cliflags.NewCommonOptions(),
           }
    
           cmd := &cobra.Command{
                  Use:           "dockerd [OPTIONS]",
                  Short:         "A self-sufficient runtime for containers.",
                  SilenceUsage:  true,
                  SilenceErrors: true,
                  Args:          cli.NoArgs,
                  RunE: func(cmd *cobra.Command, args []string) error {
                         opts.flags = cmd.Flags()
                         return runDaemon(opts)
                  },
           }
           ......
    
           return cmd
    }

1.3.1 dockerd 启动 deamon 实际运行的是 runDeamon 函数，可以使用 docker -D 启动，进入 runDeamon()后，

初始化 daemonCli，执行 start 方法，runDeamon 中除了 api service 还会 d, err := daemon.NewDaemon(cli.Config, registryService,

containerdRemote)，注册 registry，和 contanerd 等。

1.3.2defaultDaemonConfigFile默认配置文件"/etc/docker/daemon.json"

func runDaemon(opts daemonOptions) error { daemonCli := NewDaemonCli() ...... err = daemonCli.start(opts)
           notifyShutdown(err)
           return err
    }

type DaemonCli struct {
           *config.Config
           configFile *string
           flags      *pflag.FlagSet
    
           api             *apiserver.Server
           d               *daemon.Daemon
           authzMiddleware *authorization.Middleware
    }

1.4  daemonCli.start cli.Pidfile 为 /var/run/docker.pid，创建文件并写入 pid

if cli.Pidfile != "" {
           pf, err := pidfile.New(cli.Pidfile)
           if err != nil {
                  return fmt.Errorf("Error starting daemon: %v", err)
           }
           defer func() {
                  if err := pf.Remove(); err != nil {
                         logrus.Error(err)
                  }
           }()
    }

1.5  daemonCli.start 启动一个server

*  len(cli.Config.Hosts)，如果没有-H参数，长度为0，默认使用的是 /var/run/docker.sock。make一个长度为1的切片
 *  api server.New(serverConfig) 生成一个api server 对象，包含多个http服务

func (cli *DaemonCli) start(opts daemonOptions) (err error) {
           ......
    
           serverConfig := &apiserver.Config{
                  Logging:     true, SocketGroup: cli.Config.SocketGroup, Version:     dockerversion.Version, EnableCors:  cli.Config.EnableCors, CorsHeaders: cli.Config.CorsHeaders, }
    
    
           if len(cli.Config.Hosts) == 0 {
                  cli.Config.Hosts = make([]string, 1)
           }
    
           api := apiserver.New(serverConfig)
           cli.api = api
    
    }

1.6  daemonCli.start 初始化监听地址，因为启动没有指定任何 host, ParseHost 设置为默认值，unix:///var/run/docker.sock，proto 为 unix，addr 为 /var/run/docker.sock，listeners.Init 创建 socket server，accept 客户端发来的请求。其他的 TCP 协议等一样的分析。

func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
           ......
    
           for i := 0; i < len(cli.Config.Hosts); i++ {
                  var err error
                  if cli.Config.Hosts[i], err = dopts.ParseHost(cli.Config.TLS, cli.Config.Hosts[i]); err != nil {
                         return fmt.Errorf("error parsing -H %s : %v", cli.Config.Hosts[i], err)
                  }
    
                  protoAddr := cli.Config.Hosts[i]
                  protoAddrParts := strings.SplitN(protoAddr, "://", 2)
                  if len(protoAddrParts) != 2 {
                         return fmt.Errorf("bad format %s, expected PROTO://ADDR", protoAddr)
                  }
    
                  proto := protoAddrParts[0]
                  addr := protoAddrParts[1]
                  ......
                  ls, err := listeners.Init(proto, addr, serverConfig.SocketGroup, serverConfig.TLSConfig)
                  if err != nil {
                         return err
                  }
                  ls = wrapListeners(proto, ls)
                  ......
                  hosts = append(hosts, protoAddrParts[1])
                  cli.api.Accept(addr, ls...)
           }

// Accept sets a listener the server accepts connections into.
    func (s *Server) Accept(addr string, listeners ...net.Listener) {
           for _, listener := range listeners {
                  httpServer := &HTTPServer{
                         srv: &http.Server{
                                Addr: addr,
                         },
                         l: listener,
                  }
                  s.servers = append(s.servers, httpServer)
           }
    }

1.7  daemonCli.start 生成一个 DefaultService 结构体：

registryService := registry.NewService(cli.Config.ServiceOptions)

// DefaultService is a registry service. It tracks configuration data such as a list
    // of mirrors.
    type DefaultService struct {
           config *serviceConfig
           mu     sync.Mutex
    }
    
    // NewService returns a new instance of DefaultService ready to be
    // installed into an engine.
    func NewService(options ServiceOptions) *DefaultService {
           return &DefaultService{
                  config: newServiceConfig(options),
           }
    }

1.8  daemonCli.start 生成一个 libcontainerd remote 实例：创建目录 /var/run/docker/libcontainerd，目录下 rpc addr 为 docker-containerd.sock

// New creates a fresh instance of libcontainerd remote. func New(stateDir string, options ...RemoteOption) (_ Remote, err error) {
           r := &remote{
                  stateDir:    stateDir, daemonPid:   -1, eventTsPath: filepath.Join(stateDir, eventTimestampFilename), }
           for _, option := range options {
                  if err := option.Apply(r); err != nil {
                         return nil, err
                  }
           }
    
           if err := sysinfo.MkdirAll(stateDir, 0700); err != nil {
    
           if r.rpcAddr == "" {
                  r.rpcAddr = filepath.Join(stateDir, containerdSockFilename)
           }
    
           if r.startDaemon {
                  if err := r.runContainerdDaemon(); err != nil {
       
    
           return r, nil
    }

1.8.1 runContainerDaemon，pid 文件为 /var/run/docker/libcontainerd/docker-containerd.pid，打开文件查看先前进程是否存在，否则启动一个新的实例，命令大致为：docker-containerd -l unix:///var/run/docker/libcontainerd.sock --metrics-interval=0 --start-timeout 2m --state-dir /var/run/docker/libcontainerd/containerd --shim docker-containerd-shim --runtime docker-runc

写入 pid 至文件。setOOMScore 写入 /proc/$\{pid\}/oom\_score\_adj

func (r *remote) runContainerdDaemon() error {
           pidFilename := filepath.Join(r.stateDir, containerdPidFilename)
           f, err := os.OpenFile(pidFilename, os.O_RDWR|os.O_CREATE, 0600)
    
           if n > 0 {
                  pid, err := strconv.ParseUint(string(b[:n]), 10, 64)
           
                  if system.IsProcessAlive(int(pid)) {
                         logrus.Infof("libcontainerd: previous instance of containerd still alive (%d)", pid)
                         r.daemonPid = int(pid)
                         return nil
                  }
           }
    
           // Start a new instance
           args := []string{
                  "-l", fmt.Sprintf("unix://%s", r.rpcAddr),
                  "--metrics-interval=0",
                  "--start-timeout", "2m",
                  "--state-dir", filepath.Join(r.stateDir, containerdStateDir),
           }
           ......
    
           cmd := exec.Command(containerdBinary, args...)
           if err := cmd.Start(); err != nil {
      
           r.daemonPid = cmd.Process.Pid
           return nil
    }

1.8.1.1 执行可以查看 ps axf | grep docker，启动进程 docker-containerd

![Center][]

1.8.2 启动完 docker-containerd 进程，使用 rpc 连接到这个进程，

func New(stateDir string, options ...RemoteOption) (_ Remote, err error) {
           dialOpts := []grpc.DialOption{
                  grpc.WithInsecure(),
                  grpc.WithBackoffMaxDelay(2 * time.Second),
                  grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
                         return net.DialTimeout("unix", addr, timeout)
                  }),
           }
           conn, err := grpc.Dial(r.rpcAddr, dialOpts...)
    
           r.rpcConn = conn
           r.apiClient = containerd.NewAPIClient(conn)
    
           // Get the timestamp to restore from
           t := r.getLastEventTimestamp()
           tsp, err := ptypes.TimestampProto(t)
         
           r.restoreFromTimestamp = tsp
    
           go r.handleConnectionChange()
    
           if err := r.startEventsMonitor(); err != nil {
    
           return r, nil
    }

handleConnectionChange 启动 for 循环，500 毫秒一次进行检查。

1.9 daemon.NewDaemon 放入第二章节详解

d, err := daemon.NewDaemon(cli.Config, registryService, containerdRemote, pluginStore)
    if err != nil {
           return fmt.Errorf("Error starting daemon: %v", err)
    }

1.10 initRouter(routerOptions)，路由有 checkoutpoint，container，image，volume 等。POST DELETE GET 操作

func initRouter(opts routerOptions) {
           decoder := runconfig.ContainerDecoder{}
    
           routers := []router.Router{
                  // we need to add the checkpoint router before the container router or the DELETE gets masked
                  checkpointrouter.NewRouter(opts.daemon, decoder),
                  container.NewRouter(opts.daemon, decoder),
                  image.NewRouter(opts.daemon, decoder),
                  systemrouter.NewRouter(opts.daemon, opts.cluster, opts.buildCache),
                  volume.NewRouter(opts.daemon),
                  build.NewRouter(opts.buildBackend, opts.daemon),
                  sessionrouter.NewRouter(opts.sessionManager),
                  swarmrouter.NewRouter(opts.cluster),
                  pluginrouter.NewRouter(opts.daemon.PluginManager()),
                  distributionrouter.NewRouter(opts.daemon),
           }
    
           if opts.daemon.NetworkControllerEnabled() {
                  routers = append(routers, network.NewRouter(opts.daemon, opts.cluster))
           }
    
           if opts.daemon.HasExperimental() {
                  for _, r := range routers {
                         for _, route := range r.Routes() {
                                if experimental, ok := route.(router.ExperimentalRoute); ok {
                                       experimental.Enable()
                                }
                         }
                  }
           }
    
           opts.api.InitRouter(routers...)
    }

1.10.1 initRouter 主要的是 InitRouter 函数实用的是如下：

func (s *Server) InitRouter(routers ...router.Router) {
           s.routers = append(s.routers, routers...)
    
           m := s.createMux()
           s.routerSwapper = &routerSwapper{
                  router: m,
           }
    }
    
    // createMux initializes the main router the server uses.
    func (s *Server) createMux() *mux.Router {
           m := mux.NewRouter()
    
           for _, apiRouter := range s.routers {
                  for _, r := range apiRouter.Routes() {
                         f := s.makeHTTPHandler(r.Handler())
    
                         m.Path(versionMatcher + r.Path()).Methods(r.Method()).Handler(f)
                         m.Path(r.Path()).Methods(r.Method()).Handler(f)
                  }
           }
    
           debugRouter := debug.NewRouter()
           s.routers = append(s.routers, debugRouter)
           for _, r := range debugRouter.Routes() {
                  f := s.makeHTTPHandler(r.Handler())
                  m.Path("/debug" + r.Path()).Handler(f)
           }
    
           err := errors.NewRequestNotFoundError(fmt.Errorf("page not found"))
           notFoundHandler := httputils.MakeErrorHandler(err)
           m.HandleFunc(versionMatcher+"/{path:.*}", notFoundHandler)
           m.NotFoundHandler = notFoundHandler
    
           return m
    }

1.11 cli.api.Wait 等待请求在 /var/run/docker.sock

func (s *Server) Wait(waitChan chan error) {
           if err := s.serveAPI(); err != nil {
                  logrus.Errorf("ServeAPI error: %v", err)
                  waitChan <- err
                  return }
           waitChan <- nil
    }

***docker 启动总结：***

大部分是参数的检查

启动 API server 监听请求

建立 API 路由

启动进程 libcontainerd，

镜像之间关系等

# 二. NewDaemon 函数源码分析 #

位置：daemon/daemon.go

setDefaultMtu(config)

2.1 设置 MTU。若 config 中有值则使用 config 中的，否则设置为默认的 1500

func verifyDaemonSettings(conf *config.Config) error {
           // Check for mutually incompatible config options
           if conf.BridgeConfig.Iface != "" && conf.BridgeConfig.IP != "" {
                  return fmt.Errorf("You specified -b & --bip, mutually exclusive options. Please specify only one")
           }
           if !conf.BridgeConfig.EnableIPTables && !conf.BridgeConfig.InterContainerCommunication {
                  return fmt.Errorf("You specified --iptables=false with --icc=false. ICC=false uses iptables to function. Please set --icc or --iptables to true")
           }
           if !conf.BridgeConfig.EnableIPTables && conf.BridgeConfig.EnableIPMasq {
                  conf.BridgeConfig.EnableIPMasq = false
           }
           if err := VerifyCgroupDriver(conf); err != nil {
                  return err
           }
           if conf.CgroupParent != "" && UsingSystemd(conf) {
                  if len(conf.CgroupParent) <= 6 || !strings.HasSuffix(conf.CgroupParent, ".slice") {
                         return fmt.Errorf("cgroup-parent for systemd cgroup should be a valid slice named as \"xxx.slice\"")
                  }
           }
    
           if conf.DefaultRuntime == "" {
                  conf.DefaultRuntime = config.StockRuntimeName }
           if conf.Runtimes == nil {
                  conf.Runtimes = make(map[string]types.Runtime)
           }
           conf.Runtimes[config.StockRuntimeName] = types.Runtime{Path: DefaultRuntimeBinary}
    
           return nil
    }

2.2 验证 daemon 的配置

*  不能同时指定网桥和网桥的 IP。如果指定网桥，应使用网桥的当前IP地址，不能再设置 IP 地址。
 *  不能同时禁用 iptable 和 icc(容器间通信)。如果禁用 icc，docker 会在宿主机的 iptables 的 FORWARD chain 中添加一条 docker 容器间流量均 DROP 的规则，此时设置 EnableIPTables 为f alse，冲突!!!

func isBridgeNetworkDisabled(conf *config.Config) bool {
           return conf.BridgeConfig.Iface == config.DisableNetworkBridge
    }

2.3 设置是否启用网桥。disableNetworkBridge 为 none，Iface 为空，所以 DisableBridge 为false

if !platformSupported {
           return nil, errSystemNotSupported
    }
    
    if err := checkSystem(); err != nil {
           return nil, err
    }

2.4  检查系统支持和用户权限。需要 root 权限，linux 中 uid = 0 为 root 用户，检查内核版本

uidMaps, gidMaps, err := setupRemappedRoot(config)
    
    rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)

2.5 docker 支持 user namespace，主要隔离安全相关 identifiers 和 attributes，包括用户 ID、用户组 ID、root 目录、key 以及特殊权限。普通用户的进程通过 clone() 创建的新进程在新 user namespace 中拥有不同的用户和用户组，这意味着一个进程在容器外属于一个没有特权的普通用户，但是创建的容器进程却属于拥有所有权限的超级用户。

// setupDaemonProcess sets various settings for the daemon's process
    func setupDaemonProcess(config *config.Config) error {
           // setup the daemons oom_score_adj
           return setupOOMScoreAdj(config.OOMScoreAdjust)
    }

2.6 设置OOM killer值。oom killer设置/proc/self/oom\_score\_adj，值的范围为–17~15。正值易被OOM Killer选定。如果设置为–17，表示禁止被kill掉

tmp, err := prepareTempDir(config.Root, rootIDs)
    
    realTmp, err := getRealPath(tmp)
    
    os.Setenv("TMPDIR", realTmp)

2.7  prepareTempDir 设置 tmp dir。如有环境变量 DOCKER\_TMPDIR，则使用这个，否则创建 /var/lib/docker/tmp，将旧的变为 /var/lib/docker/tmp-old。getRealPath 然后创建一个指向tmp文件的符号链接realTmp，并把 realTmp 赋值给环境变量TMPDIR

// configureMaxThreads sets the Go runtime max threads threshold
    // which is 90% of the kernel setting from /proc/sys/kernel/threads-max
    func configureMaxThreads(config *config.Config) error {
           mt, err := ioutil.ReadFile("/proc/sys/kernel/threads-max")
           if err != nil {
                  return err
           }
           mtint, err := strconv.Atoi(strings.TrimSpace(string(mt)))
           if err != nil {
                  return err
           }
           maxThreads := (mtint / 100) * 90
           debug.SetMaxThreads(maxThreads)
           logrus.Debugf("Golang's threads limit set to %d", maxThreads)
           return nil
    }

2.8  设置最大线程数。从 /proc/sys/kernel/threads-max 文件读取值，然后乘以0.9

daemonRepo := filepath.Join(config.Root, "containers")
    if err := idtools.MkdirAllAs(daemonRepo, 0700, rootUID, rootGID); err != nil && !os.IsExist(err) {
           return nil, err
    }

2.9  创建容器目录，位于 docker daemon 目录下的 containers。daemon 创建容器会把容器的元数据信息放此

driverName := os.Getenv("DOCKER_DRIVER")
    if driverName == "" {
           driverName = config.GraphDriver
    }

d.stores[runtime.GOOS] = daemonStore{graphDriver: driverName}d.RegistryService = registryServiced.PluginStore = pluginStore// Plugin system initialization should happen before restore. Do not change order.d.pluginManager, err = plugin.NewManager(plugin.ManagerConfig{       Root:               filepath.Join(config.Root, "plugins"),       ExecRoot:           getPluginExecRoot(config.Root),       Store:              d.PluginStore,       Executor:           containerdRemote,       RegistryService:    registryService,       LiveRestoreEnabled: config.LiveRestoreEnabled,       LogPluginEvent:     d.LogPluginEvent, // todo: make private AuthzMiddleware:    config.AuthzMiddleware,})if err != nil {       return nil, errors.Wrap(err, "couldn't create plugin manager")}d.layerStore, err = layer.NewStoreFromOptions(layer.StoreOptions{       StorePath:                 config.Root,       MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"),       GraphDriver:               driverName,       GraphDriverOptions:        config.GraphOptions,       UIDMaps:                   uidMaps,       GIDMaps:                   gidMaps,       PluginGetter:              d.PluginStore,       ExperimentalEnabled:       config.Experimental,})if err != nil {       return nil, err}

2.10 填充 daemon 结构体

// Plugin system initialization should happen before restore. Do not change order.
    d.pluginManager, err = plugin.NewManager(plugin.ManagerConfig{
           Root:               filepath.Join(config.Root, "plugins"),
           ExecRoot:           getPluginExecRoot(config.Root),
           Store:              d.PluginStore,
           Executor:           containerdRemote,
           RegistryService:    registryService,
           LiveRestoreEnabled: config.LiveRestoreEnabled,
           LogPluginEvent:     d.LogPluginEvent, // todo: make private AuthzMiddleware:    config.AuthzMiddleware,
    })
    if err != nil {
           return nil, errors.Wrap(err, "couldn't create plugin manager")
    }

2.11 结构体如下 2.11.1 所示，Root 路径为 /var/lib/docker/plugins，ExecRoot 为 /run/docker/plugins，NewManager 生成一个plugin manager，如下 2.11.2 所示

2.11.1 结构体 ManagerConfig如下所示：

// ManagerConfig defines configuration needed to start new manager.
    type ManagerConfig struct {
           Store              *Store // remove
           Executor           libcontainerd.Remote
           RegistryService    registry.Service
           LiveRestoreEnabled bool // TODO: remove LogPluginEvent     eventLogger
           Root               string
           ExecRoot           string
           AuthzMiddleware    *authorization.Middleware
    }

2.11.2 NewManager 函数主要是生成目录 /var/lib/docker/plugins，/run/docker/plugins，/var/lib/docker/plugins/tmp，/var/lib/docker/plugins/storage/blobs/tmp ，并初始化结构体 Manager，如下所示：

func NewManager(config ManagerConfig) (*Manager, error) {
           if config.RegistryService != nil {
                  config.RegistryService = pluginRegistryService{config.RegistryService}
           }
           manager := &Manager{
                  config: config,
           }
           if err := os.MkdirAll(manager.config.Root, 0700); err != nil {
           
           if err := os.MkdirAll(manager.config.ExecRoot, 0700); err != nil {
           
           if err := os.MkdirAll(manager.tmpDir(), 0700); err != nil {
             
           var err error
           manager.containerdClient, err = config.Executor.Client(manager) // todo: move to another struct 
           manager.blobStore, err = newBasicBlobStore(filepath.Join(manager.config.Root, "storage/blobs"))
          
           manager.cMap = make(map[*v2.Plugin]*controller)
    
           manager.publisher = pubsub.NewPublisher(0, 0)
           return manager, nil
    }

2.12 如下所示，StoreOptions 结构体如下 2.12.1 所示，NewStoreFromOptions 函数 2.12.2 所示：

for platform, ds := range d.stores {
           ls, err := layer.NewStoreFromOptions(layer.StoreOptions{
                  StorePath:                 config.Root,
                  MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"),
                  GraphDriver:               ds.graphDriver,
                  GraphDriverOptions:        config.GraphOptions,
                  IDMappings:                idMappings,
                  PluginGetter:              d.PluginStore,
                  ExperimentalEnabled:       config.Experimental,
                  Platform:                  platform,
           })
           
           ds.graphDriver = ls.DriverName() // As layerstore may set the driver
           ds.layerStore = ls
           d.stores[platform] = ds
           graphDrivers = append(graphDrivers, ls.DriverName())
    }

2.12.1 StoreOptions 结构体如下所示：

// StoreOptions are the options used to create a new Store instance
    type StoreOptions struct {
           StorePath                 string
           MetadataStorePathTemplate string
           GraphDriver               string
           GraphDriverOptions        []string
           IDMappings                *idtools.IDMappings
           PluginGetter              plugingetter.PluginGetter
           ExperimentalEnabled       bool
           Platform                  string
    }

2.12.2 NewStoreFromOptions 函数中的 driver 为 aufs，创建目录 /var/lib/docker/images/aufs/layerdb，如下所示：

// NewStoreFromOptions creates a new Store instance
    func NewStoreFromOptions(options StoreOptions) (Store, error) {
           driver, err := graphdriver.New(options.GraphDriver, options.PluginGetter, graphdriver.Options{
                  Root:                options.StorePath,
                  DriverOptions:       options.GraphDriverOptions,
                  UIDMaps:             options.IDMappings.UIDs(),
                  GIDMaps:             options.IDMappings.GIDs(),
                  ExperimentalEnabled: options.ExperimentalEnabled,
           })
           if err != nil {
                  return nil, fmt.Errorf("error initializing graphdriver: %v", err)
           }
           logrus.Debugf("Using graph driver %s", driver)
    
           fms, err := NewFSMetadataStore(fmt.Sprintf(options.MetadataStorePathTemplate, driver))
           if err != nil {
                  return nil, err
           }
    
           return NewStoreFromGraphDriver(fms, driver, options.Platform)
    }

d.downloadManager = xfer.NewLayerDownloadManager(d.layerStore, *config.MaxConcurrentDownloads)
    
    d.uploadManager = xfer.NewLayerUploadManager(*config.MaxConcurrentUploads)

2.13 创建上传下载管理器。设置了上传/下载的最大并发数

for platform, ds := range d.stores {
           imageRoot := filepath.Join(config.Root, "image", ds.graphDriver)
           ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb"))
      
           var is image.Store
           is, err = image.NewImageStore(ifs, platform, ds.layerStore)
    
           ds.imageRoot = imageRoot
           ds.imageStore = is
           d.stores[platform] = ds
    }

2.14 创建镜像存储目录并restore镜像。存储镜像的根目录为 /var/lib/docker/image/aufs/imagedb，然后还会在 imagedb 目录下新建两个目录，content/sha256和metadata/sha256，存储镜像内容和镜像元数据

func (daemon *Daemon) configureVolumes(rootUID, rootGID int) (*store.VolumeStore, error) {
           volumesDriver, err := local.New(daemon.configStore.Root, rootUID, rootGID)
           if err != nil {
                  return nil, err
           }
    
           volumedrivers.RegisterPluginGetter(daemon.PluginStore)
    
           if !volumedrivers.Register(volumesDriver, volumesDriver.Name()) {
                  return nil, errors.New("local volume driver could not be registered")
           }
           return store.New(daemon.configStore.Root)
    }

2.14 配置 volumes。 设置数据卷 driver，数据卷是容器之间进行数据共享的一种手段；数据卷可以是一个本机命令（通过-v 标识挂载在到容器的某个目录下） 数据卷也可以作为数据卷容器，通过--volumes-from挂载到某个容器中；

local.New 创建目录 /var/lib/docker/volumes，容器的 volume 是可从宿主机上挂载到容器内的特定目录。一个 volume 可以被多个容器挂载共享数据。volumes 以插件形式，这里的 local.New 是创建了默认的 volume driver 的实现

trustKey, err := api.LoadOrCreateTrustKey(config.TrustKeyPath)
    if err != nil {
           return nil, err
    }

2.15 创建 key，路径在/etc/docker/key.json

trustDir := filepath.Join(config.Root, "trust")
    
    if err := system.MkdirAll(trustDir, 0700, ""); err != nil {
           return nil, err
    }

2.17 创建 trust 目录，路径在/var/lib/docker/trust

// New returns new *Events instance
    func New() *Events {
           return &Events{
                  events: make([]eventtypes.Message, 0, eventsLimit),
                  pub:    pubsub.NewPublisher(100*time.Millisecond, bufferSize),
           }
    }

2.18  创建event实例，产生的event通过订阅的方式获取

// NewReferenceStore creates a new reference store, tied to a file path where
    // the set of references are serialized in JSON format.
    func NewReferenceStore(jsonPath string) (Store, error) {
           abspath, err := filepath.Abs(jsonPath)
           if err != nil {
                  return nil, err
           }
    
           store := &store{
                  jsonPath:            abspath,
                  Repositories:        make(map[string]repository),
                  referencesByIDCache: make(map[digest.Digest]map[string]reference.Named),
           }
           // Load the json file if it exists, otherwise create it.
           if err := store.reload(); os.IsNotExist(err) {
                  if err := store.save(); err != nil {
                         return nil, err
                  }
           } else if err != nil {
                  return nil, err
           }
           return store, nil
    }

type store struct {
           mu sync.RWMutex
           // jsonPath is the path to the file where the serialized tag data is
           // stored.
           jsonPath string
           // Repositories is a map of repositories, indexed by name.
           Repositories map[string]repository
           // referencesByIDCache is a cache of references indexed by ID, to speed
           // up References.
           referencesByIDCache map[digest.Digest]map[string]reference.Named
    }

2.19  创建TagStore。用于管理存储镜像的仓库列表，Repositories记录了镜像仓库的映射，  
referencesByIDCache记录了镜像ID和镜像全名的映射

// initDiscovery initializes the discovery watcher for this daemon.
    func (daemon *Daemon) initDiscovery(conf *config.Config) error {
           advertise, err := config.ParseClusterAdvertiseSettings(conf.ClusterStore, conf.ClusterAdvertise)
           if err != nil {
                  if err == discovery.ErrDiscoveryDisabled {
                         return nil
                  }
                  return err
           }
    
           conf.ClusterAdvertise = advertise
           discoveryWatcher, err := discovery.Init(conf.ClusterStore, conf.ClusterAdvertise, conf.ClusterOpts)
           if err != nil {
                  return fmt.Errorf("discovery initialization failed (%v)", err)
           }
    
           daemon.discoveryWatcher = discoveryWatcher
           return nil
    }

2.20  服务发现。docker自带的服务发现，docker集群中使用，维护心跳、服务注册等

sysInfo := sysinfo.New(false)
    // Check if Devices cgroup is mounted, it is hard requirement for container security,
    // on Linux.
    if runtime.GOOS == "linux" && !sysInfo.CgroupDevicesEnabled {
           return nil, errors.New("Devices cgroup isn't mounted")
    }

2.21  获取系统信息。包括cgroup、网络配置(网桥/iptables)和linux安全(AppArmor/Seccomp)等。如下所示：

type SysInfo struct {
           // Whether the kernel supports AppArmor or not
           AppArmor bool
           // Whether the kernel supports Seccomp or not
           Seccomp bool
    
           cgroupMemInfo
           cgroupCPUInfo
           cgroupBlkioInfo
           cgroupCpusetInfo
           cgroupPids
    
           // Whether IPv4 forwarding is supported or not, if this was disabled, networking will not work
           IPv4ForwardingDisabled bool
    
           // Whether bridge-nf-call-iptables is supported or not
           BridgeNFCallIPTablesDisabled bool
    
           // Whether bridge-nf-call-ip6tables is supported or not
           BridgeNFCallIP6TablesDisabled bool
    
           // Whether the cgroup has the mountpoint of "devices" or not
           CgroupDevicesEnabled bool
    }

d.ID = trustKey.PublicKey().KeyID()
    d.repository = daemonRepo
    d.containers = container.NewMemoryStore()
    d.execCommands = exec.NewStore()
    d.referenceStore = referenceStore
    d.distributionMetadataStore = distributionMetadataStore
    d.trustKey = trustKey
    d.idIndex = truncindex.NewTruncIndex([]string{})
    d.statsCollector = d.newStatsCollector(1 * time.Second)
    d.defaultLogConfig = containertypes.LogConfig{
           Type:   config.LogConfig.Type,
           Config: config.LogConfig.Config,
    }
    d.EventsService = eventsService
    d.volumes = volStore
    d.root = config.Root
    d.uidMaps = uidMaps
    d.gidMaps = gidMaps
    d.seccompEnabled = sysInfo.Seccomp
    d.apparmorEnabled = sysInfo.AppArmor
    
    d.nameIndex = registrar.NewRegistrar()
    d.linkIndex = newLinkIndex()
    d.containerdRemote = containerdRemote

2.22  填充daemon结构体。

*  container.NewMemoryStore() 创建内存store，把container的名字和container的映射存储在map中
 *  exec.NewStore() 创建commands的store，把commands和config的映射，用于执行容器里面的一些任务
 *  truncindex.NewTruncIndex(\[\]string\{\}) 对contaniner或者image进行操作，截取id的一部分即可进行操作
 *  d.newStatsCollector(1 \* time.Second) 1s收集一次运行的容器的状态，包括网络和cgroup状态。

CPU： /proc/stat

func (daemon *Daemon) restore() error {
           containers := make(map[string]*container.Container)
    
           logrus.Info("Loading containers: start.")
    
           dir, err := ioutil.ReadDir(daemon.repository)
           if err != nil {
                  return err
           }
    
           for _, v := range dir {
                  id := v.Name()
                  container, err := daemon.load(id)

2.24  restore 函数。从 /var/lib/docker/containers 读取所有目录，下面的文件夹的名字就是容器的id，存入 containers map\[string\]\*container.Container 信息并注册如 daemon 结构体中。包括 start remove 操作 containers

注册容器。注册信息都在内存中，启动daemon时需要重新进行注册

***NewDaemon 总结：***

验证参数，image store，containers  等创建，读取 containers目录并重启等，填充 daemon 结构体。

[Docker]: http://lib.csdn.net/base/docker
[Center]: /images/20220705/86a6f5885c7d420ebcfa412e407fe89c.png