云原生边缘计算解决方案之openyurt

短命女 2023-01-15 14:23 224阅读 0赞

### 架构解析(openyurt) ###

组件

*  master组件：
    
     *  yurt-controller-manager
     *  yurt-tunnel-server
     *  yurt app manager
 *  边缘worker组件
    
     *  yurt-tunnel-agent
     *  yurt-hub

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIyNTQzOTkx_size_16_color_FFFFFF_t_70][]

## tunnel原理解析 ##

原生k8s：client(kubectl exec,log等) => kube-apiserver:6443 => kubelet-node:10250

边缘k8s： client(kubectl exec,log等) => kube-apiserver:6443 => yurt-tunnel-server:10263=> yurt-tunnel-agent => kubelet-node:10250

openyurt解决思路：通过iptables截取流量的方式解决 如下是iptables的链路转发的整个流程：

Chain OUTPUT:
         TUNNEL-PORT  tcp  --  0.0.0.0/0            0.0.0.0/0            /* yurttunnel server port */
         
    Chain TUNNEL-PORT:
         TUNNEL-PORT-10255  tcp  --  0.0.0.0/0            0.0.0.0/0            tcp dpt:10255 /* jump to port 10255 */
         TUNNEL-PORT-10250  tcp  --  0.0.0.0/0            0.0.0.0/0            tcp dpt:10250 /* jump to port 10250 */
         
    Chain TUNNEL-PORT-10255 
         RETURN     tcp  --  0.0.0.0/0            127.0.0.1            /* return request to access node directly */ tcp dpt:10250
         RETURN     tcp  --  0.0.0.0/0            172.16.1.199         /* return request to access node directly */ tcp dpt:10250
         DNAT       tcp  --  0.0.0.0/0            0.0.0.0/0            /* dnat to tunnel for access node */ tcp dpt:10250 to:172.16.1.199:10263
    
    Chain TUNNEL-PORT-10250
         RETURN     tcp  --  0.0.0.0/0            127.0.0.1            /* return request to access node directly */ tcp dpt:10250
         RETURN     tcp  --  0.0.0.0/0            172.16.1.199         /* return request to access node directly */ tcp dpt:10250
         DNAT       tcp  --  0.0.0.0/0            0.0.0.0/0            /* dnat to tunnel for access node */ tcp dpt:10250 to:172.16.1.199:10263

## yurt-hub原理解析 ##

作用：用作来自Kubernetes节点守护程序（Kubelet，Kubeproxy，CNI插件等）的出站流量的代理，并配有本地缓存机制，当节点无法通信apiserver的时候 不影响该节点正常提供业务达到边缘自治的功能并且yurt-hub守护程序可以帮助节点在重新启动后恢复状态。

目前版本缺陷 无法代理hostnetwork=true的pod

原因：为了保证安全性 yurthub只透出了127.0.0.1:10261端口。

解决：

1.  使用hostnetwork=true模式
2.  在无法使用hostnetwork=true模式的情况下 需要修改源码加入dummy类型网卡

代码实现：

首先创建TAP网卡，之后使用setupIfce修改网卡ip地址

func Create() (*taptun.Tap, string){
    	tap, err := taptun.NewTap(TapDeivceName)
    	if err != nil {
    		klog.Fatalf("%v:%v",TapDeivceName,err)
    	}
    	self := net.IPv4(169, 254, 2, 1)
    	mask := net.IPv4Mask(255, 255, 255, 255)
    	setupIfce(net.IPNet{IP: self, Mask: mask}, TapDeivceName)
    	return tap,self.String()
    }
    
    
    func setupIfce(ipNet net.IPNet, dev string) {
    	if err := exec.Command("ip", "link", "set", dev, "up").Run(); err != nil {
    		klog.Fatal(err)
    	}
    	if err := exec.Command("ip", "addr", "add", ipNet.String(), "dev", dev).Run(); err != nil {
    		klog.Fatal(err)
    	}
    }

//创建网卡
    	ifce, ipaddr := tap.Create()
    	tapServer := &http.Server{
    		Addr:           fmt.Sprintf("%v:10261", ipaddr),
    		Handler:        proxyHandler,
    		MaxHeaderBytes: 1 << 20,
    	}
    
        tapProxyServer := tapProxyServer{
        	ifce:    ifce,
        	tapHttp: tapServer,
    	}

将虚拟网卡绑定hub的proxy

func NewYurtHubServer(cfg *config.YurtHubConfiguration,
    	certificateMgr interfaces.YurtCertificateManager,
    	proxyHandler http.Handler) Server {
    	hubMux := mux.NewRouter()
    	registerHandlers(hubMux, cfg, certificateMgr)
    	hubServer := &http.Server{
    		Addr:           cfg.YurtHubServerAddr,
    		Handler:        hubMux,
    		MaxHeaderBytes: 1 << 20,
    	}
    
    	proxyServer := &http.Server{
    		Addr:           cfg.YurtHubProxyServerAddr,
    		Handler:        proxyHandler,
    		MaxHeaderBytes: 1 << 20,
    	}
    
    	//创建网卡
    	ifce, ipaddr := tap.Create()
    	tapServer := &http.Server{
    		Addr:           fmt.Sprintf("%v:10261", ipaddr),
    		Handler:        proxyHandler,
    		MaxHeaderBytes: 1 << 20,
    	}
    
        tapProxyServer := tapProxyServer{
        	ifce:    ifce,
        	tapHttp: tapServer,
    	}
    	return &yurtHubServer{
    		hubServer:   hubServer,
    		proxyServer: proxyServer,
    		tapServer:   tapProxyServer,
    	}
    }
    func (s *yurtHubServer) Run() {
    	go func() {
    		err := s.hubServer.ListenAndServe()
    		if err != nil {
    			panic(err)
    		}
    	}()
    
    
    	go func() {
    		err := s.proxyServer.ListenAndServe()
    		if err != nil {
    			panic(err)
    		}
    	}()
    
    	klog.Infof("tap IP:%v ifce:%v ",s.tapServer.tapHttp.Addr,s.tapServer.ifce.String())
    	err := s.tapServer.tapHttp.ListenAndServe()
    	if err != nil {
    		panic(err)
    	}
    	klog.Infof("tap1 IP:%v ifce:%v ",s.tapServer.tapHttp.Addr,s.tapServer.ifce.String())
    }

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIyNTQzOTkx_size_16_color_FFFFFF_t_70]: /images/20221022/3bd6e5a37eed42b18257b2d866206a3b.png