未完待续

港控/mmm° 2021-12-08 19:25 554阅读 0赞

1 机器规划
























机器规划 角色 部署服务
wyl01 master

kube-apiserver

kube-controller-manager
kube-scheduler
etcd
wyl02 node kube-apiserver
kube-controller-manager
kube-scheduler
etcd
wyl03 node kubelet
kube-proxy
docker
flannel
etcd

2 安装docker并启动

阿里docker镜像，下载yum源，然后进行安装，三台都要安装。

  1. [root@wyl01 opt]# yum  install docker-ce -y
  2. [root@wyl01 opt]# systemctl start docker.service

3 创建TLS证书
































组件 使用的证书
etcd ca.pem， server.pem， server-key.pem
flannel ca.pem， server.pem， server-key.pem
kube-apiserver ca.pem， server.pem， server-key.pem
kubelet ca.pem， ca-key.pem
kube-proxy ca.pem， kube-proxy.pem， kube-proxy-key.pem
kubectl ca.pem， admin.pem， admin-key.pem 
  1. mkdir /opt/kubernetes/{bin,cfg,ssl}  -p
  2. cd /opt/kubernetes/ssl
  3. # 下载证书工具
  4. wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 
  5. wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 
  6. wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 
  8. #改变指令权限
  9. chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 cfssl-certinfo_linux-amd64 
  11. # 放到bin目录下
  12. mv cfssl_linux-amd64 /usr/local/bin/cfssl 
  13. mv cfssljson_linux-amd64 /usr/local/bin/cfssljson 
  14. mv cfssl-certinfo_linux-amd64 /usr/bin/cfssl-certinfo
  16. [root@wyl01 ssl]# ls
  17. certificate.sh  cfssl-certinfo_linux-amd64  cfssljson_linux-amd64  cfssl_linux-amd64

4 生成证书

在master节点上操作

  1. [root@wyl01 ssl]# cat certificate.sh 
  2. cat > ca-config.json <<EOF
  3. {
  4.   "signing": {
  5.     "default": {
  6.       "expiry": "87600h"
  7.     },
  8.     "profiles": {
  9.       "kubernetes": {
  10.          "expiry": "87600h",
  11.          "usages": [
  12.             "signing",
  13.             "key encipherment",
  14.             "server auth",
  15.             "client auth"
  16.         ]
  17.       }
  18.     }
  19.   }
  20. }
  21. EOF
  23. cat > ca-csr.json <<EOF
  24. {
  25.     "CN": "kubernetes",
  26.     "key": {
  27.         "algo": "rsa",
  28.         "size": 2048
  29.     },
  30.     "names": [
  31.         {
  32.             "C": "CN",
  33.             "L": "Beijing",
  34.             "ST": "Beijing",
  35.               "O": "k8s",
  36.             "OU": "System"
  37.         }
  38.     ]
  39. }
  40. EOF
  42. cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
  44. #-----------------------
  46. cat > server-csr.json <<EOF
  47. {
  48.     "CN": "kubernetes",
  49.     "hosts": [
  50.       "127.0.0.1",
  51.       "192.168.52.128",
  52.       "192.168.52.129",
  53.       "192.168.52.130",
  54.       "10.10.10.1",
  55.       "kubernetes",
  56.       "kubernetes.default",
  57.       "kubernetes.default.svc",
  58.       "kubernetes.default.svc.cluster",
  59.       "kubernetes.default.svc.cluster.local"
  60.     ],
  61.     "key": {
  62.         "algo": "rsa",
  63.         "size": 2048
  64.     },
  65.     "names": [
  66.         {
  67.             "C": "CN",
  68.             "L": "BeiJing",
  69.             "ST": "BeiJing",
  70.             "O": "k8s",
  71.             "OU": "System"
  72.         }
  73.     ]
  74. }
  75. EOF
  77. cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server
  79. #-----------------------
  81. cat > admin-csr.json <<EOF
  82. {
  83.   "CN": "admin",
  84.   "hosts": [],
  85.   "key": {
  86.     "algo": "rsa",
  87.     "size": 2048
  88.   },
  89.   "names": [
  90.     {
  91.       "C": "CN",
  92.       "L": "BeiJing",
  93.       "ST": "BeiJing",
  94.       "O": "system:masters",
  95.       "OU": "System"
  96.     }
  97.   ]
  98. }
  99. EOF
  101. cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin
  103. #-----------------------
  105. cat > kube-proxy-csr.json <<EOF
  106. {
  107.   "CN": "system:kube-proxy",
  108.   "hosts": [],
  109.   "key": {
  110.     "algo": "rsa",
  111.     "size": 2048
  112.   },
  113.   "names": [
  114.     {
  115.       "C": "CN",
  116.       "L": "BeiJing",
  117.       "ST": "BeiJing",
  118.       "O": "k8s",
  119.       "OU": "System"
  120.     }
  121.   ]
  122. }
  123. EOF
  125. cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
  127. # 执行脚本
  128. [root@wyl01 ssl]# sh certificate.sh
  129. # 删除其他文件，只留下证书的文件
  130. [root@wyl01 ssl]# ls |grep -v pem|xargs -i rm {}
  131. [root@wyl01 ssl]# ll
  132. total 32
  133. -rw------- 1 root root 1679 Jul 19 09:51 admin-key.pem
  134. -rw-r--r-- 1 root root 1399 Jul 19 09:51 admin.pem
  135. -rw------- 1 root root 1679 Jul 19 09:51 ca-key.pem
  136. -rw-r--r-- 1 root root 1359 Jul 19 09:51 ca.pem
  137. -rw------- 1 root root 1679 Jul 19 09:51 kube-proxy-key.pem
  138. -rw-r--r-- 1 root root 1403 Jul 19 09:51 kube-proxy.pem
  139. -rw------- 1 root root 1679 Jul 19 09:51 server-key.pem
  140. -rw-r--r-- 1 root root 1627 Jul 19 09:51 server.pem

5 部署etcd服务（三台）

  1. [root@wyl01 software]# tar -xf etcd-v3.2.12-linux-amd64.tar.gz 
  2. [root@wyl01 software]# ls
  3. etcd.sh  etcd-v3.2.12-linux-amd64  etcd-v3.2.12-linux-amd64.tar.gz
  5. [root@wyl01 software]# cd etcd-v3.2.12-linux-amd64/
  7. [root@wyl01 etcd-v3.2.12-linux-amd64]# ls
  8. Documentation  etcd  etcdctl  README-etcdctl.md  README.md  READMEv2-etcdctl.md
  10. [root@wyl01 etcd-v3.2.12-linux-amd64]# cp -r etcd* /opt/kubernetes/bin/
  11. [root@wyl01 etcd-v3.2.12-linux-amd64]# ll /opt/kubernetes/bin/
  12. total 32284
  13. -rwxr-xr-x 1 root root 17817664 Jul 15 14:47 etcd
  14. -rwxr-xr-x 1 root root 15234432 Jul 15 14:47 etcdctl
  16. [root@wyl01 software]# vim  /opt/kubernetes/cfg/etcd 
  18. # 另外2台node节点,只要修改第2行到第9行的ip为ip为本机的ip，ETCD_NAME的值分别为etcd02，etcd03.
  19. #[Member]
  20. ETCD_NAME="etcd01"
  21. ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
  22. ETCD_LISTEN_PEER_URLS="https://192.168.52.128:2380"
  23. ETCD_LISTEN_CLIENT_URLS="https://192.168.52.128:2379"
  25. #[Clustering]
  26. ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.52.128:2380"
  27. ETCD_ADVERTISE_CLIENT_URLS="https://192.168.52.128:2379"
  28. ETCD_INITIAL_CLUSTER="etcd01=https://192.168.52.128:2380,etcd02=https://192.168.52.129:2380,etcd03=https://192.168.52.130:2380"
  29. ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
  30. ETCD_INITIAL_CLUSTER_STATE="new"
  32. [root@wyl01 software]# vim /etc/systemd/system/etcd.service
  33. [Unit]
  34. Description=Etcd Server
  35. After=network.target
  36. After=network-online.target
  37. Wants=network-online.target
  39. [Service]
  40. Type=notify
  41. EnvironmentFile=-/opt/kubernetes/cfg/etcd
  42. ExecStart=/opt/kubernetes/bin/etcd \
  43. --name=${ETCD_NAME} \
  44. --data-dir=${ETCD_DATA_DIR} \
  45. --listen-peer-urls=${ETCD_LISTEN_PEER_URLS} \
  46. --listen-client-urls=${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
  47. --advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} \
  48. --initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
  49. --initial-cluster=${ETCD_INITIAL_CLUSTER} \
  50. --initial-cluster-token=${ETCD_INITIAL_CLUSTER} \
  51. --initial-cluster-state=new \
  52. --cert-file=/opt/kubernetes/ssl/server.pem \
  53. --key-file=/opt/kubernetes/ssl/server-key.pem \
  54. --peer-cert-file=/opt/kubernetes/ssl/server.pem \
  55. --peer-key-file=/opt/kubernetes/ssl/server-key.pem \
  56. --trusted-ca-file=/opt/kubernetes/ssl/ca.pem \
  57. --peer-trusted-ca-file=/opt/kubernetes/ssl/ca.pem
  58. Restart=on-failure
  59. LimitNOFILE=65536
  61. [Install]
  62. WantedBy=multi-user.target

启动并查看服务

  1. [root@wyl01 software]# systemctl daemon-reload
  3. [root@wyl01 software]# systemctl start  etcd.service

20190719103547339.png

6 部署Flannel容器集群网络

部署master节点服务

部署kube-apiserver服务：

  1. vim kube-apiserver
  3. #!/bin/bash
  5. MASTER_ADDRESS=${1:-"192.168.52.128"}
  6. ETCD_SERVERS=${2:-"https://192.168.52.128:2379,https://192.168.52.129:2379,https://192.168.52.130:2379"}
  8. cat <<EOF >/opt/kubernetes/config/kube-apiserver
  10. KUBE_APISERVER_OPTS="--logtostderr=true \\
  11. --v=4 \\
  12. --etcd-servers=${ETCD_SERVERS} \\
  13. --insecure-bind-address=127.0.0.1 \\
  14. --bind-address=${MASTER_ADDRESS} \\
  15. --insecure-port=8080 \\
  16. --secure-port=6443 \\
  17. --advertise-address=${MASTER_ADDRESS} \\
  18. --allow-privileged=true \\
  19. --service-cluster-ip-range=10.10.10.0/24 \\
  20. --admission-control=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota,NodeRestriction \
  21. --authorization-mode=RBAC,Node \\
  22. --kubelet-https=true \\
  23. --enable-bootstrap-token-auth \\
  24. --token-auth-file=/opt/kubernetes/config/token.csv \\
  25. --service-node-port-range=30000-50000 \\
  26. --tls-cert-file=/opt/kubernetes/ssl/server.pem  \\
  27. --tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \\
  28. --client-ca-file=/opt/kubernetes/ssl/ca.pem \\
  29. --service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \\
  30. --etcd-cafile=/opt/kubernetes/ssl/ca.pem \\
  31. --etcd-certfile=/opt/kubernetes/ssl/server.pem \\
  32. --etcd-keyfile=/opt/kubernetes/ssl/server-key.pem"
  34. EOF
  36. cat <<EOF >/usr/lib/systemd/system/kube-apiserver.service
  37. [Unit]
  38. Description=Kubernetes API Server
  39. Documentation=https://github.com/kubernetes/kubernetes
  41. [Service]
  42. EnvironmentFile=/opt/kubernetes/config/kube-apiserver
  43. ExecStart=/opt/kubernetes/bin/kube-apiserver \$KUBE_APISERVER_OPTS
  44. Restart=on-failure
  46. [Install]
  47. WantedBy=multi-user.target
  48. EOF
  50. systemctl daemon-reload
  51. systemctl enable kube-apiserver
  52. systemctl restart kube-apiserver

这里启动报错，将token文件拷贝到上面/opt/kubernetes/config目录下

  1. cp /opt/kubernetes/ssl/token.csv /opt/kubernetes/config/token.csv
  3. systemctl start kube-apiserver  #启动服务

watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3d5bDk1Mjc_size_16_color_FFFFFF_t_70

部署kube-controller

  1. [root@wyl01 master]# vim controller-manager.sh   #不需要修改
  2. #!/bin/bash
  4. MASTER_ADDRESS=${1:-"127.0.0.1"}
  6. cat <<EOF >/opt/kubernetes/config/kube-controller-manager
  9. KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=true \\
  10. --v=4 \\
  11. --master=${MASTER_ADDRESS}:8080 \\
  12. --leader-elect=true \\
  13. --address=127.0.0.1 \\
  14. --service-cluster-ip-range=10.10.10.0/24 \\
  15. --cluster-name=kubernetes \\
  16. --cluster-signing-cert-file=/opt/kubernetes/ssl/ca.pem \\
  17. --cluster-signing-key-file=/opt/kubernetes/ssl/ca-key.pem  \\
  18. --service-account-private-key-file=/opt/kubernetes/ssl/ca-key.pem \\
  19. --root-ca-file=/opt/kubernetes/ssl/ca.pem"
  21. EOF
  23. cat <<EOF >/usr/lib/systemd/system/kube-controller-manager.service
  24. [Unit]
  25. Description=Kubernetes Controller Manager
  26. Documentation=https://github.com/kubernetes/kubernetes
  28. [Service]
  29. EnvironmentFile=-/opt/kubernetes/config/kube-controller-manager
  30. ExecStart=/opt/kubernetes/bin/kube-controller-manager \$KUBE_CONTROLLER_MANAGER_OPTS
  31. Restart=on-failure
  33. [Install]
  34. WantedBy=multi-user.target
  35. EOF
  37. systemctl daemon-reload
  38. systemctl enable kube-controller-manager
  39. systemctl restart kube-controller-manager
  41. [root@wyl01 master]# sh controller-manager.sh 127.0.0.1

watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3d5bDk1Mjc_size_16_color_FFFFFF_t_70 1

  1. [root@wyl01 master]# vim scheduler.sh 
  2. #!/bin/bash
  4. MASTER_ADDRESS=${1:-"127.0.0.1"}
  6. cat <<EOF >/opt/kubernetes/config/kube-scheduler
  8. KUBE_SCHEDULER_OPTS="--logtostderr=true \\
  9. --v=4 \\
  10. --master=${MASTER_ADDRESS}:8080 \\
  11. --leader-elect"
  13. EOF
  15. cat <<EOF >/usr/lib/systemd/system/kube-scheduler.service
  16. [Unit]
  17. Description=Kubernetes Scheduler
  18. Documentation=https://github.com/kubernetes/kubernetes
  20. [Service]
  21. EnvironmentFile=-/opt/kubernetes/config/kube-scheduler
  22. ExecStart=/opt/kubernetes/bin/kube-scheduler \$KUBE_SCHEDULER_OPTS
  23. Restart=on-failure
  25. [Install]
  26. WantedBy=multi-user.target
  27. EOF
  29. systemctl daemon-reload
  30. systemctl enable kube-scheduler
  31. systemctl restart kube-scheduler
  34. [root@wyl01 master]# sh scheduler.sh 127.0.0.1
  35. Created symlink from /etc/systemd/system/multi-user.target.wants/kube-scheduler.service to /usr/lib/systemd/system/kube-scheduler.service.
  36. [root@wyl01 master]#

watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3d5bDk1Mjc_size_16_color_FFFFFF_t_70 2

检查

20190715201832228.png

部署node节点组件：

在node节点上我们要部署2个服务，一个kubelet，还有一个是kube-proxy，先wyl02，wy03上运行kubelet.sh脚本，部署kubelet服务，脚本内容如下：

  1. [root@wyl03 software]# sh kubelet.sh 192.168.52.130 10.10.10.2
  3. 脚本内容：
  4. #!/bin/bash
  6. NODE_ADDRESS=${1:-"192.168.52.130"}
  7. DNS_SERVER_IP=${2:-"10.10.10.2"}
  9. cat <<EOF >/opt/kubernetes/config/kubelet
  11. KUBELET_OPTS="--logtostderr=true \\
  12. --v=4 \\
  13. --address=${NODE_ADDRESS} \\
  14. --hostname-override=${NODE_ADDRESS} \\
  15. --kubeconfig=/opt/kubernetes/config/kubelet.kubeconfig \\
  16. --experimental-bootstrap-kubeconfig=/opt/kubernetes/config/bootstrap.kubeconfig \\
  17. --cert-dir=/opt/kubernetes/ssl \\
  18. --allow-privileged=true \\
  19. --cluster-dns=${DNS_SERVER_IP} \\
  20. --cluster-domain=cluster.local \\
  21. --fail-swap-on=false \\
  22. --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0"
  24. EOF
  26. cat <<EOF >/usr/lib/systemd/system/kubelet.service
  27. [Unit]
  28. Description=Kubernetes Kubelet
  29. After=docker.service
  30. Requires=docker.service
  32. [Service]
  33. EnvironmentFile=-/opt/kubernetes/config/kubelet
  34. ExecStart=/opt/kubernetes/bin/kubelet \$KUBELET_OPTS
  35. Restart=on-failure
  36. KillMode=process
  38. [Install]
  39. WantedBy=multi-user.target
  40. EOF
  42. systemctl daemon-reload
  43. systemctl enable kubelet
  44. systemctl restart kubelet

运行中可能会报一下错误：

  1. error: failed to run Kubelet: cannot create certificate signing request: certificatesigningrequests.certificates.k8s.io is forbidden: User "kubelet-bootstrap" cannot create certificatesigningrequests.certificates.k8s.io at the cluster scope

解决方案：

创建角色用户，在master节点创建下面这个用户

watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3d5bDk1Mjc_size_16_color_FFFFFF_t_70 3

  1. [root@wyl01 ssl]# kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap
  2. clusterrolebinding "kubelet-bootstrap" created

创建完，我们再执行脚本，查看进程。

watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3d5bDk1Mjc_size_16_color_FFFFFF_t_70 4

部署kubelet-proxy服务，直接运行proxy.sh脚本即可，ps查看进程。

  1. [root@wyl02 software]# sh proxy.sh 192.168.52.129

启动后，一切都正常后，也要再wyl03节点上启动这2个服务。

master节点认证

  1. [root@wyl01 ssl]# kubectl get csr
  2. NAME                                                   AGE       REQUESTOR           CONDITION
  3. node-csr-enYniqvIBNOIX4DGLV0cL14oj-tLIvDrboLrlKdcWK8   1m        kubelet-bootstrap   Pending
  4. node-csr-siAO6abKNXjFH4bebYTW_k5T02so7BbiqdUyODPrRJY   3m        kubelet-bootstrap   Pending
  6. # 认证
  7. [root@wyl01 ssl]# kubectl certificate approve node-csr-enYniqvIBNOIX4DGLV0cL14oj-tLIvDrboLrlKdcWK8 
  8. certificatesigningrequest "node-csr-enYniqvIBNOIX4DGLV0cL14oj-tLIvDrboLrlKdcWK8" approved
  9. [root@wyl01 ssl]# kubectl certificate approve  node-csr-siAO6abKNXjFH4bebYTW_k5T02so7BbiqdUyODPrRJY
  10. certificatesigningrequest "node-csr-siAO6abKNXjFH4bebYTW_k5T02so7BbiqdUyODPrRJY" approved
  11. [root@wyl01 ssl]# kubectl get csr
  12. NAME                                                   AGE       REQUESTOR           CONDITION
  13. node-csr-enYniqvIBNOIX4DGLV0cL14oj-tLIvDrboLrlKdcWK8   3m        kubelet-bootstrap   Approved,Issued
  14. node-csr-siAO6abKNXjFH4bebYTW_k5T02so7BbiqdUyODPrRJY   4m        kubelet-bootstrap   Approved,Issued
  16. [root@wyl01 ssl]# kubectl get node
  17. NAME             STATUS    ROLES     AGE       VERSION
  18. 192.168.52.129   Ready     <none>    54s       v1.9.0
  19. 192.168.52.130   Ready     <none>    1m        v1.9.0

运行一个nginx实例

  1. [root@wyl01 ssl]# kubectl run nginx --image=nginx --replicas=3
  2. deployment "nginx" created
  4. [root@wyl01 ssl]# kubectl get pod
  5. NAME                   READY     STATUS    RESTARTS   AGE
  6. nginx-8586cf59-c7bmj   1/1       Running   0          1m
  7. nginx-8586cf59-ntwgf   1/1       Running   0          1m
  8. nginx-8586cf59-pj45w   1/1       Running   0          1m
  11. [root@wyl01 ssl]# kubectl get pod -o wide
  12. NAME                   READY     STATUS    RESTARTS   AGE       IP            NODE
  13. nginx-8586cf59-c7bmj   1/1       Running   0          1m        172.17.57.2   192.168.52.130
  14. nginx-8586cf59-ntwgf   1/1       Running   0          1m        172.17.51.2   192.168.52.129
  15. nginx-8586cf59-pj45w   1/1       Running   0          1m        172.17.57.3   192.168.52.130
  17. [root@wyl01 ssl]#  kubectl expose deployment nginx --port=88 --target-port=80 --type=NodePort
  18. service "nginx" exposed
  21. [root@wyl01 ssl]# kubectl get svc nginx
  22. NAME      TYPE       CLUSTER-IP     EXTERNAL-IP   PORT(S)        AGE
  23. nginx     NodePort   10.10.10.132   <none>        88:39327/TCP   1h
  25. 在node节点输入curl 10.10.10.132:88或者在pc端输入本机node节点ip:39327访问

部署dashboard管理页面

  1. [root@wyl01 ui]# ll
  2. total 12
  3. -rw-r--r-- 1 root root 1148 Jul 13 17:16 dashboard-deployment.yaml
  4. -rw-r--r-- 1 root root  612 Jul 13 17:16 dashboard-rbac.yaml
  5. -rw-r--r-- 1 root root  338 Jul 13 17:16 dashboard-service.yaml
  7. [root@wyl01 ui]# kubectl create -f dashboard-rbac.yaml 
  8. serviceaccount "kubernetes-dashboard" created
  9. clusterrolebinding "kubernetes-dashboard-minimal" created
  11. [root@wyl01 ui]# kubectl create -f dashboard-deployment.yaml 
  12. deployment "kubernetes-dashboard" created
  14. [root@wyl01 ui]# kubectl create -f dashboard-service.yaml 
  15. service "kubernetes-dashboard" created

发表评论

表情：
评论列表 （有 0 条评论，554人围观）

还没有评论，来说两句吧...

相关阅读

    相关 17未完，18待续

    一个很短的序 昨天，在 2017 年最后一天跑了人生中第一次 10 公里，留下了 17 年最后的纪念。今天，在 2018 年第一天写下第一篇博客文章，开始新的人生旅程。

    Love The Way You LieLove The Way You Lie/ 0 赞/ 342 阅读

    相关 List接口（未完待续）

    List接口 List集合代表一个有序集合，集合中的每个元素都有其对应的顺序索引。 List集合允许使用重复元素，可以通过索引来访问指定位置的集合的元素。List集合默认按

    心已赠人心已赠人/ 0 赞/ 456 阅读