九、Swift对象存储服务(双节点搭建)

梦里梦外; 2023-06-18 02:58 15阅读 0赞

# 九、Swift对象存储服务(双节点搭建) #

# 要求:Controoler节点需要2块空盘 #

# Compute节点需要再加2块空盘![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70] #

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 1]

# 本次搭建采用Controller 和 Compute双节点节点做swift组件 #

# 1.Controller安装并配置控制节点 #

# 2.Compute安装和配置存储节点 #

# 3.Compute节点安装并配置组件 #

# 4.Controller节点创建，分发并初始化rings #

# 5.启动服务 #

# 6.Controller执行验证操作 #

## 1.Controller安装并配置控制节点 ##

\#\#\#\# 1.获得凭证并创建身份认证服务的凭证

# . /root/admin-openrc
    # openstack user create --domain default --password 000000 swift
    # openstack role add --project service --user swift admin
    # openstack service create --name swift --description "OpenStack Object Storage" object-store

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 2]

### 创建对象存储服务 API 端点 ###

# openstack endpoint create --region RegionOne object-store public http://controller:8080/v1/AUTH_%\(tenant_id\)s
    
    # openstack endpoint create --region RegionOne object-store internal http://controller:8080/v1/AUTH_%\(tenant_id\)s
    
    # openstack endpoint create --region RegionOne object-store admin http://controller:8080/v1

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 3]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 4]

#### 3.安装并配置组件 ####

\[root@controller ~\]\# yum -y install openstack-swift-proxy python-swiftclient python-keystoneclient python-keystonemiddleware memcached xfsprogs rsync openstack-swift-account openstack-swift-container openstack-swift-object

### 编辑文件 /etc/swift/proxy-server.conf,`删除原有内容`，直接添加如下内容 ###

# vi /etc/swift/proxy-server.conf
    
    
    [DEFAULT]
    bind_port = 8080
    swift_dir = /etc/swift
    user = swift
    [pipeline:main]
    pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk ratelimit authtoken keystoneauth container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server
    [app:proxy-server]
    use = egg:swift#proxy
    account_autocreate = True
    [filter:tempauth]
    use = egg:swift#tempauth
    user_admin_admin = admin .admin .reseller_admin
    user_test_tester = testing .admin
    user_test2_tester2 = testing2 .admin
    user_test_tester3 = testing3
    user_test5_tester5 = testing5 service
    [filter:authtoken]
    paste.filter_factory = keystonemiddleware.auth_token:filter_factory
    auth_uri = http://controller:5000
    auth_url = http://controller:35357
    memcached_servers = controller:11211
    auth_type = password
    project_domain_name = default
    user_domain_name = default
    project_name = service
    username = swift
    password = 000000
    delay_auth_decision = True
    [filter:keystoneauth]
    use = egg:swift#keystoneauth
    operator_roles = admin,user
    [filter:healthcheck]
    use = egg:swift#healthcheck
    [filter:cache]
    memcache_servers = controller:11211
    use = egg:swift#memcache
    [filter:ratelimit]
    use = egg:swift#ratelimit
    [filter:domain_remap]
    use = egg:swift#domain_remap
    [filter:catch_errors]
    use = egg:swift#catch_errors
    [filter:cname_lookup]
    use = egg:swift#cname_lookup
    [filter:staticweb]
    use = egg:swift#staticweb
    [filter:tempurl]
    use = egg:swift#tempurl
    [filter:formpost]
    use = egg:swift#formpost
    [filter:name_check]
    use = egg:swift#name_check
    [filter:list-endpoints]
    use = egg:swift#list_endpoints
    [filter:proxy-logging]
    use = egg:swift#proxy_logging
    [filter:bulk]
    use = egg:swift#bulk
    [filter:slo]
    use = egg:swift#slo
    [filter:dlo]
    use = egg:swift#dlo
    [filter:container-quotas]
    use = egg:swift#container_quotas
    [filter:account-quotas]
    use = egg:swift#account_quotas
    [filter:gatekeeper]
    use = egg:swift#gatekeeper
    [filter:container_sync]
    use = egg:swift#container_sync
    [filter:xprofile]
    use = egg:swift#xprofile
    [filter:versioned_writes]
    use = egg:swift#versioned_writes

### 编辑文件/etc/swift/swift.conf `删除原有内容`，添加如下内容 ###

# vi /etc/swift/swift.conf
    
    [swift-hash]
    swift_hash_path_suffix = changeme
    swift_hash_path_prefix = changeme
    [storage-policy:0]
    name = Policy-0
    default = yes
    aliases = yellow, orange
    [swift-constraints]

##### 查看空盘 #####

lsblk

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 5]

### 每个盘划分一个空白分区 ###

# fdisk /dev/sdb
    # fdisk /dev/sdc

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 6]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 7]

### 创建挂载目录 ###

[root@controller ~]# mkdir -p /swift/sdb
    [root@controller ~]# mkdir -p /swift/sdc

![在这里插入图片描述][20191129201922976.png]

### 指定分区格式 ###

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 8]

### 挂载分区到目录 ###

![在这里插入图片描述][20191129201947356.png]

### 添加到/etc/fstab配置文件中 ###

[root@controller ~]# vi /etc/fstab 
    /dev/sdb1 /swift/sdb xfs noatime,nodiratime,nobarrier,logbufs=8 0 0
    /dev/sdc1 /swift/sdc xfs noatime,nodiratime,nobarrier,logbufs=8 0 0

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 9]

### 编辑配置文件 /etc/rsyncd.conf ###

[root@controller ~]# vi /etc/rsyncd.conf

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 10]

### 启动服务 ###

[root@controller ~]# systemctl start rsyncd.service
    [root@controller ~]# systemctl enable rsyncd.service

![在这里插入图片描述][20191129202059457.png]

## 6.赋权： ##

# mkdir -p /var/cache/swift
    # chown -R root:swift /var/cache/swift
    # chmod -R 775 /var/cache/swift
    # chown -R swift:swift /swift
    # chown -R root:swift /etc/swift

# 编辑三个配置文件account container object #

1.编辑 /etc/swift/account-server.conf 文件，删除原有内容，直接添加以下内容
    #vi /etc/swift/account-server.conf
    
    [DEFAULT]
    bind_ip = 192.168.100.10
    bind_port = 6002
    user = swift
    swift_dir = /etc/swift
    devices = /swift
    mount_check = false
    [pipeline:main]
    pipeline = healthcheck recon account-server
    [app:account-server]
    use = egg:swift#account
    [filter:healthcheck]
    use = egg:swift#healthcheck
    [filter:recon]
    use = egg:swift#recon
    recon_cache_path = /var/cache/swift
    [account-replicator]
    [account-auditor]
    [account-reaper]
    [filter:xprofile]
    use = egg:swift#xprofile
    
    
    2.编辑``/etc/swift/container-server.conf``文件，删除原有内容，直接添加以下内容：
    # vi /etc/swift/container-server.conf
    
    [DEFAULT]
    bind_ip = 192.168.100.10
    bind_port = 6001
    user = swift
    swift_dir = /etc/swift
    devices = /swift
    mount_check = false
    [pipeline:main]
    pipeline = healthcheck recon container-server
    [app:container-server]
    use = egg:swift#container
    [filter:healthcheck]
    use = egg:swift#healthcheck
    [filter:recon]
    use = egg:swift#recon
    recon_cache_path = /var/cache/swift
    [container-replicator]
    [container-updater]
    [container-auditor]
    [container-sync]
    [filter:xprofile]
    use = egg:swift#xprofile
    
    
    3.编辑``/etc/swift/object-server.conf``文件，删除原有内容，直接添加以下内容：
    # vi /etc/swift/object-server.conf
    
    [DEFAULT]
    bind_ip = 192.168.100.10
    bind_port = 6000
    user = swift
    swift_dir = /etc/swift
    devices = /swift
    mount_check = false
    [pipeline:main]
    pipeline = healthcheck recon object-server
    [app:object-server]
    use = egg:swift#object
    [filter:healthcheck]
    use = egg:swift#healthcheck
    [filter:recon]
    use = egg:swift#recon
    recon_cache_path = /var/cache/swift
    recon_lock_path = /var/lock
    [object-replicator]
    [object-reconstructor]
    [object-updater]
    [object-auditor]
    [filter:xprofile]
    use = egg:swift#xprofile

## 2.Compute安装和配置存储节点 ##

1.安装软件包

[root@compute ~]# yum -y install xfsprogs rsync openstack-swift-account openstack-swift-container openstack-swift-object

# 搭建Swift Compute节点需要两块空盘，此处我们查看添加的空盘 #

# lsblk

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 11]

### 创建分区 ###

# fdisk /dev/sdc
    # fdisk /dev/sdd

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 12]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 13]

### 指定文件格式 ###

# mkfs.xfs /dev/sdc1
    # mkfs.xfs /dev/sdd1

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 14]

### 3.创建挂载点目录结构： ###

[root@compute ~]# mkdir -p /swift/sdc
    [root@compute ~]# mkdir -p /swift/sdd

![在这里插入图片描述][201911292025378.png]

### 挂载分区到目录 ###

[root@compute ~]# mount /dev/sdc1 /swift/sdc/
    [root@compute ~]# mount /dev/sdd1 /swift/sdd/

![在这里插入图片描述][20191129202547676.png]

#### 4.编辑`/etc/fstab`文件并添加以下内容： ####

vi /etc/fstab
    
    /dev/sdc1 /swift/sdc xfs noatime,nodiratime,nobarrier,logbufs=8 0 0
    /dev/sdd1 /swift/sdd xfs noatime,nodiratime,nobarrier,logbufs=8 0 0

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 15]

### 5.编辑`/etc/rsyncd.conf`文件并添加以下内容： ###

[root@compute ~]#vi /etc/rsyncd.conf
    
    pid file = /var/run/rsyncd.pid
    log file = /var/log/rsyncd.log
    uid = swift
    gid = swift
    address = 192.168.100.20
    [account]
    path            = /swift
    read only       = false
    write only      = no
    list            = yes
    incoming chmod  = 0644
    outgoing chmod  = 0644
    max connections = 25
    lock file =     /var/lock/account.lock
    [container]
    path            = /swift
    read only       = false
    write only      = no
    list            = yes
    incoming chmod  = 0644
    outgoing chmod  = 0644
    max connections = 25
    lock file =     /var/lock/container.lock
    [object]
    path            = /swift
    read only       = false
    write only      = no
    list            = yes
    incoming chmod  = 0644
    outgoing chmod  = 0644
    max connections = 25
    lock file =     /var/lock/object.lock

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 16]

### 6.启动 “rsyncd” 服务和配置它随系统启动： ###

# systemctl start rsyncd.service
    # systemctl enable rsyncd.service

## 6.赋权： ##

# mkdir -p /var/cache/swift
    # chown -R root:swift /var/cache/swift
    # chmod -R 775 /var/cache/swift
    # chown -R swift:swift /swift
    # chown -R root:swift /etc/swift

## 编辑配置文件 account container object ##

1.编辑 /etc/swift/account-server.conf 文件，删除原有内容，直接添加以下内容
    #vi /etc/swift/account-server.conf
    
    [DEFAULT]
    bind_ip = 192.168.100.20
    bind_port = 6002
    user = swift
    swift_dir = /etc/swift
    devices = /swift
    mount_check = false
    [pipeline:main]
    pipeline = healthcheck recon account-server
    [app:account-server]
    use = egg:swift#account
    [filter:healthcheck]
    use = egg:swift#healthcheck
    [filter:recon]
    use = egg:swift#recon
    recon_cache_path = /var/cache/swift
    [account-replicator]
    [account-auditor]
    [account-reaper]
    [filter:xprofile]
    use = egg:swift#xprofile
    
    
    2.编辑``/etc/swift/container-server.conf``文件，删除原有内容，直接添加以下内容：
    # vi /etc/swift/container-server.conf
    
    [DEFAULT]
    bind_ip = 192.168.100.20
    bind_port = 6001
    user = swift
    swift_dir = /etc/swift
    devices = /swift
    mount_check = false
    [pipeline:main]
    pipeline = healthcheck recon container-server
    [app:container-server]
    use = egg:swift#container
    [filter:healthcheck]
    use = egg:swift#healthcheck
    [filter:recon]
    use = egg:swift#recon
    recon_cache_path = /var/cache/swift
    [container-replicator]
    [container-updater]
    [container-auditor]
    [container-sync]
    [filter:xprofile]
    use = egg:swift#xprofile
    
    
    3.编辑``/etc/swift/object-server.conf``文件，删除原有内容，直接添加以下内容：
    # vi /etc/swift/object-server.conf
    
    [DEFAULT]
    bind_ip = 192.168.100.20
    bind_port = 6000
    user = swift
    swift_dir = /etc/swift
    devices = /swift
    mount_check = false
    [pipeline:main]
    pipeline = healthcheck recon object-server
    [app:object-server]
    use = egg:swift#object
    [filter:healthcheck]
    use = egg:swift#healthcheck
    [filter:recon]
    use = egg:swift#recon
    recon_cache_path = /var/cache/swift
    recon_lock_path = /var/lock
    [object-replicator]
    [object-reconstructor]
    [object-updater]
    [object-auditor]
    [filter:xprofile]
    use = egg:swift#xprofile

## 4.编辑文件/etc/swift/swift.conf 删除原有内容，添加如下内容 ##

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 17]

## 3.Controller 和 Compute节点安装并配置组件 ##

## 4.Controller节点创建，分发并初始化rings ##

### 1.创建账户ring ###

1.切换到 ``/etc/swift``目录。创建基本 account.builder 文件：
    
    # cd /etc/swift
    # swift-ring-builder account.builder create 18 1 1
    2.添加每个节点到 ring 中：
    # swift-ring-builder account.builder add --region 1 --zone 1 --ip 192.168.100.20 --port 6002 --device sdc1 --weight 100
    # swift-ring-builder account.builder add --region 1 --zone 1 --ip 192.168.100.20 --port 6002 --device sdd1 --weight 100
    # swift-ring-builder account.builder add --region 1 --zone 1 --ip 192.168.100.10 --port 6002 --device sdc1 --weight 100
    # swift-ring-builder account.builder add --region 1 --zone 1 --ip 192.168.100.10 --port 6002 --device sdb1 --weight 100
    2.验证ring内容
    # swift-ring-builder account.builder
    3.平衡ring
    # swift-ring-builder account.builder rebalance

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 18]

2.创建ring容器

1.切换到 ``/etc/swift``目录。创建基本``container.builder``文件：
    # cd /etc/swift
    # swift-ring-builder container.builder create 10 1 1
    2.添加每个节点到 ring 中：
    # swift-ring-builder container.builder add --region 1 --zone 1 --ip 192.168.100.20 --port 6001 --device sdc1 --weight 100
    # swift-ring-builder container.builder add --region 1 --zone 1 --ip 192.168.100.20 --port 6001 --device sdd1 --weight 100
    # swift-ring-builder container.builder add --region 1 --zone 1 --ip 192.168.100.10 --port 6001 --device sdc1 --weight 100
    # swift-ring-builder container.builder add --region 1 --zone 1 --ip 192.168.100.10 --port 6001 --device sdb1 --weight 100
    3.验证 ring 的内容：
    # swift-ring-builder container.builder
    4.平衡 ring：
    # swift-ring-builder container.builder rebalance

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 19]

### 3.创建Ring对象 ###

1.切换到 ``/etc/swift``目录。创建基本``object.builder``文件：
    # swift-ring-builder object.builder create 10 1 1
    2.添加每个节点到 ring 中：
    # swift-ring-builder object.builder add --region 1 --zone 1 --ip 192.168.100.20 --port 6000 --device sdc1 --weight 100
    # swift-ring-builder object.builder add --region 1 --zone 1 --ip 192.168.100.20 --port 6000 --device sdd1 --weight 100
    # swift-ring-builder object.builder add --region 1 --zone 1 --ip 192.168.100.10 --port 6000 --device sdc1 --weight 100
    # swift-ring-builder object.builder add --region 1 --zone 1 --ip 192.168.100.10 --port 6000 --device sdb1 --weight 100
    3.验证 ring 的内容：
    # swift-ring-builder object.builder
    4.平衡 ring：
    # swift-ring-builder object.builder rebalance

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 20]

#### 4.Compute节点复制Controller的配置文件 ####

复制``account.ring.gz``，container.ring.gz``和``object.ring.gz 文件到每个存储节点和其他运行了代理服务的额外节点的 /etc/swift 目录。
    [root@compute ~]# scp controller:/etc/swift/*.ring.gz /etc/swift/

![在这里插入图片描述][20191125013045545.png]

## 5.启动服务 ##

\#\#\#\# 1.在控制节点\`\`Controller\`\`上，启动对象存储代理服务及其依赖服务，并将它们配置为随系统启动：

# systemctl start openstack-swift-proxy.service memcached.service
    # systemctl enable openstack-swift-proxy.service memcached.service

![在这里插入图片描述][20191125013130479.png]

#### 2.在存储节点`Controller 和 Compute 两个节点`上，启动对象存储服务，并将其设置为随系统启动 ####

# 两个节点均需操作 #

# systemctl start openstack-swift-account.service openstack-swift-account-auditor.service openstack-swift-account-reaper.service openstack-swift-account-replicator.service
    # systemctl enable openstack-swift-account.service openstack-swift-account-auditor.service openstack-swift-account-reaper.service openstack-swift-account-replicator.service
    
    # systemctl start openstack-swift-container.service openstack-swift-container-auditor.service openstack-swift-container-replicator.service openstack-swift-container-updater.service
    # systemctl enable openstack-swift-container.service openstack-swift-container-auditor.service openstack-swift-container-replicator.service openstack-swift-container-updater.service
    
    # systemctl start openstack-swift-object.service openstack-swift-object-auditor.service openstack-swift-object-replicator.service openstack-swift-object-updater.service
    # systemctl enable openstack-swift-object.service openstack-swift-object-auditor.service openstack-swift-object-replicator.service openstack-swift-object-updater.service

## 6.Controller执行验证操作 ##

\#\#\#\# 1.导入demo凭证

# . /root/admin-openrc

#### 2. 显示服务状态： ####

# swift stat

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 21]

#### 3.创建容器 container1 ####

# openstack container create container1

![在这里插入图片描述][20191125013307356.png]

#### 4.上传一个测试文件到`container1`容器 ####

# openstack object create container1 /root/demo-openrc

![在这里插入图片描述][20191125013320660.png]

#### 5.列出`container1`容器里的所有文件 ####

# openstack object list container1

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 22]

#### 6.从`container1`容器里下载一个测试文件 ####

# openstack object save container1 /root/demo-openrc

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 23]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70]: https://img-blog.csdnimg.cn/20191129201459909.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 1]: https://img-blog.csdnimg.cn/2019112920150545.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 2]: https://img-blog.csdnimg.cn/20191129202314194.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 3]: https://img-blog.csdnimg.cn/20191129202340631.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 4]: https://img-blog.csdnimg.cn/20191129202346126.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 5]: https://img-blog.csdnimg.cn/20191129201839328.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 6]: https://img-blog.csdnimg.cn/20191129201858103.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 7]: https://img-blog.csdnimg.cn/2019112920190480.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[20191129201922976.png]: https://img-blog.csdnimg.cn/20191129201922976.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 8]: https://img-blog.csdnimg.cn/20191129201933173.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[20191129201947356.png]: https://img-blog.csdnimg.cn/20191129201947356.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 9]: https://img-blog.csdnimg.cn/20191129202029865.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 10]: https://img-blog.csdnimg.cn/20191129202043809.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[20191129202059457.png]: https://img-blog.csdnimg.cn/20191129202059457.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 11]: https://img-blog.csdnimg.cn/20191129202447617.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 12]: https://img-blog.csdnimg.cn/20191129202502511.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 13]: https://img-blog.csdnimg.cn/20191129202506308.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 14]: https://img-blog.csdnimg.cn/20191129202523567.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[201911292025378.png]: https://img-blog.csdnimg.cn/201911292025378.png
[20191129202547676.png]: https://img-blog.csdnimg.cn/20191129202547676.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 15]: https://img-blog.csdnimg.cn/20191129202613648.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 16]: https://img-blog.csdnimg.cn/20191129202646250.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 17]: https://img-blog.csdnimg.cn/20191129203042582.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 18]: https://img-blog.csdnimg.cn/20191129203143234.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 19]: https://img-blog.csdnimg.cn/20191129203256598.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 20]: https://img-blog.csdnimg.cn/20191129203250203.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[20191125013045545.png]: https://img-blog.csdnimg.cn/20191125013045545.png
[20191125013130479.png]: https://img-blog.csdnimg.cn/20191125013130479.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 21]: https://img-blog.csdnimg.cn/20191125013240185.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[20191125013307356.png]: https://img-blog.csdnimg.cn/20191125013307356.png
[20191125013320660.png]: https://img-blog.csdnimg.cn/20191125013320660.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 22]: https://img-blog.csdnimg.cn/20191125013343240.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ_size_16_color_FFFFFF_t_70 23]: https://img-blog.csdnimg.cn/20191125013359564.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2ZseTE1NzQ=,size_16,color_FFFFFF,t_70