MFS高可用部署（pacemaker+corosync+iscsi+fence）（待补充）

àì夳堔傛蜴生んèń 2021-10-19 23:20 215阅读 0赞

实验环境：

<table> 
 <thead> 
  <tr> 
   <th>server1：172.25.16.1</th> 
   <th>master</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td>server2：172.25.16.2</td> 
   <td>chunk server</td> 
  </tr> 
  <tr> 
   <td>server3：172.25.16.3</td> 
   <td>chunk server</td> 
  </tr> 
  <tr> 
   <td>server4:172.25.16.4</td> 
   <td>backup-mfsmaster</td> 
  </tr> 
 </tbody> 
</table>

server4:

[root@server4 ~]# yum install moosefs-master-3.0.103-1.rhsystemd.x86_64.rpm -y
    [root@server4 ~]# vim /etc/hosts
    [root@server4 ~]# cd /usr/lib/systemd/system
    [root@server4 system]# vim moosefs-master.service 
    8 ExecStart=/usr/sbin/mfsmaster -a
    [root@server4 system]# systemctl daemon-reload 
    [root@server4 system]# systemctl start moosefs-master
    [root@server4 system]# netstat -antlp

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 1]  
2.高可用配置

[root@server1 ~]# cd /etc/yum.repos.d/
    [root@server1 yum.repos.d]# vim westos.repo
    [westos]
    name=westos
    baseurl=http://172.25.16.250/westos73
    gpgcheck=0
    
    
    [HighAvailability]
    name=HighAvailability
    baseurl=http://172.25.16.250/westos73/addons/HighAvailability
    gpgcheck=0
    
    
    [ResilientStorage]
    name=ResilientStorage
    baseurl=http://172.16.250/westos73/addons/ResilientStorage
    gpgcheck=0

[root@server1 yum.repos.d]# yum clean all
    [root@server1 yum.repos.d]# yum repolist

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 2]

3.server1和server4上安装pacemaker、corosync

[root@server1 yum.repos.d]# yum install pacemaker corosync -y
    [root@server1 yum.repos.d]# scp westos.repo server4:/etc/yum.repos.d/
    
    [root@server4 ~]# yum install pacemaker corosync -y

4.server1和server4之间实现免密  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 3]![在这里插入图片描述][20190817231321481.png]

5.在server1和server4上安装资源管理工具并且开启相应服务

//server1
    [root@server1 ~]# yum install -y pcs
    [root@server1 ~]# systemctl start pcsd
    [root@server1 ~]# systemctl enable  pcsd  ##开启该服务之后会在/etc/passwd目录下生成hacluster用户，及高可用管理用户
    [root@server1 ~]# passwd hacluster   ##配置高可用集群管理用户的密码
    
    //server4
    [root@server4 ~]# yum install -y pcs
    [root@server4 ~]# systemctl start pcsd
    [root@server4 ~]# systemctl enable  pcsd  ##开启该服务之后会在/etc/passwd目录下生成hacluster用户，及高可用管理用户
    [root@server4 ~]# passwd hacluster   ##配置高可用集群管理用户的密码

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 4]![在这里插入图片描述][20190817232008715.png]

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 5]  
![在这里插入图片描述][20190817232020256.png]  
6.在server1上使用集群管理工具创建mfsmaster集群并且启动集群

[root@server1 ~]# pcs cluster auth server1 server4 		##创建集群
    [root@server1 ~]# pcs cluster setup --name mycluster server1 server4		##为集群命名
    [root@server1 ~]# pcs cluster start --all		##启动集群
    [root@server1 ~]# pcs status 			##查看集群状态
    [root@server1 ~]# corosync-cfgtool -s
    [root@server1 ~]# pcs status corosync

![在这里插入图片描述][20190817232054249.png]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 6]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 7]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 8]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 9]  
7.检查集群并解决报错

[root@server1 ~]# crm_verify -L -V
    [root@server1 ~]# pcs property set stonith-enabled=false
    [root@server1 ~]# crm_verify -L -V

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 10]  
注意：查看集群状态时出现错误是因为默认配置fence模块的这里先不配置fence所以先将fence设置关掉，等后面配置fence的时候再将其打开即可，这里的stonith指的是fence设备

8.给mfs高可用集群添加资源，此次资源添加vip

在这里插入代码片//查看资源
    [root@server1 ~]# pcs resource standards  ##查看集群资源
    [root@server1 ~]# pcs resource agents ocf:heartbeat  ##查看该资源下支持的资源类型
    
    //创建资源
    [root@server1 ~]# pcs resource create vip ocf:heartbeat:IPaddr2 ip=172.25.16.100 cidr_netmask=32 op monitor interval=30s
    [root@server1 ~]# pcs resource show

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 11]![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 12]

[root@server4 ~]# crm_mon 	##查看集群信息

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 13]  
高可用测试：  
关闭cluster集群中的server1，查看vip的变化  
发现vip由server1漂移到server4  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 14]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 15]  
server4上查看监控信息

[root@server4 ~]# crm_mon

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 16]开启cluster集群里的server1，查看监控信息

[root@server1 ~]# pcs cluster start server1
    [root@server1 ~]# crm_mon

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 17]

## pacemaker+corosync实现高可用后的iscsi实现数据共享 ##

1.关闭master及其chunkserver服务为iscsi的实验环境做准备

[root@server1 ~]# systemctl stop moosefs-master
    [root@server2 ~]# systemctl stop moosefs-chunkserver
    [root@server3 ~]# systemctl stop moosefs-chunkserver
    [root@server4 ~]# systemctl stop  moosefs-master

2.配置iscsi

server3(targetcli):  
1.添加共享磁盘  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 18]  
fdisk -l:  
![在这里插入图片描述][20190821160411535.png]

2.安装targetcli并做相应配置

//安装targetcli
    [root@server3 ~]# yum install -y targetcli
    
    //登录targetcli并且将添加的硬盘当作targetcli磁盘共享存储的磁盘
    [root@server3 ~]# targetcli
    /> ls   ##查看targetcli的相关信息，里面会有目录及其相应目录下的文件
    
    //添加磁盘：
    /> cd backstores/block 
    /backstores/block> create my_disk1 /dev/vda
    在iscis目录下：
    /backstores/block> cd /
    /> cd iscsi 
    创建磁盘目录，进入目录下的相应目录创建已添加磁盘：
    /iscsi> create  iqn.2019-05.com.example:server3
    /iscsi> cd iqn.2019-05.com.example:server3/
    /iscsi/iqn.20...ample:server3> cd tpg1/luns 
    /iscsi/iqn.20...er2/tpg1/luns> 
    /iscsi/iqn.20...er2/tpg1/luns> create /backstores/block/my_disk1 
    创建client磁盘共享目录：
    /iscsi/iqn.20...er2/tpg1/luns> cd /iscsi/iqn.2019-05.com.example:server3/tpg1/acls 
    /iscsi/iqn.20...er2/tpg1/acls> create iqn.2019-05.com.example:client

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 19]  
server1(iscsi):

1.下载iscsi客户端并且做相应配置

[root@server1 ~]# yum install -y iscsi-*
    [root@server1 ~]# vim /etc/iscsi/initiatorname.iscsi 
    [root@server1 ~]# iscsiadm -m discovery -t st -p 172.25.31.3
    172.25.31.3:3260,1 iqn.2019-05.com.example:server3
    [root@server1 ~]# systemctl restart iscsid
    [root@server1 ~]# iscsiadm -m node -l
    Logging in to [iface: default, target: iqn.2019-05.com.example:server3, portal: 172.25.31.3,3260] (multiple)
    Login to [iface: default, target: iqn.2019-05.com.example:server3, portal: 172.25.31.3,3260] successful.

fdisk -l查看  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 20]  
挂载创建分区并进行格式化

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 21]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 22]

//因为mfs文件存储的相应信息都在/var/lib/mfs/ ，要创建mfs文件系统必须将其信息写到已经共享成功的磁盘里面，实现mfs文件系统的相应操作：
    [root@server1 ~]# mount /dev/sdb1 /mnt
    [root@server1 ~]# cd /var/lib/mfs/  这是mfs的数据目录
    [root@server1 mfs]# ls
    changelog.2.mfs  changelog.4.mfs  changelog.6.mfs  metadata.mfs         metadata.mfs.empty
    changelog.3.mfs  changelog.5.mfs  metadata.crc     metadata.mfs.back.1  stats.mfs
    [root@server1 mfs]# cp -p * /mnt/   带权限拷贝/var/lib/mfs的所有数据文件到/dev/sdb1上
    [root@server1 mfs]# cd /mnt/
    [root@server1 mnt]# ls
    changelog.2.mfs  changelog.4.mfs  changelog.6.mfs  metadata.crc  metadata.mfs.back.1  stats.mfs
    changelog.3.mfs  changelog.5.mfs  lost+found       metadata.mfs  metadata.mfs.empty
    [root@server1 mnt]# chown mfs.mfs /mnt   当目录属于mfs用户和组时，才能正常使用
    [root@server1 mnt]# cd
    [root@server1 ~]# umount /mnt
    [root@server1 ~]# mount /dev/sdb1 /var/lib/mfs/   # 使用分区，测试是否可以使用共享磁盘
    [root@server1 ~]# systemctl start moosefs-master  服务开启成功，就说明数据文件拷贝成功，共享磁盘可以正常使用
    [root@server1 ~]# systemctl stop moosefs-master  关闭服务因为要进行高可用测试，是集群开启服务而不能手动开启

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 23]

server4(iscsi)同server1操作

[root@server4 system]# yum install -y iscsi-*
    [root@server4 system]# vim /etc/iscsi/initiatorname.iscsi 
    文件编辑内容如下：
    InitiatorName=iqn.2019-05.com.example:client
    [root@server4 system]#  iscsiadm -m discovery -t st -p 172.25.31.3
    [root@server4 system]# iscsiadm -m node -l

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 24]  
注意：在mfs文件系统创建完成之后，对于在主从及server1上和server4上的mfs文件系统用户的所属组及其相应id都要相同，如果不同可能会导致共享出错

![在这里插入图片描述][20190821162838265.png]  
![在这里插入图片描述][20190821162909698.png]

卸载之前挂载的/var/lib/mfs

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 25]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 26]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70]: /images/20211019/9d63bffce2184120a7c55e3b3dc3b879.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 1]: /images/20211019/4c3ebedb7e0a4b6a97e11e7117c58874.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 2]: /images/20211019/d32fbc666d5b43479e68a27506b51de7.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 3]: https://img-blog.csdnimg.cn/20190817231316837.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg==,size_16,color_FFFFFF,t_70
[20190817231321481.png]: /images/20211019/d892e1f728ee4e71b6aa5ac88b82c9b5.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 4]: https://img-blog.csdnimg.cn/20190817231529900.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg==,size_16,color_FFFFFF,t_70
[20190817232008715.png]: /images/20211019/949bc7d5fc724c048a5ea088729491d1.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 5]: /images/20211019/1fd6fa00d496443e8bde60bd1f4352b9.png
[20190817232020256.png]: /images/20211019/448cd70bef9447239c1f7a234dcf8e22.png
[20190817232054249.png]: /images/20211019/901e051906c74a7ea34fed597ff51323.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 6]: /images/20211019/da169f5546b7404ead15995f03c61e80.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 7]: /images/20211019/9750c997de384d84ada280d0eb5c3965.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 8]: /images/20211019/c4c8908856da4f25b7cd749dc7406810.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 9]: /images/20211019/c5f801ec3b7f451a9bcf755ad36f7832.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 10]: /images/20211019/2f6ee6c76d3241568524fc40358fa9b6.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 11]: https://img-blog.csdnimg.cn/20190817233506698.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg==,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 12]: /images/20211019/13df5b6764eb4c4cb968b14044060881.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 13]: /images/20211019/40e90f76edb2484684c477d9916f5fe2.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 14]: /images/20211019/60d25f81d4454ae4a021448f8601e9a0.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 15]: /images/20211019/820bff2115104a94939aeb467cfa49f3.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 16]: /images/20211019/a2dee25eac1a475797f75d89fecf6174.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 17]: /images/20211019/33309c27aaae44149f80bb46aa712b92.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 18]: /images/20211019/a6a01b6f941d40e1b04831e2bae60cb4.png
[20190821160411535.png]: /images/20211019/6250fec1693647719d2b0a6004dd0864.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 19]: /images/20211019/0da38bb649804d52ba4145d8fe441ccf.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 20]: /images/20211019/48f519bf2df448c3b4cd7e456e0cd989.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 21]: /images/20211019/40010bf83dd34055848b5b8fe041ea09.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 22]: /images/20211019/1844fa85eb8541d880dba2abd2cafdab.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 23]: /images/20211019/2b45548656054a08ba9863960bb17d2b.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 24]: /images/20211019/ac06c097740b427ba4fba4dbcbed8429.png
[20190821162838265.png]: /images/20211019/baae6493d30846bcb4f5efffb3fd874e.png
[20190821162909698.png]: /images/20211019/e5959fd334e546a187743fbfa592ab14.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 25]: /images/20211019/d59ca166b1344630abf5571eab3229ca.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDg4OTYxNg_size_16_color_FFFFFF_t_70 26]: /images/20211019/ff870b80f7a04e5aaa989d1c59952f6c.png