Hystrix

﹏ヽ暗。殇╰゛Y 2024-03-22 19:55 53阅读 0赞

## 简介 ##

![在这里插入图片描述][075266d455eb4303ba7094db68850413.png]  
![在这里插入图片描述][13e1d1d6d0a7435ca8c28ed3f4e6b04c.png]

## 服务降级、服务熔断、服务限流 ##

### 服务降级 ###

![在这里插入图片描述][30918557afad438295c01ed30bb4d108.png]  
![在这里插入图片描述][4c51aab2015145098039b2c60764ce13.png]

### 服务熔断 ###

![在这里插入图片描述][51c45403baa64b5f95747f36ecc2555e.png]  
![在这里插入图片描述][85286a95bf804744b16545030a908296.png]

### 服务限流 ###

![在这里插入图片描述][89e0ef974a304e998b71f71aee3fcd66.png]

## 案例 ##

### 构建项目 ###

1.  构建 cloud-provider-hystrix-payment8001 项目
2.  pom.xml

<?xml version="1.0" encoding="UTF-8"?>
    <project xmlns="http://maven.apache.org/POM/4.0.0"
             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
             xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
        <modelVersion>4.0.0</modelVersion>
        <parent>
            <groupId>com.atguigu.springcloud</groupId>
            <artifactId>cloud2020</artifactId>
            <version>1.0-SNAPSHOT</version>
        </parent>
    
        <artifactId>cloud-provider-hystrix-payment8001</artifactId>
    
        <dependencies>
            
            <dependency>
                <groupId>org.springframework.cloud</groupId>
                <artifactId>spring-cloud-starter-netflix-hystrix</artifactId>
            </dependency>
            
            <dependency>
                <groupId>org.springframework.cloud</groupId>
                <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
            </dependency>
            
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-web</artifactId>
            </dependency>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-actuator</artifactId>
            </dependency>
            <dependency>
                <groupId>com.atguigu.springcloud</groupId>
                <artifactId>cloud-api-commons</artifactId>
                <version>${project.version}</version>
            </dependency>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-devtools</artifactId>
                <scope>runtime</scope>
                <optional>true</optional>
            </dependency>
            <dependency>
                <groupId>org.projectlombok</groupId>
                <artifactId>lombok</artifactId>
                <optional>true</optional>
            </dependency>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-test</artifactId>
                <scope>test</scope>
            </dependency>
        </dependencies>
    
    </project>

1.  application.yml

server:
      port: 8001
    
    spring:
      application:
        name: cloud-provider-hystrix-payment
    
    eureka:
      client:
        register-with-eureka: true
        fetch-registry: true
        service-url:
          #defaultZone: http://eureka7001.com:7001/eureka,http://eureka7002.com:7002/eureka
          defaultZone: http://eureka7001.com:7001/eureka

1.  主启动类

package com.atguigu.springcloud;
    
    import org.springframework.boot.SpringApplication;
    import org.springframework.boot.autoconfigure.SpringBootApplication;
    import org.springframework.cloud.netflix.eureka.EnableEurekaClient;
    
    @SpringBootApplication
    @EnableEurekaClient
    public class PaymentHystrixMain8001 {
        
        public static void main(String[] args) {
        
            SpringApplication.run(PaymentHystrixMain8001.class, args);
        }
    }

1.  创建 service.PaymentService

package com.atguigu.springcloud.service;
    
    import lombok.extern.slf4j.Slf4j;
    import org.springframework.stereotype.Service;
    
    import java.util.concurrent.TimeUnit;
    
    @Service
    @Slf4j
    public class PaymentService {
        
        /**
         * 正常访问
         * @param id
         * @return
         */
        public String paymentInfo_OK(Integer id) {
        
            return "线程池： " + Thread.currentThread().getName() + " paymentInfo_OK, id: " + id + "\t" + "哈哈哈哈";
        }
    
        public String paymentInfo_TimeOut(Integer id) {
        
            int timeNumber = 3;
            try {
        
                TimeUnit.SECONDS.sleep(timeNumber);
            } catch (InterruptedException e) {
        
                e.printStackTrace();
            }
            return "线程池： " + Thread.currentThread().getName() + " paymentInfo_TimeOut, id: " + id + "\t" + "哈哈哈哈~~~" + " 耗时" + timeNumber + "秒钟";
        }
    }

1.  创建 controller.PaymenrController

package com.atguigu.springcloud.controller;
    
    import com.atguigu.springcloud.service.PaymentService;
    import lombok.extern.slf4j.Slf4j;
    import org.springframework.beans.factory.annotation.Value;
    import org.springframework.web.bind.annotation.PathVariable;
    import org.springframework.web.bind.annotation.RequestMapping;
    import org.springframework.web.bind.annotation.RestController;
    
    import javax.annotation.Resource;
    
    @RestController
    @Slf4j
    public class PaymentController {
        
        @Resource
        private PaymentService paymentService;
    
        @Value("${server.port}")
        private String serverPort;
    
        @RequestMapping("/payment/hystrix/ok/{id}")
        public String payment_OK(@PathVariable("id") Integer id) {
        
            String result = paymentService.paymentInfo_OK(id);
            log.info("***** result: " + result);
            return result;
        }
    
        @RequestMapping("/payment/hystrix/timeout/{id}")
        public String payment_TimeOut(@PathVariable("id") Integer id) {
        
            String result = paymentService.paymentInfo_TimeOut(id);
            log.info("***** result: " + result);
            return result;
        }
    }

1.  运行测试，先启动 cloud-eureka-server7001 （这个是服务注册中心服务端），然后启动该项目 cloud-provider-hystrix-payment8001  
    7.1 http://localhost:8001/payment/hystrix/ok/1  
    ![在这里插入图片描述][bc5b62f331084ccc9c4f6ecfab6eb3c9.png]  
    7.2 http://localhost:8001/payment/hystrix/timeout/1  
    ![在这里插入图片描述][08f8773275b748e48529aeedf55581bf.png]  
    ![在这里插入图片描述][4050ada7561f4d0a86b28c18c9ddd83b.png]

### 高并发测试 ###

JMeter官网 ： https://jmeter.apache.org/index.html  
![在这里插入图片描述][3fdb65ab8ab74b5c87ee4d1be5d6067d.png]  
1.测试计划中右键添加-》线程-》线程组（线程组202102，线程数：200，线程数：100，其他参数默认）  
![在这里插入图片描述][921b8cfe0b134d1a9b741d92a7d0c34e.png]

2.刚刚新建线程组202102，右键它-》添加-》取样器-》Http请求-》基本 输入http://localhost:8001/payment/hystrix/timeout/31  
![在这里插入图片描述][e68678f86a924fd491808272a7abe6d4.png]  
![01804733386a47d8a53617440d0fc27e.png][]  
3.点击绿色三角形图标启动。  
![在这里插入图片描述][29725129334d439bbe1706ee42fb9ee2.png]  
原因：  
![在这里插入图片描述][bfa09fcb171a4d8c85887402d0bd41a6.png]  
结论：  
![在这里插入图片描述][64e1f27c102d4bc58404a369a7eec315.png]

### 构建项目80端 cloud-consumer-feign-hystrix-order80 ###

1.  pom.xml

<?xml version="1.0" encoding="UTF-8"?>
    <project xmlns="http://maven.apache.org/POM/4.0.0"
             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
             xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
        <modelVersion>4.0.0</modelVersion>
        <parent>
            <groupId>com.atguigu.springcloud</groupId>
            <artifactId>cloud2020</artifactId>
            <version>1.0-SNAPSHOT</version>
        </parent>
    
        <artifactId>cloud-consumer-feign-hystrix-order80</artifactId>
    
        <dependencies>
            
            <dependency>
                <groupId>org.springframework.cloud</groupId>
                <artifactId>spring-cloud-starter-openfeign</artifactId>
            </dependency>
            
            <dependency>
                <groupId>org.springframework.cloud</groupId>
                <artifactId>spring-cloud-starter-netflix-hystrix</artifactId>
            </dependency>
            
            <dependency>
                <groupId>org.springframework.cloud</groupId>
                <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
            </dependency>
            
            <dependency>
                <groupId>com.atguigu.springcloud</groupId>
                <artifactId>cloud-api-commons</artifactId>
                <version>1.0-SNAPSHOT</version>
            </dependency>
            
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-web</artifactId>
            </dependency>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-actuator</artifactId>
            </dependency>
            
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-devtools</artifactId>
                <scope>runtime</scope>
                <optional>true</optional>
            </dependency>
            <dependency>
                <groupId>org.projectlombok</groupId>
                <artifactId>lombok</artifactId>
                <optional>true</optional>
            </dependency>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-test</artifactId>
                <scope>test</scope>
            </dependency>
        </dependencies>
    
    </project>

1.  appilcation.yml

server:
      port: 80
    
    eureka:
      client:
        register-with-eureka: false
        service-url:
          defaultZone: http://eureka7001.com:7001/eureka/

1.  主启动类

package com.atguigu.springcloud;
    
    import org.springframework.boot.SpringApplication;
    import org.springframework.boot.autoconfigure.SpringBootApplication;
    import org.springframework.cloud.netflix.eureka.EnableEurekaClient;
    import org.springframework.cloud.openfeign.EnableFeignClients;
    
    @SpringBootApplication
    @EnableFeignClients
    public class OrderHystrixMain80 {
        
        public static void main(String[] args) {
        
            SpringApplication.run(OrderHystrixMain80.class, args);
        }
    }

1.  业务类 service.PaymentHystrixService，PaymentHystrixService是一个接口

package com.atguigu.springcloud.service;
    
    import org.springframework.cloud.openfeign.FeignClient;
    import org.springframework.stereotype.Component;
    import org.springframework.web.bind.annotation.PathVariable;
    import org.springframework.web.bind.annotation.RequestMapping;
    
    @Component
    @FeignClient(value = "CLOUD-PROVIDER-HYSTRIX-PAYMENT")
    public interface PaymentHystrixService{
        
        @RequestMapping("/payment/hystrix/ok/{id}")
        public String payment_OK(@PathVariable("id") Integer id);
    
        @RequestMapping("/payment/hystrix/timeout/{id}")
        public String payment_TimeOut(@PathVariable("id") Integer id);
    }

1.  创建 controller.OrderHystrixController

package com.atguigu.springcloud.controller;
    
    import com.atguigu.springcloud.service.PaymentHystrixService;
    import lombok.extern.slf4j.Slf4j;
    import org.springframework.web.bind.annotation.GetMapping;
    import org.springframework.web.bind.annotation.PathVariable;
    import org.springframework.web.bind.annotation.RestController;
    
    import javax.annotation.Resource;
    
    @RestController
    @Slf4j
    public class OrderHystrixController {
        
        @Resource
        private PaymentHystrixService paymentHystrixService;
    
        @GetMapping("/consumer/payment/hystrix/ok/{id}")
        public String paymentInfo_OK(@PathVariable("id") Integer id)
        {
        
            String result = paymentHystrixService.payment_OK(id);
            return result;
        }
    
        @GetMapping("/consumer/payment/hystrix/timeout/{id}")
        public String paymentInfo_TimeOut(@PathVariable("id") Integer id) {
        
            String result = paymentHystrixService.payment_TimeOut(id);
            return result;
        }
    }

1.  运行测试  
    6.1 正常测试： http://localhost/consumer/payment/hystrix/ok/1  
    ![在这里插入图片描述][5d328d05345e4fe9a01cbc0bf98a785c.png]

6.2 高并发测试  
![在这里插入图片描述][c945bbb74d1945f6b0213da017107bb7.png]  
消费端超时错误：  
![在这里插入图片描述][0db09c3c5d164044b67e21c1197071ee.png]  
消费者80被拖慢：  
![在这里插入图片描述][89fdf6c2f61c4b019ea6c0e737666420.png]  
![在这里插入图片描述][a7953354797d42b1ad38bd03934b535d.png]

### 降级容错解决的维度要求 ###

![在这里插入图片描述][de706a084bb34bc5a6d2a48235257dca.png]  
![在这里插入图片描述][a6d00453aeb0421b962644818574876e.png]  
超时导致服务器变慢(转圈) - 超时不再等待

出错(宕机或程序运行出错) - 出错要有兜底

解决：

*  对方服务(8001)超时了，调用者(80)不能一直卡死等待，必须有服务降级。
 *  对方服务(8001)down机了，调用者(80)不能一直卡死等待，必须有服务降级。
 *  对方服务(8001)OK，调用者(80)自己出故障或有自我要求(自己的等待时间小于服务提供者)，自己处理降级。

### 服务降级的配置 ###

#### 先从 8001 服务提供者找问题 ####

![在这里插入图片描述][aeb5bbf66fb4436ea93667fcacd67116.png]  
![在这里插入图片描述][eee5fb10fb074ba8bbe4f560160c24ca.png]

package com.atguigu.springcloud.service;
    
    import com.netflix.hystrix.contrib.javanica.annotation.HystrixCommand;
    import com.netflix.hystrix.contrib.javanica.annotation.HystrixProperty;
    import lombok.extern.slf4j.Slf4j;
    import org.springframework.stereotype.Service;
    
    import java.util.concurrent.TimeUnit;
    
    @Service
    @Slf4j
    public class PaymentService {
        
        /**
         * 正常访问
         * @param id
         * @return
         */
        public String paymentInfo_OK(Integer id) {
        
            return "线程池： " + Thread.currentThread().getName() + " paymentInfo_OK, id: " + id + "\t" + "哈哈哈哈";
        }
    
        // 兜底的注解，paymentInfo_TimeOutHandler 是兜底的方法名
        // 线程名称 execution.isolation.thread.timeoutInMilliseconds
        // value : 设置超时时间
        @HystrixCommand(fallbackMethod = "paymentInfo_TimeOutHandler", commandProperties = {
        
                @HystrixProperty(name = "execution.isolation.thread.timeoutInMilliseconds", value = "3000")
        })
        public String paymentInfo_TimeOut(Integer id) {
        
            int timeNumber = 5;
            try {
        
                TimeUnit.SECONDS.sleep(timeNumber);
            } catch (InterruptedException e) {
        
                e.printStackTrace();
            }
            return "线程池： " + Thread.currentThread().getName() + " paymentInfo_TimeOut, id: " + id + "\t" + "哈哈哈哈~~~" + " 耗时" + timeNumber + "秒钟";
        }
    
        public String paymentInfo_TimeOutHandler(Integer id) {
        
            return "线程池： " + Thread.currentThread().getName() + " 系统繁忙，请稍后再试, id: " + id + "\t" + "哈哈哈哈";
        }
    }

![在这里插入图片描述][b41e6de9b2944b688f23a30f1ade275b.png]

package com.atguigu.springcloud;
    
    import org.springframework.boot.SpringApplication;
    import org.springframework.boot.autoconfigure.SpringBootApplication;
    import org.springframework.cloud.client.circuitbreaker.EnableCircuitBreaker;
    import org.springframework.cloud.netflix.eureka.EnableEurekaClient;
    
    @SpringBootApplication
    @EnableEurekaClient
    // @EnableCircuitBreaker 激活 hystrix 的 服务降级方法
    @EnableCircuitBreaker
    public class PaymentHystrixMain8001 {
        
        public static void main(String[] args) {
        
            SpringApplication.run(PaymentHystrixMain8001.class, args);
        }
    }

运行检测  
![在这里插入图片描述][6d1e82c7ac744b55972449ef2d2d429d.png]  
![在这里插入图片描述][98c1082f0c73453d87ae5521d4808c10.png]

#### 从 80 端服务消费者找问题 ####

![在这里插入图片描述][58b2c178f0704b9c86af7f0e328f05a8.png]  
在 application.yml

server:
      port: 80
    
    eureka:
      client:
        register-with-eureka: false
        service-url:
          defaultZone: http://eureka7001.com:7001/eureka/
    
    # 在 feign 中开启 hystrix
    feign:
      hystrix:
        enabled: true

在主启动类  
![在这里插入图片描述][04480185ee13448582bb0613e3810259.png]

package com.atguigu.springcloud;
    
    import org.springframework.boot.SpringApplication;
    import org.springframework.boot.autoconfigure.SpringBootApplication;
    import org.springframework.cloud.netflix.eureka.EnableEurekaClient;
    import org.springframework.cloud.netflix.hystrix.EnableHystrix;
    import org.springframework.cloud.openfeign.EnableFeignClients;
    
    @SpringBootApplication
    @EnableFeignClients
    // @EnableHystrix 在 feign 中开启 hystrix 服务降级
    @EnableHystrix
    public class OrderHystrixMain80 {
        public static void main(String[] args) {
            SpringApplication.run(OrderHystrixMain80.class, args);
        }
    }

在业务类

package com.atguigu.springcloud.controller;
    
    import com.atguigu.springcloud.service.PaymentHystrixService;
    import com.netflix.hystrix.contrib.javanica.annotation.HystrixCommand;
    import com.netflix.hystrix.contrib.javanica.annotation.HystrixProperty;
    import lombok.extern.slf4j.Slf4j;
    import org.springframework.web.bind.annotation.GetMapping;
    import org.springframework.web.bind.annotation.PathVariable;
    import org.springframework.web.bind.annotation.RestController;
    
    import javax.annotation.Resource;
    
    @RestController
    @Slf4j
    public class OrderHystrixController {
        
        @Resource
        private PaymentHystrixService paymentHystrixService;
    
        @GetMapping("/consumer/payment/hystrix/ok/{id}")
        public String paymentInfo_OK(@PathVariable("id") Integer id)
        {
        
            String result = paymentHystrixService.payment_OK(id);
            return result;
        }
    
        @GetMapping("/consumer/payment/hystrix/timeout/{id}")
        @HystrixCommand(fallbackMethod = "paymentTimeOutFallbackMethod",commandProperties = {
        
                @HystrixProperty(name="execution.isolation.thread.timeoutInMilliseconds",value="1500")
        })
        public String paymentInfo_TimeOut(@PathVariable("id") Integer id) {
        
            String result = paymentHystrixService.payment_TimeOut(id);
            return result;
        }
    
        public String paymentTimeOutFallbackMethod(@PathVariable("id") Integer id){
        
            return "我是消费者80,对方支付系统繁忙请10秒钟后再试或者自己运行出错请检查自己,o(╥﹏╥)o";
        }
    }

运行检测结果  
![在这里插入图片描述][109c2b8f89ef4bf5955c8bd8088d9fa2.png]

#### 目前出现的问题 ####

##### 问题1 #####

**目前问题**： 每个业务方法对应一个兜底的方法，代码膨胀  
**解决方法**：

*  1:1每个方法配置一个服务降级方法，技术上可以，但是不聪明
 *  1:N 除了个别重要核心业务有专属，其它普通的可以通过@DefaultProperties(defaultFallback = “”)统一跳转到统一处理结果页面

@DefaultProperties(defaultFallback = “”) 配置在类上，全局通用

通用的和独享的各自分开，避免了代码膨胀，合理减少了代码量  
![在这里插入图片描述][626bfa1d13e14a2daefc0894b8b9c6d3.png]

package com.atguigu.springcloud.controller;
    
    import com.atguigu.springcloud.service.PaymentHystrixService;
    import com.netflix.hystrix.contrib.javanica.annotation.DefaultProperties;
    import com.netflix.hystrix.contrib.javanica.annotation.HystrixCommand;
    import com.netflix.hystrix.contrib.javanica.annotation.HystrixProperty;
    import lombok.extern.slf4j.Slf4j;
    import org.springframework.web.bind.annotation.GetMapping;
    import org.springframework.web.bind.annotation.PathVariable;
    import org.springframework.web.bind.annotation.RestController;
    
    import javax.annotation.Resource;
    
    @RestController
    @Slf4j
    @DefaultProperties(defaultFallback = "payment_Global_FallbackMethod")
    public class OrderHystrixController {
        
        @Resource
        private PaymentHystrixService paymentHystrixService;
    
        @GetMapping("/consumer/payment/hystrix/ok/{id}")
        public String paymentInfo_OK(@PathVariable("id") Integer id)
        {
        
            String result = paymentHystrixService.payment_OK(id);
            return result;
        }
    
        @GetMapping("/consumer/payment/hystrix/timeout/{id}")
    //    @HystrixCommand(fallbackMethod = "paymentTimeOutFallbackMethod",commandProperties = {
        
    //            @HystrixProperty(name="execution.isolation.thread.timeoutInMilliseconds",value="1500")
    //    })
        @HystrixCommand
        public String paymentInfo_TimeOut(@PathVariable("id") Integer id) {
        
            String result = paymentHystrixService.payment_TimeOut(id);
            return result;
        }
    
        // 下面是全局fallback方法
        public String payment_Global_FallbackMethod()
        {
        
            return "Global异常处理信息，请稍后再试，/(ㄒoㄒ)/~~";
        }
    
        public String paymentTimeOutFallbackMethod(@PathVariable("id") Integer id){
        
            return "我是消费者80,对方支付系统繁忙请10秒钟后再试或者自己运行出错请检查自己,o(╥﹏╥)o";
        }
    
    }

##### 问题2 #####

\*\*目前问题：\*\*服务降级方法和业务逻辑代码混在一起，代码混乱  
![在这里插入图片描述][0f1f3b67b76f4329b5b26d35bc190177.png]  
![在这里插入图片描述][e04538fb12ff44ccbd01663e75a01627.png]  
**再看我们的业务类 PaymentController :**  
![在这里插入图片描述][cbc55a25599146fdbb8d4ac2f77f5266.png]

1.  修改 cloud-consumer-feign-hystrix-order80  
    ![在这里插入图片描述][dbcb78b84ed6475db37e70eb8a22ad15.png]  
    PaymentHystrixService 接口

package com.atguigu.springcloud.service;
    
    import org.springframework.cloud.openfeign.FeignClient;
    import org.springframework.stereotype.Component;
    import org.springframework.web.bind.annotation.PathVariable;
    import org.springframework.web.bind.annotation.RequestMapping;
    
    @Component
    // fallback = PaymentFallbackService.class 指定为 PaymentFallbackService 类
    @FeignClient(value = "CLOUD-PROVIDER-HYSTRIX-PAYMENT", fallback = PaymentFallbackService.class)
    public interface PaymentHystrixService {
        
        @RequestMapping("/payment/hystrix/ok/{id}")
        public String payment_OK(@PathVariable("id") Integer id);
    
        @RequestMapping("/payment/hystrix/timeout/{id}")
        public String payment_TimeOut(@PathVariable("id") Integer id);
    }

PaymentFallbackService 类

package com.atguigu.springcloud.service;
    
    import org.springframework.stereotype.Component;
    
    @Component
    public class PaymentFallbackService implements PaymentHystrixService{
        
        @Override
        public String payment_OK(Integer id) {
        
            return "-----PaymentFallbackService fall back-paymentInfo_OK ,o(╥﹏╥)o";
        }
    
        @Override
        public String payment_TimeOut(Integer id) {
        
            return "-----PaymentFallbackService fall back-paymentInfo_TimeOut ,o(╥﹏╥)o";
        }
    }

1.  在 application.yml 添加以下代码

server:
      port: 80
    
    eureka:
      client:
        register-with-eureka: false
        service-url:
          defaultZone: http://eureka7001.com:7001/eureka/
    
    # 新添加的
    # 在 feign 中开启 hystrix
    feign:
      hystrix:
        enabled: true

1.  运行测试  
    3.1 正常测试访问: http://localhost/consumer/payment/hystrix/ok/1  
    ![在这里插入图片描述][7bd8389707bc42b9a69b402055299b97.png]  
    3.2 将 cloud-provider-hystrix-payment8001 暂停，再访问： http://localhost/consumer/payment/hystrix/ok/1  
    ![在这里插入图片描述][46ec611c3f7441d0a2be521a574314b3.png]  
    ![在这里插入图片描述][81d2b003da3046719978091c7b995530.png]

### 服务熔断 ###

![在这里插入图片描述][05e4d80dafe54cf4bdc730dd1d021881.png]

#### 实操 ####

1.  修改 cloud-provider-hystrix-payment8001 的 PaymentService

package com.atguigu.springcloud.service;
    
    import cn.hutool.core.util.IdUtil;
    import com.netflix.hystrix.contrib.javanica.annotation.HystrixCommand;
    import com.netflix.hystrix.contrib.javanica.annotation.HystrixProperty;
    import lombok.extern.slf4j.Slf4j;
    import org.springframework.stereotype.Service;
    import org.springframework.web.bind.annotation.PathVariable;
    
    import java.util.concurrent.TimeUnit;
    
    @Service
    @Slf4j
    public class PaymentService {
        
        /**
         * 正常访问
         * @param id
         * @return
         */
        public String paymentInfo_OK(Integer id) {
        
            return "线程池： " + Thread.currentThread().getName() + " paymentInfo_OK, id: " + id + "\t" + "哈哈哈哈";
        }
    
        // 兜底的注解，paymentInfo_TimeOutHandler 是兜底的方法名
        // 线程名称 execution.isolation.thread.timeoutInMilliseconds
        // value : 设置超时时间
        @HystrixCommand(fallbackMethod = "paymentInfo_TimeOutHandler", commandProperties = {
        
                @HystrixProperty(name = "execution.isolation.thread.timeoutInMilliseconds", value = "5000")
        })
        public String paymentInfo_TimeOut(Integer id) {
        
            int timeNumber = 3;
            try {
        
                TimeUnit.SECONDS.sleep(timeNumber);
            } catch (InterruptedException e) {
        
                e.printStackTrace();
            }
            return "线程池： " + Thread.currentThread().getName() + " paymentInfo_TimeOut, id: " + id + "\t" + "哈哈哈哈~~~" + " 耗时" + timeNumber + "秒钟";
        }
    
        public String paymentInfo_TimeOutHandler(Integer id) {
        
            return "线程池： " + Thread.currentThread().getName() + " 8001系统繁忙，请稍后再试, id: " + id + "\t" + "哈哈哈哈";
        }
    
        // ===== 服务熔断
        @HystrixCommand(fallbackMethod = "paymentCircuitBreaker_fallback", commandProperties = {
        
                @HystrixProperty(name = "circuitBreaker.enabled",value = "true"),// 是否开启断路器
                @HystrixProperty(name = "circuitBreaker.requestVolumeThreshold",value = "10"),// 请求次数
                @HystrixProperty(name = "circuitBreaker.sleepWindowInMilliseconds",value = "10000"), // 时间窗口期
                @HystrixProperty(name = "circuitBreaker.errorThresholdPercentage",value = "60"),// 失败率达到多少后跳闸
        })
        public String paymentCircuitBreaker(@PathVariable("id") Integer id) {
        
            if(id < 0) {
        
                throw new RuntimeException("******id 不能负数");
            }
    
            // IdUtil.simpleUUID() 等价于 UUID.randomUUID().toString()
            String serialNumber = IdUtil.simpleUUID();
    
            return Thread.currentThread().getName()+"\t"+"调用成功，流水号: " + serialNumber;
        }
        public String paymentCircuitBreaker_fallback(@PathVariable("id") Integer id) {
        
            return "id 不能负数，请稍后再试，/(ㄒoㄒ)/~~   id: " +id;
        }
    }

1.  修改 cloud-provider-hystrix-payment8001 的 PaymentController

package com.atguigu.springcloud.controller;
    
    import com.atguigu.springcloud.service.PaymentService;
    import com.netflix.hystrix.contrib.javanica.annotation.DefaultProperties;
    import com.netflix.hystrix.contrib.javanica.annotation.HystrixCommand;
    import lombok.extern.slf4j.Slf4j;
    import org.springframework.beans.factory.annotation.Value;
    import org.springframework.web.bind.annotation.GetMapping;
    import org.springframework.web.bind.annotation.PathVariable;
    import org.springframework.web.bind.annotation.RequestMapping;
    import org.springframework.web.bind.annotation.RestController;
    
    import javax.annotation.Resource;
    
    @RestController
    @Slf4j
    public class PaymentController {
        
        @Resource
        private PaymentService paymentService;
    
        @Value("${server.port}")
        private String serverPort;
    
        @RequestMapping("/payment/hystrix/ok/{id}")
        public String payment_OK(@PathVariable("id") Integer id) {
        
            String result = paymentService.paymentInfo_OK(id);
            log.info("***** result: " + result);
            return result;
        }
    
        @RequestMapping("/payment/hystrix/timeout/{id}")
        public String payment_TimeOut(@PathVariable("id") Integer id) {
        
            String result = paymentService.paymentInfo_TimeOut(id);
            log.info("***** result: " + result);
            return result;
        }
    
        // ====== 服务熔断
        @GetMapping("/payment/circuit/{id}")
        public String paymentCircuitBreaker(@PathVariable("id") Integer id)
        {
        
            String result = paymentService.paymentCircuitBreaker(id);
            log.info("****result: "+result);
            return result;
        }
    }

1.  运行测试  
    3.1 正确的运行测试： http://localhost:8001/payment/circuit/1  
    ![在这里插入图片描述][42abd903e5884488a260faa61d7d92a5.png]

3.2 错误的运行测试： http://localhost:8001/payment/circuit/-1  
![在这里插入图片描述][ddbf6ed536684dc493179d85e48114a8.png]

#### 服务熔断总结 ####

![在这里插入图片描述][d99dd988f45b438a96d80d160859db18.png]  
![在这里插入图片描述][16b6eaae51854905b5f1e149f9cfc68a.png]

##### 断路器开启或关闭的条件 #####

![在这里插入图片描述][bc2c08875e0642f78a52dc2724bd9cb9.png]

##### 断路器打开后 #####

![在这里插入图片描述][eeb088e67eac467a9dc53fb1729cef6c.png]

##### All 配置 #####

@HystrixCommand(fallbackMethod = "fallbackMethod", 
                    groupKey = "strGroupCommand", 
                    commandKey = "strCommand", 
                    threadPoolKey = "strThreadPool",
                    
                    commandProperties = {
        
                        // 设置隔离策略，THREAD 表示线程池 SEMAPHORE：信号池隔离
                        @HystrixProperty(name = "execution.isolation.strategy", value = "THREAD"),
                        // 当隔离策略选择信号池隔离的时候，用来设置信号池的大小（最大并发数）
                        @HystrixProperty(name = "execution.isolation.semaphore.maxConcurrentRequests", value = "10"),
                        // 配置命令执行的超时时间
                        @HystrixProperty(name = "execution.isolation.thread.timeoutinMilliseconds", value = "10"),
                        // 是否启用超时时间
                        @HystrixProperty(name = "execution.timeout.enabled", value = "true"),
                        // 执行超时的时候是否中断
                        @HystrixProperty(name = "execution.isolation.thread.interruptOnTimeout", value = "true"),
                        
                        // 执行被取消的时候是否中断
                        @HystrixProperty(name = "execution.isolation.thread.interruptOnCancel", value = "true"),
                        // 允许回调方法执行的最大并发数
                        @HystrixProperty(name = "fallback.isolation.semaphore.maxConcurrentRequests", value = "10"),
                        // 服务降级是否启用，是否执行回调函数
                        @HystrixProperty(name = "fallback.enabled", value = "true"),
                        // 是否启用断路器
                        @HystrixProperty(name = "circuitBreaker.enabled", value = "true"),
                        // 该属性用来设置在滚动时间窗中，断路器熔断的最小请求数。例如，默认该值为 20 的时候，如果滚动时间窗（默认10秒）内仅收到了19个请求， 即使这19个请求都失败了，断路器也不会打开。
                        @HystrixProperty(name = "circuitBreaker.requestVolumeThreshold", value = "20"),
                        
                        // 该属性用来设置在滚动时间窗中，表示在滚动时间窗中，在请求数量超过 circuitBreaker.requestVolumeThreshold 的情况下，如果错误请求数的百分比超过50, 就把断路器设置为 "打开" 状态，否则就设置为 "关闭" 状态。
                        @HystrixProperty(name = "circuitBreaker.errorThresholdPercentage", value = "50"),
                        // 该属性用来设置当断路器打开之后的休眠时间窗。 休眠时间窗结束之后，会将断路器置为 "半开" 状态，尝试熔断的请求命令，如果依然失败就将断路器继续设置为 "打开" 状态，如果成功就设置为 "关闭" 状态。
                        @HystrixProperty(name = "circuitBreaker.sleepWindowinMilliseconds", value = "5000"),
                        // 断路器强制打开
                        @HystrixProperty(name = "circuitBreaker.forceOpen", value = "false"),
                        // 断路器强制关闭
                        @HystrixProperty(name = "circuitBreaker.forceClosed", value = "false"),
                        // 滚动时间窗设置，该时间用于断路器判断健康度时需要收集信息的持续时间
                        @HystrixProperty(name = "metrics.rollingStats.timeinMilliseconds", value = "10000"),
                        
                        // 该属性用来设置滚动时间窗统计指标信息时划分"桶"的数量，断路器在收集指标信息的时候会根据设置的时间窗长度拆分成多个 "桶" 来累计各度量值，每个"桶"记录了一段时间内的采集指标。
                        // 比如 10 秒内拆分成 10 个"桶"收集这样，所以 timeinMilliseconds 必须能被 numBuckets 整除。否则会抛异常
                        @HystrixProperty(name = "metrics.rollingStats.numBuckets", value = "10"),
                        // 该属性用来设置对命令执行的延迟是否使用百分位数来跟踪和计算。如果设置为 false, 那么所有的概要统计都将返回 -1。
                        @HystrixProperty(name = "metrics.rollingPercentile.enabled", value = "false"),
                        // 该属性用来设置百分位统计的滚动窗口的持续时间，单位为毫秒。
                        @HystrixProperty(name = "metrics.rollingPercentile.timeInMilliseconds", value = "60000"),
                        // 该属性用来设置百分位统计滚动窗口中使用 “ 桶 ”的数量。
                        @HystrixProperty(name = "metrics.rollingPercentile.numBuckets", value = "60000"),
                        // 该属性用来设置在执行过程中每个 “桶” 中保留的最大执行次数。如果在滚动时间窗内发生超过该设定值的执行次数，
                        // 就从最初的位置开始重写。例如，将该值设置为100, 滚动窗口为10秒，若在10秒内一个 “桶 ”中发生了500次执行，
                        // 那么该 “桶” 中只保留 最后的100次执行的统计。另外，增加该值的大小将会增加内存量的消耗，并增加排序百分位数所需的计算时间。
                        @HystrixProperty(name = "metrics.rollingPercentile.bucketSize", value = "100"),
                        
                        // 该属性用来设置采集影响断路器状态的健康快照（请求的成功、 错误百分比）的间隔等待时间。
                        @HystrixProperty(name = "metrics.healthSnapshot.intervalinMilliseconds", value = "500"),
                        // 是否开启请求缓存
                        @HystrixProperty(name = "requestCache.enabled", value = "true"),
                        // HystrixCommand的执行和事件是否打印日志到 HystrixRequestLog 中
                        @HystrixProperty(name = "requestLog.enabled", value = "true"),
    
                    },
                    threadPoolProperties = {
        
                        // 该参数用来设置执行命令线程池的核心线程数，该值也就是命令执行的最大并发量
                        @HystrixProperty(name = "coreSize", value = "10"),
                        // 该参数用来设置线程池的最大队列大小。当设置为 -1 时，线程池将使用 SynchronousQueue 实现的队列，否则将使用 LinkedBlockingQueue 实现的队列。
                        @HystrixProperty(name = "maxQueueSize", value = "-1"),
                        // 该参数用来为队列设置拒绝阈值。 通过该参数， 即使队列没有达到最大值也能拒绝请求。
                        // 该参数主要是对 LinkedBlockingQueue 队列的补充,因为 LinkedBlockingQueue 队列不能动态修改它的对象大小，而通过该属性就可以调整拒绝请求的队列大小了。
                        @HystrixProperty(name = "queueSizeRejectionThreshold", value = "5"),
                    }
                   )
    public String doSomething() {
        
    	...
    }

## Hystrix 的工作流程 ##

![在这里插入图片描述][c187d71704714ce38dab7f38cf3fc922.png]

## Hystrix图形化Dashboard搭建 ##

### 概念 ###

![在这里插入图片描述][a1a7c0df0b864c3eab699088f84b4d26.png]

### 搭建项目 cloud-consumer-hystrix-dashboard9001 仪表盘 ###

1.  pom.xml

1.  application.yml

server:
      port: 9001

1.  主启动类，添加 @EnableHystrixDashboard 注解

package com.atguigu.springcloud;
    
    import org.springframework.boot.SpringApplication;
    import org.springframework.boot.autoconfigure.SpringBootApplication;
    import org.springframework.cloud.netflix.hystrix.dashboard.EnableHystrixDashboard;
    
    @SpringBootApplication
    // @EnableHystrixDashboard 开启仪表盘注解
    @EnableHystrixDashboard
    public class HystrixDashboardMain9001 {
        
        public static void main(String[] args) {
        
            SpringApplication.run(HystrixDashboardMain9001.class, args);
        }
    }

1.  所有Provider微服务提供类(8001/8002/8003)都需要监控依赖配置

<dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-actuator</artifactId>
    </dependency>

1.  启动 cloud-consumer-hystrix-dashboard9001，监控微服务8001，访问： http://localhost:9001/hystrix  
    ![在这里插入图片描述][73673d2c0708469ba435e2986933696e.png]

### Hystrix图形化Dashboard监控实战 ###

1.  修改cloud-provider-hystrix-payment8001，在主启动类中  
    ![在这里插入图片描述][779529531b1e4775b51e999c5748b7a8.png]

package com.atguigu.springcloud;
    
    import com.netflix.hystrix.contrib.metrics.eventstream.HystrixMetricsStreamServlet;
    import org.springframework.boot.SpringApplication;
    import org.springframework.boot.autoconfigure.SpringBootApplication;
    import org.springframework.boot.web.servlet.ServletRegistrationBean;
    import org.springframework.cloud.client.circuitbreaker.EnableCircuitBreaker;
    import org.springframework.cloud.netflix.eureka.EnableEurekaClient;
    import org.springframework.context.annotation.Bean;
    
    @SpringBootApplication
    @EnableEurekaClient
    // @EnableCircuitBreaker 激活 hystrix 的 服务降级方法
    @EnableCircuitBreaker
    public class PaymentHystrixMain8001 {
        
        public static void main(String[] args) {
        
            SpringApplication.run(PaymentHystrixMain8001.class, args);
        }
    
        /**
         *此配置是为了服务监控而配置，与服务容错本身无关，springcloud升级后的坑
         *ServletRegistrationBean因为springboot的默认路径不是"/hystrix.stream"，
         *只要在自己的项目里配置上下面的servlet就可以了
         *否则，Unable to connect to Command Metric Stream 404
         */
        @Bean
        public ServletRegistrationBean getServlet() {
        
            HystrixMetricsStreamServlet streamServlet = new HystrixMetricsStreamServlet();
            ServletRegistrationBean registrationBean = new ServletRegistrationBean(streamServlet);
            registrationBean.setLoadOnStartup(1);
            registrationBean.addUrlMappings("/hystrix.stream");
            registrationBean.setName("HystrixMetricsStreamServlet");
            return registrationBean;
        }
    }

1.  监控测试，端口9001检测端口8001  
    ![在这里插入图片描述][f718aee4988f49a08e073d2aaecedc35.png]  
    测试地址

*  http://localhost:8001/payment/circuit/1
 *  http://localhost:8001/payment/circuit/-1
 *  测试通过
 *  先访问正确地址，再访问错误地址，再正确地址，会发现图示断路器都是慢慢放开的。  
    ![在这里插入图片描述][70ab9c16e992479989d283c87dfd25da.png]  
    ![在这里插入图片描述][44d076ab202c47c9ba8c9cd9d3592f41.png]  
    ![在这里插入图片描述][414c00aac7ed4a829457161c31324a8a.png]  
    ![在这里插入图片描述][8f04160331b047ef85e372b2e47f9e20.png]  
    ![在这里插入图片描述][99e0ecb1779043fc864544a0082b4305.png]

[075266d455eb4303ba7094db68850413.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/5f6dbd6c27e8438a841c880aece0ef56.png
[13e1d1d6d0a7435ca8c28ed3f4e6b04c.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/e13bd49387c74f7eb810dd96b0ce4d43.png
[30918557afad438295c01ed30bb4d108.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/ea6c0b7f5ce047f99eb7c4e2aa903e7a.png
[4c51aab2015145098039b2c60764ce13.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/7932a9d0befd4954b8cdb3d7c815edfa.png
[51c45403baa64b5f95747f36ecc2555e.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/68163518a57c48338d4a2ef30a440a86.png
[85286a95bf804744b16545030a908296.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/8dbbabf26be8490e8226a88f9079a2d5.png
[89e0ef974a304e998b71f71aee3fcd66.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/4f2bc0c5b4af41b9a913ab533e05bea8.png
[bc5b62f331084ccc9c4f6ecfab6eb3c9.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/7d730a5a3e5e4a0884090b8b0d1ca210.png
[08f8773275b748e48529aeedf55581bf.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/e2c64422b91a44a99f62ed04bd739574.png
[4050ada7561f4d0a86b28c18c9ddd83b.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/3889aebb161143e1882286dfa820953d.png
[3fdb65ab8ab74b5c87ee4d1be5d6067d.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/d61ff3249d194c6aae8a59a064f9a116.png
[921b8cfe0b134d1a9b741d92a7d0c34e.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/338ad240bf384d1fbf50f61dd6243047.png
[e68678f86a924fd491808272a7abe6d4.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/fd1d06ddfc364f0f8c1d66750114f064.png
[01804733386a47d8a53617440d0fc27e.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/bc2a816ae60c475086e6af5a3044d4ee.png
[29725129334d439bbe1706ee42fb9ee2.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/965765021a314c2e9f8cde58a08a957a.png
[bfa09fcb171a4d8c85887402d0bd41a6.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/b61bc8ddb3d442ccaa4ffd946c84462e.png
[64e1f27c102d4bc58404a369a7eec315.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/dc086952ceea44d5bbfafa94c4077dbc.png
[5d328d05345e4fe9a01cbc0bf98a785c.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/77bfa036b45d41fbb205879a98e3065d.png
[c945bbb74d1945f6b0213da017107bb7.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/d528c4677f684c59bf6f4d9e64f05f07.png
[0db09c3c5d164044b67e21c1197071ee.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/bf02781c0f57418db62dc1ab9c805e09.png
[89fdf6c2f61c4b019ea6c0e737666420.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/d49308d590ee4bc49306bffd675342a5.png
[a7953354797d42b1ad38bd03934b535d.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/33c5bcac2c5543e1a5d6306518f92210.png
[de706a084bb34bc5a6d2a48235257dca.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/e24aa7d4b92e475486be58fe52602afb.png
[a6d00453aeb0421b962644818574876e.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/4dcde532390c4de8a91ffd9892a7fab9.png
[aeb5bbf66fb4436ea93667fcacd67116.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/16cac7c48c8c41d8b93cf5689d387c12.png
[eee5fb10fb074ba8bbe4f560160c24ca.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/7e1f16ff2dfe4bdea290fa21127fdde4.png
[b41e6de9b2944b688f23a30f1ade275b.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/d9f9014c0c2a484db6ff9cdb9a543e63.png
[6d1e82c7ac744b55972449ef2d2d429d.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/03228a3482634394b3ddac9a2a480b76.png
[98c1082f0c73453d87ae5521d4808c10.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/1b21c4a4f895472d82d1f220e9c0cc7c.png
[58b2c178f0704b9c86af7f0e328f05a8.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/c47524fec4f94d528fc17889fc96500e.png
[04480185ee13448582bb0613e3810259.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/629fab2632454ba28f5d1db70d98dc06.png
[109c2b8f89ef4bf5955c8bd8088d9fa2.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/ec105cb7111d48c09752366057e153d1.png
[626bfa1d13e14a2daefc0894b8b9c6d3.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/f6e073090d65432398fe3d19ef7be9f2.png
[0f1f3b67b76f4329b5b26d35bc190177.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/31c6e5d26da1403b9b95ca28b308d62d.png
[e04538fb12ff44ccbd01663e75a01627.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/eef6652ff302488892281a59bececc65.png
[cbc55a25599146fdbb8d4ac2f77f5266.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/dcd2584a27c049dbba4a490800859670.png
[dbcb78b84ed6475db37e70eb8a22ad15.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/dd398ef399874f7d843640072c914b88.png
[7bd8389707bc42b9a69b402055299b97.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/14f5a60bd554432182f44476da087f72.png
[46ec611c3f7441d0a2be521a574314b3.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/9586e5d7d0214b63bb8fe2fcdafde2d2.png
[81d2b003da3046719978091c7b995530.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/0710d97090c14d1a82af2049b0dde959.png
[05e4d80dafe54cf4bdc730dd1d021881.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/374ee44e19534c598e9cddb69d81143f.png
[42abd903e5884488a260faa61d7d92a5.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/090d923578934f6e942c689770ef4f2c.png
[ddbf6ed536684dc493179d85e48114a8.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/6575a668b3214edeb56bc208b29d1ec1.png
[d99dd988f45b438a96d80d160859db18.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/d2c7e6c3784741669b2c6988acd23f02.png
[16b6eaae51854905b5f1e149f9cfc68a.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/03d1cacb5fe94c97b0eaca90ee9123c9.png
[bc2c08875e0642f78a52dc2724bd9cb9.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/6cc4622076fa4b0bbc38c1963453aa52.png
[eeb088e67eac467a9dc53fb1729cef6c.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/4bafa9384b4945788400784fb5570526.png
[c187d71704714ce38dab7f38cf3fc922.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/2a89d9513fc24dd3a3126262c434de58.png
[a1a7c0df0b864c3eab699088f84b4d26.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/bf2a06d50978489598a862143f43628b.png
[73673d2c0708469ba435e2986933696e.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/8954f5b31c5841cbbd0bfdce168c28e2.png
[779529531b1e4775b51e999c5748b7a8.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/2384d8f1dd734ca08dfb6ddb6c6c8d07.png
[f718aee4988f49a08e073d2aaecedc35.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/ee6014b4e51f4c48b35da94e38ec040f.png
[70ab9c16e992479989d283c87dfd25da.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/3147ffff2cf44218af2e98b3c04bdf4c.png
[44d076ab202c47c9ba8c9cd9d3592f41.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/4b6385d41c1c4dfb9706f08cb7a01171.png
[414c00aac7ed4a829457161c31324a8a.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/02e5bd421b1c4659af233f02e71e192a.png
[8f04160331b047ef85e372b2e47f9e20.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/7cba3b9fb6de42f7ba9dcc2a367de853.png
[99e0ecb1779043fc864544a0082b4305.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/14/310e56571d3641fd81ff08674d9bdc67.png