Springboot定时采集Hadoop的JMX信息，定制化监控

柔情只为你懂 2023-02-17 01:18 3阅读 0赞

## 背景： ##

Haddop自身会定时将元数据信息保存起来，通过WebUI的方式进行展现，例如：blocks、files......，通过WebUI的方式显然是不方便随时进行查看，并且对其进行监控，所以我们要对这些关键的元数据指标进行采集、存储，然后封装成服务，进行定制化监控。

## 实现： ##

[http://localhost:50070/jmx?qry=Hadoop:service=NameNode,name=FSNamesystem][http_localhost_50070_jmx_qry_Hadoop_service_NameNode_name_FSNamesystem]

创建Springboot项目，并添加几个依赖：

<dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-web</artifactId>
            </dependency>
    
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-data-jpa</artifactId>
            </dependency>
    
            <dependency>
                <groupId>mysql</groupId>
                <artifactId>mysql-connector-java</artifactId>
                <version>5.1.48</version>
            </dependency>
    
            <dependency>
                <groupId>com.alibaba</groupId>
                <artifactId>fastjson</artifactId>
                <version>1.2.68</version>
            </dependency>

在application.yml中配置mysql地址等信息

spring:
      datasource:
        url: jdbc:mysql://localhost:3306/hadoopplatform?characterEncoding=utf8&useSSL=false
        username: root
        password: "000000"
        driver-class-name: com.mysql.jdbc.Driver
    
      jpa:
        show-sql: true
        hibernate:
          ddl-auto: update

import com.fasterxml.jackson.annotation.JsonIgnore;
    
    import javax.persistence.MappedSuperclass;
    
    @MappedSuperclass
    public class BaseEntity {
    
    	private Integer createTime;
    
    	@JsonIgnore
    	private boolean isTrash = false;
        
        // 省略getter/setter
    }

import java.util.ArrayList;
    import java.util.List;
    import java.util.Map;
    
    public class HadoopMetics {
    
    	List<Map<String, Object>> beans = new ArrayList<>();
    
    	public Object getValue(String name) {
    		if (beans.isEmpty()) {
    			return null;
    		}
    		return beans.get(0).getOrDefault(name, null);
    	}
    }

import com.example.hadoopplatform.domain.Summary;
    import org.springframework.data.jpa.repository.JpaRepository;
    
    public interface SummaryRepository extends JpaRepository<Summary, Long> {
    }

import java.util.List;
    
    public interface MeticsService {
    
    	void addHDFSSummary(Summary summary);
    
    	Summary findSummary(int time);
    
    	List<Summary> Summary(int start, int end);
    }

import com.example.hadoopplatform.domain.Summary;
    import com.example.hadoopplatform.repository.SummaryRepository;
    import org.springframework.stereotype.Service;
    
    import javax.annotation.Resource;
    import java.util.List;
    
    @Service
    public class MeticsServiceImpl implements MeticsService{
    
    	@Resource
    	SummaryRepository summaryRepository;
    
    
    	@Override
    	public void addHDFSSummary(Summary summary) {
    		summaryRepository.save(summary);
    	}
    
    	@Override
    	public Summary findSummary(int time) {
    		return null;
    	}
    
    	@Override
    	public List<Summary> Summary(int start, int end) {
    		return null;
    	}
    }

import com.example.hadoopplatform.domain.HadoopMetics;
    import com.example.hadoopplatform.domain.Summary;
    import com.example.hadoopplatform.utils.HttpClientUtils;
    import org.springframework.scheduling.annotation.Scheduled;
    import org.springframework.stereotype.Component;
    
    import javax.annotation.Resource;
    
    @Component
    public class JmxScheduler {
    
    	HttpClientUtils client = new HttpClientUtils();
    
    	//	@Value("${hadoop.host}")
    //	private String url = "http://" + host + ':' + port + "/jmx?qry=" + qry;
    	private String url = "http://localhost:50070/jmx?qry=Hadoop:service=NameNode,name=FSNamesystem";
    
    	@Resource
    	MeticsService meticsService;
    
    	@Scheduled(cron = "0/2 * * * * *")
    	public void metics() {
    
    		HadoopMetics metics = client.get(url, HadoopMetics.class);
    
    		Summary s = new Summary();
    		s.setCreateTime((int) (System.currentTimeMillis() / 1000));
    		s.setTrash(false);
    		s.setBlocks(Long.parseLong(metics.getValue("BlocksTotal").toString()));
    		s.setFiles(Long.parseLong(metics.getValue("TotalFiles").toString()));
    
    		meticsService.addHDFSSummary(s);
    		System.out.println(s.toString());
    	}
    }

到此就能将jmx信息采集到MySQL中，定制化监控根据自身的需要实现服务，进行查询即可。

[http_localhost_50070_jmx_qry_Hadoop_service_NameNode_name_FSNamesystem]: http://localhost:50070/jmx?qry=Hadoop:service=NameNode,name=FSNamesystem