SpringBoot+log4j+logstash+Elasticsearch架构的日志分析

布满荆棘的人生 2022-06-08 09:39 762阅读 0赞

在用SpringBoot做微服务项目时，需要对程序的处理逻辑进行细化分析以及输出分析过程，基于此需求，所以选择采用log4j+logstash+Elasticsearch架构方式存储项目的详细处理业务以及逻辑的过程。具体的做法如下：

第一步：搭建SpringBoot Maven项目，日志收集采用log4j，pom.xml配置如下：

<?xml version="1.0" encoding="UTF-8"?>
    <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    
    <groupId>media_docment</groupId>
    <artifactId>media_docment</artifactId>
    <version>1.0</version>
    
    <properties>
    <java.version>1.8</java.version>
    <spring.boot.version>1.5.4.RELEASE</spring.boot.version>
    </properties>
    
    <dependencyManagement>
    <dependencies>
    <dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-dependencies</artifactId>
    <version>${spring.boot.version}</version>
    <type>pom</type>
    <scope>import</scope>
    </dependency>
    </dependencies>
    </dependencyManagement>
    
    <dependencies>
    
    <dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-web</artifactId>
    <exclusions>
    <exclusion>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-logging</artifactId>
    </exclusion>
    </exclusions>
    </dependency>
    
    <dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-log4j</artifactId>
    <version>1.3.8.RELEASE</version>
    </dependency>
    
    <dependency>
    <groupId>org.freemarker</groupId>
    <artifactId>freemarker</artifactId>
    <version>2.3.23</version>
    <scope>test</scope>
    </dependency>
    <dependency>
    <groupId>org.mybatis.generator</groupId>
    <artifactId>mybatis-generator-core</artifactId>
    <version>1.3.5</version>
    <scope>test</scope>
    </dependency>
    
    <dependency>
    <groupId>org.apache.mina</groupId>
    <artifactId>mina-core</artifactId>
    <version>2.0.16</version>
    </dependency>
    
    <dependency>
    <groupId>org.apache.mina</groupId>
    <artifactId>mina-integration-beans</artifactId>
    <version>2.0.16</version>
    <exclusions>
    <exclusion>
    <groupId>org.apache.mina</groupId>
    <artifactId>mina-core</artifactId>
    </exclusion>
    </exclusions>
    </dependency>
    
    
    
    <dependency>
    <groupId>tv.cntt</groupId>
    <artifactId>annovention</artifactId>
    <version>1.2</version>
    </dependency>
    
    <dependency>
    <groupId>org.elasticsearch</groupId>
    <artifactId>elasticsearch</artifactId>
    <version>2.2.0</version>
    </dependency>
    
    </dependencies>
    
    <build>
    <plugins>
    <plugin>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-maven-plugin</artifactId>
    <version>${spring.boot.version}</version>
    </plugin>
    <plugin>
    <artifactId>maven-compiler-plugin</artifactId>
    <configuration>
    <source>${java.version}</source>
    <target>${java.version}</target>
    <encoding>UTF-8</encoding>
    </configuration>
    </plugin>
    </plugins>
    </build>
    
    <repositories>
    <repository>
    <id>aliyun-repos</id>
    <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
    <snapshots>
    <enabled>false</enabled>
    </snapshots>
    </repository>
    </repositories>
    
    <pluginRepositories>
    <pluginRepository>
    <id>aliyun-plugin</id>
    <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
    <snapshots>
    <enabled>false</enabled>
    </snapshots>
    </pluginRepository>
    </pluginRepositories>
    
    </project>

项目中应用log4j的jar包：

import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    
    private static Logger logger = LoggerFactory.getLogger(MediaDocumentApplication.class);
    Map<String,String> map=new HashMap<String,String>();
     logger.info(JSON.toJSONString(map));

map的内容自己添加就行了。

log4j的配置文件内容如下：

# LOG4J  
    log4j.rootCategory=INFO,stdout,file,logstash
     
     
    # \u63a7\u5236\u53f0\u8f93\u51fa
    log4j.appender.stdout=org.apache.log4j.ConsoleAppender
    log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
    log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %5p %c{1}:%L - %m%n
     
    # root\u65e5\u5fd7\u8f93\u51fa
    log4j.appender.file=org.apache.log4j.RollingFileAppender 
    log4j.appender.file.file=logs/media.log
    log4j.appender.file.MaxFileSize=10240KB 
    log4j.appender.file.MaxBackupIndex=20
    log4j.appender.file.layout=org.apache.log4j.PatternLayout 
    log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH\:mm\:ss,SSS} [%t]  %-5p %c{1} %x - %m%n
    log4j.appender.file.append=true
     
    # logstash\u65e5\u5fd7\u8f93\u51fa\u5230ES
    log4j.appender.logstash=org.apache.log4j.net.SocketAppender
    log4j.appender.logstash.port=56789
    #log4j.appender.logstash.remoteHost=192.168.10.4
    log4j.appender.logstash.remoteHost=117.34.118.31
    log4j.appender.Logstash.Threshold=INFO  
    log4j.appender.Logstash.ReconnectionDelay=60000  
    log4j.appender.Logstash.LocationInfo=true

至此：SpringBoot项目的log4j日志就完成了。

第二步：安装ElasticSearch，本人是Linux上安装调试的，安装过程百度搜索一下或者在本人的博客中查找详细过程，这里不详细写了。然后配置插件，主要的有2个，head和sql插件。配置比较简单，下载相关的程序包，加压到 /usr/local/elasticsearch/plugins路径下即可使用：结果如下所示：

![Image 1][]

第三步：logstash的安装配置：

下载与ElasticSearch版本一致的logstash版本，解压：然后写配置信息：

input {
      stdin { }
      log4j {
        mode => "server"
            host  => "192.168.0.77"
            port => 56789
            type => "log4j"
          }
    }
    filter {
    
          
     json{
        source => "message"
        remove_field =>["message"] 
        remove_field =>["priority"] 
        remove_field =>["logger_name"] 
        remove_field =>["thread"] 
        remove_field =>["class"] 
        remove_field =>["file"] 
        remove_field =>["method"] 
         }
    
     if([esname]=~ "mid-log" )
           {
    
           }else {
           drop {}
           }
    
    }
    output {
      elasticsearch {
      hosts => "192.168.0.77:9200"
      index => "%{esname}-s-%{+YYYY.MM.dd}"
      flush_size => 1000 
      idle_flush_time => 5  
      }
    }

这是本人写的，具体按需求改动：完成之后在logstash根目录下运行启动logstash：

bin/logstash lib/log4j2.conf

log4j2.conf我放在了lib目录下；然后会输出启动成功的信息；

至此，改日志收集项目就完成了，经测试，项目运行正常，日志收集正常。

[Image 1]: