Elasticsearch-核心篇(12)-SpringBoot整合Logback&ES进行日志采集

小灰灰 2022-09-07 14:50 686阅读 0赞

### 文章目录 ###

*  一、Logback整合Elasticsearch
 *  二、logback-elasticsearch-appender

# 一、Logback整合Elasticsearch #

*  基础环境信息参考上一节文章配置好

1.  添加logback和Elasticsearch整合日志依赖
    
     *  源码地址：[https://gitee.com/tianxincoord/logback-elasticsearch-appender][https_gitee.com_tianxincoord_logback-elasticsearch-appender]

<dependency>
      <groupId>com.internetitem</groupId>
      <artifactId>logback-elasticsearch-appender</artifactId>
      <version>1.6</version>
    </dependency>

1.  创建定时日志类，用于定时写入日志，同时使用MDC记录需要核心信息

import java.time.LocalDateTime;
    import java.util.Random;
    import java.util.UUID;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    import org.slf4j.MDC;
    import org.springframework.scheduling.annotation.EnableScheduling;
    import org.springframework.scheduling.annotation.Scheduled;
    import org.springframework.stereotype.Component;
    
    @EnableScheduling
    @Component
    public class LogstashLogbackAggregation { 
    
        private static final Logger LOGGER = LoggerFactory.getLogger(LogstashLogbackAggregation.class);
        private static final Random RANDOM = new Random();
    
        @Scheduled(cron = "0/3 * * * * ?")
        public void logbackMdc() { 
            MDC.put("code", UUID.randomUUID().toString());
            MDC.put("createTime", LocalDateTime.now().toString());
            MDC.put("content", "logstash日志采集错误" + LocalDateTime.now());
            LOGGER.info("logstash日志采集，当前时间【{}】", LocalDateTime.now());
            MDC.clear();
            System.out.println(String.format("logstash日志采集，当前时间【%s】", LocalDateTime.now()));
        }
    }

1.  添加logback配置文件配置es相关信息

<?xml version="1.0" encoding="UTF-8"?>
    <configuration debug="true" scan="true" scanPeriod="1 seconds">
    
        <contextName>logback</contextName>
        
        <property name="log_pattern" value="%d{HH:mm:ss.SSS} %-5level [%thread] %logger{36} - %msg%n%ex"/>
    
        
        <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
            <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
                <level>INFO</level>
            </filter>
            <encoder>
                <pattern>${log_pattern}</pattern>
            </encoder>
        </appender>
    
        
        <appender name="LOGSTASH_MDC" class="com.internetitem.logback.elasticsearch.ElasticsearchAppender">
            <url>http://127.0.0.1:9200/_bulk</url>
            <index>logstash-logs{yyyy-MM}</index>
            <errorLoggerName>es-error</errorLoggerName>
            <connectTimeout>60000</connectTimeout>
            <authentication class="com.internetitem.logback.elasticsearch.config.BasicAuthentication"/>
            
            <properties>
                
                <property>
                    <name>host</name>
                    <value>${HOSTNAME}</value>
                </property>
                
                <property>
                    <name>code</name>
                    <value>%X{code}</value>
                </property>
                <property>
                    <name>createTime</name>
                    <value>%X{createTime}</value>
                </property>
                <property>
                    <name>content</name>
                    <value>%X{content}</value>
                </property>
                
                <property>
                    <name>logger</name>
                    <value>%logger</value>
                </property>
            </properties>
            
            <headers>
                <header>
                    <name>Content-Type</name>
                    <value>application/json</value>
                </header>
            </headers>
        </appender>
    
        
        <logger name="com.codecoord.springboot.elasticsearch.logstash" level="INFO" additivity="false">
            <appender-ref ref="LOGSTASH_MDC"/>
        </logger>
    
        <root level="INFO">
            <appender-ref ref="STDOUT"/>
        </root>
    </configuration>

1.  整合结果

{ 
      "_index" : "logstash-logs-2021-06",
      "_type" : "_doc",
      "_id" : "fcumGHoBUgXCDHMjqHTK",
      "_score" : 1.0,
      "_source" : { 
        "@timestamp" : "2021-06-17T14:26:53.002+0800",
        "message" : "logstash日志采集，当前时间【2021-06-17T14:26:53.002】",
        "host" : "DESKTOP-MBL60LI",
        "code" : "1baa76f6-815b-4004-8d5a-34243446ec69",
        "createTime" : "2021-06-17T14:26:53.002",
        "content" : "logstash日志采集错误2021-06-17T14:26:53.002",
        "logger" : "com.codecoord.springboot.elasticsearch.logstash.LogstashLogbackAggregation"
      }
    }

# 二、logback-elasticsearch-appender #

1.  Logback Elasticsearch Appender配置示例

<appender name="ELASTIC" class="com.internetitem.logback.elasticsearch.ElasticsearchAppender">
       <url>http://yourserver/_bulk</url>
       <index>logs-%date{yyyy-MM-dd}</index>
       <type>tester</type>
       <loggerName>es-logger</loggerName> 
       <errorLoggerName>es-error-logger</errorLoggerName> 
       <connectTimeout>30000</connectTimeout> 
       <errorsToStderr>false</errorsToStderr> 
       <includeCallerData>false</includeCallerData> 
       <logsToStderr>false</logsToStderr> 
       <maxQueueSize>104857600</maxQueueSize> 
       <maxRetries>3</maxRetries> 
       <readTimeout>30000</readTimeout> 
       <sleepTime>250</sleepTime> 
       <rawJsonMessage>false</rawJsonMessage> 
       <includeMdc>false</includeMdc> 
       <maxMessageSize>100</maxMessageSize> 
       <authentication class="com.internetitem.logback.elasticsearch.config.BasicAuthentication" /> 
       <properties>
         <property>
           <name>host</name>
           <value>${HOSTNAME}</value>
           <allowEmpty>false</allowEmpty>
         </property>
         <property>
           <name>severity</name>
           <value>%level</value>
         </property>
         <property>
           <name>thread</name>
           <value>%thread</value>
         </property>
         <property>
           <name>stacktrace</name>
           <value>%ex</value>
         </property>
         <property>
           <name>logger</name>
           <value>%logger</value>
         </property>
       </properties>
       <headers>
         <header>
           <name>Content-Type</name>
           <value>application/json</value>
         </header>
       </headers>
    </appender>
    
    <root level="info">
      <appender-ref ref="FILELOGGER" />
      <appender-ref ref="ELASTIC" />
    </root>
    
    <logger name="es-error-logger" level="INFO" additivity="false">
      <appender-ref ref="FILELOGGER" />
    </logger>
    
    <logger name="es-logger" level="INFO" additivity="false">
      <appender name="ES_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
        
        <encoder>
          <pattern>%msg</pattern> 
        </encoder>
      </appender>
    </logger>

1.  配置说明

*  `url` (required): The URL to your Elasticsearch bulk API endpoint
 *  `index` (required): Name if the index to publish to (populated using PatternLayout just like individual properties - see below)
 *  `type` (optional): Elasticsearch `_type` field for records. Although this library does not require `type` to be populated, Elasticsearch may, unless the configured URL includes the type (i.e. `{index}/{type}/_bulk` as opposed to `/_bulk` and `/{index}/_bulk`). See the Elasticsearch [Bulk API][] documentation for more information
 *  `sleepTime` (optional, default 250): Time (in ms) to sleep between attempts at delivering a message
 *  `maxRetries` (optional, default 3): Number of times to attempt retrying a message on failure. Note that subsequent log messages reset the retry count to 0. This value is important if your program is about to exit (i.e. it is not producing any more log lines) but is unable to deliver some messages to ES
 *  `connectTimeout` (optional, default 30000): Elasticsearch connect timeout (in ms)
 *  `readTimeout` (optional, default 30000): Elasticsearch read timeout (in ms)
 *  `includeCallerData` (optional, default false): If set to `true`, save the caller data (identical to the [AsyncAppender’s includeCallerData][AsyncAppender_s includeCallerData])
 *  `errorsToStderr` (optional, default false): If set to `true`, any errors in communicating with Elasticsearch will also be dumped to stderr (normally they are only reported to the internal Logback Status system, in order to prevent a feedback loop)
 *  `logsToStderr` (optional, default false): If set to `true`, dump the raw Elasticsearch messages to stderr
 *  `maxQueueSize` (optional, default 104,857,600 = 200MB): Maximum size (in characters) of the send buffer. After this point, *logs will be dropped*. This should only happen if Elasticsearch is down, but this is a self-protection mechanism to ensure that the logging system doesn’t cause the main process to run out of memory. Note that this maximum is approximate; once the maximum is hit, no new logs will be accepted until it shrinks, but any logs already accepted to be processed will still be added to the buffer
 *  `loggerName` (optional): If set, raw ES-formatted log data will be sent to this logger
 *  `errorLoggerName` (optional): If set, any internal errors or problems will be logged to this logger
 *  `rawJsonMessage` (optional, default false): If set to `true`, the log message is interpreted as pre-formatted raw JSON message.
 *  `includeMdc` (optional, default false): If set to `true`, then all [MDC][] values will be mapped to properties on the JSON payload.
 *  `maxMessageSize` (optional, default -1): If set to a number greater than 0, truncate messages larger than this length, then append “`..`” to denote that the message was truncated
 *  `authentication` (optional): Add the ability to send authentication headers (see below)

The fields `@timestamp` and `message` are always sent and can not currently be configured. Additional fields can be sent by adding `<property>` elements to the `<properties>` set.

*  `name` (required): Key to be used in the log event
 *  `value` (required): Text string to be sent. Internally, the value is populated using a Logback PatternLayout, so all [Conversion Words][] can be used (in addition to the standard static variable interpolations like `${HOSTNAME}`).
 *  `allowEmpty` (optional, default `false`): Normally, if the `value` results in a `null` or empty string, the field will not be sent. If `allowEmpty` is set to `true` then the field will be sent regardless
 *  `type` (optional, default `String`): type of the field on the resulting JSON message. Possible values are: `String`, `int`, `float` and `boolean`.

[https_gitee.com_tianxincoord_logback-elasticsearch-appender]: https://gitee.com/tianxincoord/logback-elasticsearch-appender
[Bulk API]: https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
[AsyncAppender_s includeCallerData]: http://logback.qos.ch/manual/appenders.html#asyncIncludeCallerData
[MDC]: http://www.slf4j.org/api/org/slf4j/MDC.html
[Conversion Words]: http://logback.qos.ch/manual/layouts.html#conversionWord