ELK 之 logstash 利用 IP 获取地理位置

灰太狼 2022-10-30 13:25 225阅读 0赞

### ELK 之 logstash 利用 IP 获取地理位置 geoip ###

*   *   *  摘要
         *  解决方案
         *  解析结果

### 摘要 ###

在平常使用时，利用`logstash`获取日志信息，有时需要将日志中的IP地址转换成坐标或是实际的位置，这里提供一种可行的解决方案。（笔者实际生产环境中使用这种方案，仅供参考）

### 解决方案 ###

将原始IP，利用`Maxmind GeoLite2 databases`数据库转换成`geoip`。[可以参考官方文档][Link 1]

这里利用两种过滤器，首先利用`dissect`解析日志，其次利用`geoip`将IP转换成坐标地址。

input {
        
        file {
        
            path => "/home/server/logs/web_log.log*" #要监听的日志文件路径 
            type => web_log #TODO 日志类型
            exclude => "*.gz"       #不想监听的文件规则，基于glob匹配语法
            start_position => "end"   #第一次丛头开始读取文件 beginning or end
            stat_interval => "3"    #定时检查文件是否更新，默认1s
        }
    }
    
    filter {
        
        dissect {
        
          mapping => {
        
            message => "%{username}###%{url}###%{method}###%{start_time}###%{ip}"
          }
        }
       geoip {
        
         source => "ip"
         target => "geoip"
         # 可以指定IP数据库 database => "/server/GeoIPCity.dat"
         add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
         add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
        }
    }
    output {
        
      elasticsearch {
        
        hosts => ["http://localhost:9200"]
        index => "web_log-%{+YYYY.MM.dd}"
      }
    }

### 解析结果 ###

{
        
      "took" : 0,
      "timed_out" : false,
      "_shards" : {
        
        "total" : 1,
        "successful" : 1,
        "skipped" : 0,
        "failed" : 0
      },
      "hits" : {
        
        "total" : {
        
          "value" : 1,
          "relation" : "eq"
        },
        "max_score" : 1.0,
        "hits" : [
          {
        
            "_index" : "web_log-2021.02.19",
            "_type" : "_doc",
            "_id" : "TkLBuXcBjl6ifEWWpyXP",
            "_score" : 1.0,
            "_source" : {
        
              "@timestamp" : "2021-02-19T10:06:51.719Z",
              "path" : "/home/server/logs/web_log.log",
              "geoip" : {
        
                "country_code3" : "CN",
                "timezone" : "Asia/Shanghai",
                "continent_code" : "AS",
                "country_name" : "China",
                "longitude" : 16.3889,
                "country_code2" : "CN",
                "latitude" : 9.9288,
                "ip" : "123.456.789.0",
                "city_name" : "Beijing",
                "region_name" : "Beijing",
                "location" : {
        
                  "lon" : 16.3889,
                  "lat" : 9.9288
                },
                "region_code" : "BJ",
                "coordinates" : [
                  "16.3889",
                  "9.9288"
                ]
              },
              "username" : "weicai",
              "url" : "/login/",
              "@version" : "1",
              "type" : "web_log",
              "message" : """weicai###/login/###GET###136###123.456.789.0""",
              "deal_time" : "136",
              "ip" : "123.456.789.0",
              "method" : "GET",
              "host" : "weicai-PC"
            }
          }
        ]
      }
    }

[Link 1]: https://www.elastic.co/guide/en/logstash/current/plugins-filters-geoip.html