我正在使用logstash来处理来自dnsmasq的日志。 在DNS响应中，我使用'geoip'过滤器来丰富请求和位置信息。不幸的是，一些请求充满了地理定位信息而其他请求却没有。
geoip {
source => "serverip"
}
104.156.81.217
104.156.85.217
104.16.92.65
104.16.93.65
104.16.94.65
104.16.95.65
104.16.96.65
104.20.5.131
104.20.6.131
104.20.77.18
104.20.78.18
104.244.43.135
104.244.43.167
104.244.43.231
104.244.43.39
104.244.43.7
104.28.30.27
104.28.31.27
104.40.196.5
104.41.231.130
104.45.95.112
104.47.151.128
104.71.97.80
104.84.200.206
104.90.129.122
104.90.176.199
104.90.176.77
104.94.60.210
104.98.119.204
104.98.150.212
162.255.119.124
185.118.208.20
185.19.196.101
185.54.150.54
185.63.147.12
191.232.139.13
191.233.80.151
191.239.8.125
192.229.233.25
23.101.51.170
23.196.235.245
23.196.247.114
23.196.249.86
23.196.255.139
23.197.0.60
23.199.209.223
23.235.33.217
23.235.37.217
23.97.173.24
{
"message" => "May 27 18:17:16 dnsmasq[385]: reply www.google.com is 216.58.213.228",
"@version" => "1",
"@timestamp" => "2016-05-27T18:17:17.147Z",
"path" => "/var/log/dnsmasq.log",
"host" => "dns",
"type" => "dnsmasq",
"reqtimestamp" => "May 27 18:17:16",
"program" => "dnsmasq",
"pid" => "385",
"action" => "reply",
"domain" => "www.google.com",
"function" => "is",
"serverip" => "216.58.213.228",
"geoip" => {
"ip" => "216.58.213.228",
"country_code2" => "US",
"country_code3" => "USA",
"country_name" => "United States",
"continent_code" => "NA",
"region_name" => "CA",
"city_name" => "Mountain View",
"postal_code" => "94043",
"latitude" => 37.41919999999999,
"longitude" => -122.0574,
"dma_code" => 807,
"area_code" => 650,
"timezone" => "America/Los_Angeles",
"real_region_name" => "California",
"location" => [
[0] -122.0574,
[1] 37.41919999999999
]
}
}
{
"message" => "May 27 18:15:50 dnsmasq[385]: reply e5884.d.akamaiedge.net is 23.197.8.251",
"@version" => "1",
"@timestamp" => "2016-05-27T18:15:51.697Z",
"path" => "/var/log/dnsmasq.log",
"host" => "dns",
"type" => "dnsmasq",
"reqtimestamp" => "May 27 18:15:50",
"program" => "dnsmasq",
"pid" => "385",
"action" => "reply",
"domain" => "e5884.d.akamaiedge.net",
"function" => "is",
"serverip" => "23.197.8.251"
}
input {
file {
path => "/var/log/dnsmasq.log"
start_position => "beginning"
type => "dnsmasq"
}
}
# Mar 15 20:13:05 dnsmasq[346]: query[A] imap.gmail.com from 192.168.0.140
# Mar 2 20:38:45 dnsmasq-dhcp[11856]: DHCPACK(eth0) 192.168.0.152 60:67:20:72:df:00 E0199149
# Mar 15 21:55:34 dnsmasq-dhcp[346]: 3806132383 DHCPACK(eth0) 192.168.0.80 04:0c:ce:d1:af:18 Air-de-irobot
# Mar 16 08:54:31 dnsmasq-dhcp[346]: 4280587370 DHCPACK(eth0) 192.168.0.158 48:9d:24:ae:0e:00 BB-JP
# Mar 16 08:18:49 dnsmasq[346]: /etc/pihole/gravity.list ssl.google-analytics.com is 192.168.0.2
filter {
if [type] == "dnsmasq" {
grok {
match => [ "message", "%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: ?(%{NONNEGINT:num} )?%{NOTSPACE:action} %{IP:clientip} %{MAC:clientmac} ?(%{HOSTNAME:clientname})?"]
match => [ "message", "%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: ?(%{NONNEGINT:num} )?%{USER:action}?(\[%{USER:subaction}\])? %{NOTSPACE:domain} %{NOTSPACE:function} %{IP:clientip}"]
match => [ "message", "%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: %{NOTSPACE:action} %{DATA:data}"]
}
if [action] =~ "DHCPACK" {
if ![clientname] {
mutate {
add_field => { "clientname" => "No name" }
}
}
aggregate {
task_id => "%{clientip}"
code => "map['clientmac'] = event['clientmac']; map['clientname'] = event['clientname'];"
map_action => "create_or_update"
# timeout = 0 sets the timeout to the default value 1800 seconds.
timeout => 172800
}
} else if [action] == "query" {
aggregate {
task_id => "%{clientip}"
code => "event['clientmac'] = map['clientmac']; event['clientname'] = map['clientname']"
map_action => "update"
}
if ![clientname] {
mutate {
add_field => { "clientname" => "%{clientip}" }
}
}
if ![clientmac] {
mutate {
add_field => { "clientmac" => "%{clientip}" }
}
}
} else if [action] == "reply" {
mutate {
rename => { "clientip" => "serverip" }
}
geoip {
source => "serverip"
}
} else
{
drop{}
}
}
}
output {
# elasticsearch { hosts => ["localhost:9200"] }
stdout { codec => rubydebug }
}
答案 0 :(得分：3)
最新的Logstash版本（2.3.2）捆绑了logstash geoip filter 2.0.7，后者又包含了2013年（GeoLiteCity-2013-01-18.dat）旧的Maxmind的GeoIP数据库，从调试日志中可以看出< / p>
Using geoip database {:path=>"/usr/local/Cellar/logstash/2.3.2/vendor/bundle/jruby/1.9/gems/logstash-filter-geoip-2.0.7/vendor/GeoLiteCity-2013-01-18.dat", :level=>:info, :file=>"logstash/filters/geoip.rb", :line=>"97", :method=>"register"}
Maxmind有一项新的GeoIP2服务，你可以试用here，它会正确地对上面列表中的所有IP进行地理编码。
就Logstash geoip而言，从GeoIP切换到GeoIP2 has been made in March并将在next Logstash 5.0 version中提供。