ELK Stack - Turn boring logfiles into sexy dashboard
0 likes5,416 views
The document outlines the setup and configuration of a logging system using Logstash, Elasticsearch, and Kibana to analyze Apache access log files. It provides detailed input configurations for Logstash, including data filtering and geographic IP resolution, and describes dashboard elements in Kibana for visualizing the logged data. Additionally, examples of queries and visualizations for effective data representation and analysis are included.
![Boring Logfiles
178.236.7.219 - - [27/Oct/2014:06:43:13 +0100] "GET
/ps/amazon.de/index.php?shopkey=DEADBEEF&shopurl=https
%3A%2F%2Fwww.amazon.de
%2F&multishop_id=0&userip=173.194.112.2&referer=https
%3A%2F%2Fwww.amazon.de%2FTechnik
%2FFernseher&count=0&group[]=2&service=search&query=le
d HTTP/1.1" 200 12363 "-" "-"](https://image.slidesharecdn.com/elkstack-141027144413-conversion-gate02/85/ELK-Stack-Turn-boring-logfiles-into-sexy-dashboard-2-320.jpg)
![Logstash.conf
input {
file {
path => "/home/georg/coding/logstash-apache/access.log"
start_position => beginning
}
#pipe {
# command => "ssh frontend1.findologic.com tail -f /var/log/apache2/access.log"
#}
}
filter {
grok {
match => { "message" => "%{IPORHOST:clientip} %{USER:ident} %{USER:auth} [%{HTTPDATE:timestamp}] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent}" }
}
# split the query string: query=test&count=0&first=10 → request_query: test, request_count: 0, request_first: 10
kv {
field_split => "&?"
source => "request"
prefix => "request_"
}
# urldecode does not decode the + into whitespace so do this manually
mutate {
gsub => [
"request_query", "+", " "
]
}
# URL-decode the query so it is easier to read. We don't really about the other request_* parameters
urldecode {
field => "request_query"
}
# extract the script file, eg. index.php
mutate {
gsub => [
# Remove the query string including the ? : /path/index.php?key=value -> /path/index.php
"request", "?.*$", "",
# Remove everything up to the last / : /path/index.php -> index.php
"request", "^.*/", ""
]
}
# Default to index.php if no script file is given
if [request] == "" {
mutate {
replace => [ "request", "index.php" ]
}
}
# If the IP is syntactically valid resolve it geographically
if [request_userip] =~ "^d+.d+.d+.d+$" {
geoip {
source => "request_userip"
target => "client_geoip"
}
}
}
output {
elasticsearch {
host => localhost
}
#stdout { codec => rubydebug }
}](https://image.slidesharecdn.com/elkstack-141027144413-conversion-gate02/85/ELK-Stack-Turn-boring-logfiles-into-sexy-dashboard-11-320.jpg)
![Kibana Dashboard
{
"title": "Logstash Search",
"services": {
"query": {
"list": {
"0": {
"query": "*",
"alias": "",
"color": "#7EB26D",
"id": 0,
"pin": false,
"type": "lucene",
"enable": true
},
"1": {
"id": 1,
"color": "#EAB839",
"alias": "Suche",
"pin": false,
"type": "lucene",
"enable": true,
"query": "request:index.php"
},
"2": {
"id": 2,
"color": "#6ED0E0",
"alias": "Autocomplete",
"pin": false,
"type": "lucene",
"enable": true,
"query": "request:autocomplete.php"
},
"3": {
"id": 3,
"color": "#EF843C",
"alias": "",
"pin": false,
"type": "lucene",
"enable": true,
"query": "client_geoip.country_code2:DE OR client_geoip.country_code2:AT OR client_geoip.country_code2:CH"
}
},
"ids": [
0,
1,
2,
3
]
},
"filter": {
"list": {
"0": {
"type": "time",
"field": "@timestamp",
"from": "now-24h",
"to": "now",
"mandate": "must",
"active": true,
"alias": "",
"id": 0
},
"1": {
"type": "time",
"from": "2014-10-27T18:04:33.357Z",
"to": "2014-10-27T19:18:47.459Z",
"field": "@timestamp",
"mandate": "must",
"active": true,
"alias": "",
"id": 1
}
},
"ids": [
0,
1
]
}
},
"rows": [
{
"title": "Graph",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "histogram",
"mode": "count",
"time_field": "@timestamp",
"value_field": null,
"auto_int": true,
"resolution": 100,
"interval": "30s",
"fill": 3,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": true,
"stack": true,
"points": false,
"lines": false,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"queries": {
"mode": "selected",
"ids": [
1,
2
]
},
"title": "Events over time",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": true
},
"scale": 1,
"y_format": "none",
"grid": {
"max": null,
"min": 0
},
"annotate": {
"enable": false,
"query": "*",
"size": 20,
"field": "_type",
"sort": [
"_score",
"desc"
]
},
"pointradius": 5,
"show_query": true,
"legend_counts": true,
"zerofill": true,
"derivative": false
}
],
"notice": false
},
{
"title": "",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"error": false,
"span": 6,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "request_query",
"exclude": [],
"missing": false,
"other": false,
"size": 10,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "pie",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3
]
},
"tmode": "terms",
"tstat": "total",
"valuefield": ""
},
{
"error": false,
"span": 6,
"editable": true,
"type": "bettermap",
"loadingEditor": false,
"field": "client_geoip.location",
"size": 1000,
"spyable": true,
"tooltip": "_id",
"queries": {
"mode": "selected",
"ids": [
3
]
}
}
],
"notice": false
},
{
"title": "Events",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"title": "All events",
"error": false,
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "table",
"size": 100,
"pages": 5,
"offset": 0,
"sort": [
"@timestamp",
"desc"
],
"style": {
"font-size": "9pt"
},
"overflow": "min-height",
"fields": [],
"localTime": true,
"timeField": "@timestamp",
"highlight": [],
"sortable": true,
"header": true,
"paging": true,
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3
]
},
"field_list": true,
"status": "Stable",
"trimFactor": 300,
"normTimes": true,
"all_fields": false
}
],
"notice": false
}
],
"editable": true,
"failover": false,
"index": {
"interval": "day",
"pattern": "[logstash-]YYYY.MM.DD",
"default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED",
"warm_fields": true
},
"style": "light",
"panel_hints": true,
"pulldowns": [
{
"type": "query",
"collapse": false,
"notice": false,
"query": "*",
"pinned": true,
"history": [
"client_geoip.country_code2:DE OR client_geoip.country_code2:AT OR client_geoip.country_code2:CH",
"request:autocomplete.php",
"request:index.php",
"*"
],
"remember": 10,
"enable": true
},
{
"type": "filtering",
"collapse": false,
"notice": true,
"enable": true
}
],
"nav": [
{
"type": "timepicker",
"collapse": false,
"notice": false,
"status": "Stable",
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
],
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"timefield": "@timestamp",
"now": false,
"filter_id": 0,
"enable": true
}
],
"loader": {
"save_gist": false,
"save_elasticsearch": true,
"save_local": true,
"save_default": true,
"save_temp": true,
"save_temp_ttl_enable": true,
"save_temp_ttl": "30d",
"load_gist": true,
"load_elasticsearch": true,
"load_elasticsearch_size": 20,
"load_local": true,
"hide": false
},
"refresh": false
}
Vergrößern zum Anzeigen](https://image.slidesharecdn.com/elkstack-141027144413-conversion-gate02/85/ELK-Stack-Turn-boring-logfiles-into-sexy-dashboard-12-320.jpg)
ELK Stack - Turn boring logfiles into sexy dashboard
- 1. FINDOLOGIC GmbH Boring Logfiles → Sexy Dashboards
- 2. Boring Logfiles 178.236.7.219 - - [27/Oct/2014:06:43:13 +0100] "GET /ps/amazon.de/index.php?shopkey=DEADBEEF&shopurl=https %3A%2F%2Fwww.amazon.de %2F&multishop_id=0&userip=173.194.112.2&referer=https %3A%2F%2Fwww.amazon.de%2FTechnik %2FFernseher&count=0&group[]=2&service=search&query=le d HTTP/1.1" 200 12363 "-" "-"
- 4. Stack ● Elasticsearch ● Logstash ● Kibana
- 5. Infrastruktur App 1 App 2 App 3 Logstash 1 Logstash 2 Logstash 3 ElasticSearch Kibana
- 6. Logstash Architektur Input Filter Output ● Grok ● Search / Replace ● Split ● GeoIP ● DNS ● URL-Decode ● ... ● ElasticSearch ● Redis ● CouchDB ● ... ● Pipe ● Socket ● File ● ...
- 7. Logstash stand-alone ● Logs zentralisieren ● In DB speichern ● Durchsuchbar machen
- 8. Setup ● Logstash herunterladen, extrahieren ● Elasticsearch installieren ● fertig
- 9. Georg? ● CTO bei FINDOLOGIC GmbH ● http://www.findologic.com ● g.sorst@findologic.com ● @piefke_schorsch
- 10. Hands on ● Grok ● kv ● Query ● Endpoint ● GeoIP
- 11. Logstash.conf input { file { path => "/home/georg/coding/logstash-apache/access.log" start_position => beginning } #pipe { # command => "ssh frontend1.findologic.com tail -f /var/log/apache2/access.log" #} } filter { grok { match => { "message" => "%{IPORHOST:clientip} %{USER:ident} %{USER:auth} [%{HTTPDATE:timestamp}] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent}" } } # split the query string: query=test&count=0&first=10 → request_query: test, request_count: 0, request_first: 10 kv { field_split => "&?" source => "request" prefix => "request_" } # urldecode does not decode the + into whitespace so do this manually mutate { gsub => [ "request_query", "+", " " ] } # URL-decode the query so it is easier to read. We don't really about the other request_* parameters urldecode { field => "request_query" } # extract the script file, eg. index.php mutate { gsub => [ # Remove the query string including the ? : /path/index.php?key=value -> /path/index.php "request", "?.*$", "", # Remove everything up to the last / : /path/index.php -> index.php "request", "^.*/", "" ] } # Default to index.php if no script file is given if [request] == "" { mutate { replace => [ "request", "index.php" ] } } # If the IP is syntactically valid resolve it geographically if [request_userip] =~ "^d+.d+.d+.d+$" { geoip { source => "request_userip" target => "client_geoip" } } } output { elasticsearch { host => localhost } #stdout { codec => rubydebug } }
- 12. Kibana Dashboard { "title": "Logstash Search", "services": { "query": { "list": { "0": { "query": "*", "alias": "", "color": "#7EB26D", "id": 0, "pin": false, "type": "lucene", "enable": true }, "1": { "id": 1, "color": "#EAB839", "alias": "Suche", "pin": false, "type": "lucene", "enable": true, "query": "request:index.php" }, "2": { "id": 2, "color": "#6ED0E0", "alias": "Autocomplete", "pin": false, "type": "lucene", "enable": true, "query": "request:autocomplete.php" }, "3": { "id": 3, "color": "#EF843C", "alias": "", "pin": false, "type": "lucene", "enable": true, "query": "client_geoip.country_code2:DE OR client_geoip.country_code2:AT OR client_geoip.country_code2:CH" } }, "ids": [ 0, 1, 2, 3 ] }, "filter": { "list": { "0": { "type": "time", "field": "@timestamp", "from": "now-24h", "to": "now", "mandate": "must", "active": true, "alias": "", "id": 0 }, "1": { "type": "time", "from": "2014-10-27T18:04:33.357Z", "to": "2014-10-27T19:18:47.459Z", "field": "@timestamp", "mandate": "must", "active": true, "alias": "", "id": 1 } }, "ids": [ 0, 1 ] } }, "rows": [ { "title": "Graph", "height": "350px", "editable": true, "collapse": false, "collapsable": true, "panels": [ { "span": 12, "editable": true, "group": [ "default" ], "type": "histogram", "mode": "count", "time_field": "@timestamp", "value_field": null, "auto_int": true, "resolution": 100, "interval": "30s", "fill": 3, "linewidth": 3, "timezone": "browser", "spyable": true, "zoomlinks": true, "bars": true, "stack": true, "points": false, "lines": false, "legend": true, "x-axis": true, "y-axis": true, "percentage": false, "interactive": true, "queries": { "mode": "selected", "ids": [ 1, 2 ] }, "title": "Events over time", "intervals": [ "auto", "1s", "1m", "5m", "10m", "30m", "1h", "3h", "12h", "1d", "1w", "1M", "1y" ], "options": true, "tooltip": { "value_type": "cumulative", "query_as_alias": true }, "scale": 1, "y_format": "none", "grid": { "max": null, "min": 0 }, "annotate": { "enable": false, "query": "*", "size": 20, "field": "_type", "sort": [ "_score", "desc" ] }, "pointradius": 5, "show_query": true, "legend_counts": true, "zerofill": true, "derivative": false } ], "notice": false }, { "title": "", "height": "350px", "editable": true, "collapse": false, "collapsable": true, "panels": [ { "error": false, "span": 6, "editable": true, "type": "terms", "loadingEditor": false, "field": "request_query", "exclude": [], "missing": false, "other": false, "size": 10, "order": "count", "style": { "font-size": "10pt" }, "donut": false, "tilt": false, "labels": true, "arrangement": "horizontal", "chart": "pie", "counter_pos": "above", "spyable": true, "queries": { "mode": "all", "ids": [ 0, 1, 2, 3 ] }, "tmode": "terms", "tstat": "total", "valuefield": "" }, { "error": false, "span": 6, "editable": true, "type": "bettermap", "loadingEditor": false, "field": "client_geoip.location", "size": 1000, "spyable": true, "tooltip": "_id", "queries": { "mode": "selected", "ids": [ 3 ] } } ], "notice": false }, { "title": "Events", "height": "350px", "editable": true, "collapse": false, "collapsable": true, "panels": [ { "title": "All events", "error": false, "span": 12, "editable": true, "group": [ "default" ], "type": "table", "size": 100, "pages": 5, "offset": 0, "sort": [ "@timestamp", "desc" ], "style": { "font-size": "9pt" }, "overflow": "min-height", "fields": [], "localTime": true, "timeField": "@timestamp", "highlight": [], "sortable": true, "header": true, "paging": true, "spyable": true, "queries": { "mode": "all", "ids": [ 0, 1, 2, 3 ] }, "field_list": true, "status": "Stable", "trimFactor": 300, "normTimes": true, "all_fields": false } ], "notice": false } ], "editable": true, "failover": false, "index": { "interval": "day", "pattern": "[logstash-]YYYY.MM.DD", "default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED", "warm_fields": true }, "style": "light", "panel_hints": true, "pulldowns": [ { "type": "query", "collapse": false, "notice": false, "query": "*", "pinned": true, "history": [ "client_geoip.country_code2:DE OR client_geoip.country_code2:AT OR client_geoip.country_code2:CH", "request:autocomplete.php", "request:index.php", "*" ], "remember": 10, "enable": true }, { "type": "filtering", "collapse": false, "notice": true, "enable": true } ], "nav": [ { "type": "timepicker", "collapse": false, "notice": false, "status": "Stable", "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ], "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "timefield": "@timestamp", "now": false, "filter_id": 0, "enable": true } ], "loader": { "save_gist": false, "save_elasticsearch": true, "save_local": true, "save_default": true, "save_temp": true, "save_temp_ttl_enable": true, "save_temp_ttl": "30d", "load_gist": true, "load_elasticsearch": true, "load_elasticsearch_size": 20, "load_local": true, "hide": false }, "refresh": false } Vergrößern zum Anzeigen