Febrian12345
Reputation: 1
How to parse glastopf honeypot json logs in logstash?
I have a raw data in json format that I want to visualize in Kibana, but I'm having trouble parsing the data in logstash, previously I was able to visualize raw data for honeypot dionaea and cowrie. please help me in doing this task. Here I show the configuration from the file /etc/logstash/conf.d/myconfig.conf and an example of the raw data to be parsed. Thank you
/etc/logstash/conf.d/myconfig.conf
input {
beats {
port => 5044
}
}
filter {
if [document_type] == "cowrie" {
json {
source => "message"
}
mutate {
copy => { "_id" => "[@metadata][_id]"}
remove_field => ["_id"]
rename => { "[payload][peerIP]" => "peerIP"}
rename => { "[payload][hostPort]" => "local_port"}
rename => { "[payload][protocol]" => "protocol"}
}
grok {
match => {
"message" => [
".*credentials\":\[\[\"%{DATA:userfailed}\",\"%{DATA:passfailed}\"\]\]",
".*loggedin\":\[\"%{DATA:usersuccess}\",\"%{DATA:passsuccess}\"\]",
".*loggedin\":\[%{DATA:unpassuccess}\]",
".*credentials\"\:\[\[%{DATA:unpassfailed}\]\]"
]
}
}
geoip { source => "peerIP"
target => "geoip"
}
}
if [document_type] == "dionaea" {
json {
source => "message"
}
mutate {
copy => { "_id" => "[@metadata][_id]"}
remove_field => ["_id"]
rename => { "[payload][remote_host]" => "peerIP"}
rename => { "[payload][local_port]" => "local_port"}
rename => { "[payload][connection_protocol]" => "protocol"}
}
geoip {
source => "peerIP"
target => "geoip"
}
}
if [document_type] == "glastopf" {
json {
source => "message"
}
mutate {
copy => { "_id" => "[@metadata][_id]"}
remove_field => ["_id"]
}
grok {
match => {
"message" => [
".*pattern\":\"%{DATA:attack_technique}\"\,",
".*request_raw\"\:\"%{DATA:attack_method}\/",
".*User-Agent: %{DATA:user_agent}\/",
".*source_ip\"\:\"%{IP:peerIP}\"\,",
".*source_port\"\:%{DATA:source_port}\,",
".*http_host\"\:\"%{IP:destination_ip}\"\,",
".*request_url\"\:\"%{DATA:request_url}\"\,"
]
}
}
geoip { source => "peerIP"
target => "geoip"
}
}
if [document_type] == "honeypot" {
json {
source => "message"
}
mutate {
copy => { "_id" => "[@metadata][_id]"}
remove_field => ["_id"]
rename => { "[payload][remote_host]" => "peerIP"}
rename => { "[payload][local_port]" => "local_port"}
rename => { "[payload][connection_protocol]" => "protocol"}
rename => { "[payload][peerIP]" => "peerIP"}
rename => { "[payload][hostPort]" => "local_port"}
rename => { "[payload][protocol]" => "protocol"}
}
grok {
match => {
"message" => [
".*credentials\":\[\[\"%{DATA:userfailed}\",\"%{DATA:passfailed}\"\]\]",
".*loggedin\":\[\"%{DATA:usersuccess}\",\"%{DATA:passsuccess}\"\]",
".*loggedin\":\[%{DATA:unpassuccess}\]",
".*credentials\"\:\[\[%{DATA:unpassfailed}\]\]",
".*pattern\":\"%{DATA:attack_technique}\"\,"
]
}
}
geoip { source => "peerIP"
target => "geoip"
}
}
}
output {
if [document_type] == "cowrie" {
elasticsearch {
hosts => ["http://localhost:9200"]
index => "logstash-cowrie-%{+YYYY.MM.dd}"
document_id => "%{[@metadata][_id]}"
}
}
if [document_type] == "dionaea" {
elasticsearch {
hosts => ["http://localhost:9200"]
index => "logstash-dionaea-%{+YYYY.MM.dd}"
document_id => "%{[@metadata][_id]}"
}
}
if [document_type] == "glastopf" {
elasticsearch {
hosts => ["http://localhost:9200"]
index => "logstash-glastopf-%{+YYYY.MM.dd}"
document_id => "%{[@metadata][_id]}"
}
}
if [document_type] == "honeypot" {
elasticsearch {
hosts => ["http://localhost:9200"]
index => "logstash-honeypot-%{+YYYY.MM.dd}"
document_id => "%{[@metadata][_id]}"
}
}
}
RAW data Glastopf
{"_id":{"$oid":"631930a04e016a52d1af9008"},"ident":"honeypot3-glast","timestamp":{"$date":"2022-09-08T00:00:32.314Z"},"normalized":true,"payload":{"pattern":"unknown","request_raw":"GET /cgi-bin/voteList.asp?item_ID=%29%2F%2A%2A%2FaS%2F%2A%2A%2FGmeF%2F%2A%2A%2FwHerE%2F%2A%2A%2F7374%3D7374%2F%2A%2A%2FanD%2F%2A%2A%2FELt%286133%2F%2A%2A%2FBeTweEN%2F%2A%2A%2F6133%2F%2A%2A%2FanD%2F%2A%2A%2F6133%2C4322%29--%2F%2A%2A%2FAqHq HTTP/1.1\r\nAccept: */*\r\nAccept-Encoding: gzip,deflate\r\nCache-Control: no-cache\r\nConnection: close\r\nHost: 103.219.114.117\r\nUser-Agent: Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1)","source_ip":"216.131.82.64","filename":null,"known_file":false,"source":["216.131.88.64",57540],"version":"3.1.3-dev","http_host":"103.219.112.117","time":"2022-09-08 07:00:30","sensor_ip":"192.168.81.12","source_port":57540,"sensorid":"6ea7fbee-f343-4db4-9dbd-55b1926ae53e","request_url":"/cgi-bin/voteList.asp?item_ID=%29%2F%2A%2A%2FaS%2F%2A%2A%2FGmeF%2F%2A%2A%2FwHerE%2F%2A%2A%2F7374%3D7374%2F%2A%2A%2FanD%2F%2A%2A%2FELt%286133%2F%2A%2A%2FBeTweEN%2F%2A%2A%2F6133%2F%2A%2A%2FanD%2F%2A%2A%2F6133%2C4322%29--%2F%2A%2A%2FAqHq","file_sha256":null,"sensor_port":80},"channel":"glastopf.events"}
{"_id":{"$oid":"631930be4e016a52d1af911c"},"ident":"honeypot3-glast","timestamp":{"$date":"2022-09-08T00:01:02.762Z"},"normalized":true,"payload":{"pattern":"unknown","request_raw":"GET /cgi-bin/voteList.asp?item_ID=%60%2F%2A%2A%2FWhere%2F%2A%2A%2F8781%3D8781%2F%2A%2A%2FaNd%2F%2A%2A%2FeLt%282062%2F%2A%2A%2FbetWEen%2F%2A%2A%2F6788%2F%2A%2A%2FaNd%2F%2A%2A%2F6788%2C6788%29--%2F%2A%2A%2FXZeM HTTP/1.1\r\nAccept: */*\r\nAccept-Encoding: gzip,deflate\r\nCache-Control: no-cache\r\nConnection: close\r\nHost: 103.219.112.117\r\nUser-Agent: Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1)","source_ip":"216.131.88.64","filename":null,"known_file":false,"source":["216.131.88.64",51719],"version":"3.1.3-dev","http_host":"103.219.112.117","time":"2022-09-08 07:01:00","sensor_ip":"192.168.81.12","source_port":51719,"sensorid":"6ea7fbee-f343-4db4-9dbd-55b1926ae53e","request_url":"/cgi-bin/voteList.asp?item_ID=%60%2F%2A%2A%2FWhere%2F%2A%2A%2F8781%3D8781%2F%2A%2A%2FaNd%2F%2A%2A%2FeLt%282062%2F%2A%2A%2FbetWEen%2F%2A%2A%2F6788%2F%2A%2A%2FaNd%2F%2A%2A%2F6788%2C6788%29--%2F%2A%2A%2FXZeM","file_sha256":null,"sensor_port":80},"channel":"glastopf.events"}
{"_id":{"$oid":"6319310a4e016a52d1af9415"},"ident":"honeypot3-glast","timestamp":{"$date":"2022-09-08T00:02:18.730Z"},"normalized":true,"payload":{"pattern":"unknown","request_raw":"GET /cgi-bin/voteList.asp?item_ID=%60%2F%2A%2A%2FwherE%2F%2A%2A%2F6471%3D6471%2F%2A%2A%2FAnD%2F%2A%2A%2FElT%286133%2F%2A%2A%2FbEtween%2F%2A%2A%2F6133%2F%2A%2A%2FAnD%2F%2A%2A%2F6133%2C4322%29--%2F%2A%2A%2FnpTT HTTP/1.1\r\nAccept: */*\r\nAccept-Encoding: gzip,deflate\r\nCache-Control: no-cache\r\nConnection: close\r\nHost: 103.219.112.119\r\nUser-Agent: Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1)","source_ip":"216.131.88.69","filename":null,"known_file":false,"source":["216.131.88.64",55560],"version":"3.1.3-dev","http_host":"103.219.112.117","time":"2022-09-08 07:02:16","sensor_ip":"192.168.81.12","source_port":55560,"sensorid":"6ea7fbee-f343-4db4-9dbd-55b1926ae53e","request_url":"/cgi-bin/voteList.asp?item_ID=%60%2F%2A%2A%2FwherE%2F%2A%2A%2F6471%3D6471%2F%2A%2A%2FAnD%2F%2A%2A%2FElT%286133%2F%2A%2A%2FbEtween%2F%2A%2A%2F6133%2F%2A%2A%2FAnD%2F%2A%2A%2F6133%2C4322%29--%2F%2A%2A%2FnpTT","file_sha256":null,"sensor_port":80},"channel":"glastopf.events"}
I want to visualize raw json data from glastopf in Kibana.
Upvotes: 0
Views: 114
Answers (1)
Paulo
Reputation: 10346
Tldr;
Elasticsearch has released the Elastic Common Schema a.k.a ECS.
Which give may provide some guide lines on how to map the fields.
To combine multiple index in the same "view". You could either map them under the same data view(in v8.x) or index pattern (prior v8).
You could also create a dashboard with multiple visualisation, one for each index.
Upvotes: 0
Related Questions
- Parsing JSON to Kibana using Grok Filter
- Parsing out awkward JSON in Logstash
- grok pattern to parse the logs in logstash
- Extract from LogStash message field in Kibana dashboard
- Logstash - parse data with grok
- Parsing my json file by using grok pattern in logstash?
- Logstash not parsing JSON
- Querying Kibana using grok pattern
- Logstash Grok Parsing Issue
- Logstash does not parse json