AbhishekChalke
AbhishekChalke

Reputation: 11

How to parse nested json by configuring fluent td-agent.conf to add json values to databse

I am having logs from suricata in json format, the json logs contains various json objects depending on the value of event_type key. I want to separately parse each type of json object and add into separate database respectively. I am able to achieve upto this. The issue i am facing is one of the json object of event_type:"alert" has key containing nest json object and i am unable to fetch add the nested json object key values to the database.

Below i have shared the json logs and also the td-agent.conf, Please let me know what changes i have to make. Thank you.

JSON logs (eve.json is the file). The first object contains nested json
----------------------------------------------------------------------------------

# alerts blocked
{
"timestamp":"2024-05-03T18:51:34.438018+0530",
"flow_id":1758064090296381,
"event_type":"alert",
"src_ip":"192.168.0.13",
"src_port":38886,
"dest_ip":"128.199.21.18",
"dest_port":443,
"proto":"TCP",
"pkt_src":"wire/pcap",
"community_id":"1:A41J5o4e0N34stx1DwREiJbobt8=",
"alert":{"action":"blocked","gid":1,"signature_id":3005,"rev":1,"signature":"Block IP 128.199.21.18","category":"","severity":3},
"tls":{"sni":"abc.com","version":"UNDETERMINED","ja3":{"hash":"feed44cabd35129b172a17","string":"771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,11-16-65281-45-13-10-23-35-18-51,29-23-24,0"}},
"app_proto":"tls",
"direction":"to_server",
"flow":{"pkts_toserver":3,"pkts_toclient":1,"bytes_toserver":696,"bytes_toclient":60,"start":"2024-05-03T18:51:34.409331+0530","src_ip":"192.168.0.13","dest_ip":"128.199.21.18","src_port":38886,"dest_port":443}
}


# http
{
"timestamp":"2024-05-04T11:11:02.867940+0530",
"flow_id":1158110809570877,
"event_type":"http",
"src_ip":"192.168.0.13",
"src_port":39814,
"dest_ip":"142.250.196.46",
"dest_port":80,
"proto":"TCP",
"pkt_src":"wire/pcap",
"community_id":"1:PA7cqvS66klAis5kFxNrLRwitzA=",
"tx_id":1,
"http":{"hostname":"translate.google.com","url":"/translate_a/element.js?cb=googleTranslateElementInit","http_user_agent":"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36","http_content_type":"application/binary","http_refer":"http://www.softwareqatest.com/","http_method":"GET","protocol":"HTTP/1.1","status":301,"redirect":"https://translate.google.com/translate_a/element.js?cb=googleTranslateElementInit","length":0}

}
-------------------------------------------------------------------------------------------------
Below is the td-agent.conf file
-----------------------------------------------------------------------------------
<source>
    @type tail
    path /var/log/suricata/eve.json
    pos_file /var/log/suricata/eve.pos
    tag suricata_eve_logs
    <parse>
    @type json
    </parse>
    read_from_head true
    refresh_interval 2
</source>


<match suricata_eve_logs>
    @type rewrite_tag_filter
    <rule>
    key event_type
    pattern "tls"
    tag suricata_https
    </rule>

    <rule>
    key event_type
    pattern "http"
    tag suricata_http
    </rule>

    <rule>
    key event_type
    pattern "alert"
    tag suricata_alert
    </rule>
</match>



<match suricata_https>
    type stdout
    type postgres
    host localhost
    port 5432
    database abc
    username ****
    password ******
    key_names timestamp,src_ip,src_port,dest_ip,dest_port
    sql INSERT INTO https2 (time,src_ip,src_port,dest_ip,dest_port) VALUES ($1,$2,$3,$4,$5,$6)
    flush_intervals 5s
</match>

<match suricata_http>
    type stdout
    type postgres
    host localhost
    port 5432
    database abc
    username ****
    password ******
    key_names timestamp,src_ip,src_port,dest_ip,dest_port
    sql INSERT INTO http2 (time,src_ip,src_port,dest_ip,dest_port) VALUES ($1,$2,$3,$4,$5,$6)
    flush_intervals 5s
</match>

<match suricata_alert>
    type stdout
    type postgres
    host localhost
    port 5432
    database abc
    username ****
    password ******
    key_names timestamp,src_ip,src_port,dest_ip,dest_port
    sql INSERT INTO alerts2 (time,src_ip,src_port,dest_ip,dest_port) VALUES ($1,$2,$3,$4,$5,$6)
    flush_intervals 5s
</match>

-----------------------------------------------------------------------------------------------


i want to access the "alert":{"action":"blocked"} action key's value blocked from the alerts blocked json

i tried using an underscore as alert_action and using dot notaion alert.action to fetch the action key, but could not get the value to add to the database.

Upvotes: 0

Views: 89

Answers (0)

Related Questions