CODEWITHSUNDEEP
logstashlogstash-grok
JaneD
JaneD

Reputation: 199

Logstash multiple unable to parse and add new date

I'm trying to rewrite the timestamp for the logs that come through logstash this way :

filter {
    grok {
      match => { "message" => "(?<timestamp>%{DAY:day} %{MONTH:month} *%{MONTHDAY:mday} %{TIME:time} %{YEAR:year})(?<message>.*$)" }
      overwrite => [ "message" ]
      add_tag => [ "oracle_audit" ]
    }
grok {
     match => { "message" => "ACTION :\[[0-9]*\] '(?<ora_audit_action>.*)'.*DATABASE USER:\[[0-9]*\] '(?<ora_audit_dbuser>.*)'.*PRIVILEGE :\[[0-9]*\] '(?<ora_audit_priv>.*)'.*CLIENT USER:\[[0-9]*\] '(?<ora_audit_osuser>.*)'.*CLIENT TERMINAL:\[[0-9]*\] '(?<ora_audit_term>.*)'.*STATUS:\[[0-9]*\] '(?<ora_audit_status>.*)'.*DBID:\[[0-9]*\] '(?<ora_audit_dbid>.*)'.*SESSIONID:\[[0-9]*\] '(?<ora_audit_sessionid>.*)'.*USERHOST:\[[0-9]*\] '(?<ora_audit_dbhost>.*)'.*CLIENT ADDRESS:\[[0-9]*\] '(?<ora_audit_clientaddr>.*)'.*ACTION NUMBER:\[[0-9]*\] '(?<ora_audit_actionnum>.*)'" }
}
prune {
     whitelist_names => ["ora_audit_action", "ora_audit_dbuser", "ora_audit_dbid", "ora_audit_status", "ora_audit_osuser", "ora_audit_priv", "ora_audit_term", "ora_audit_sessionid", "ora_audit_dbhost", "ora_audit_clientaddr", "ora_audit_actionnum", "host", "@timestamp", "@version", "message"]
}

mutate {
       add_field => {
        "timestamp" => "%{year} %{month} %{mday} %{time}"
       }
  }

  date {
      locale => "en"
      match => [ "timestamp" , "yyyy MMM dd HH:mm:ss", "yyyy MMM d HH:mm:ss", "yyyy MMM  d HH:mm:ss" ]
      timezone => "UTC"
    }
}

However it kept giving me parser error :

{
         "ora_audit_term" => "pts/0",
    "ora_audit_actionnum" => "3",
               "@version" => "1",
       "ora_audit_dbuser" => "/",
    "ora_audit_sessionid" => "4294967295",
       "ora_audit_action" => "SELECT STATUS FROM V$INSTANCE",
                   "tags" => [
        [0] "_dateparsefailure"
    ],
       "ora_audit_osuser" => "test_user",
         "ora_audit_dbid" => "1762369616",
              "timestamp" => "%{year} %{month} %{mday} %{time}",
             "@timestamp" => 2020-10-05T13:43:38.238Z,
         "ora_audit_priv" => "SYSDBA",
                "message" => " +00:00\nLENGTH : '296'\nACTION :[29] 'SELECT STATUS FROM V$INSTANCE'\nDATABASE USER:[1] '/'\nPRIVILEGE :[6] 'SYSDBA'\nCLIENT USER:[9] 'test_user'\nCLIENT TERMINAL:[5] 'pts/0'\nSTATUS:[1] '0'\nDBID:[10] '1762369616'\nSESSIONID:[10] '4294967295'\nUSERHOST:[21] 'testdevserver'\nCLIENT ADDRESS:[0] ''\nACTION NUMBER:[1] '3'",
       "ora_audit_dbhost" => "testdevserver",
       "ora_audit_status" => "0"
}

Below is a sample log:


Thu Oct  1 23:01:00 2020 +00:00
LENGTH : '296'
ACTION :[29] 'SELECT STATUS FROM V$INSTANCE'
DATABASE USER:[1] '/'
PRIVILEGE :[6] 'SYSDBA'
CLIENT USER:[9] 'test_user'
CLIENT TERMINAL:[5] 'pts/0'
STATUS:[1] '0'
DBID:[10] '1762369616'
SESSIONID:[10] '4294967295'
USERHOST:[21] 'testdevserver'
CLIENT ADDRESS:[0] ''
ACTION NUMBER:[1] '3'

What can I do to get it to parse the timestamp properly so it will match the timestamp of the logs? :(

Thanks J

Upvotes: 0

Views: 67

Answers (1)

Badger
Badger

Reputation: 4072

"timestamp" => "%{year} %{month} %{mday} %{time}",

The date filter fails because it cannot parse that. The mutate+add_field creates that because none of those fields exists. Even if the grok created those fields the prune+whitelist_names would delete them!

Upvotes: 1

Related Questions