How to get TimeStamp using logstash from Json file? There multiple date fields in the JSON

Question

I have my JSON input as follows which has date field and need to extract the date time field from Json,

{
  "Properties": {
         "Client Name": "Chubb",
         "Portfolio": "Chubb-Transfer"
  },
"Capture": [
         {
            "CaptureGUID": "caa1f5ba-1e93-4926-b3ac-e30d0d9d4cbb",
            "HTMLPath": "Captures\\C:\\",
            "ScreenName": "Amdocs CRM - ClearCallCenter - [Console]",
            "TimeStamp": "20170926110036"
          },
         {
            "CaptureGUID": "0faf6b54-999f-4bfd-b8d0-e81a589f9185",
            "HTMLPath": "Captures\\C:\\",
            "ScreenName": "Microsoft Excel - 1.0.1 1.0.6 1.0.8 Match 3.0.6 Hit NAIC Optimized.xlsx",
            "TimeStamp": "20170926105418"
          }
     ]
}

and My Logstash Config is as below, how to convert the string date ("TimeStamp": "20170926105418") to date format.Have updated with full Logstash file

input {
    file { 
        type => "json"
        path => "C:/ELK/data/Recordings/*.json"
        start_position => beginning
        codec => multiline {
           pattern => "^{"
           negate => "true"
           what => "previous"
           multiline_tag => "multi_tagged"
           max_lines => 30000
       }
    }
}
filter{
    date {
        match => ["Capture.TimeStamp", "yyyyMMddHHmmss"]
        target => "TimeStamp"
    }

    mutate { 
    replace => { "message" => "%{message}}" }
    gsub => [ 'message','\n','']
    }

    json { 
        source => "message" 
        remove_field => ["message"]
    }


}

output {
    elasticsearch {
    index => "test10"
    }
    stdout { codec => rubydebug }
}

Answer 1

Have solved by the following,

input {
    file { 
        type => "json"
        path => "C:/ELK/data/Recordings/*.json"
        start_position => beginning
        codec => multiline {
           pattern => "^{"
           negate => "true"
           what => "previous"
           max_lines => 30000
       }
    }
}
filter{

    mutate { 
    replace => { "message" => "%{message}}" }
    gsub => [ 'message','\n','']
    }

    json { 
    source => "message" 
    remove_field => ["message"]
    }

    date {
    match => ["[Capture][0][TimeStamp]", "yyyyMMddHHmmss"]
    target=> "[Capture][0]StartTime"
    timezone => "Africa/Lome"
    locale => "en" 
    }


}

output {
    elasticsearch {
    index => "test15"
    }
    stdout { codec => rubydebug } 
}

Answer 2

Remove the date filter from logstash config file. Handle the date parsing while mapping the index. Below is the mapping for your use case.

PUT json
{
  "mappings": {
    "json": {
      "properties": {
        "Capture": {
          "type": "nested",
          "properties": {
            "CaptureGUID": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "HTMLPath": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "ScreenName": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "TimeStamp": {
              "type": "date",
              "format": "yyyyMMddHHmmss"
            }
          }
        },
        "Properties": {
          "properties": {
            "Client Name": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "Portfolio": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            }
          }
        }
      }
    }
  }
}