CODEWITHSUNDEEP
datetimeelasticsearchlogstashjodatime
niting
niting

Reputation: 55

ElasticSearch indexing issue ,failed to parse timestamp

i am new to ELK . i have created index in Elasticsearch

{
  "logstash": {
    "aliases": {},
    "mappings": {
      "log": {
        "dynamic_templates": [
          {
            "message_field": {
              "path_match": "message",
              "match_mapping_type": "string",
              "mapping": {
                "norms": false,
                "type": "text"
              }
            }
          },
          {
            "string_fields": {
              "match": "*",
              "match_mapping_type": "string",
              "mapping": {
                "fields": {
                  "keyword": {
                    "type": "keyword"
                  }
                },
                "norms": false,
                "type": "text"
              }
            }
          }
        ],
        "properties": {
          "@timestamp": {
            "type": "date"
          },
          "@version": {
            "type": "keyword",
            "include_in_all": false
          },
          "activity": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword"
              }
            }
          },
          "beat": {
            "properties": {
              "hostname": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword"
                  }
                }
              },
              "name": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword"
                  }
                }
              },
              "version": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword"
                  }
                }
              }
            }
          },
          "filename": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword"
              }
            }
          },
          "host": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword"
              }
            }
          },
          "input_type": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword"
              }
            }
          },
          "message": {
            "type": "text",
            "norms": false
          },
          "offset": {
            "type": "long"
          },
          "source": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword"
              }
            }
          },
          "tags": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword"
              }
            }
          },
          "timestamp": {
            "type": "date",
            "include_in_all": false,
            "format": "YYYY-MM-DD HH:mm:ss.SSS"
          },
          "type": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword"
              }
            }
          },
          "user": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword"
              }
            }
          }
        }
      }
    },
    "settings": {
      "index": {
        "creation_date": "1488805244467",
        "number_of_shards": "1",
        "number_of_replicas": "0",
        "uuid": "5ijhh193Tr6y_hxaQrW9kg",
        "version": {
          "created": "5020199"
        },
        "provided_name": "logstash"
      }
    }
  }
}

Below is my logstash configuration

input{
    beats{
        port=>5044
    }
}filter{
    grok{
        match=>{"message" => "\[%{TIMESTAMP_ISO8601:timestamp}\] ALL AUDIT: User \[%{GREEDYDATA:user}\] is %{GREEDYDATA:activity} \[%{GREEDYDATA:filename}\] for transfer."}
    }
}output{
    elasticsearch{
        hosts=>"localhost:9200"
        index=> "logstash"
    }

Sample Data

[2017-03-05 12:37:21.465] ALL AUDIT: User [user1] is opening file [filename1] for transfer.

but when i am loading file through filebeat > logstash > elasticsearch in elasticsearch i am getting below error

org.elasticsearch.index.mapper.MapperParsingException: failed to parse [timestamp]
Caused by: java.lang.IllegalArgumentException: Invalid format: "2017-03-05T12:36:33.606" is malformed at "12:36:33.606"
    at org.joda.time.format.DateTimeParserBucket.doParseMillis(DateTimeParserBucket.java:187) ~[joda-time-2.9.5.jar:2.9.5]

Please help , what timestamp format should i configure ?

Upvotes: 1

Views: 4247

Answers (1)

Adonis
Adonis

Reputation: 4818

In your timestamp mapping you indicate the format as "format": "YYYY-MM-DD HH:mm:ss.SSS" Here the format you are sending through beats is not the same, check: 2017-03-05T12:36:33.606

That's why Elastic is complaining about the format. Your format should be: "YYYY-MM-DD'T'HH:mm:ss.SSS" (notice the capital T)

See the documentation for more details: https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html

Upvotes: 2

Related Questions