Elasticsearch mapping configuration from logstash

Question

I am trying to configure index template management for Elasticsearch from Logstash on Windows.

I have c:\ulyaoth\logstash-2.3.1\bin\logstash.json file:

    input {
      beats {
       port => 5044
       type => "log"
      }
    }

    filter {
        grok {
            match => ["message","%{TIMESTAMP_ISO8601:timestamp_match}"]
            remove_field => ["_id","_index","_score","_type","beat.hostname","beat.name","count","fileds","host","input_type","offset","tags","type"]
        }

        mutate {
            remove_field => ["_id","_index","_score","_type","beat.hostname","beat.name","count","fileds","host","input_type","offset","tags","type"]
        }

        date {
            match => ["timestamp_match","YYYY-MM-dd HH:mm:ss.SSS"]
            target => "timestamp_match"
        }
    }

    output {
      elasticsearch {
        hosts => "localhost:9200"
        index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
        document_type => "%{[@metadata][type]}"
        template => "c:/ulyaoth/logstash-2.3.1/bin/elasticsearch-template.custom.json"
        template_name => "elasticsearch-template"
        manage_template => true
        template_overwrite => true
      }
    }

and the template file c:/ulyaoth/logstash-2.3.1/bin/elasticsearch-template.custom.json. I dug up this file from c:\ulyaoth\logstash-2.3.1\vendor\bundle\jruby\1.9\gems\logstash-output-elasticsearch-2.5.5-java\lib\logstash\outputs\elasticsearch\ and edited it so that:

    "source":{"index": "not_analyzed"}

This is the whole file:

    {
      "template" : "logstash-*",
      "settings" : {
        "index.refresh_interval" : "5s"
      },
      "mappings" : {
        "_default_" : {
          "_all" : {"enabled" : true, "omit_norms" : true},
          "dynamic_templates" : [ {
            "message_field" : {
              "match" : "message",
              "match_mapping_type" : "string",
              "mapping" : {
                "type" : "string", "index" : "analyzed", "omit_norms" : true,
                "fielddata" : { "format" : "disabled" }
              }
            }
          }, {
            "string_fields" : {
              "match" : "*",
              "match_mapping_type" : "string",
              "mapping" : {
                "type" : "string", "index" : "analyzed", "omit_norms" : true,
                "fielddata" : { "format" : "disabled" },
                "fields" : {
                  "raw" : {"type": "string", "index" : "not_analyzed", "doc_values" : true, "ignore_above" : 256}
                }
              }
            }
          }, {
            "float_fields" : {
              "match" : "*",
              "match_mapping_type" : "float",
              "mapping" : { "type" : "float", "doc_values" : true }
            }
          }, {
            "double_fields" : {
              "match" : "*",
              "match_mapping_type" : "double",
              "mapping" : { "type" : "double", "doc_values" : true }
            }
          }, {
            "byte_fields" : {
              "match" : "*",
              "match_mapping_type" : "byte",
              "mapping" : { "type" : "byte", "doc_values" : true }
            }
          }, {
            "short_fields" : {
              "match" : "*",
              "match_mapping_type" : "short",
              "mapping" : { "type" : "short", "doc_values" : true }
            }
          }, {
            "integer_fields" : {
              "match" : "*",
              "match_mapping_type" : "integer",
              "mapping" : { "type" : "integer", "doc_values" : true }
            }
          }, {
            "long_fields" : {
              "match" : "*",
              "match_mapping_type" : "long",
              "mapping" : { "type" : "long", "doc_values" : true }
            }
          }, {
            "date_fields" : {
              "match" : "*",
              "match_mapping_type" : "date",
              "mapping" : { "type" : "date", "doc_values" : true }
            }
          }, {
            "geo_point_fields" : {
              "match" : "*",
              "match_mapping_type" : "geo_point",
              "mapping" : { "type" : "geo_point", "doc_values" : true }
            }
          } ],
          "properties" : {
            "@timestamp": { "type": "date", "doc_values" : true },
            "@version": { "type": "string", "index": "not_analyzed", "doc_values" : true },
            "source":{"index": "not_analyzed"}
            "geoip"  : {
              "type" : "object",
              "dynamic": true,
              "properties" : {
                "ip": { "type": "ip", "doc_values" : true },
                "location" : { "type" : "geo_point", "doc_values" : true },
                "latitude" : { "type" : "float", "doc_values" : true },
                "longitude" : { "type" : "float", "doc_values" : true }
              }
            }
          }
        }
      }
    }

My problem is that the template is not registered. The REST query returns empty object, and also I see the field still analyzed in Kibana.

    GET /_template HTTP/1.1
    Host: 127.0.0.1:9200

Additional problem is that remove_field also does not work - I still see all these fields.

    remove_field => ["_id","_index","_score","_type","beat.hostname","beat.name","count","fileds","host","input_type","offset","tags","type"]

I don't see any logstash logs (ironically:) and in ES logs I don't see any errors or template issues.

How can these problemse be sloved?

EDIT:

The final working configuration is:

    {
      "template" : "filebeat-*",
      "settings" : {
        "index.refresh_interval" : "5s"
      },
      "mappings" : {
        "_default_" : {
          "_all" : {"enabled" : true, "omit_norms" : true},
          "dynamic_templates" : [ {
            "message_field" : {
              "match" : "message",
              "match_mapping_type" : "string",
              "mapping" : {
                "type" : "string", "index" : "analyzed", "omit_norms" : true,
                "fielddata" : { "format" : "disabled" }
              }
            }
          }, {
            "string_fields" : {
              "match" : "*",
              "match_mapping_type" : "string",
              "mapping" : {
                "type" : "string", "index" : "analyzed", "omit_norms" : true,
                "fielddata" : { "format" : "disabled" },
                "fields" : {
                  "raw" : {"type": "string", "index" : "not_analyzed", "doc_values" : true, "ignore_above" : 256}
                }
              }
            }
          }, {
            "float_fields" : {
              "match" : "*",
              "match_mapping_type" : "float",
              "mapping" : { "type" : "float", "doc_values" : true }
            }
          }, {
            "double_fields" : {
              "match" : "*",
              "match_mapping_type" : "double",
              "mapping" : { "type" : "double", "doc_values" : true }
            }
          }, {
            "byte_fields" : {
              "match" : "*",
              "match_mapping_type" : "byte",
              "mapping" : { "type" : "byte", "doc_values" : true }
            }
          }, {
            "short_fields" : {
              "match" : "*",
              "match_mapping_type" : "short",
              "mapping" : { "type" : "short", "doc_values" : true }
            }
          }, {
            "integer_fields" : {
              "match" : "*",
              "match_mapping_type" : "integer",
              "mapping" : { "type" : "integer", "doc_values" : true }
            }
          }, {
            "long_fields" : {
              "match" : "*",
              "match_mapping_type" : "long",
              "mapping" : { "type" : "long", "doc_values" : true }
            }
          }, {
            "date_fields" : {
              "match" : "*",
              "match_mapping_type" : "date",
              "mapping" : { "type" : "date", "doc_values" : true }
            }
          }, {
            "geo_point_fields" : {
              "match" : "*",
              "match_mapping_type" : "geo_point",
              "mapping" : { "type" : "geo_point", "doc_values" : true }
            }
          } ],
          "properties" : {
            "@timestamp": { "type": "date", "doc_values" : true },
            "@version": { "type": "string", "index": "not_analyzed", "doc_values" : true },
            "source":{ "type": "string", "index": "not_analyzed"}
            "geoip"  : {
              "type" : "object",
              "dynamic": true,
              "properties" : {
                "ip": { "type": "ip", "doc_values" : true },
                "location" : { "type" : "geo_point", "doc_values" : true },
                "latitude" : { "type" : "float", "doc_values" : true },
                "longitude" : { "type" : "float", "doc_values" : true }
              }
            }
          }
        }
      }
    }

• changed
  "template" : "filebeat-*", and "source":{ "type": "string", "index": "not_analyzed"}

Answer 1

The source field doesn't have a type. Maybe you meant:

"source":{ "type": "string", "index": "not_analyzed"},