CODEWITHSUNDEEP
node.jselasticsearchloggingfluentd
fedor-sg
fedor-sg

Reputation: 229

Creating mapping in Elasticsearch with Fluentd

I'm trying to create mapping in Elasticsearch using Fluentd, which Node.js connects to.

Elasticsearch mapping example:

PUT http://host:9200/test_mapping
{
  "mappings": {
    "properties": {
        "response_code": {
            "type": "text",
            "fielddata": true
        },
        "response_text": {
            "type": "text",
            "fielddata": true
        },
        "status": {
            "type": "boolean"
        },
        "ip": {
            "type": "ip"
        },
        "login": {
            "type": "text",
            "fielddata": true
        }
    }
  }
}

Fluentd configuration example:

<source>
  @type forward
  port 24225
</source>

<match mapping.doc>
  @type elasticsearch
  logstash_format "#{ENV['LOGSTASH_FORMAT']}"
  scheme "#{ENV['SCHEME']}"
  host "#{ENV['HOST']}"
  port "#{ENV['PORT']}"
  write_operation index
  index_name "#{ENV['INDEX_NAME']}"
  flush_interval "#{ENV['FLUSH_INTERVAL']}"
</match>

Sample code on Node.js:

// ...
require('dotenv').config();
const env = process.env;
const loggerfluentd = require('fluent-logger');
loggerfluentd.configure('mapping', {
            host: env.FLUENTD_HOST,
            port: Number.parseInt(env.FLUENTD_PORT),
            timeout: 3.0,
            reconnectInterval: 10000 // 10 sec
        });

function EmitMapping(data) {
    loggerfluentd.emit(env.INDEX_NAME, data);
}

exports.EmitMapping = EmitMapping;

This configuration does not create mapping, but simply adds new documents to Elasticsearch.

Is it possible to change the configuration so that by executing the EmitMapping () function not to add new documents (with automatically assigned data types in mapping), namely to create your own mapping with your own data types?

Upvotes: 0

Views: 1276

Answers (1)

fedor-sg
fedor-sg

Reputation: 229

There is a possibility that the elasticsearch plugin does not create and does not change the index, but simply writes to the index, therefore I used the http plugin:

<match mapping.doc>
  @type http
  endpoint "#{ENV['SCHEME']}://#{ENV['HOST']}:#{ENV['PORT']}/#{ENV['INDEX_NAME']}"
  http_method put
  headers {"Content-Type":"application/json"}
  open_timeout 2
  <buffer>
    flush_interval "#{ENV['FLUSH_INTERVAL']}"
  </buffer>
</match>

Upvotes: 1

Related Questions