Elastic Search when to add dynamic mappings

Question

I've been having troubles with Elastic Search (ES) dynamic mappings. Seems like I'm in a catch-22. https://www.elastic.co/guide/en/elasticsearch/guide/current/custom-dynamic-mapping.html

The main goal is to store everything as a string that comes into ES.

What I've tried:

1. In ES you can't create a dynamic mapping until the index has been created. Okay, makes sense.

2. I can't create an empty index, so if the first item sent into the index is not a string, I can't re-assign it... I won't know what type of object with be the first item in the index, it could be any type, due to how the the app accepts a variety of objects/events.

So if I can't create the mapping ahead of time, and I can't insert an empty index to create the mapping, and I can't change the mapping after the fact, how do I deal with the first item if its NOT a string???

Here's what I'm currently doing (using the Javascript Client).

createESIndex = function (esClient){
    esClient.index({
        index: 'timeline-2015-11-21',
        type: 'event',
        body: event
    },function (error, response) {
        if (error) {
            logger.log(logger.SEVERITY.ERROR, 'acceptEvent elasticsearch create failed with: '+ error + " req:" + JSON.stringify(event));
            console.log(logger.SEVERITY.ERROR, 'acceptEvent elasticsearch create failed with: '+ error + " req:" + JSON.stringify(event));
            res.status(500).send('Error saving document');
        } else {
            res.status(200).send('Accepted');
        }
    });
}

esClientLookup.getClient( function(esClient) {

    esClient.indices.putTemplate({
        name: "timeline-mapping-template",
        body:{
            "template": "timeline-*",
            "mappings": {
                "event": {
                    "dynamic_templates": [
                        { "timestamp-only": {
                              "match":              "@timestamp",
                              "match_mapping_type": "date",
                              "mapping": {
                                  "type":           "date",
                              }
                        }},
                        { "all-others": {
                              "match":              "*",
                              "match_mapping_type": "string",
                              "mapping": {
                                  "type":           "string",
                              }
                            }
                        }
                    ]
                }
            }
        }
    }).then(function(res){
        console.log("put template response: " + JSON.stringify(res));
        createESIndex(esClient);

    }, function(error){
        console.log(error);
        res.status(500).send('Error saving document');
    });
});

Answer 1

export const user = {
  email: {
    type: 'text',
  },
};
export const activity = {
  date: {
    type: 'text',
  },
};
export const common = {
  name: {
    type: 'text',
  },
};

import { Client } from '@elastic/elasticsearch';
import { user } from './user';
import { activity } from './activity';
import { common } from './common';

export class UserDataFactory {
  private schema = {
    ...user,
    ...activity,
    ...common,
    relation_type: {
      type: 'join',
      eager_global_ordinals: true,
      relations: {
        parent: ['activity'],
      },
    },
  };
  constructor(private client: Client) {
    Object.setPrototypeOf(this, UserDataFactory.prototype);
  }
  async create() {
    const settings = {
      settings: {
        analysis: {
          normalizer: {
            useLowercase: {
              filter: ['lowercase'],
            },
          },
        },
      },
      mappings: {
        properties: this.schema,
      },
    };

    const { body } = await this.client.indices.exists({
      index: ElasticIndex.UserDataFactory,
    });

    await Promise.all([
      await (async (client) => {
        await new Promise(async function (resolve, reject) {
          if (!body) {
            await client.indices.create({
              index: ElasticIndex.UserDataFactory,
            });
          }
          resolve({ body });
        });
      })(this.client),
    ]);

    await this.client.indices.close({ index: ElasticIndex.UserDataFactory });

    await this.client.indices.putSettings({
      index: ElasticIndex.UserDataFactory,
      body: settings,
    });

    await this.client.indices.open({
      index: ElasticIndex.UserDataFactory,
    });

    await this.client.indices.putMapping({
      index: ElasticIndex.UserDataFactory,
      body: {
        dynamic: 'strict',
        properties: {
          ...this.schema,
        },
      },
    });
  }
}

wrapper.ts

class ElasticWrapper {
  private _client: Client = new Client({
    node: process.env.elasticsearch_node,
    auth: {
      username: 'elastic',
      password: process.env.elasticsearch_password || 'changeme',
    },
    ssl: {
      ca: process.env.elasticsearch_certificate,
      rejectUnauthorized: false,
    },
  });
  get client() {
    return this._client;
  }
}

export const elasticWrapper = new ElasticWrapper();

index.ts

new UserDataFactory(elasticWrapper.client).create();

Answer 2

Index templates to the rescue !! That's exactly what you need, the idea is to create a template of your index and as soon as you wish to store a document in that index, ES will create it for you with the mapping you gave (even dynamic ones)

curl -XPUT localhost:9200/_template/my_template -d '{
  "template": "index_name_*",
  "settings": {
    "number_of_shards": 1
  },
  "mappings": {
    "type_name": {
      "dynamic_templates": [
        {
          "strings": {
            "match": "*",
            "match_mapping_type": "*",
            "mapping": {
              "type": "string"
            }
          }
        }
      ],
      "properties": {}
    }
  }
}'

Then when you index anything in an index whose name matches index_name_*, the index will be created with the dynamic mapping above.

For instance:

curl -XPUT localhost:9200/index_name_1/type_name/1 -d '{
  "one": 1,
  "two": "two", 
  "three": true
}'

That will create a new index called index_name_1 with a mapping type for type_name where all properties are string. You can verify that with

curl -XGET localhost:9200/index_name_1/_mapping/type_name

Response:

{
  "index_name_1" : {
    "mappings" : {
      "type_name" : {
        "dynamic_templates" : [ {
          "strings" : {
            "mapping" : {
              "type" : "string"
            },
            "match" : "*",
            "match_mapping_type" : "*"
          }
        } ],
        "properties" : {
          "one" : {
            "type" : "string"
          },
          "three" : {
            "type" : "string"
          },
          "two" : {
            "type" : "string"
          }
        }
      }
    }
  }
}

Note that if you're willing to do this via the Javascript API, you can use the indices.putTemplate call.