Reputation: 6758
ElasticSearch/Kibana fields on update of document
I'm very new to ES. I'm using https://github.com/dariusk/corpora/blob/master/data/humans/us_presidents.json as a learning set.
At first I entered this in Dev Tools tab in Kibana:
POST /presidents/president/1
{ "bo" :
{
"website":"",
"startdate":"2009-01-20",
"role_type_label":"President",
"enddate":"2013-01-20",
"description":"President",
"district":null,
"phone":null,
"title":"President",
"congress_numbers":[
111,
112,
113
],
"title_long":"President",
"current":false,
"person":{
"name":"President Barack Obama [D]",
"firstname":"Barack",
"twitterid":null,
"middlename":"",
"gender":"male",
"bioguideid":"O000167",
"namemod":"",
"birthday":"1961-08-04",
"link":"https://www.govtrack.us/congress/members/barack_obama/400629",
"youtubeid":null,
"sortname":"Obama, Barack (President) [D]",
"lastname":"Obama",
"gender_label":"Male",
"osid":"N00009638",
"pvsid":"9490",
"nickname":"",
"id":400629,
"cspanid":null
}
...
}
}
Then I realized that if I want add more data about individual presidents, I should rather do this:
POST /presidents/president/1
{
"website":"",
"startdate":"2009-01-20",
"role_type_label":"President",
"enddate":"2013-01-20",
"description":"President",
"district":null,
"phone":null,
"title":"President",
"congress_numbers":[
111,
112,
113
],
"title_long":"President",
"current":false,
"person":{
"name":"President Barack Obama [D]",
"firstname":"Barack",
"twitterid":null,
"middlename":"",
"gender":"male",
"bioguideid":"O000167",
"namemod":"",
"birthday":"1961-08-04",
"link":"https://www.govtrack.us/congress/members/barack_obama/400629",
"youtubeid":null,
"sortname":"Obama, Barack (President) [D]",
"lastname":"Obama",
"gender_label":"Male",
"osid":"N00009638",
"pvsid":"9490",
"nickname":"",
"id":400629,
"cspanid":null
}
}
OK, so ES accepted the update fine.
But now, when I go to Management / Index Patterns in Kibana, I see both person.lastname and bo.person.lastname as fields.
Why the previous field remained? Is this normal for ES to retain fields that are no longer in the updated document?
And obviously, apart from exceptionally funny ones, please no quips about today's election results.
Upvotes: 0
Views: 276
Answers (1)
Reputation: 2787
This is normal, expected, behaviour for Elasticsearch.
ES, by default, will dynamically map the data you are inserting. When you index multiple objects under the same type in an index, all of these objects share the same mapping. The intention is to allow objects to be inserted that do not necessarily carry all of the potential fields of its type, and be inserted into the index anyways.
You can define a mapping yourself, either upon index creation, or by defining a new type for an index. Mappings can also be updated, with some caveats.
To see the mapping of the type in your index, execute the following:
GET /tk_file.2016/TK_FILE/_mapping
Your response will look like this:
{
"presidents": {
"mappings": {
"president": {
"properties": {
"bo": {
"properties": {
"congress_numbers": {
"type": "long"
},
"current": {
"type": "boolean"
},
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"enddate": {
"type": "date"
},
"person": {
"properties": {
"bioguideid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"birthday": {
"type": "date"
},
"firstname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"gender": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"gender_label": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "long"
},
"lastname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"middlename": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"namemod": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"nickname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"osid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"pvsid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sortname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"role_type_label": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"startdate": {
"type": "date"
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title_long": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"website": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"congress_numbers": {
"type": "long"
},
"current": {
"type": "boolean"
},
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"enddate": {
"type": "date"
},
"person": {
"properties": {
"bioguideid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"birthday": {
"type": "date"
},
"firstname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"gender": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"gender_label": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"id": {
"type": "long"
},
"lastname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"link": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"middlename": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"namemod": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"nickname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"osid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"pvsid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"sortname": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"role_type_label": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"startdate": {
"type": "date"
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title_long": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"website": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
Notice here that you have a set of mappings for the bo object and its related sub-fields, as well as having a mapping for each field of your subsequent document. This is the effect of the dynamic mapping.
To disable this mapping flexibility, you can disable dynamic mappings explicitly, and defining the structure of the document yourself, including its datatypes. Perhaps you want the bioguideid to be an integer instead of text, as it is defined in the current mapping? I direct you to the Mappings API.
Upvotes: 1
Related Questions
- How to update value from a document in elasticsearch through Kibana
- elasticsearch return document on update
- add new field to existing documents by update in elasticsearch
- Update all documents field entry
- Update field in a document based on the condition in Kibana/Elasticsearch
- Update elastic search doc field value for specific fields in all documents
- elasticsearch script update based on existing doc value
- Updating document and adding new field in elastic search
- Return document on update elasticsearch
- elasticsearch update document frequently