ahmokhtari
Reputation: 71
Apply mappings in ElasticSearch 7
I have been using ElasticSearch 6 for a long time and managed to apply a mapping for my most used fields. Recently, I've created another cluster (v7.7), but cannot apply the same mappings as the default has been deprecated. I have some nested objects in my mappings. When I replace default with _doc, I can apply the mapping, however all the incoming requests to ES will be rejected and will go to Logstash dead letter queue. (I'm using logstash and sending my application logs to ElasticSearch. Here is my mapping json:
"index_patterns": [
"logstash-*"
],
"settings": {
"index": {
"mapping": {
"total_fields": {
"limit": "50000"
}
},
"refresh_interval": "5s"
}
},
"mappings": {
"_default_": {
"_all": {
"norms": false
},
"properties": {
"@timestamp": {
"type": "date"
},
"name": {
"ignore_above": 64,
"type": "keyword"
},
"logger": {
"ignore_above": 64,
"type": "keyword"
},
"hostname": {
"ignore_above": 256,
"type": "keyword"
},
"pid": {
"type": "integer"
},
"level": {
"type": "short"
},
"v": {
"type": "short"
},
"env": {
"ignore_above": 64,
"type": "keyword"
},
"service": {
"ignore_above": 256,
"type": "keyword"
},
"message": {
"type": "text"
},
"module": {
"type": "keyword"
},
"subModule": {
"type": "keyword"
},
"docker": {
"properties": {
"container_id": {
"ignore_above": 512,
"type": "keyword"
},
"container_name": {
"ignore_above": 512,
"type": "keyword"
},
"image_id": {
"ignore_above": 512,
"type": "keyword"
},
"image_name": {
"ignore_above": 512,
"type": "keyword"
},
"command": {
"ignore_above": 512,
"type": "keyword"
},
"tag": {
"ignore_above": 512,
"type": "keyword"
},
"created": {
"type": "date"
}
}
},
"data": {
"type": "text"
},
"context": {
"properties": {
"trade": {
"properties": {
"id": {
"ignore_above": 64,
"type": "keyword"
}
}
},
"trader": {
"properties": {
"id": {
"ignore_above": 64,
"type": "keyword"
},
"email": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"payment": {
"properties": {
"id": {
"ignore_above": 64,
"type": "keyword"
}
}
},
"err": {
"properties": {
"message": {
"type": "text"
},
"name": {
"ignore_above": 256,
"type": "keyword"
},
"stack": {
"type": "text"
},
"code": {
"ignore_above": 256,
"type": "keyword"
},
"file": {
"ignore_above": 256,
"type": "keyword"
},
"line": {
"type": "integer"
},
"cause": {
"properties": {
"message": {
"type": "text"
},
"name": {
"ignore_above": 256,
"type": "keyword"
},
"stack": {
"type": "text"
},
"code": {
"ignore_above": 256,
"type": "keyword"
},
"file": {
"ignore_above": 256,
"type": "keyword"
},
"line": {
"type": "integer"
},
"cause": {
"properties": {
"message": {
"type": "text"
},
"name": {
"ignore_above": 256,
"type": "keyword"
},
"stack": {
"type": "text"
},
"code": {
"ignore_above": 256,
"type": "keyword"
},
"file": {
"ignore_above": 256,
"type": "keyword"
},
"line": {
"type": "integer"
}
}
}
}
}
}
},
"req": {
"properties": {
"id": {
"ignore_above": 64,
"type": "keyword"
},
"remoteAddress": {
"type": "ip"
},
"remotePort": {
"type": "integer"
},
"method": {
"ignore_above": 32,
"type": "keyword"
},
"path": {
"type": "text"
},
"body": {
"type": "text"
},
"bodyLength": {
"type": "integer"
},
"headers": {
"type": "text"
}
}
},
"res": {
"properties": {
"statusCode": {
"type": "short"
},
"body": {
"type": "text"
},
"bodyLength": {
"type": "integer"
},
"headers": {
"type": "text"
},
"latency": {
"type": "integer"
}
}
},
"event": {
"properties": {
"eventName": {
"ignore_above": 256,
"type": "keyword"
},
"context": {
"type": "text"
},
"uuid": {
"ignore_above": 64,
"type": "keyword"
},
"time": {
"type": "long"
},
"attempts": {
"type": "short"
}
}
},
"task": {
"properties": {
"taskName": {
"ignore_above": 256,
"type": "keyword"
},
"context": {
"type": "text"
},
"uuid": {
"ignore_above": 64,
"type": "keyword"
},
"time": {
"type": "long"
},
"attempts": {
"type": "short"
},
"origin": {
"ignore_above": 128,
"type": "keyword"
}
}
}
}
},
"dlq": {
"properties": {
"eventFields": {
"type": "text"
},
"reason": {
"type": "text"
}
}
},
"tags": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
}
I'm wondering how can I apply this to elasticSearch7?
Here is the ElasticSearch error:
[o.e.a.a.i.m.p.TransportPutMappingAction] [0c85334cb2d41154383d0174502e13f2] failed to put mappings on indices [[[__PATH__]]], type [logs]
java.lang.IllegalArgumentException: Rejecting mapping update to [logstash-2020.10.02] as the final mapping would have more than 1 type: [_doc, logs]
Upvotes: 0
Views: 1496
Answers (4)
Diwakar Kumar Keshri
Reputation: 1
Elasticsearch 7.0 using php 7.4 version. TO mapping the database table in Elasticsearch 7.0 in server
<?php
require 'includes/config.php';
require 'includes/connection.php';
require 'vendor/autoload.php';
class SearchElastic {
private $elasticclient = null;
public function __construct(){
$db = new Connection();
$this->con = $db->connect();
//echo "<pre>";print_r($this->con); die;
$elasticUrls = 'http://'.ELASTICSEARCH_HOST.':'.ELASTICSEARCH_PORT;
$hosts = [$elasticUrls]; // SSL to localhost
$this->elasticclient = Elasticsearch\ClientBuilder::create()->setHosts($hosts)->build();
//echo "<pre>"; print_r($this->elasticclient); die;
}
public function Mapping(){
$params = ['index' => 'employee'];
$response = $this->elasticclient->indices()->delete($params);
$params = [
'index' => 'employee',
'body' => [
'products' => [
'mappings' => [
'properties' => [
'code' => [
'type' => 'integer'
],
'name' => [
'type' => 'text'
],
'created_at' => [
'type' => 'text'
],
'last_updated' => [
'type' => 'text'
],
'rank' => [
'type' => 'integer'
],
]
]
]
]
];
//echo "<pre>"; print_r($params); die;
$this->elasticclient->indices()->create($params);
//echo "<pre>"; print_r($var); die;
}
public function Search($query){
$client = $this->elasticclient;
$result = array();
$i = 0;
$params = [
'index' => 'employee',
'type' => 'doc',
'body' => [
'query' => [
'match' => ['name' => $query],
],
'size' => 9,
'sort' => [
['rank' => 'desc'],
],
],
];
$query = $client->search($params);
//echo "<pre>"; print_r($query);die;
$hits = sizeof($query['hits']['hits']);
$hit = $query['hits']['hits'];
$result['searchfound'] = $hits;
while ($i < $hits) {
$result['result'][$i] = $query['hits']['hits'][$i]['_source'];
$i++;
}
return $result;
}
public function InsertData(){
$this->Mapping();
$client = $this->elasticclient;
$stmt = "SELECT * FROM `table_name` where id=1";
$result = $this->con->query($stmt);
$params = null;
while ($row = $result->fetch_assoc()){
$params['body'][] = array(
'index' => array(
'_index' => 'employee',
'_type' => 'doc',
'_id' => $row['id'],
) ,
);
$params['body'][] = [
'id' => $row['id'],
'name' => $row['name'],
'created_at' => $row['created_at'],
'last_updated' => $row['last_updated'],
'rank' => $row['rank'],
];
}
$responses = $client->bulk($params);
//echo "<pre>"; print_r($responses); die;
return true;
}
public function UpdateData(){
$client = $this->elasticclient;
$stmt = "SELECT * FROM `table_name` limit 1, 100";
$result = $this->con->query($stmt);
$params = null;
while ($row = $result->fetch_assoc()){
$params['body'][] = array(
'index' => array(
'_index' => 'employee',
'_type' => 'doc',
'_id' => $row['id'],
) ,
);
$params['body'][] = [
'id' => $row['id'],
'name' => $row['name'],
'created_at' => $row['created_at'],
'last_updated' => $row['last_updated'],
'rank' => $row['rank'],
];
}
$responses = $client->bulk($params);
//echo "<pre>"; print_r($responses); die;
return true;
}
}
?>
Upvotes: 0
ahmokhtari
Reputation: 71
I found the answer!
Logstash sends the logs to ElasticSearch with type logs. Replacing the default with logs fixed the issue. However, for applying the mappings in version7, you need to use PUT /_template/template_1?include_type_name=true
Upvotes: 1
Bhavya
Reputation: 16192
The answer given by @YLR is almost correct, but there are some parsing errors in the mapping provided by him.
_all field is deprecated in 6.0
_all may no longer be enabled for indices created in 6.0+, use a custom field and the mapping copy_to parameter
Refer to this Elasticsearch documentation, to know more about this.
Indices created in Elasticsearch 7.0.0 or later no longer accept a default mapping. Indices created in 6.x will continue to function as before in Elasticsearch 6.x. Types are deprecated in APIs in 7.0, with breaking changes to the index creation, put mapping, get mapping, put template, get template and get field mappings APIs.
The modified mapping is :
{
"index_patterns": [
"logstash-*"
],
"settings": {
"index": {
"mapping": {
"total_fields": {
"limit": "50000"
}
},
"refresh_interval": "5s"
}
},
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"name": {
"ignore_above": 64,
"type": "keyword"
},
"logger": {
"ignore_above": 64,
"type": "keyword"
},
"hostname": {
"ignore_above": 256,
"type": "keyword"
},
"pid": {
"type": "integer"
},
"level": {
"type": "short"
},
"v": {
"type": "short"
},
"env": {
"ignore_above": 64,
"type": "keyword"
},
"service": {
"ignore_above": 256,
"type": "keyword"
},
"message": {
"type": "text"
},
"module": {
"type": "keyword"
},
"subModule": {
"type": "keyword"
},
"docker": {
"properties": {
"container_id": {
"ignore_above": 512,
"type": "keyword"
},
"container_name": {
"ignore_above": 512,
"type": "keyword"
},
"image_id": {
"ignore_above": 512,
"type": "keyword"
},
"image_name": {
"ignore_above": 512,
"type": "keyword"
},
"command": {
"ignore_above": 512,
"type": "keyword"
},
"tag": {
"ignore_above": 512,
"type": "keyword"
},
"created": {
"type": "date"
}
}
},
"data": {
"type": "text"
},
"context": {
"properties": {
"trade": {
"properties": {
"id": {
"ignore_above": 64,
"type": "keyword"
}
}
},
"trader": {
"properties": {
"id": {
"ignore_above": 64,
"type": "keyword"
},
"email": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"payment": {
"properties": {
"id": {
"ignore_above": 64,
"type": "keyword"
}
}
},
"err": {
"properties": {
"message": {
"type": "text"
},
"name": {
"ignore_above": 256,
"type": "keyword"
},
"stack": {
"type": "text"
},
"code": {
"ignore_above": 256,
"type": "keyword"
},
"file": {
"ignore_above": 256,
"type": "keyword"
},
"line": {
"type": "integer"
},
"cause": {
"properties": {
"message": {
"type": "text"
},
"name": {
"ignore_above": 256,
"type": "keyword"
},
"stack": {
"type": "text"
},
"code": {
"ignore_above": 256,
"type": "keyword"
},
"file": {
"ignore_above": 256,
"type": "keyword"
},
"line": {
"type": "integer"
},
"cause": {
"properties": {
"message": {
"type": "text"
},
"name": {
"ignore_above": 256,
"type": "keyword"
},
"stack": {
"type": "text"
},
"code": {
"ignore_above": 256,
"type": "keyword"
},
"file": {
"ignore_above": 256,
"type": "keyword"
},
"line": {
"type": "integer"
}
}
}
}
}
}
},
"req": {
"properties": {
"id": {
"ignore_above": 64,
"type": "keyword"
},
"remoteAddress": {
"type": "ip"
},
"remotePort": {
"type": "integer"
},
"method": {
"ignore_above": 32,
"type": "keyword"
},
"path": {
"type": "text"
},
"body": {
"type": "text"
},
"bodyLength": {
"type": "integer"
},
"headers": {
"type": "text"
}
}
},
"res": {
"properties": {
"statusCode": {
"type": "short"
},
"body": {
"type": "text"
},
"bodyLength": {
"type": "integer"
},
"headers": {
"type": "text"
},
"latency": {
"type": "integer"
}
}
},
"event": {
"properties": {
"eventName": {
"ignore_above": 256,
"type": "keyword"
},
"context": {
"type": "text"
},
"uuid": {
"ignore_above": 64,
"type": "keyword"
},
"time": {
"type": "long"
},
"attempts": {
"type": "short"
}
}
},
"task": {
"properties": {
"taskName": {
"ignore_above": 256,
"type": "keyword"
},
"context": {
"type": "text"
},
"uuid": {
"ignore_above": 64,
"type": "keyword"
},
"time": {
"type": "long"
},
"attempts": {
"type": "short"
},
"origin": {
"ignore_above": 128,
"type": "keyword"
}
}
}
}
},
"dlq": {
"properties": {
"eventFields": {
"type": "text"
},
"reason": {
"type": "text"
}
}
},
"tags": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
Upvotes: 1
YLR
Reputation: 1540
You must update some field that doesn't been supporte in v7.
This request is an example with remove / update unsupported field :
PUT _index_template/template_1
{
"index_patterns": [
"logstash-*"
],
"template": {
"settings": {
"index": {
"mapping": {
"total_fields": {
"limit": "50000"
}
},
"refresh_interval": "5s"
}
},
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"name": {
"ignore_above": 64,
"type": "keyword"
},
"logger": {
"ignore_above": 64,
"type": "keyword"
},
"hostname": {
"ignore_above": 256,
"type": "keyword"
},
"pid": {
"type": "integer"
},
"level": {
"type": "short"
},
"v": {
"type": "short"
},
"env": {
"ignore_above": 64,
"type": "keyword"
},
"service": {
"ignore_above": 256,
"type": "keyword"
},
"message": {
"type": "text"
},
"module": {
"type": "keyword"
},
"subModule": {
"type": "keyword"
},
"docker": {
"properties": {
"container_id": {
"ignore_above": 512,
"type": "keyword"
},
"container_name": {
"ignore_above": 512,
"type": "keyword"
},
"image_id": {
"ignore_above": 512,
"type": "keyword"
},
"image_name": {
"ignore_above": 512,
"type": "keyword"
},
"command": {
"ignore_above": 512,
"type": "keyword"
},
"tag": {
"ignore_above": 512,
"type": "keyword"
},
"created": {
"type": "date"
}
}
},
"data": {
"type": "text"
},
"context": {
"properties": {
"trade": {
"properties": {
"id": {
"ignore_above": 64,
"type": "keyword"
}
}
},
"trader": {
"properties": {
"id": {
"ignore_above": 64,
"type": "keyword"
},
"email": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"payment": {
"properties": {
"id": {
"ignore_above": 64,
"type": "keyword"
}
}
},
"err": {
"properties": {
"message": {
"type": "text"
},
"name": {
"ignore_above": 256,
"type": "keyword"
},
"stack": {
"type": "text"
},
"code": {
"ignore_above": 256,
"type": "keyword"
},
"file": {
"ignore_above": 256,
"type": "keyword"
},
"line": {
"type": "integer"
},
"cause": {
"properties": {
"message": {
"type": "text"
},
"name": {
"ignore_above": 256,
"type": "keyword"
},
"stack": {
"type": "text"
},
"code": {
"ignore_above": 256,
"type": "keyword"
},
"file": {
"ignore_above": 256,
"type": "keyword"
},
"line": {
"type": "integer"
},
"cause": {
"properties": {
"message": {
"type": "text"
},
"name": {
"ignore_above": 256,
"type": "keyword"
},
"stack": {
"type": "text"
},
"code": {
"ignore_above": 256,
"type": "keyword"
},
"file": {
"ignore_above": 256,
"type": "keyword"
},
"line": {
"type": "integer"
}
}
}
}
}
}
},
"req": {
"properties": {
"id": {
"ignore_above": 64,
"type": "keyword"
},
"remoteAddress": {
"type": "ip"
},
"remotePort": {
"type": "integer"
},
"method": {
"ignore_above": 32,
"type": "keyword"
},
"path": {
"type": "text"
},
"body": {
"type": "text"
},
"bodyLength": {
"type": "integer"
},
"headers": {
"type": "text"
}
}
},
"res": {
"properties": {
"statusCode": {
"type": "short"
},
"body": {
"type": "text"
},
"bodyLength": {
"type": "integer"
},
"headers": {
"type": "text"
},
"latency": {
"type": "integer"
}
}
},
"event": {
"properties": {
"eventName": {
"ignore_above": 256,
"type": "keyword"
},
"context": {
"type": "text"
},
"uuid": {
"ignore_above": 64,
"type": "keyword"
},
"time": {
"type": "long"
},
"attempts": {
"type": "short"
}
}
},
"task": {
"properties": {
"taskName": {
"ignore_above": 256,
"type": "keyword"
},
"context": {
"type": "text"
},
"uuid": {
"ignore_above": 64,
"type": "keyword"
},
"time": {
"type": "long"
},
"attempts": {
"type": "short"
},
"origin": {
"ignore_above": 128,
"type": "keyword"
}
}
}
}
},
"dlq": {
"properties": {
"eventFields": {
"type": "text"
},
"reason": {
"type": "text"
}
}
},
"tags": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
}
}
Upvotes: 0
Related Questions
- In which part of logstash can I put the mapping of an elasticsearch index?
- Configuring logstash
- Elasticsearch Mapping Template
- can i set logstash default elasticsearch mapping through elasticsearch-template.json
- logstash output to elasticsearch index and mapping
- Elasticsearch mapping configuration from logstash
- add custom mapping for elasticsearch in logstash
- Change Mapping for Field for ALL OF LOGSTASH Created indexes
- Is it possible to set custom mapping for index in logstash but not in elasticsearch?
- How to use mapping in elasticsearch?