Nested sort causing CircuitBreakingException using icu_collation for Japanese text

Question

I am using Elasticsearch 2.4, with the icu_analysis plugin added to provide sorting on Japanese text. It works well enough on my local environment, which has a limited number of documents, but when I attempt it on a more realistic dataset, queries fail with the following CircuitBreakingException:

"CircuitBreakingException[[fielddata] Data too large, data for [translations.name.jp_sort] would be larger than limit of [10239895142/9.5gb]]"

I understand that this occurs when attempting to sort on fielddata with large document counts and that docvalues should be used instead - but I am not sure if that can be done in this situation or why it isn't happening already.

There are about 470 million documents in the index, which are storing translations as a nested document - only about 35 million out of the full set contain a Japanese translation. Here is the mapping for the documents:

{
  "settings" : {
    "number_of_shards" : 6,
    "number_of_replicas": 0,
    "analysis": {
        "filter": {
            "trigrams_filter": {
                "type":     "ngram",
                "min_gram": 3,
                "max_gram": 3
            },
          "japanese_ordering": {
            "type":     "icu_collation",
            "language": "ja",
            "country":  "JP"
          }
        },
      "analyzer": {
        "trigrams": {
          "tokenizer": "my_ngram_tokenizer",
          "filter": "lowercase"
        },
        "japanese_ordering": {
          "tokenizer": "keyword",
          "filter":  [ "japanese_ordering" ]
        }
      },
      "tokenizer": {
        "my_ngram_tokenizer": {
          "type": "nGram",
          "min_gram": "3",
          "max_gram": "3",
          "token_chars": [
            "letter",
            "digit",
            "symbol",
            "punctuation"
          ]
        }
      }
    }
  },
  "mappings" : {
    "product" : {
      "_all" : {
        "enabled" : false
      },
      "properties" : {
        "name" : {
          "type" : "string",
          "analyzer": "trigrams",
          "fields": {
            "value" : {
              "type": "string",
              "index": "not_analyzed"
            }
          }
        },
        "record_status" : {
          "type" : "integer"
        },
        "categories" : {
          "type" : "integer"
        },
        "variant_status" : {
          "type" : "integer"
        },
        "visit_count" : {
          "type" : "integer"
        },
        "translations": {
          "type": "nested",
          "properties": {
            "name": {
              "type": "string",
              "fields": {
                "jp_sort": { 
                  "type":     "string",
                  "analyzer": "japanese_ordering"
                }
              }
            },
            "language_id": {
              "type": "short"
            }
          }
        }
      }
    }
  }
}

and this is the query that is CircuitBreaking:

{
    "from": 0,
    "size": 20,
    "query": {
        "bool": {
            "should": [],
            "must_not": [],
            "must": [{
                "nested": {
                    "path": "translations",
                    "score_mode": "max",
                    "query": {
                        "bool": {
                            "must": [{
                                "match": {
                                    "translations.name": {
                                        "query": "\u30C6\u30B9\u30C8",
                                        "boost": 5
                                    }
                                }
                            }]
                        }
                    }
                }
            }]
        }
    },
    "filter": {
        "bool": {
            "must": [{
                "terms": {
                    "variant_status": ["1"],
                    "_cache": true
                }
            }, {
                "nested": {
                    "path": "translations",
                    "query": {
                        "bool": {
                            "must": [{
                                "term": {
                                    "translations.language_id": 9,
                                    "_cache": true
                                }
                            }]
                        }
                    }
                }
            }, {
                "term": {
                    "record_status": 1,
                    "_cache": true
                }
            }],
            "must_not": [{
                "term": {
                    "product_collections": 0
                }
            }]
        }
    },
    "sort": [{
        "translations.name.jp_sort": {
            "order": "asc",
            "nested_path": "translations"
        }
    }]
}

Answer 1

The ES 5.5 release has introduced a new field type called icu_collation_keyword which solves the issue you're facing.

You can read more here: https://www.elastic.co/blog/elasticsearch-5-5-0-released