Sorting in Elasticsearch make consistency problem

Question

I'm sending my data to elasticsearch with index_number of documents. Its unique identifier. When i try to sort it with this, from python client i get this consistency problem as you see in the picture.

This is my query dsl

"size": 1,
"query": {
    "match_all": {}
},
"sort": [
    {
        "index_number.keyword": {
            "order": "asc",
            "missing": "_last",
            "unmapped_type": "String"
        }
    }
]

In logstash output

output{
    elasticsearch {
        hosts => ["localhost:9200"]
        index => "logstash_%{+yyyy-MM-dd}"
        manage_template => true
        template_name => "logstash_template"
        template => "..../logstash_template.json"
        http_compression => true
    }
}

In my logstash template.json
...

{
"index_patterns": ["logstash_*"],
"template": {
  "settings":{
    "number_of_shards": 1,
    "number_of_replicas": 0,
    "index": {
      "sort.field": "index_number", 
      "sort.order": "asc"  
    }
  },
  "mappings": {
    "dynamic_templates":{
      "string_fields": {
        "match": "*",
        "match_mapping_type": "string",
        "mapping": {"type":"keyword"}
      }
    },
    "properties": {
      "index_number": {
        "type": "keyword",
        "fields": {
          "numeric": {
            "type": "double"
          }
        }
      }
    }
  }
}
  }

....

Mapping on elasticsearch

{
  "logstash_2020-03-12" : {
    "mappings" : {
      "properties" : {
        .....
        "index_number" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "city" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "country" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        -----
      }
    }
  }
}

How can i solve it? Thanks for answering.

Answer 1

You need to add template_overwrite to your Logstash output configuration otherwise the logstash_template is not overridden if it already exists:

output{
    elasticsearch {
        hosts => ["localhost:9200"]
        index => "logstash_%{+yyyy-MM-dd}"
        manage_template => true
        template_override => true                         <-- add this
        template_name => "logstash_template"
        template => "..../logstash_template.json"
        http_compression => true
    }
}

Make sure that your logstash_template.json file has the following format:

{
  "index_patterns": [
    "logstash_*"
  ],
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 0,
    "index": {
      "sort.field": "index_number",
      "sort.order": "asc"
    }
  },
  "mappings": {
    "dynamic_templates": {
      "string_fields": {
        "match": "*",
        "match_mapping_type": "string",
        "mapping": {
          "type": "keyword"
        }
      }
    },
    "properties": {
      "index_number": {
        "type": "keyword",
        "fields": {
          "numeric": {
            "type": "double"
          }
        }
      }
    }
  }
}

You had mappings and settings enclosed within the template section, but this is only for the new index templates which the elasticsearch Logstash output doesn't support yet. You need to use the legacy index templates.