[BUG] Error processing multiple range aggregations in a single query #20473
Description
Describe the bug
Bonsai.io is no longer supporting Elasticsearch 7.10.x on sandbox free mode. So, I decided to try OpenSearch (still offered in Sandbox mode), version 2.19.2.
I created the same index structure (mappings, etc) I had on Elasticsearch but on OpenSearch, adding several documents on the index. Then I ran a set of regression tests to ensure that there's backwards compatibility (I am using very basic search and agg operations) and one test failed, where I was using two range aggregations.
Related component
Search:Aggregations
To Reproduce
File where I have index data -> Employees_raw.json
- Provision a new Opensearch server (version 2.19.2) and an Elasticsearch server (version 7.10.2)
- Create a new index called
employeeswith the following mapping in both servers.
{
"mappings":{
"properties":{
"Address":{
"type":"search_as_you_type"
},
"Age":{
"type":"integer"
},
"DateOfJoining":{
"type":"date",
"format":"yyyy-MM-dd"
},
"Designation":{
"type":"keyword"
},
"FirstName":{
"type":"text"
},
"Gender":{
"type":"keyword"
},
"Interests":{
"type":"text"
},
"LastName":{
"type":"text"
},
"MaritalStatus":{
"type":"keyword"
},
"Salary":{
"type":"double"
}
}
}
}-
Use the attached file to perform a bulk upload using
curl -XPUT 'your-server-url/_bulk' --data-binary @Employees_raw.json -H 'Content-Type: application/json'or any other mechanism. -
Perform the following query on both servers:
{
"from":0,
"size":5,
"query":{
"bool":{
"must":[
{
"match_all":{
"boost":1.0
}
}
],
"adjust_pure_negative":true,
"boost":1.0
}
},
"version":true,
"explain":false,
"aggregations":{
"ageValues":{
"range":{
"field":"Age",
"ranges":[
{
"key":"-29",
"to":29.0
},
{
"key":"29-33",
"from":29.0,
"to":33.0
},
{
"key":"33-",
"from":33.0
}
],
"keyed":false
}
},
"genderValues":{
"terms":{
"field":"Gender",
"size":10000,
"min_doc_count":1,
"shard_min_doc_count":0,
"show_term_doc_count_error":false,
"order":[
{
"_count":"desc"
},
{
"_key":"asc"
}
]
}
},
"designationValues":{
"terms":{
"field":"Designation",
"size":10000,
"min_doc_count":1,
"shard_min_doc_count":0,
"show_term_doc_count_error":false,
"order":[
{
"_count":"desc"
},
{
"_key":"asc"
}
]
}
},
"civilStatusValues":{
"terms":{
"field":"MaritalStatus",
"size":10000,
"min_doc_count":1,
"shard_min_doc_count":0,
"show_term_doc_count_error":false,
"order":[
{
"_count":"desc"
},
{
"_key":"asc"
}
]
}
},
"salaryValues":{
"range":{
"field":"Salary",
"ranges":[
{
"key":"-62000",
"to":62000.0
},
{
"key":"62000-68000",
"from":62000.0,
"to":68000.0
},
{
"key":"68000-",
"from":68000.0
}
],
"keyed":false
}
}
}
}It will be succesfull on ElasticSearch with this output:
{
"took": 4,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 9999,
"relation": "eq"
},
"max_score": 1.0,
"hits": [
{
"_index": "employees",
"_type": "_doc",
"_id": "KuEJ9ZsB-alOqON0CNnp",
"_version": 1,
"_score": 1.0,
"_source": {
"FirstName": "CIERRA",
"LastName": "TOOLS",
"Designation": "Project Manager",
"Salary": "61000",
"DateOfJoining": "2014-01-13",
"Address": "8445 Green Street Morristown, NJ 07960",
"Gender": "Female",
"Age": 35,
"MaritalStatus": "Married",
"Interests": "R/C Boats,Dolls,Cloud Watching,Animals/pets/dogs,Crocheting,Casino Gambling"
}
},
{
"_index": "employees",
"_type": "_doc",
"_id": "K-EJ9ZsB-alOqON0CNnp",
"_version": 1,
"_score": 1.0,
"_source": {
"FirstName": "JEWELL",
"LastName": "URESTI",
"Designation": "QA Engineer",
"Salary": "66000",
"DateOfJoining": "2003-02-19",
"Address": "70 East Gainsway Lane West Deptford, NJ 08096",
"Gender": "Male",
"Age": 33,
"MaritalStatus": "Unmarried",
"Interests": "Running,Aircraft Spotting,Pottery,TV watching,YoYo,Juggling"
}
},
{
"_index": "employees",
"_type": "_doc",
"_id": "LOEJ9ZsB-alOqON0CNnp",
"_version": 1,
"_score": 1.0,
"_source": {
"FirstName": "HERSCHEL",
"LastName": "BARTOLOME",
"Designation": "QA Engineer",
"Salary": "70000",
"DateOfJoining": "2008-06-02",
"Address": "9261 Andover St. De Pere, WI 54115",
"Gender": "Male",
"Age": 35,
"MaritalStatus": "Unmarried",
"Interests": "Cheerleading,Snorkeling,Reading To The Elderly,Writing Music,Darts"
}
},
{
"_index": "employees",
"_type": "_doc",
"_id": "LeEJ9ZsB-alOqON0CNnp",
"_version": 1,
"_score": 1.0,
"_source": {
"FirstName": "DORSEY",
"LastName": "CHANNER",
"Designation": "Project Manager",
"Salary": "61000",
"DateOfJoining": "2003-02-10",
"Address": "43 Carson Street East Haven, CT 06512",
"Gender": "Male",
"Age": 32,
"MaritalStatus": "Unmarried",
"Interests": "Airsofting"
}
},
{
"_index": "employees",
"_type": "_doc",
"_id": "LuEJ9ZsB-alOqON0CNnp",
"_version": 1,
"_score": 1.0,
"_source": {
"FirstName": "TAI",
"LastName": "GOTTSHALL",
"Designation": "QA Engineer",
"Salary": "63000",
"DateOfJoining": "2003-09-30",
"Address": "9956 Bradford Circle Piqua, OH 45356",
"Gender": "Female",
"Age": 34,
"MaritalStatus": "Unmarried",
"Interests": "R/C Helicopters,Spending time with family/kids,Working on cars"
}
}
]
},
"aggregations": {
"civilStatusValues": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "Married",
"doc_count": 5022
},
{
"key": "Unmarried",
"doc_count": 4977
}
]
},
"genderValues": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "Male",
"doc_count": 5026
},
{
"key": "Female",
"doc_count": 4973
}
]
},
"ageValues": {
"buckets": [
{
"key": "-29",
"to": 29.0,
"doc_count": 2815
},
{
"key": "29-33",
"from": 29.0,
"to": 33.0,
"doc_count": 4373
},
{
"key": "33-",
"from": 33.0,
"doc_count": 2811
}
]
},
"salaryValues": {
"buckets": [
{
"key": "-62000",
"to": 62000.0,
"doc_count": 5289
},
{
"key": "62000-68000",
"from": 62000.0,
"to": 68000.0,
"doc_count": 3119
},
{
"key": "68000-",
"from": 68000.0,
"doc_count": 1591
}
]
},
"designationValues": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "Senior Software Engineer",
"doc_count": 4735
},
{
"key": "Software Engineer",
"doc_count": 4264
},
{
"key": "Cloud Engineer",
"doc_count": 100
},
{
"key": "Data Scientist",
"doc_count": 100
},
{
"key": "Database Administrator",
"doc_count": 100
},
{
"key": "DevOps Engineer",
"doc_count": 100
},
{
"key": "Network Engineer",
"doc_count": 100
},
{
"key": "Project Manager",
"doc_count": 100
},
{
"key": "QA Engineer",
"doc_count": 100
},
{
"key": "Security Analyst",
"doc_count": 100
},
{
"key": "System Administrator",
"doc_count": 100
},
{
"key": "UI/UX Designer",
"doc_count": 100
}
]
}
}
}It will fail on OpenSearch, with this output:
{
"error": {
"root_cause": [
{
"type": "array_index_out_of_bounds_exception",
"reason": "Index 0 out of bounds for length -3"
}
],
"type": "search_phase_execution_exception",
"reason": "all shards failed",
"phase": "query",
"grouped": true,
"failed_shards": [
{
"shard": 0,
"index": "employees",
"node": "kqHnCCPaR5ikYHdjt7XBPg",
"reason": {
"type": "array_index_out_of_bounds_exception",
"reason": "Index 0 out of bounds for length -3"
}
}
],
"caused_by": {
"type": "array_index_out_of_bounds_exception",
"reason": "Index 0 out of bounds for length -3",
"caused_by": {
"type": "array_index_out_of_bounds_exception",
"reason": "Index 0 out of bounds for length -3"
}
}
},
"status": 500
}I assume there's backwards compatibility for this specific case. I have tried removing one of the two range aggregations (any of them) and then the query is handled properly. But with two range aggs together OpenSearch fails.
Expected behavior
OpenSearch should return something similar to:
{
"took": 4,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 9999,
"relation": "eq"
},
"max_score": 1.0,
"hits": [
{
"_index": "employees",
"_type": "_doc",
"_id": "KuEJ9ZsB-alOqON0CNnp",
"_version": 1,
"_score": 1.0,
"_source": {
"FirstName": "CIERRA",
"LastName": "TOOLS",
"Designation": "Project Manager",
"Salary": "61000",
"DateOfJoining": "2014-01-13",
"Address": "8445 Green Street Morristown, NJ 07960",
"Gender": "Female",
"Age": 35,
"MaritalStatus": "Married",
"Interests": "R/C Boats,Dolls,Cloud Watching,Animals/pets/dogs,Crocheting,Casino Gambling"
}
},
{
"_index": "employees",
"_type": "_doc",
"_id": "K-EJ9ZsB-alOqON0CNnp",
"_version": 1,
"_score": 1.0,
"_source": {
"FirstName": "JEWELL",
"LastName": "URESTI",
"Designation": "QA Engineer",
"Salary": "66000",
"DateOfJoining": "2003-02-19",
"Address": "70 East Gainsway Lane West Deptford, NJ 08096",
"Gender": "Male",
"Age": 33,
"MaritalStatus": "Unmarried",
"Interests": "Running,Aircraft Spotting,Pottery,TV watching,YoYo,Juggling"
}
},
{
"_index": "employees",
"_type": "_doc",
"_id": "LOEJ9ZsB-alOqON0CNnp",
"_version": 1,
"_score": 1.0,
"_source": {
"FirstName": "HERSCHEL",
"LastName": "BARTOLOME",
"Designation": "QA Engineer",
"Salary": "70000",
"DateOfJoining": "2008-06-02",
"Address": "9261 Andover St. De Pere, WI 54115",
"Gender": "Male",
"Age": 35,
"MaritalStatus": "Unmarried",
"Interests": "Cheerleading,Snorkeling,Reading To The Elderly,Writing Music,Darts"
}
},
{
"_index": "employees",
"_type": "_doc",
"_id": "LeEJ9ZsB-alOqON0CNnp",
"_version": 1,
"_score": 1.0,
"_source": {
"FirstName": "DORSEY",
"LastName": "CHANNER",
"Designation": "Project Manager",
"Salary": "61000",
"DateOfJoining": "2003-02-10",
"Address": "43 Carson Street East Haven, CT 06512",
"Gender": "Male",
"Age": 32,
"MaritalStatus": "Unmarried",
"Interests": "Airsofting"
}
},
{
"_index": "employees",
"_type": "_doc",
"_id": "LuEJ9ZsB-alOqON0CNnp",
"_version": 1,
"_score": 1.0,
"_source": {
"FirstName": "TAI",
"LastName": "GOTTSHALL",
"Designation": "QA Engineer",
"Salary": "63000",
"DateOfJoining": "2003-09-30",
"Address": "9956 Bradford Circle Piqua, OH 45356",
"Gender": "Female",
"Age": 34,
"MaritalStatus": "Unmarried",
"Interests": "R/C Helicopters,Spending time with family/kids,Working on cars"
}
}
]
},
"aggregations": {
"civilStatusValues": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "Married",
"doc_count": 5022
},
{
"key": "Unmarried",
"doc_count": 4977
}
]
},
"genderValues": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "Male",
"doc_count": 5026
},
{
"key": "Female",
"doc_count": 4973
}
]
},
"ageValues": {
"buckets": [
{
"key": "-29",
"to": 29.0,
"doc_count": 2815
},
{
"key": "29-33",
"from": 29.0,
"to": 33.0,
"doc_count": 4373
},
{
"key": "33-",
"from": 33.0,
"doc_count": 2811
}
]
},
"salaryValues": {
"buckets": [
{
"key": "-62000",
"to": 62000.0,
"doc_count": 5289
},
{
"key": "62000-68000",
"from": 62000.0,
"to": 68000.0,
"doc_count": 3119
},
{
"key": "68000-",
"from": 68000.0,
"doc_count": 1591
}
]
},
"designationValues": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "Senior Software Engineer",
"doc_count": 4735
},
{
"key": "Software Engineer",
"doc_count": 4264
},
{
"key": "Cloud Engineer",
"doc_count": 100
},
{
"key": "Data Scientist",
"doc_count": 100
},
{
"key": "Database Administrator",
"doc_count": 100
},
{
"key": "DevOps Engineer",
"doc_count": 100
},
{
"key": "Network Engineer",
"doc_count": 100
},
{
"key": "Project Manager",
"doc_count": 100
},
{
"key": "QA Engineer",
"doc_count": 100
},
{
"key": "Security Analyst",
"doc_count": 100
},
{
"key": "System Administrator",
"doc_count": 100
},
{
"key": "UI/UX Designer",
"doc_count": 100
}
]
}
}
}Additional Details
Plugins
Fresh install in Bonsai.io platform.
Host/Environment (please complete the following information):
- Unavailable information (Bonsai.io deployed on AWS, I assume a Linux instance is being used).