From c6ed018c1246068573640c664afb7098a5ba009a Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Mon, 4 Nov 2024 15:41:26 +0100 Subject: [PATCH] Fix sometimes not adding empty buckets (#948) --- quesma/model/metrics_aggregations/quantile.go | 12 ++++-- quesma/queryparser/pancake_json_rendering.go | 17 ++------ quesma/testdata/aggregation_requests_2.go | 25 +++++++++--- quesma/testdata/clients/clover.go | 15 +++++-- .../dashboard-1/aggregation_requests.go | 26 ++++++++++++- quesma/testdata/dates.go | 39 +++++++++++++++++++ .../pipeline_aggregation_requests.go | 6 +++ .../pipeline_aggregation_requests.go | 15 +++++++ 8 files changed, 129 insertions(+), 26 deletions(-) diff --git a/quesma/model/metrics_aggregations/quantile.go b/quesma/model/metrics_aggregations/quantile.go index 00d6a1c5f..f7da9180c 100644 --- a/quesma/model/metrics_aggregations/quantile.go +++ b/quesma/model/metrics_aggregations/quantile.go @@ -34,10 +34,10 @@ func (query Quantile) TranslateSqlResponseToJson(rows []model.QueryResultRow) mo valueAsStringMap := make(model.JsonMap) if len(rows) == 0 { - return emptyPercentilesResult + return query.emptyPercentilesResult() } if len(rows[0].Cols) == 0 { - return emptyPercentilesResult + return query.emptyPercentilesResult() } percentileIdx := -1 @@ -151,8 +151,12 @@ func (query Quantile) processResult(colName string, percentileReturnedByClickhou return percentile, percentileAsString, percentileIsNanOrInvalid } -var emptyPercentilesResult = model.JsonMap{ - "values": 0, +func (query Quantile) emptyPercentilesResult() model.JsonMap { + result := make(model.JsonMap, len(query.percentileNames)) + for _, percentileName := range query.percentileNames { + result[query.createPercentileNameToReturn(percentileName)] = nil + } + return model.JsonMap{"values": result} } // Kibana requires .0 at the end of the percentile name if it's an integer. diff --git a/quesma/queryparser/pancake_json_rendering.go b/quesma/queryparser/pancake_json_rendering.go index 9aa5a3889..660b4aa49 100644 --- a/quesma/queryparser/pancake_json_rendering.go +++ b/quesma/queryparser/pancake_json_rendering.go @@ -316,11 +316,6 @@ func (p *pancakeJSONRenderer) layerToJSON(remainingLayers []*pancakeModelLayer, bucketArr[i][pipelineAggrName] = pipelineAggrResult[i] } - if docCount, ok := bucket["doc_count"]; ok && fmt.Sprintf("%v", docCount) == "0" { - // Not sure, but it does the trick. - continue - } - subAggr, err := p.layerToJSON(remainingLayers[1:], subAggrRows[i]) if err != nil { return nil, err @@ -337,11 +332,6 @@ func (p *pancakeJSONRenderer) layerToJSON(remainingLayers []*pancakeModelLayer, bucketArr[i][pipelineAggrName] = pipelineAggrResult[i] } - if docCount, ok := bucket["doc_count"]; ok && fmt.Sprintf("%v", docCount) == "0" { - // Not sure, but it does the trick. - continue - } - // if our bucket aggregation is a date_histogram, we need original key, not processed one, which is "key" key, exists := bucket[bucket_aggregations.OriginalKeyName] if !exists { @@ -372,11 +362,12 @@ func (p *pancakeJSONRenderer) layerToJSON(remainingLayers []*pancakeModelLayer, return nil, err } bucketArr[i] = util.MergeMaps(p.ctx, bucket, subAggr) - if _, exists = bucketArr[i][bucket_aggregations.OriginalKeyName]; exists { - delete(bucketArr[i], bucket_aggregations.OriginalKeyName) - } } } + + for i := 0; i < len(bucketArr); i++ { + delete(bucketArr[i], bucket_aggregations.OriginalKeyName) + } } if layer.nextBucketAggregation.metadata != nil { diff --git a/quesma/testdata/aggregation_requests_2.go b/quesma/testdata/aggregation_requests_2.go index 102a9621d..6d37eac93 100644 --- a/quesma/testdata/aggregation_requests_2.go +++ b/quesma/testdata/aggregation_requests_2.go @@ -1780,23 +1780,38 @@ var AggregationTests2 = []AggregationTestCase{ }, { "doc_count": 0, - "key": 9200.0 + "key": 9200.0, + "3": { + "buckets": [] + } }, { "doc_count": 0, - "key": 9300.0 + "key": 9300.0, + "3": { + "buckets": [] + } }, { "doc_count": 0, - "key": 9400.0 + "key": 9400.0, + "3": { + "buckets": [] + } }, { "doc_count": 0, - "key": 9500.0 + "key": 9500.0, + "3": { + "buckets": [] + } }, { "doc_count": 0, - "key": 9600.0 + "key": 9600.0, + "3": { + "buckets": [] + } }, { "doc_count": 2, diff --git a/quesma/testdata/clients/clover.go b/quesma/testdata/clients/clover.go index cfb255b2b..37864686f 100644 --- a/quesma/testdata/clients/clover.go +++ b/quesma/testdata/clients/clover.go @@ -876,17 +876,26 @@ var CloverTests = []testdata.AggregationTestCase{ { "doc_count": 0, "key": 1727308800000, - "key_as_string": "2024-09-26T00:00:00.000" + "key_as_string": "2024-09-26T00:00:00.000", + "count": { + "value": 0 + } }, { "doc_count": 0, "key": 1727913600000, - "key_as_string": "2024-10-03T00:00:00.000" + "key_as_string": "2024-10-03T00:00:00.000", + "count": { + "value": 0 + } }, { "doc_count": 0, "key": 1728518400000, - "key_as_string": "2024-10-10T00:00:00.000" + "key_as_string": "2024-10-10T00:00:00.000", + "count": { + "value": 0 + } }, { "doc_count": 1, diff --git a/quesma/testdata/dashboard-1/aggregation_requests.go b/quesma/testdata/dashboard-1/aggregation_requests.go index a60b31530..c758c4a3e 100644 --- a/quesma/testdata/dashboard-1/aggregation_requests.go +++ b/quesma/testdata/dashboard-1/aggregation_requests.go @@ -134,10 +134,16 @@ var AggregationTests = []testdata.AggregationTestCase{ "key": 0 }, { + "1": { + "buckets": [] + }, "doc_count": 0, "key": 2000000 }, { + "1": { + "buckets": [] + }, "doc_count": 0, "key": 4000000 }, @@ -310,7 +316,12 @@ var AggregationTests = []testdata.AggregationTestCase{ }, { "doc_count": 0, - "key": 1 + "key": 1, + "2": { + "values": { + "95.0": null + } + } }, { "2": { @@ -328,6 +339,9 @@ var AggregationTests = []testdata.AggregationTestCase{ "key_as_string": "2024-04-24T11:15:30.000" }, { + "1": { + "buckets": [] + }, "doc_count": 0, "key": 1713957360000, "key_as_string": "2024-04-24T11:16:00.000" @@ -345,6 +359,11 @@ var AggregationTests = []testdata.AggregationTestCase{ "key": 1 }, { + "2": { + "values": { + "95.0": null + } + }, "doc_count": 0, "key": 1.5 }, @@ -358,6 +377,11 @@ var AggregationTests = []testdata.AggregationTestCase{ "key": 2 }, { + "2": { + "values": { + "95.0": null + } + }, "doc_count": 0, "key": 2.5 }, diff --git a/quesma/testdata/dates.go b/quesma/testdata/dates.go index 3b5c753ce..eecfdff9a 100644 --- a/quesma/testdata/dates.go +++ b/quesma/testdata/dates.go @@ -198,21 +198,33 @@ var AggregationTestsWithDates = []AggregationTestCase{ "timeseries": { "buckets": [ { + "c4a48962-08e6-4791-ae9e-b50f1b111488": { + "value": 0 + }, "doc_count": 0, "key": 1730370420000, "key_as_string": "2024-10-31T10:27:00.000" }, { + "c4a48962-08e6-4791-ae9e-b50f1b111488": { + "value": 0 + }, "doc_count": 0, "key": 1730370430000, "key_as_string": "2024-10-31T10:27:10.000" }, { + "c4a48962-08e6-4791-ae9e-b50f1b111488": { + "value": 0 + }, "doc_count": 0, "key": 1730370440000, "key_as_string": "2024-10-31T10:27:20.000" }, { + "c4a48962-08e6-4791-ae9e-b50f1b111488": { + "value": 0 + }, "doc_count": 0, "key": 1730370450000, "key_as_string": "2024-10-31T10:27:30.000" @@ -226,21 +238,33 @@ var AggregationTestsWithDates = []AggregationTestCase{ "key_as_string": "2024-10-31T10:27:40.000" }, { + "c4a48962-08e6-4791-ae9e-b50f1b111488": { + "value": 0 + }, "doc_count": 0, "key": 1730370470000, "key_as_string": "2024-10-31T10:27:50.000" }, { + "c4a48962-08e6-4791-ae9e-b50f1b111488": { + "value": 0 + }, "doc_count": 0, "key": 1730370480000, "key_as_string": "2024-10-31T10:28:00.000" }, { + "c4a48962-08e6-4791-ae9e-b50f1b111488": { + "value": 0 + }, "doc_count": 0, "key": 1730370490000, "key_as_string": "2024-10-31T10:28:10.000" }, { + "c4a48962-08e6-4791-ae9e-b50f1b111488": { + "value": 0 + }, "doc_count": 0, "key": 1730370500000, "key_as_string": "2024-10-31T10:28:20.000" @@ -387,21 +411,33 @@ var AggregationTestsWithDates = []AggregationTestCase{ "key_as_string": "2024-10-31T10:27:40.000" }, { + "c4a48962-08e6-4791-ae9e-b50f1b111488": { + "value": 0 + }, "doc_count": 0, "key": 1730370470000, "key_as_string": "2024-10-31T10:27:50.000" }, { + "c4a48962-08e6-4791-ae9e-b50f1b111488": { + "value": 0 + }, "doc_count": 0, "key": 1730370480000, "key_as_string": "2024-10-31T10:28:00.000" }, { + "c4a48962-08e6-4791-ae9e-b50f1b111488": { + "value": 0 + }, "doc_count": 0, "key": 1730370490000, "key_as_string": "2024-10-31T10:28:10.000" }, { + "c4a48962-08e6-4791-ae9e-b50f1b111488": { + "value": 0 + }, "doc_count": 0, "key": 1730370500000, "key_as_string": "2024-10-31T10:28:20.000" @@ -415,6 +451,9 @@ var AggregationTestsWithDates = []AggregationTestCase{ "key_as_string": "2024-10-31T10:28:30.000" }, { + "c4a48962-08e6-4791-ae9e-b50f1b111488": { + "value": 0 + }, "doc_count": 0, "key": 1730370520000, "key_as_string": "2024-10-31T10:28:40.000" diff --git a/quesma/testdata/kibana-visualize/pipeline_aggregation_requests.go b/quesma/testdata/kibana-visualize/pipeline_aggregation_requests.go index ccda65c50..e08fbaf6e 100644 --- a/quesma/testdata/kibana-visualize/pipeline_aggregation_requests.go +++ b/quesma/testdata/kibana-visualize/pipeline_aggregation_requests.go @@ -283,6 +283,9 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ "1": { "value": 5.0 }, + "1-metric": { + "value": null + }, "doc_count": 0, "key": 1728144360000, "key_as_string": "2024-10-05T16:06:00.000" @@ -291,6 +294,9 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ "1": { "value": 5.0 }, + "1-metric": { + "value": null + }, "doc_count": 0, "key": 1728144420000, "key_as_string": "2024-10-05T16:07:00.000" diff --git a/quesma/testdata/opensearch-visualize/pipeline_aggregation_requests.go b/quesma/testdata/opensearch-visualize/pipeline_aggregation_requests.go index a2902569a..bc3bff162 100644 --- a/quesma/testdata/opensearch-visualize/pipeline_aggregation_requests.go +++ b/quesma/testdata/opensearch-visualize/pipeline_aggregation_requests.go @@ -755,6 +755,9 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ "1": { "value": null }, + "1-metric": { + "value": null + }, "doc_count": 0, "key": 1715197200000, "key_as_string": "2024-05-08T19:40:00.000" @@ -763,6 +766,9 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ "1": { "value": null }, + "1-metric": { + "value": null + }, "doc_count": 0, "key": 1715197800000, "key_as_string": "2024-05-08T19:50:00.000" @@ -1625,6 +1631,9 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ "1": { "value": null }, + "1-metric": { + "value": null + }, "doc_count": 0, "key": 1715197200000, "key_as_string": "2024-05-08T19:40:00.000" @@ -1633,6 +1642,9 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ "1": { "value": null }, + "1-metric": { + "value": null + }, "doc_count": 0, "key": 1715197800000, "key_as_string": "2024-05-08T19:50:00.000" @@ -3691,6 +3703,9 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ "key": 3.0 }, { + "1-metric": { + "value": null + }, "3": { "value": null },