Skip to content

Commit

Permalink
Fix sometimes not adding empty buckets (#948)
Browse files Browse the repository at this point in the history
  • Loading branch information
@trzysiek
trzysiek authored Nov 4, 2024
1 parent 4415532 commit c6ed018
Show file tree
Hide file tree
Showing 8 changed files with 129 additions and 26 deletions.
12 changes: 8 additions & 4 deletions quesma/model/metrics_aggregations/quantile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,10 @@ func (query Quantile) TranslateSqlResponseToJson(rows []model.QueryResultRow) mo
valueAsStringMap := make(model.JsonMap)

if len(rows) == 0 {
return emptyPercentilesResult
return query.emptyPercentilesResult()
}
if len(rows[0].Cols) == 0 {
return emptyPercentilesResult
return query.emptyPercentilesResult()
}

percentileIdx := -1
Expand Down Expand Up @@ -151,8 +151,12 @@ func (query Quantile) processResult(colName string, percentileReturnedByClickhou
return percentile, percentileAsString, percentileIsNanOrInvalid
}

var emptyPercentilesResult = model.JsonMap{
"values": 0,
func (query Quantile) emptyPercentilesResult() model.JsonMap {
result := make(model.JsonMap, len(query.percentileNames))
for _, percentileName := range query.percentileNames {
result[query.createPercentileNameToReturn(percentileName)] = nil
}
return model.JsonMap{"values": result}
}

// Kibana requires .0 at the end of the percentile name if it's an integer.
Expand Down
17 changes: 4 additions & 13 deletions quesma/queryparser/pancake_json_rendering.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -316,11 +316,6 @@ func (p *pancakeJSONRenderer) layerToJSON(remainingLayers []*pancakeModelLayer,
bucketArr[i][pipelineAggrName] = pipelineAggrResult[i]
}

if docCount, ok := bucket["doc_count"]; ok && fmt.Sprintf("%v", docCount) == "0" {
// Not sure, but it does the trick.
continue
}

subAggr, err := p.layerToJSON(remainingLayers[1:], subAggrRows[i])
if err != nil {
return nil, err
Expand All @@ -337,11 +332,6 @@ func (p *pancakeJSONRenderer) layerToJSON(remainingLayers []*pancakeModelLayer,
bucketArr[i][pipelineAggrName] = pipelineAggrResult[i]
}

if docCount, ok := bucket["doc_count"]; ok && fmt.Sprintf("%v", docCount) == "0" {
// Not sure, but it does the trick.
continue
}

// if our bucket aggregation is a date_histogram, we need original key, not processed one, which is "key"
key, exists := bucket[bucket_aggregations.OriginalKeyName]
if !exists {
Expand Down Expand Up @@ -372,11 +362,12 @@ func (p *pancakeJSONRenderer) layerToJSON(remainingLayers []*pancakeModelLayer,
return nil, err
}
bucketArr[i] = util.MergeMaps(p.ctx, bucket, subAggr)
if _, exists = bucketArr[i][bucket_aggregations.OriginalKeyName]; exists {
delete(bucketArr[i], bucket_aggregations.OriginalKeyName)
}
}
}

for i := 0; i < len(bucketArr); i++ {
delete(bucketArr[i], bucket_aggregations.OriginalKeyName)
}
}

if layer.nextBucketAggregation.metadata != nil {
Expand Down
25 changes: 20 additions & 5 deletions quesma/testdata/aggregation_requests_2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1780,23 +1780,38 @@ var AggregationTests2 = []AggregationTestCase{
},
{
"doc_count": 0,
"key": 9200.0
"key": 9200.0,
"3": {
"buckets": []
}
},
{
"doc_count": 0,
"key": 9300.0
"key": 9300.0,
"3": {
"buckets": []
}
},
{
"doc_count": 0,
"key": 9400.0
"key": 9400.0,
"3": {
"buckets": []
}
},
{
"doc_count": 0,
"key": 9500.0
"key": 9500.0,
"3": {
"buckets": []
}
},
{
"doc_count": 0,
"key": 9600.0
"key": 9600.0,
"3": {
"buckets": []
}
},
{
"doc_count": 2,
Expand Down
15 changes: 12 additions & 3 deletions quesma/testdata/clients/clover.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -876,17 +876,26 @@ var CloverTests = []testdata.AggregationTestCase{
{
"doc_count": 0,
"key": 1727308800000,
"key_as_string": "2024-09-26T00:00:00.000"
"key_as_string": "2024-09-26T00:00:00.000",
"count": {
"value": 0
}
},
{
"doc_count": 0,
"key": 1727913600000,
"key_as_string": "2024-10-03T00:00:00.000"
"key_as_string": "2024-10-03T00:00:00.000",
"count": {
"value": 0
}
},
{
"doc_count": 0,
"key": 1728518400000,
"key_as_string": "2024-10-10T00:00:00.000"
"key_as_string": "2024-10-10T00:00:00.000",
"count": {
"value": 0
}
},
{
"doc_count": 1,
Expand Down
26 changes: 25 additions & 1 deletion quesma/testdata/dashboard-1/aggregation_requests.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,10 +134,16 @@ var AggregationTests = []testdata.AggregationTestCase{
"key": 0
},
{
"1": {
"buckets": []
},
"doc_count": 0,
"key": 2000000
},
{
"1": {
"buckets": []
},
"doc_count": 0,
"key": 4000000
},
Expand Down Expand Up @@ -310,7 +316,12 @@ var AggregationTests = []testdata.AggregationTestCase{
},
{
"doc_count": 0,
"key": 1
"key": 1,
"2": {
"values": {
"95.0": null
}
}
},
{
"2": {
Expand All @@ -328,6 +339,9 @@ var AggregationTests = []testdata.AggregationTestCase{
"key_as_string": "2024-04-24T11:15:30.000"
},
{
"1": {
"buckets": []
},
"doc_count": 0,
"key": 1713957360000,
"key_as_string": "2024-04-24T11:16:00.000"
Expand All @@ -345,6 +359,11 @@ var AggregationTests = []testdata.AggregationTestCase{
"key": 1
},
{
"2": {
"values": {
"95.0": null
}
},
"doc_count": 0,
"key": 1.5
},
Expand All @@ -358,6 +377,11 @@ var AggregationTests = []testdata.AggregationTestCase{
"key": 2
},
{
"2": {
"values": {
"95.0": null
}
},
"doc_count": 0,
"key": 2.5
},
Expand Down
39 changes: 39 additions & 0 deletions quesma/testdata/dates.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,21 +198,33 @@ var AggregationTestsWithDates = []AggregationTestCase{
"timeseries": {
"buckets": [
{
"c4a48962-08e6-4791-ae9e-b50f1b111488": {
"value": 0
},
"doc_count": 0,
"key": 1730370420000,
"key_as_string": "2024-10-31T10:27:00.000"
},
{
"c4a48962-08e6-4791-ae9e-b50f1b111488": {
"value": 0
},
"doc_count": 0,
"key": 1730370430000,
"key_as_string": "2024-10-31T10:27:10.000"
},
{
"c4a48962-08e6-4791-ae9e-b50f1b111488": {
"value": 0
},
"doc_count": 0,
"key": 1730370440000,
"key_as_string": "2024-10-31T10:27:20.000"
},
{
"c4a48962-08e6-4791-ae9e-b50f1b111488": {
"value": 0
},
"doc_count": 0,
"key": 1730370450000,
"key_as_string": "2024-10-31T10:27:30.000"
Expand All @@ -226,21 +238,33 @@ var AggregationTestsWithDates = []AggregationTestCase{
"key_as_string": "2024-10-31T10:27:40.000"
},
{
"c4a48962-08e6-4791-ae9e-b50f1b111488": {
"value": 0
},
"doc_count": 0,
"key": 1730370470000,
"key_as_string": "2024-10-31T10:27:50.000"
},
{
"c4a48962-08e6-4791-ae9e-b50f1b111488": {
"value": 0
},
"doc_count": 0,
"key": 1730370480000,
"key_as_string": "2024-10-31T10:28:00.000"
},
{
"c4a48962-08e6-4791-ae9e-b50f1b111488": {
"value": 0
},
"doc_count": 0,
"key": 1730370490000,
"key_as_string": "2024-10-31T10:28:10.000"
},
{
"c4a48962-08e6-4791-ae9e-b50f1b111488": {
"value": 0
},
"doc_count": 0,
"key": 1730370500000,
"key_as_string": "2024-10-31T10:28:20.000"
Expand Down Expand Up @@ -387,21 +411,33 @@ var AggregationTestsWithDates = []AggregationTestCase{
"key_as_string": "2024-10-31T10:27:40.000"
},
{
"c4a48962-08e6-4791-ae9e-b50f1b111488": {
"value": 0
},
"doc_count": 0,
"key": 1730370470000,
"key_as_string": "2024-10-31T10:27:50.000"
},
{
"c4a48962-08e6-4791-ae9e-b50f1b111488": {
"value": 0
},
"doc_count": 0,
"key": 1730370480000,
"key_as_string": "2024-10-31T10:28:00.000"
},
{
"c4a48962-08e6-4791-ae9e-b50f1b111488": {
"value": 0
},
"doc_count": 0,
"key": 1730370490000,
"key_as_string": "2024-10-31T10:28:10.000"
},
{
"c4a48962-08e6-4791-ae9e-b50f1b111488": {
"value": 0
},
"doc_count": 0,
"key": 1730370500000,
"key_as_string": "2024-10-31T10:28:20.000"
Expand All @@ -415,6 +451,9 @@ var AggregationTestsWithDates = []AggregationTestCase{
"key_as_string": "2024-10-31T10:28:30.000"
},
{
"c4a48962-08e6-4791-ae9e-b50f1b111488": {
"value": 0
},
"doc_count": 0,
"key": 1730370520000,
"key_as_string": "2024-10-31T10:28:40.000"
Expand Down
6 changes: 6 additions & 0 deletions quesma/testdata/kibana-visualize/pipeline_aggregation_requests.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,9 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{
"1": {
"value": 5.0
},
"1-metric": {
"value": null
},
"doc_count": 0,
"key": 1728144360000,
"key_as_string": "2024-10-05T16:06:00.000"
Expand All @@ -291,6 +294,9 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{
"1": {
"value": 5.0
},
"1-metric": {
"value": null
},
"doc_count": 0,
"key": 1728144420000,
"key_as_string": "2024-10-05T16:07:00.000"
Expand Down
15 changes: 15 additions & 0 deletions quesma/testdata/opensearch-visualize/pipeline_aggregation_requests.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -755,6 +755,9 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{
"1": {
"value": null
},
"1-metric": {
"value": null
},
"doc_count": 0,
"key": 1715197200000,
"key_as_string": "2024-05-08T19:40:00.000"
Expand All @@ -763,6 +766,9 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{
"1": {
"value": null
},
"1-metric": {
"value": null
},
"doc_count": 0,
"key": 1715197800000,
"key_as_string": "2024-05-08T19:50:00.000"
Expand Down Expand Up @@ -1625,6 +1631,9 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{
"1": {
"value": null
},
"1-metric": {
"value": null
},
"doc_count": 0,
"key": 1715197200000,
"key_as_string": "2024-05-08T19:40:00.000"
Expand All @@ -1633,6 +1642,9 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{
"1": {
"value": null
},
"1-metric": {
"value": null
},
"doc_count": 0,
"key": 1715197800000,
"key_as_string": "2024-05-08T19:50:00.000"
Expand Down Expand Up @@ -3691,6 +3703,9 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{
"key": 3.0
},
{
"1-metric": {
"value": null
},
"3": {
"value": null
},
Expand Down

0 comments on commit c6ed018

Please sign in to comment.