diff --git a/quesma/model/metrics_aggregations/stats.go b/quesma/model/metrics_aggregations/stats.go index fd8b118e9..c7d96b1af 100644 --- a/quesma/model/metrics_aggregations/stats.go +++ b/quesma/model/metrics_aggregations/stats.go @@ -24,13 +24,17 @@ func (query Stats) TranslateSqlResponseToJson(rows []model.QueryResultRow, level if len(rows) > 0 { resultMap = make(model.JsonMap) for _, v := range rows[0].Cols[level:] { - // v.ColName = e.g. avg(...). We need to extract only 'avg'. + // v.ColName = e.g. avgOrNull(...). We need to extract only 'avg'. + // first: avgOrNull(..) -> avgOrNull firstLeftBracketIndex := strings.Index(v.ColName, "(") if firstLeftBracketIndex == -1 { logger.Error().Msgf("invalid column name in stats aggregation: %s. Skipping", v.ColName) continue } - resultMap[v.ColName[:firstLeftBracketIndex]] = v.Value + fullName := v.ColName[:firstLeftBracketIndex] + // second: if ends with OrNull, then avgOrNull -> avg + withoutOrNull, _ := strings.CutSuffix(fullName, "OrNull") + resultMap[withoutOrNull] = v.Value } } return []model.JsonMap{resultMap} diff --git a/quesma/queryparser/aggregation_parser.go b/quesma/queryparser/aggregation_parser.go index 8b61b71a0..bfdffe31f 100644 --- a/quesma/queryparser/aggregation_parser.go +++ b/quesma/queryparser/aggregation_parser.go @@ -85,7 +85,7 @@ func (b *aggrQueryBuilder) buildMetricsAggregation(metricsAggr metricsAggregatio query := b.buildAggregationCommon(metadata) switch metricsAggr.AggrType { case "sum", "min", "max", "avg": - query.NonSchemaFields = append(query.NonSchemaFields, metricsAggr.AggrType+`("`+getFirstFieldName()+`")`) + query.NonSchemaFields = append(query.NonSchemaFields, metricsAggr.AggrType+`OrNull("`+getFirstFieldName()+`")`) case "quantile": for usersPercent, percentAsFloat := range metricsAggr.Percentiles { query.NonSchemaFields = append(query.NonSchemaFields, fmt.Sprintf( @@ -100,10 +100,10 @@ func (b *aggrQueryBuilder) buildMetricsAggregation(metricsAggr metricsAggregatio query.NonSchemaFields = append( query.NonSchemaFields, "count(`"+fieldName+"`)", - "min(`"+fieldName+"`)", - "max(`"+fieldName+"`)", - "avg(`"+fieldName+"`)", - "sum(`"+fieldName+"`)", + "minOrNull(`"+fieldName+"`)", + "maxOrNull(`"+fieldName+"`)", + "avgOrNull(`"+fieldName+"`)", + "sumOrNull(`"+fieldName+"`)", ) case "top_hits": query.Fields = append(query.Fields, metricsAggr.FieldNames...) diff --git a/quesma/queryparser/aggregation_parser_test.go b/quesma/queryparser/aggregation_parser_test.go index f254a169e..e6ca256ff 100644 --- a/quesma/queryparser/aggregation_parser_test.go +++ b/quesma/queryparser/aggregation_parser_test.go @@ -43,8 +43,8 @@ var aggregationTests = []struct { }`, []string{ `SELECT count() FROM ` + tableNameQuoted + ` `, - `SELECT max("AvgTicketPrice") FROM ` + tableNameQuoted + ` `, - `SELECT min("AvgTicketPrice") FROM ` + tableNameQuoted + ` `, + `SELECT maxOrNull("AvgTicketPrice") FROM ` + tableNameQuoted + ` `, + `SELECT minOrNull("AvgTicketPrice") FROM ` + tableNameQuoted + ` `, }, }, { // [1] @@ -340,7 +340,7 @@ var aggregationTests = []struct { }`, []string{ `SELECT count() FROM ` + tableNameQuoted + ` `, - `SELECT sum("taxful_total_price") FROM ` + tableNameQuoted + " ", + `SELECT sumOrNull("taxful_total_price") FROM ` + tableNameQuoted + " ", }, }, { // [9] @@ -375,7 +375,7 @@ var aggregationTests = []struct { }`, []string{ `SELECT count() FROM ` + tableNameQuoted + ` `, - `SELECT avg("total_quantity") FROM ` + tableNameQuoted + " ", + `SELECT avgOrNull("total_quantity") FROM ` + tableNameQuoted + " ", }, }, { // [11] diff --git a/quesma/testdata/aggregation_requests.go b/quesma/testdata/aggregation_requests.go index dd0256351..6001b683c 100644 --- a/quesma/testdata/aggregation_requests.go +++ b/quesma/testdata/aggregation_requests.go @@ -106,8 +106,8 @@ var AggregationTests = []AggregationTestCase{ }, []string{ `SELECT count() FROM "` + TableName + `" WHERE "timestamp">=parseDateTime64BestEffort('2024-02-02T13:47:16.029Z') AND "timestamp"<=parseDateTime64BestEffort('2024-02-09T13:47:16.029Z') `, - `SELECT max("AvgTicketPrice") FROM "` + TableName + `" WHERE "timestamp">=parseDateTime64BestEffort('2024-02-02T13:47:16.029Z') AND "timestamp"<=parseDateTime64BestEffort('2024-02-09T13:47:16.029Z') `, - `SELECT min("AvgTicketPrice") FROM "` + TableName + `" WHERE "timestamp">=parseDateTime64BestEffort('2024-02-02T13:47:16.029Z') AND "timestamp"<=parseDateTime64BestEffort('2024-02-09T13:47:16.029Z') `, + `SELECT maxOrNull("AvgTicketPrice") FROM "` + TableName + `" WHERE "timestamp">=parseDateTime64BestEffort('2024-02-02T13:47:16.029Z') AND "timestamp"<=parseDateTime64BestEffort('2024-02-09T13:47:16.029Z') `, + `SELECT minOrNull("AvgTicketPrice") FROM "` + TableName + `" WHERE "timestamp">=parseDateTime64BestEffort('2024-02-02T13:47:16.029Z') AND "timestamp"<=parseDateTime64BestEffort('2024-02-09T13:47:16.029Z') `, }, }, { // [1] @@ -601,7 +601,7 @@ var AggregationTests = []AggregationTestCase{ }, []string{ `SELECT count() FROM "` + TableName + `" WHERE "order_date">=parseDateTime64BestEffort('2024-02-06T09:59:57.034Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-13T09:59:57.034Z') `, - `SELECT sum("taxful_total_price") FROM "` + TableName + `" WHERE "order_date">=parseDateTime64BestEffort('2024-02-06T09:59:57.034Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-13T09:59:57.034Z') `, + `SELECT sumOrNull("taxful_total_price") FROM "` + TableName + `" WHERE "order_date">=parseDateTime64BestEffort('2024-02-06T09:59:57.034Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-13T09:59:57.034Z') `, }, }, { // [4] @@ -2340,6 +2340,11 @@ var AggregationTests = []AggregationTestCase{ "max": { "field": "@timestamp" } + }, + "average_timestamp": { + "avg": { + "field": "@timestamp" + } } }, "query": { @@ -2389,6 +2394,9 @@ var AggregationTests = []AggregationTestCase{ }, "latest_timestamp": { "value": null + }, + "average_timestamp": { + "value": null } }, "hits": { @@ -2406,13 +2414,15 @@ var AggregationTests = []AggregationTestCase{ }`, [][]model.QueryResultRow{ {{Cols: []model.QueryResultCol{model.NewQueryResultCol("hits", uint64(0))}}}, - {}, // on purpose, simulates no rows returned - {}, // on purpose, simulates no rows returned + {{Cols: []model.QueryResultCol{model.NewQueryResultCol(`minOrNull("@timestamp")`, nil)}}}, + {{Cols: []model.QueryResultCol{model.NewQueryResultCol(`maxOrNull("@timestamp")`, nil)}}}, + {{Cols: []model.QueryResultCol{model.NewQueryResultCol(`maxOrNull("@timestamp")`, nil)}}}, }, []string{ `SELECT count() FROM "` + TableName + `" WHERE "message" iLIKE '%posei%' AND "message" iLIKE '%User logged out%' AND "host.name" iLIKE '%poseidon%' `, - `SELECT min("@timestamp") FROM "` + TableName + `" WHERE "message" iLIKE '%posei%' AND "message" iLIKE '%User logged out%' AND "host.name" iLIKE '%poseidon%' `, - `SELECT max("@timestamp") FROM "` + TableName + `" WHERE "message" iLIKE '%posei%' AND "message" iLIKE '%User logged out%' AND "host.name" iLIKE '%poseidon%' `, + `SELECT avgOrNull("@timestamp") FROM "` + TableName + `" WHERE "message" iLIKE '%posei%' AND "message" iLIKE '%User logged out%' AND "host.name" iLIKE '%poseidon%' `, + `SELECT minOrNull("@timestamp") FROM "` + TableName + `" WHERE "message" iLIKE '%posei%' AND "message" iLIKE '%User logged out%' AND "host.name" iLIKE '%poseidon%' `, + `SELECT maxOrNull("@timestamp") FROM "` + TableName + `" WHERE "message" iLIKE '%posei%' AND "message" iLIKE '%User logged out%' AND "host.name" iLIKE '%poseidon%' `, }, }, { // [15] @@ -2540,7 +2550,7 @@ var AggregationTests = []AggregationTestCase{ }, []string{ `SELECT count() FROM "` + TableName + `" WHERE "order_date">=parseDateTime64BestEffort('2024-02-19T17:40:56.351Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-26T17:40:56.351Z') `, - `SELECT ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 24*time.Hour) + `, sum("taxful_total_price") FROM "` + TableName + `" WHERE "order_date">=parseDateTime64BestEffort('2024-02-19T17:40:56.351Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-26T17:40:56.351Z') GROUP BY (` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 24*time.Hour) + ") ORDER BY (" + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 24*time.Hour) + ")", + `SELECT ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 24*time.Hour) + `, sumOrNull("taxful_total_price") FROM "` + TableName + `" WHERE "order_date">=parseDateTime64BestEffort('2024-02-19T17:40:56.351Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-26T17:40:56.351Z') GROUP BY (` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 24*time.Hour) + ") ORDER BY (" + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 24*time.Hour) + ")", `SELECT ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 24*time.Hour) + `, count() FROM "` + TableName + `" WHERE "order_date">=parseDateTime64BestEffort('2024-02-19T17:40:56.351Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-26T17:40:56.351Z') GROUP BY (` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 24*time.Hour) + ") ORDER BY (" + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 24*time.Hour) + ")", }, }, @@ -2817,7 +2827,7 @@ var AggregationTests = []AggregationTestCase{ }, []string{ `SELECT count() FROM "` + TableName + `" WHERE "order_date">=parseDateTime64BestEffort('2024-02-22T18:47:34.149Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T18:47:34.149Z') `, - `SELECT ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour) + `, sum("taxful_total_price") FROM "` + TableName + `" WHERE ("order_date">=parseDateTime64BestEffort('2024-02-22T18:47:34.149Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T18:47:34.149Z')) AND "products.product_name" ILIKE '%watch%' GROUP BY (` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour) + ") ORDER BY (" + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour) + ")", + `SELECT ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour) + `, sumOrNull("taxful_total_price") FROM "` + TableName + `" WHERE ("order_date">=parseDateTime64BestEffort('2024-02-22T18:47:34.149Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T18:47:34.149Z')) AND "products.product_name" ILIKE '%watch%' GROUP BY (` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour) + ") ORDER BY (" + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour) + ")", `SELECT ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour) + `, count() FROM "` + TableName + `" WHERE ("order_date">=parseDateTime64BestEffort('2024-02-22T18:47:34.149Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T18:47:34.149Z')) AND "products.product_name" ILIKE '%watch%' GROUP BY (` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour) + ") ORDER BY (" + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour) + ")", `SELECT ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour) + `, count() FROM "` + TableName + `" WHERE "order_date">=parseDateTime64BestEffort('2024-02-22T18:47:34.149Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T18:47:34.149Z') GROUP BY (` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour) + ") ORDER BY (" + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour) + ")", }, @@ -3044,12 +3054,12 @@ var AggregationTests = []AggregationTestCase{ }, []string{ `SELECT count() FROM ` + QuotedTableName + ` WHERE ("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) OR ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z')) `, - "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/86400000), " + `sum("taxful_total_price") FROM ` + QuotedTableName + ` WHERE (("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) OR ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z'))) AND ("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) GROUP BY ` + "(toInt64(toUnixTimestamp64Milli(`order_date`)/86400000)) ORDER BY (toInt64(toUnixTimestamp64Milli(`order_date`)/86400000))", - "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/86400000), " + `sum("taxful_total_price") FROM ` + QuotedTableName + ` WHERE (("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) OR ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z'))) AND ("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) GROUP BY ` + "(toInt64(toUnixTimestamp64Milli(`order_date`)/86400000)) ORDER BY (toInt64(toUnixTimestamp64Milli(`order_date`)/86400000))", + "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/86400000), " + `sumOrNull("taxful_total_price") FROM ` + QuotedTableName + ` WHERE (("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) OR ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z'))) AND ("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) GROUP BY ` + "(toInt64(toUnixTimestamp64Milli(`order_date`)/86400000)) ORDER BY (toInt64(toUnixTimestamp64Milli(`order_date`)/86400000))", + "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/86400000), " + `sumOrNull("taxful_total_price") FROM ` + QuotedTableName + ` WHERE (("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) OR ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z'))) AND ("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) GROUP BY ` + "(toInt64(toUnixTimestamp64Milli(`order_date`)/86400000)) ORDER BY (toInt64(toUnixTimestamp64Milli(`order_date`)/86400000))", "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/86400000), count() FROM " + QuotedTableName + ` WHERE (("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) OR ("order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') AND "order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z'))) AND ("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) GROUP BY ` + "(toInt64(toUnixTimestamp64Milli(`order_date`)/86400000)) ORDER BY (toInt64(toUnixTimestamp64Milli(`order_date`)/86400000))", `SELECT count() FROM ` + QuotedTableName + ` WHERE (("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) OR ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z'))) AND ("order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z') AND "order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z')) `, - "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/86400000), " + `sum("taxful_total_price") FROM ` + QuotedTableName + ` WHERE (("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) OR ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z'))) AND ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z')) GROUP BY ` + "(toInt64(toUnixTimestamp64Milli(`order_date`)/86400000)) ORDER BY (toInt64(toUnixTimestamp64Milli(`order_date`)/86400000))", - "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/86400000), " + `sum("taxful_total_price") FROM ` + QuotedTableName + ` WHERE (("order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z') AND "order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z')) OR ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z'))) AND ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z')) GROUP BY ` + "(toInt64(toUnixTimestamp64Milli(`order_date`)/86400000)) ORDER BY (toInt64(toUnixTimestamp64Milli(`order_date`)/86400000))", + "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/86400000), " + `sumOrNull("taxful_total_price") FROM ` + QuotedTableName + ` WHERE (("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) OR ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z'))) AND ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z')) GROUP BY ` + "(toInt64(toUnixTimestamp64Milli(`order_date`)/86400000)) ORDER BY (toInt64(toUnixTimestamp64Milli(`order_date`)/86400000))", + "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/86400000), " + `sumOrNull("taxful_total_price") FROM ` + QuotedTableName + ` WHERE (("order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z') AND "order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z')) OR ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z'))) AND ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z')) GROUP BY ` + "(toInt64(toUnixTimestamp64Milli(`order_date`)/86400000)) ORDER BY (toInt64(toUnixTimestamp64Milli(`order_date`)/86400000))", "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/86400000), count() FROM " + QuotedTableName + ` WHERE (("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) OR ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z'))) AND ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z')) GROUP BY ` + "(toInt64(toUnixTimestamp64Milli(`order_date`)/86400000)) ORDER BY (toInt64(toUnixTimestamp64Milli(`order_date`)/86400000))", `SELECT count() FROM ` + QuotedTableName + ` WHERE (("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) OR ("order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') AND "order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z'))) AND ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z')) `, }, @@ -3364,10 +3374,10 @@ var AggregationTests = []AggregationTestCase{ {{Cols: []model.QueryResultCol{model.NewQueryResultCol("hits", uint64(1634))}}}, {{Cols: []model.QueryResultCol{ model.NewQueryResultCol("count(bytes_gauge)", 1634), - model.NewQueryResultCol("min(bytes_gauge)", 0), - model.NewQueryResultCol("max(bytes_gauge)", 19742), - model.NewQueryResultCol("avg(bytes_gauge)", 5750.900856793146), - model.NewQueryResultCol("sum(bytes_gauge)", 9396972), + model.NewQueryResultCol("minOrNull(bytes_gauge)", 0), + model.NewQueryResultCol("maxOrNull(bytes_gauge)", 19742), + model.NewQueryResultCol("avgOrNull(bytes_gauge)", 5750.900856793146), + model.NewQueryResultCol("sumOrNull(bytes_gauge)", 9396972), }}}, {{Cols: []model.QueryResultCol{model.NewQueryResultCol("hits", uint64(1634))}}}, {{Cols: []model.QueryResultCol{ @@ -3423,7 +3433,7 @@ var AggregationTests = []AggregationTestCase{ }, []string{ `SELECT count() FROM ` + QuotedTableName + ` WHERE toUnixTimestamp64Milli("timestamp")<=1.711228426749e+12 AND toUnixTimestamp64Milli("timestamp")>=1.709932426749e+12 `, - "SELECT count(`bytes_gauge`), min(`bytes_gauge`), max(`bytes_gauge`), avg(`bytes_gauge`), sum(`bytes_gauge`) FROM " + QuotedTableName + ` WHERE (toUnixTimestamp64Milli("timestamp")>=1.709932426749e+12 AND toUnixTimestamp64Milli("timestamp")<=1.711228426749e+12) AND "bytes_gauge" IS NOT NULL `, + "SELECT count(`bytes_gauge`), minOrNull(`bytes_gauge`), maxOrNull(`bytes_gauge`), avgOrNull(`bytes_gauge`), sumOrNull(`bytes_gauge`) FROM " + QuotedTableName + ` WHERE (toUnixTimestamp64Milli("timestamp")>=1.709932426749e+12 AND toUnixTimestamp64Milli("timestamp")<=1.711228426749e+12) AND "bytes_gauge" IS NOT NULL `, `SELECT count() FROM ` + QuotedTableName + ` WHERE (toUnixTimestamp64Milli("timestamp")>=1.709932426749e+12 AND toUnixTimestamp64Milli("timestamp")<=1.711228426749e+12) AND "bytes_gauge" IS NOT NULL `, "TODO", // too tiresome to implement the check, so for now this SQL for quantiles isn't tested "TODO", // too tiresome to implement the check, so for now this SQL for quantiles isn't tested diff --git a/quesma/testdata/dashboard-1/aggregation_requests.go b/quesma/testdata/dashboard-1/aggregation_requests.go index 628104e96..2b69dd7c9 100644 --- a/quesma/testdata/dashboard-1/aggregation_requests.go +++ b/quesma/testdata/dashboard-1/aggregation_requests.go @@ -143,12 +143,12 @@ var AggregationTests = []testdata.AggregationTestCase{ {Cols: []model.QueryResultCol{ model.NewQueryResultCol("floor(rspContentLen / 2000000.000000) * 2000000.000000", 0), model.NewQueryResultCol("floor(rspContentLen / 2000000.000000) * 2000000.000000", 0), - model.NewQueryResultCol("avg(rspContentLen)", 42516.52153947081), + model.NewQueryResultCol("avgOrNull(rspContentLen)", 42516.52153947081), }}, {Cols: []model.QueryResultCol{ model.NewQueryResultCol("floor(rspContentLen / 2000000.000000) * 2000000.000000", 658000000), model.NewQueryResultCol("floor(rspContentLen / 2000000.000000) * 2000000.000000", 658000000), - model.NewQueryResultCol("avg(rspContentLen)", 658654099), + model.NewQueryResultCol("avgOrNull(rspContentLen)", 658654099), }}, }, { @@ -176,7 +176,7 @@ var AggregationTests = []testdata.AggregationTestCase{ }, ExpectedSQLs: []string{ `SELECT count() FROM ` + testdata.QuotedTableName + ` WHERE "reqTimeSec">='2024-04-24T10:55:23.606Z' AND "reqTimeSec"<='2024-04-24T11:10:23.606Z' `, - `SELECT floor("rspContentLen" / 2000000.000000) * 2000000.000000, floor("rspContentLen" / 2000000.000000) * 2000000.000000, avg("rspContentLen") ` + + `SELECT floor("rspContentLen" / 2000000.000000) * 2000000.000000, floor("rspContentLen" / 2000000.000000) * 2000000.000000, avgOrNull("rspContentLen") ` + `FROM ` + testdata.QuotedTableName + ` WHERE "reqTimeSec">='2024-04-24T10:55:23.606Z' AND "reqTimeSec"<='2024-04-24T11:10:23.606Z' ` + `GROUP BY (floor("rspContentLen" / 2000000.000000) * 2000000.000000, floor("rspContentLen" / 2000000.000000) * 2000000.000000) ` + `ORDER BY (floor("rspContentLen" / 2000000.000000) * 2000000.000000, floor("rspContentLen" / 2000000.000000) * 2000000.000000)`, diff --git a/quesma/testdata/opensearch-visualize/aggregation_requests.go b/quesma/testdata/opensearch-visualize/aggregation_requests.go index 61cec663d..a94c0d903 100644 --- a/quesma/testdata/opensearch-visualize/aggregation_requests.go +++ b/quesma/testdata/opensearch-visualize/aggregation_requests.go @@ -463,10 +463,10 @@ var AggregationTests = []testdata.AggregationTestCase{ // TODO after merge of some PR, change logs-generic-default to testdata.QuotedTableName `SELECT count() FROM "logs-generic-default" ` + `WHERE "epoch_time">='2024-04-28T14:34:22.674Z' AND "epoch_time"<='2024-04-28T14:49:22.674Z' `, - `SELECT sum("properties.entry_time") FROM "logs-generic-default" ` + + `SELECT sumOrNull("properties.entry_time") FROM "logs-generic-default" ` + `WHERE ("epoch_time">='2024-04-28T14:34:22.674Z' AND "epoch_time"<='2024-04-28T14:49:22.674Z') ` + `AND "epoch_time_original">=0 AND "epoch_time_original"<1000 `, - `SELECT sum("properties.entry_time") FROM "logs-generic-default" ` + + `SELECT sumOrNull("properties.entry_time") FROM "logs-generic-default" ` + `WHERE ("epoch_time">='2024-04-28T14:34:22.674Z' AND "epoch_time"<='2024-04-28T14:49:22.674Z') ` + `AND "epoch_time_original">=1000 `, `SELECT count(if("epoch_time_original">=0 AND "epoch_time_original"<1000, 1, NULL)), ` + diff --git a/quesma/testdata/requests.go b/quesma/testdata/requests.go index cc9ba7730..54c7e8d07 100644 --- a/quesma/testdata/requests.go +++ b/quesma/testdata/requests.go @@ -789,8 +789,8 @@ var TestsAsyncSearch = []AsyncSearchTestCase{ model.SearchQueryInfo{Typ: model.Normal}, []string{ `SELECT count() FROM "logs-generic-default" WHERE "message" iLIKE '%posei%' AND "message" iLIKE '%User logged out%' AND "host.name" iLIKE '%poseidon%' `, - `SELECT m..("@timestamp") FROM "logs-generic-default" WHERE "message" iLIKE '%posei%' AND "message" iLIKE '%User logged out%' AND "host.name" iLIKE '%poseidon%' `, - `SELECT m..("@timestamp") FROM "logs-generic-default" WHERE "message" iLIKE '%posei%' AND "message" iLIKE '%User logged out%' AND "host.name" iLIKE '%poseidon%' `, + `SELECT m..OrNull("@timestamp") FROM "logs-generic-default" WHERE "message" iLIKE '%posei%' AND "message" iLIKE '%User logged out%' AND "host.name" iLIKE '%poseidon%' `, + `SELECT m..OrNull("@timestamp") FROM "logs-generic-default" WHERE "message" iLIKE '%posei%' AND "message" iLIKE '%User logged out%' AND "host.name" iLIKE '%poseidon%' `, }, true, },