Skip to content

Commit

Permalink
fix tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@pivovarit
pivovarit committed May 6, 2024
1 parent 2aaa123 commit 8cb2b3c
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 21 deletions.
10 changes: 6 additions & 4 deletions quesma/quesma/search_opensearch_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
)

func TestSearchOpensearch(t *testing.T) {
cfg := config.QuesmaConfiguration{IndexConfig: map[string]config.IndexConfiguration{tableName: {Enabled: true}}}
table := clickhouse.Table{
Name: tableName,
Config: clickhouse.NewDefaultCHConfig(),
Expand All @@ -36,7 +37,7 @@ func TestSearchOpensearch(t *testing.T) {
assert.NoError(t, err)
defer db.Close()
lm := clickhouse.NewLogManagerWithConnection(db, concurrent.NewMapWith(tableName, &table))
managementConsole := ui.NewQuesmaManagementConsole(config.QuesmaConfiguration{}, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent())
managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent())
cw := queryparser.ClickhouseQueryTranslator{ClickhouseLM: lm, Table: &table, Ctx: context.Background()}

simpleQuery, queryInfo, _ := cw.ParseQuery(tt.QueryJson)
Expand All @@ -49,7 +50,7 @@ func TestSearchOpensearch(t *testing.T) {
}

queryRunner := NewQueryRunner()
_, err = queryRunner.handleSearch(ctx, tableName, []byte(tt.QueryJson), config.QuesmaConfiguration{}, lm, nil, managementConsole)
_, err = queryRunner.handleSearch(ctx, tableName, []byte(tt.QueryJson), cfg, lm, nil, managementConsole)
assert.NoError(t, err)

if err = mock.ExpectationsWereMet(); err != nil {
Expand Down Expand Up @@ -152,6 +153,7 @@ func TestHighlighter(t *testing.T) {
],
"version": true
}`
cfg := config.QuesmaConfiguration{IndexConfig: map[string]config.IndexConfiguration{tableName: {Enabled: true}}}
table := clickhouse.Table{
Name: tableName,
Config: clickhouse.NewDefaultCHConfig(),
Expand All @@ -167,7 +169,7 @@ func TestHighlighter(t *testing.T) {
assert.NoError(t, err)
defer db.Close()
lm := clickhouse.NewLogManagerWithConnection(db, concurrent.NewMapWith(tableName, &table))
managementConsole := ui.NewQuesmaManagementConsole(config.QuesmaConfiguration{}, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent())
managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent())

mock.ExpectQuery("").WillReturnRows(sqlmock.NewRows([]string{"message$*%:;", "host.name", "@timestamp"}). // careful, it's not always in this order, order is nondeterministic
AddRow("abcd", "abcd", "abcd").
Expand All @@ -177,7 +179,7 @@ func TestHighlighter(t *testing.T) {
AddRow("text", "text", "text"))

queryRunner := NewQueryRunner()
response, err := queryRunner.handleSearch(ctx, tableName, []byte(query), config.QuesmaConfiguration{}, lm, nil, managementConsole)
response, err := queryRunner.handleSearch(ctx, tableName, []byte(query), cfg, lm, nil, managementConsole)
assert.NoError(t, err)
if err = mock.ExpectationsWereMet(); err != nil {
t.Fatal("there were unfulfilled expections:", err)
Expand Down
41 changes: 24 additions & 17 deletions quesma/quesma/search_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ var ctx = context.WithValue(context.TODO(), tracing.RequestIdCtxKey, tracing.Get
const tableName = `logs-generic-default`

func TestAsyncSearchHandler(t *testing.T) {
cfg := config.QuesmaConfiguration{IndexConfig: map[string]config.IndexConfiguration{tableName: {Enabled: true}}}
table := concurrent.NewMapWith(tableName, &clickhouse.Table{
Name: tableName,
Config: clickhouse.NewDefaultCHConfig(),
Expand Down Expand Up @@ -82,7 +83,7 @@ func TestAsyncSearchHandler(t *testing.T) {
defer db.Close()
assert.NoError(t, err)
lm := clickhouse.NewLogManagerWithConnection(db, table)
managementConsole := ui.NewQuesmaManagementConsole(config.QuesmaConfiguration{}, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent())
managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent())

for _, wantedRegex := range tt.WantedRegexes {
if tt.WantedParseResult.Typ == model.ListAllFields {
Expand All @@ -99,7 +100,7 @@ func TestAsyncSearchHandler(t *testing.T) {
mock.ExpectQuery(wantedRegex).WillReturnRows(sqlmock.NewRows([]string{"@timestamp", "host.name"}))
}
queryRunner := NewQueryRunner()
_, err = queryRunner.handleAsyncSearch(ctx, config.QuesmaConfiguration{}, tableName, []byte(tt.QueryJson), lm, nil, managementConsole, defaultAsyncSearchTimeout, true)
_, err = queryRunner.handleAsyncSearch(ctx, cfg, tableName, []byte(tt.QueryJson), lm, nil, managementConsole, defaultAsyncSearchTimeout, true)
assert.NoError(t, err)

if err := mock.ExpectationsWereMet(); err != nil {
Expand All @@ -110,6 +111,7 @@ func TestAsyncSearchHandler(t *testing.T) {
}

func TestAsyncSearchHandlerSpecialCharacters(t *testing.T) {
cfg := config.QuesmaConfiguration{IndexConfig: map[string]config.IndexConfiguration{tableName: {Enabled: true}}}
table := clickhouse.Table{
Name: tableName,
Config: clickhouse.NewDefaultCHConfig(),
Expand All @@ -130,14 +132,14 @@ func TestAsyncSearchHandlerSpecialCharacters(t *testing.T) {
defer db.Close()
assert.NoError(t, err)
lm := clickhouse.NewLogManagerWithConnection(db, concurrent.NewMapWith(tableName, &table))
managementConsole := ui.NewQuesmaManagementConsole(config.QuesmaConfiguration{}, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent())
managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent())

for _, expectedSql := range tt.ExpectedSQLs {
mock.ExpectQuery(testdata.EscapeBrackets(expectedSql)).WillReturnRows(sqlmock.NewRows([]string{"@timestamp", "host.name"}))
}

queryRunner := NewQueryRunner()
_, err = queryRunner.handleAsyncSearch(ctx, config.QuesmaConfiguration{}, tableName, []byte(tt.QueryRequestJson), lm, nil, managementConsole, defaultAsyncSearchTimeout, true)
_, err = queryRunner.handleAsyncSearch(ctx, cfg, tableName, []byte(tt.QueryRequestJson), lm, nil, managementConsole, defaultAsyncSearchTimeout, true)
assert.NoError(t, err)

if err = mock.ExpectationsWereMet(); err != nil {
Expand All @@ -163,6 +165,7 @@ var table = concurrent.NewMapWith(tableName, &clickhouse.Table{
})

func TestSearchHandler(t *testing.T) {
cfg := config.QuesmaConfiguration{IndexConfig: map[string]config.IndexConfiguration{tableName: {Enabled: true}}}
for _, tt := range testdata.TestsSearch {
t.Run(tt.Name, func(t *testing.T) {
db, mock, err := sqlmock.New()
Expand All @@ -173,13 +176,13 @@ func TestSearchHandler(t *testing.T) {
assert.NoError(t, err)

lm := clickhouse.NewLogManagerWithConnection(db, table)
managementConsole := ui.NewQuesmaManagementConsole(config.QuesmaConfiguration{}, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent())
managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent())
for _, wantedRegex := range tt.WantedRegexes {
mock.ExpectQuery(testdata.EscapeWildcard(testdata.EscapeBrackets(wantedRegex))).
WillReturnRows(sqlmock.NewRows([]string{"@timestamp", "host.name"}))
}
queryRunner := NewQueryRunner()
_, _ = queryRunner.handleSearch(ctx, tableName, []byte(tt.QueryJson), config.QuesmaConfiguration{}, lm, nil, managementConsole)
_, _ = queryRunner.handleSearch(ctx, tableName, []byte(tt.QueryJson), cfg, lm, nil, managementConsole)

if err := mock.ExpectationsWereMet(); err != nil {
t.Fatal("there were unfulfilled expections:", err)
Expand All @@ -190,6 +193,7 @@ func TestSearchHandler(t *testing.T) {

// TODO this test gives wrong results??
func TestSearchHandlerNoAttrsConfig(t *testing.T) {
cfg := config.QuesmaConfiguration{IndexConfig: map[string]config.IndexConfiguration{tableName: {Enabled: true}}}
for _, tt := range testdata.TestsSearchNoAttrs {
t.Run(tt.Name, func(t *testing.T) {
db, mock, err := sqlmock.New()
Expand All @@ -200,12 +204,12 @@ func TestSearchHandlerNoAttrsConfig(t *testing.T) {
assert.NoError(t, err)

lm := clickhouse.NewLogManagerWithConnection(db, table)
managementConsole := ui.NewQuesmaManagementConsole(config.QuesmaConfiguration{}, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent())
managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent())
for _, wantedRegex := range tt.WantedRegexes {
mock.ExpectQuery(testdata.EscapeBrackets(wantedRegex)).WillReturnRows(sqlmock.NewRows([]string{"@timestamp", "host.name"}))
}
queryRunner := NewQueryRunner()
_, _ = queryRunner.handleSearch(ctx, tableName, []byte(tt.QueryJson), config.QuesmaConfiguration{}, lm, nil, managementConsole)
_, _ = queryRunner.handleSearch(ctx, tableName, []byte(tt.QueryJson), cfg, lm, nil, managementConsole)

if err := mock.ExpectationsWereMet(); err != nil {
t.Fatal("there were unfulfilled expections:", err)
Expand All @@ -215,6 +219,7 @@ func TestSearchHandlerNoAttrsConfig(t *testing.T) {
}

func TestAsyncSearchFilter(t *testing.T) {
cfg := config.QuesmaConfiguration{IndexConfig: map[string]config.IndexConfiguration{tableName: {Enabled: true}}}
for _, tt := range testdata.TestSearchFilter {
t.Run(tt.Name, func(t *testing.T) {
db, mock, err := sqlmock.New()
Expand All @@ -225,12 +230,12 @@ func TestAsyncSearchFilter(t *testing.T) {
assert.NoError(t, err)

lm := clickhouse.NewLogManagerWithConnection(db, table)
managementConsole := ui.NewQuesmaManagementConsole(config.QuesmaConfiguration{}, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent())
managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent())
for _, wantedRegex := range tt.WantedRegexes {
mock.ExpectQuery(testdata.EscapeBrackets(wantedRegex)).WillReturnRows(sqlmock.NewRows([]string{"@timestamp", "host.name"}))
}
queryRunner := NewQueryRunner()
_, _ = queryRunner.handleAsyncSearch(ctx, config.QuesmaConfiguration{}, tableName, []byte(tt.QueryJson), lm, nil, managementConsole, defaultAsyncSearchTimeout, true)
_, _ = queryRunner.handleAsyncSearch(ctx, cfg, tableName, []byte(tt.QueryJson), lm, nil, managementConsole, defaultAsyncSearchTimeout, true)
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatal("there were unfulfilled expections:", err)
}
Expand All @@ -243,6 +248,7 @@ func TestAsyncSearchFilter(t *testing.T) {
// (testing of creating response is lacking), because of `sqlmock` limitation.
// It can't return uint64, thus creating response code panics because of that.
func TestHandlingDateTimeFields(t *testing.T) {
cfg := config.QuesmaConfiguration{IndexConfig: map[string]config.IndexConfiguration{tableName: {Enabled: true}}}
// I'm testing querying for all 3 types of fields that we support right now.
const dateTimeTimestampField = "timestamp"
const dateTime64TimestampField = "timestamp64"
Expand Down Expand Up @@ -294,7 +300,7 @@ func TestHandlingDateTimeFields(t *testing.T) {
}
defer db.Close()
lm := clickhouse.NewLogManagerWithConnection(db, concurrent.NewMapWith(tableName, &table))
managementConsole := ui.NewQuesmaManagementConsole(config.QuesmaConfiguration{}, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent())
managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent())

for _, fieldName := range []string{dateTimeTimestampField, dateTime64TimestampField, dateTime64OurTimestampField} {
mock.ExpectQuery(testdata.EscapeBrackets(`SELECT count() FROM "logs-generic-default" WHERE `)).
Expand All @@ -303,7 +309,7 @@ func TestHandlingDateTimeFields(t *testing.T) {
WillReturnRows(sqlmock.NewRows([]string{"key", "doc_count"}))
// .AddRow(1000, uint64(10)).AddRow(1001, uint64(20))) // here rows should be added if uint64 were supported
queryRunner := NewQueryRunner()
response, err := queryRunner.handleAsyncSearch(ctx, config.QuesmaConfiguration{}, tableName, []byte(query(fieldName)), lm, nil, managementConsole, defaultAsyncSearchTimeout, true)
response, err := queryRunner.handleAsyncSearch(ctx, cfg, tableName, []byte(query(fieldName)), lm, nil, managementConsole, defaultAsyncSearchTimeout, true)
assert.NoError(t, err)

var responseMap model.JsonMap
Expand All @@ -320,6 +326,7 @@ func TestHandlingDateTimeFields(t *testing.T) {
// (top 10 values, "other" value, min/max).
// Both `_search`, and `_async_search` handlers are tested.
func TestNumericFacetsQueries(t *testing.T) {
cfg := config.QuesmaConfiguration{IndexConfig: map[string]config.IndexConfiguration{tableName: {Enabled: true}}}
table := concurrent.NewMapWith(tableName, &clickhouse.Table{
Name: tableName,
Config: clickhouse.NewDefaultCHConfig(),
Expand All @@ -346,7 +353,7 @@ func TestNumericFacetsQueries(t *testing.T) {
defer db.Close()
assert.NoError(t, err)
lm := clickhouse.NewLogManagerWithConnection(db, table)
managementConsole := ui.NewQuesmaManagementConsole(config.QuesmaConfiguration{}, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent())
managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent())

returnedBuckets := sqlmock.NewRows([]string{"", ""})
for _, row := range tt.ResultRows {
Expand All @@ -359,9 +366,9 @@ func TestNumericFacetsQueries(t *testing.T) {
queryRunner := NewQueryRunner()
var response []byte
if handlerName == "handleSearch" {
response, err = queryRunner.handleSearch(ctx, tableName, []byte(tt.QueryJson), config.QuesmaConfiguration{}, lm, nil, managementConsole)
response, err = queryRunner.handleSearch(ctx, tableName, []byte(tt.QueryJson), cfg, lm, nil, managementConsole)
} else if handlerName == "handleAsyncSearch" {
response, err = queryRunner.handleAsyncSearch(ctx, config.QuesmaConfiguration{}, tableName, []byte(tt.QueryJson), lm, nil, managementConsole, defaultAsyncSearchTimeout, true)
response, err = queryRunner.handleAsyncSearch(ctx, cfg, tableName, []byte(tt.QueryJson), lm, nil, managementConsole, defaultAsyncSearchTimeout, true)
}
assert.NoError(t, err)

Expand Down Expand Up @@ -408,7 +415,7 @@ func TestAllUnsupportedQueryTypesAreProperlyRecorded(t *testing.T) {
assert.NoError(t, err)

lm := clickhouse.NewLogManagerWithConnection(db, table)
cfg := config.QuesmaConfiguration{}
cfg := config.QuesmaConfiguration{IndexConfig: map[string]config.IndexConfiguration{tableName: {Enabled: true}}}
logChan := logger.InitOnlyChannelLoggerForTests(cfg, &tracing.AsyncTraceLogger{AsyncQueryTrace: concurrent.NewMap[string, tracing.TraceCtx]()})
managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, logChan, telemetry.NewPhoneHomeEmptyAgent())
go managementConsole.RunOnlyChannelProcessor()
Expand Down Expand Up @@ -459,7 +466,7 @@ func TestDifferentUnsupportedQueries(t *testing.T) {
assert.NoError(t, err)

lm := clickhouse.NewLogManagerWithConnection(db, table)
cfg := config.QuesmaConfiguration{}
cfg := config.QuesmaConfiguration{IndexConfig: map[string]config.IndexConfiguration{tableName: {Enabled: true}}}
logChan := logger.InitOnlyChannelLoggerForTests(cfg, &tracing.AsyncTraceLogger{AsyncQueryTrace: concurrent.NewMap[string, tracing.TraceCtx]()})
managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, logChan, telemetry.NewPhoneHomeEmptyAgent())
go managementConsole.RunOnlyChannelProcessor()
Expand Down

0 comments on commit 8cb2b3c

Please sign in to comment.