Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement more operators #7

Closed
wants to merge 1 commit into from
Closed

Implement more operators #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 117 additions & 13 deletions filter/converter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"strings"
)

var BasicOperatorMap = map[string]string{
var basicOperatorMap = map[string]string{
"$gt": ">",
"$gte": ">=",
"$lt": "<",
Expand All @@ -19,6 +19,11 @@ var BasicOperatorMap = map[string]string{
"$regex": "~*",
}

// ReservedColumnName is a reserved column name used internally for nested $elemMatch queries.
// This column name should not be used in the database or any JSONB column.
// You can set this to a different value as long as it's a valid Postgres identifier.
var ReservedColumnName = "__placeholder"

type Converter struct {
nestedColumn string
nestedExemptions []string
Expand Down Expand Up @@ -53,14 +58,18 @@ func (c *Converter) Convert(query []byte, startAtParameterIndex int) (conditions
return "", nil, fmt.Errorf("startAtParameterIndex must be greater than 0")
}

if len(query) == 0 {
return c.emptyCondition, nil, nil
}

var mongoFilter map[string]any
err = json.Unmarshal(query, &mongoFilter)
if err != nil {
return "", nil, err
}

if len(mongoFilter) == 0 {
return c.emptyCondition, []any{}, nil
return c.emptyCondition, nil, nil
}

conditions, values, err = c.convertFilter(mongoFilter, startAtParameterIndex)
Expand Down Expand Up @@ -89,7 +98,7 @@ func (c *Converter) convertFilter(filter map[string]any, paramIndex int) (string
value := filter[key]

switch key {
case "$or", "$and":
case "$or", "$and", "$nor":
opConditions, ok := anyToSliceMapAny(value)
if !ok {
return "", nil, fmt.Errorf("invalid value for $or operator (must be array of objects): %v", value)
Expand All @@ -108,15 +117,31 @@ func (c *Converter) convertFilter(filter map[string]any, paramIndex int) (string
inner = append(inner, innerConditions)
values = append(values, innerValues...)
}
op := "AND"
if key == "$or" {
op = "OR"
}
if len(inner) > 1 {
conditions = append(conditions, "("+strings.Join(inner, " "+op+" ")+")")
if key == "$nor" {
conditions = append(conditions, "NOT ("+strings.Join(inner, " OR ")+")")
} else {
conditions = append(conditions, strings.Join(inner, " "+op+" "))
op := "AND"
if key == "$or" {
op = "OR"
}
if len(inner) > 1 {
conditions = append(conditions, "("+strings.Join(inner, " "+op+" ")+")")
} else {
conditions = append(conditions, strings.Join(inner, " "+op+" "))
}
}
case "$not":
vv, ok := value.(map[string]any)
if !ok {
return "", nil, fmt.Errorf("invalid value for $not operator (must be object): %v", value)
}
innerConditions, innerValues, err := c.convertFilter(vv, paramIndex)
if err != nil {
return "", nil, err
}
paramIndex += len(innerValues)
conditions = append(conditions, "NOT "+innerConditions)
values = append(values, innerValues...)
default:
if !isValidPostgresIdentifier(key) {
return "", nil, fmt.Errorf("invalid column name: %s", key)
Expand All @@ -140,19 +165,73 @@ func (c *Converter) convertFilter(filter map[string]any, paramIndex int) (string
return "", nil, fmt.Errorf("$or as scalar operator not supported")
case "$and":
return "", nil, fmt.Errorf("$and as scalar operator not supported")
case "$in":
case "$not":
return "", nil, fmt.Errorf("$not as scalar operator not supported")
case "$in", "$nin":
if !isScalarSlice(v[operator]) {
return "", nil, fmt.Errorf("invalid value for $in operator (must array of primatives): %v", v[operator])
}
inner = append(inner, fmt.Sprintf("(%s = ANY($%d))", c.columnName(key), paramIndex))
op := ""
if operator == "$nin" {
op = "NOT "
}
// `column != ANY(...)` does not work, so we need to do `NOT column = ANY(...)` instead.
inner = append(inner, fmt.Sprintf("(%s%s = ANY($%d))", op, c.columnName(key), paramIndex))
paramIndex++
if c.arrayDriver != nil {
v[operator] = c.arrayDriver(v[operator])
}
values = append(values, v[operator])
case "$exists":
// $exists only works on jsonb columns, so we need to check if the key is in the JSONB data first.
isNestedColumn := c.nestedColumn != ""
for _, exemption := range c.nestedExemptions {
if exemption == key {
isNestedColumn = false
break
}
}
if !isNestedColumn {
// There is no way in Postgres to check if a column exists on a table.
return "", nil, fmt.Errorf("$exists operator not supported on non-nested jsonb columns")
}
neg := ""
if v[operator] == false {
neg = "NOT "
}
inner = append(inner, fmt.Sprintf("(%sjsonb_path_match(%s, 'exists($.%s)'))", neg, c.nestedColumn, key))
case "$elemMatch":
// $elemMatch needs a different implementation depending on if the column is in JSONB or not.
isNestedColumn := c.nestedColumn != ""
for _, exemption := range c.nestedExemptions {
if exemption == key {
isNestedColumn = false
break
}
}
innerConditions, innerValues, err := c.convertFilter(map[string]any{ReservedColumnName: v[operator]}, paramIndex)
if err != nil {
return "", nil, err
}
paramIndex += len(innerValues)
if isNestedColumn {
// This will for example become:
//
// EXISTS (SELECT 1 FROM jsonb_array_elements("meta"->'foo') AS __placeholder WHERE ("__placeholder"::text = $1))
//
// We can't use c.columnName here because we need `->` to get the jsonb value instead of `->>` which gets the text value.
inner = append(inner, fmt.Sprintf("EXISTS (SELECT 1 FROM jsonb_array_elements(%q->'%s') AS %s WHERE %s)", c.nestedColumn, key, ReservedColumnName, innerConditions))
} else {
// This will for example become:
//
// EXISTS (SELECT 1 FROM unnest("foo") AS __placeholder WHERE ("__placeholder"::text = $1))
//
inner = append(inner, fmt.Sprintf("EXISTS (SELECT 1 FROM unnest(%s) AS %s WHERE %s)", c.columnName(key), ReservedColumnName, innerConditions))
}
values = append(values, innerValues...)
default:
value := v[operator]
op, ok := BasicOperatorMap[operator]
op, ok := basicOperatorMap[operator]
if !ok {
return "", nil, fmt.Errorf("unknown operator: %s", operator)
}
Expand All @@ -166,6 +245,28 @@ func (c *Converter) convertFilter(filter map[string]any, paramIndex int) (string
innerResult = "(" + innerResult + ")"
}
conditions = append(conditions, innerResult)
case []string, []float64, []bool, []any:
conditions = append(conditions, fmt.Sprintf("(%s = ANY($%d))", c.columnName(key), paramIndex))
paramIndex++
if c.arrayDriver != nil {
value = c.arrayDriver(value)
}
values = append(values, value)
case nil:
// Comparing a column to NULL needs a different implementation depending on if the column is in JSONB or not.
// JSONB columns are NULL even if they don't exist, so we need to check if the column exists first.
isNestedColumn := c.nestedColumn != ""
for _, exemption := range c.nestedExemptions {
if exemption == key {
isNestedColumn = false
break
}
}
if isNestedColumn {
conditions = append(conditions, fmt.Sprintf("(jsonb_path_match(%s, 'exists($.%s)') AND %s IS NULL)", c.nestedColumn, key, c.columnName(key)))
} else {
conditions = append(conditions, fmt.Sprintf("(%s IS NULL)", c.columnName(key)))
}
default:
conditions = append(conditions, fmt.Sprintf("(%s = $%d)", c.columnName(key), paramIndex))
paramIndex++
Expand All @@ -182,6 +283,9 @@ func (c *Converter) convertFilter(filter map[string]any, paramIndex int) (string
}

func (c *Converter) columnName(column string) string {
if column == ReservedColumnName {
return fmt.Sprintf(`%q::text`, column)
}
if c.nestedColumn == "" {
return fmt.Sprintf("%q", column)
}
Expand Down
112 changes: 104 additions & 8 deletions filter/converter_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,31 +74,39 @@ func TestConverter_Convert(t *testing.T) {
nil,
},
{
"in-array operator simple",
"simple $in",
nil,
`{"status": {"$in": ["NEW", "OPEN"]}}`,
`("status" = ANY($1))`,
[]any{[]any{"NEW", "OPEN"}},
nil,
},
{
"in-array operator invalid value",
"simple $nin",
nil,
`{"status": {"$nin": ["NEW", "OPEN"]}}`,
`(NOT "status" = ANY($1))`,
[]any{[]any{"NEW", "OPEN"}},
nil,
},
{
"$in invalid value",
nil,
`{"status": {"$in": [{"hacker": 1}, "OPEN"]}}`,
``,
nil,
fmt.Errorf("invalid value for $in operator (must array of primatives): [map[hacker:1] OPEN]"),
},
{
"in-array operator scalar value",
"$in scalar value",
nil,
`{"status": {"$in": "text"}}`,
``,
nil,
fmt.Errorf("invalid value for $in operator (must array of primatives): text"),
},
{
"in-array operator with null value",
"$in with null value",
nil,
`{"status": {"$in": ["guest", null]}}`,
`("status" = ANY($1))`,
Expand All @@ -115,9 +123,9 @@ func TestConverter_Convert(t *testing.T) {
},
{
"or operator complex",
nil,
`{"$or": [{"org": "poki", "admin": true}, {"age": {"$gte": 18}}]}`,
`((("admin" = $1) AND ("org" = $2)) OR ("age" >= $3))`,
filter.WithNestedJSONB("meta", "org", "admin", "age"),
`{"$or": [{"org": "poki", "admin": true}, {"age": {"$gte": 18}}, {"picture": {"$exists": false}}]}`,
`((("admin" = $1) AND ("org" = $2)) OR ("age" >= $3) OR (NOT jsonb_path_match(meta, 'exists($.picture)')))`,
[]any{true, "poki", float64(18)},
nil,
},
Expand All @@ -137,6 +145,14 @@ func TestConverter_Convert(t *testing.T) {
nil,
fmt.Errorf("$or as scalar operator not supported"),
},
{
"$nor operator basic",
nil,
`{"$nor": [{"name": "John"}, {"name": "Doe"}]}`,
`NOT (("name" = $1) OR ("name" = $2))`,
[]any{"John", "Doe"},
nil,
},
{
"and operator basic",
nil,
Expand Down Expand Up @@ -198,7 +214,7 @@ func TestConverter_Convert(t *testing.T) {
nil,
`{}`,
`FALSE`,
[]any{},
nil,
nil,
}, {
"empty or conditions",
Expand All @@ -208,6 +224,86 @@ func TestConverter_Convert(t *testing.T) {
nil,
fmt.Errorf("empty objects not allowed"),
},
{
"sql injection",
nil,
`{"\"bla = 1 --": 1}`,
``,
nil,
fmt.Errorf("invalid column name: \"bla = 1 --"),
},
{
"$exists on normal column",
nil,
`{"name": {"$exists": false}}`,
``,
nil,
fmt.Errorf("$exists operator not supported on non-nested jsonb columns"),
},
{
"$exists jsonb column",
filter.WithNestedJSONB("meta"),
`{"name": {"$exists": false}}`,
`(NOT jsonb_path_match(meta, 'exists($.name)'))`,
nil,
nil,
},
{
"null nornal column",
nil,
`{"name": null}`,
`("name" IS NULL)`,
nil,
nil,
},
{
"null jsonb column",
filter.WithNestedJSONB("meta"),
`{"name": null}`,
`(jsonb_path_match(meta, 'exists($.name)') AND "meta"->>'name' IS NULL)`,
nil,
nil,
},
{
"$elemMatch on normal column",
nil,
`{"name": {"$elemMatch": {"$eq": "John"}}}`,
`EXISTS (SELECT 1 FROM unnest("name") AS __placeholder WHERE ("__placeholder"::text = $1))`,
[]any{"John"},
nil,
},
{
"$elemMatch on jsonb column",
filter.WithNestedJSONB("meta"),
`{"name": {"$elemMatch": {"$eq": "John"}}}`,
`EXISTS (SELECT 1 FROM jsonb_array_elements("meta"->'name') AS __placeholder WHERE ("__placeholder"::text = $1))`,
[]any{"John"},
nil,
},
{
"$elemMatch with $gt",
nil,
`{"age": {"$elemMatch": {"$gt": 18}}}`,
`EXISTS (SELECT 1 FROM unnest("age") AS __placeholder WHERE ("__placeholder"::text > $1))`,
[]any{float64(18)},
nil,
},
{
"$not operator",
nil,
`{"$not": {"name": "John"}}`,
`NOT ("name" = $1)`,
[]any{"John"},
nil,
},
{
"$not in the wrong place",
nil,
`{"name": {"$not": {"$eq": "John"}}}`,
``,
nil,
fmt.Errorf("$not as scalar operator not supported"),
},
}

for _, tt := range tests {
Expand Down
Loading
Loading