EQL - Final (#69)

This is a "final" change. Changes in parser: - add pipe command (parsed but not executed) - floats - optional field names (parsed but not evaluated) - add a full list of functions, antlr can give a nicer error message Some bug fixes.
QuesmaOrg · May 10, 2024 · fa23264 · fa23264
1 parent ee7fc1d
commit fa23264
Show file tree

Hide file tree

Showing 17 changed files with 1,660 additions and 567 deletions.
diff --git a/quesma/eql/README.md b/quesma/eql/README.md
@@ -0,0 +1,93 @@
+EQL support
+---
+
+
+This package contains the EQL parser and query transformer. 
+
+- The parser is generated using ANTLR4. The grammar is defined in `EQL.g4` file. The generated code is in `parser` directory. Do not review the generated code.
+- HTTP endpoint is implemented in `FIXME`
+- `query_translator.go` is the glue code that connects the parser with the Quesma search engine.
+- Sample EQL query as an HTTP request is in `http_request/eql_search.http` file.
+- A simple command line client is implemented in `playground` directory.
+- End-to-End tests are implemented in `e2e` directory. See file `e2e/eql_test.go` for more details.
+
+
+What is supported?
+---
+
+Comparison operators
+
+| operator | supported          | comment |
+|----------|--------------------|---------|
+| `==`     | :heavy_check_mark: |         |
+| `!=`     | :heavy_check_mark: |         |
+| `>`      | :heavy_check_mark: |         |
+| `>=`     | :heavy_check_mark: |         |
+| `<`      | :heavy_check_mark: |         |
+| `<=`     | :heavy_check_mark: |         | 
+| `:`      | :heavy_check_mark: |         |
+
+
+Lookup operators
+
+| operator  | supported          | comment |
+|-----------|--------------------|---------|
+| `in`      | :heavy_check_mark: |         |
+| `not in`  | :heavy_check_mark: |         |
+| `in~`     | :heavy_check_mark: |         |
+| `not in~` | :heavy_check_mark: |         |
+| `:`       | :heavy_check_mark: |         |
+| `like`    | :heavy_check_mark: |         |
+| `like~`   | :heavy_check_mark: |         |
+| `regex`   | :heavy_check_mark: |         |
+| `regex~`  | :heavy_check_mark: |         |
+
+
+Logical operators
+
+| operator | supported          | comment |
+|----------|--------------------|---------|
+| `and`    | :heavy_check_mark: |         |
+| `or`     | :heavy_check_mark: |         |
+| `not`    | :heavy_check_mark: |         |
+
+
+
+Supported functions
+
+
+| function          | supported          | comment                                |
+|-------------------|--------------------|----------------------------------------|
+| `add`             | :heavy_check_mark: |                                        |
+| `between`         | :x:                |                                        |
+| `cidrMatch`       | :cockroach:        |                                        |
+| `concat`          | :heavy_check_mark: |                                        |
+| `divide`          | :cockroach:        | division of integers should be rounded |
+| `endsWith`        | :heavy_check_mark: |                                        |
+| `endsWith~`       | :heavy_check_mark: |                                        |
+| `indexOf`         | :cockroach:        |                                        |
+| `indexOf~`        | :cockroach:        |                                        |
+| `length`          | :heavy_check_mark: |                                        |
+| `modulo`          | :heavy_check_mark: |                                        |
+| `multiply`        | :heavy_check_mark: |                                        |
+| `number`          | :cockroach:        |                                        |
+| `startsWith`      | :heavy_check_mark: |                                        |
+| `startsWith~`     | :heavy_check_mark: |                                        |
+| `string`          | :heavy_check_mark: |                                        |
+| `stringContains`  | :cockroach:        |                                        |
+| `stringContains~` | :cockroach:        |                                        |
+| `substring`       | :cockroach:        |                                        |
+| `subtract`        | :heavy_check_mark: |                                        |
+
+
+
+
+Known limitations
+---
+
+1. We support only simple EQL queries. Sequence and sample queries are not supported.
+2. Pipe operators are not supported. Syntax is parsed. Error is returned if pipe operator is used in the query. (https://www.elastic.co/guide/en/elasticsearch/reference/current/eql-syntax.html#eql-pipes)
+3. Optional fields are not supported. Field names are parsed. Error is returned if that field is used in the query. (https://www.elastic.co/guide/en/elasticsearch/reference/current/eql-syntax.html#eql-syntax-optional-fields)
+4. Backtick escaping is not supported. (https://www.elastic.co/guide/en/elasticsearch/reference/current/eql-syntax.html#eql-syntax-escape-a-field-name)
+5. Error handling is missing. Every error will be returned as na internal server error.
+
diff --git a/quesma/eql/e2e/end2end_test.go b/quesma/eql/e2e/end2end_test.go
@@ -24,8 +24,12 @@ func TestE2E(t *testing.T) {
 	// Queries start with a "--" are skipped.
 	var eqlQueries = []string{
 		`any where false`,
+		`any where false and true`,
 		`not_existing where true`,
 		"process where true",
+		"process where false and true",
+		"process where not false and true",
+
 		"process where process.pid == 1",
 		"process where process.pid > 0",
 		"process where process.pid >= 0",
@@ -35,21 +39,23 @@ func TestE2E(t *testing.T) {
 		`process where process.pid == 2 / 2`,
 		`process where process.pid == 3 % 2`,
 		`process where process.pid == 2 * 3 / 6`,
-		`-- process where process.pid < 4.0 / 2`, // TODO add floats
+		`process where process.pid < 4.0 / 2`,
+
 		`process where not false`,
 		`process where not (event.type == "start")`,
 		`process where process.pid == 1 and event.type == "start"`,
+		`process where process.pid == 1 and event.type != "start"`,
 		`process where event.type : "start"`,
 		`process where event.type : "st*"`,
 		`process where event.type :  ("start", "stop")`,
 		`process where process.pid == 1 and event.type like "st*"`,
-		`-- process where process.pid == 1 and event.type like "st%"`, // FIXME this is a bug, we should escape % in like
+		`process where process.pid == 1 and event.type like "st%"`,
 		`process where process.name like~ "test"`,
 		`process where process.name like ("test", "test2")`,
 		`process where event.type in ("start", "stop")`,
 		`process where event.type in~ ("STaRT", "StOP")`,
 		`process where event.type not in ("start", "stop")`,
-		`-- process where event.type not in~ ("STaRT", "StOP")`, // FIXME THIS IS A BUG,  quesma retured: 3 but elastic returned: 1
+		`process where event.type not in~ ("STaRT", "StOP")`,
 
 		`process where process.name != string(1)`,
 		`process where process.name == null`,
@@ -68,17 +74,17 @@ func TestE2E(t *testing.T) {
 		`process where process.name like "Te"`,
 		`process where process.name like "T*t"`,
 
-		`-- process where process.name : "_est"`, //FIXME we should escace _ in like,  quesma retured: 3 but elastic returned: 0
-		`-- process where process.name : "Te_t"`, // FIXME quesma retured: 3 but elastic returned: 0
+		`process where process.name : "_est"`,
+		`process where process.name : "Te_t"`,
 		`process where process.name : "Te_"`,
 
-		`-- process where process.name : "?est"`, // FIXME support ? wildcard , quesma retured: 0 but elastic returned: 3
-		`-- process where process.name : "Te?t"`,
+		`process where process.name : "?est"`,
+		`process where process.name : "Te?t"`,
 		`process where process.name : "Te?"`,
 
 		`process where process.pid == add(0,1)`,
-		`-- process where process.pid == add(-2,3)`, // FIXME this is a bug, we should support negative numbers
-		`-- process where process.pid == add(-2,3)`,
+		`process where process.pid == add(-2,3)`,
+		`process where process.pid == add(-2,3)`,
 
 		// FIXME this is an  elastic limitation
 		// elastic fail response: {"error":{"root_cause":[{"type":"ql_illegal_argument_exception","reason":"Line 1:40: Comparisons against fields are not (currently) supported; offender [add(process.pid,0)] in [==]"}],"type":"ql_illegal_argument_exception","reason":"Line 1:40: Comparisons against fields are not (currently) supported; offender [add(process.pid,0)] in [==]"},"status":500}
@@ -180,7 +186,10 @@ func TestE2E(t *testing.T) {
 		`process where subtract(null, 2) == null`,
 		`process where subtract(2, null) == null`,
 
-		`process where ?not_existing == null`, // FIXME this is a bug, optional fields are not supported yet
+		`-- process where ?not_existing == null`, // FIXME this is a bug, optional fields are not supported yet
+
+		`process where process.name != "foo"`,
+		`process where process.name != "';delete from table;"`,
 	}
 
 	// This our category name. Each test runs in a separate category.

diff --git a/quesma/eql/parser.go b/quesma/eql/parser.go
@@ -75,5 +75,5 @@ func (s *EQL) Parse(query string) (parser.IQueryContext, error) {
 }
 
 func (s *EQL) IsSupported(ast parser.IQueryContext) bool {
-	return ast.SimpleQuery() != nil
+	return ast.SimpleQuery() != nil && len(ast.AllPipe()) == 0
 }
diff --git a/quesma/eql/parser/EQL.g4 b/quesma/eql/parser/EQL.g4
@@ -1,12 +1,9 @@
 grammar EQL;
 
-query: ( simpleQuery
-     | sequenceQuery
-     | sampleQuery ) EOF
+query: ( simpleQuery | sequenceQuery | sampleQuery ) ('|' pipe)* EOF
     ;
 
 simpleQuery: category 'where' condition
-// TODO add support for pipe opertor  '|'
 ;
 
 sequenceQuery: 'sequence' ( 'by' fieldList )? ( 'with' 'maxspan' '=' interval )?
@@ -35,13 +32,12 @@ category
        | STRING
        ;
 
-field: ID;
-// TODO add optional field names: `?field_name`
+field: ID | ('?' ID);
+// TODO add optional field names: '?field_name'
 // TODO add backtick escape for field names
 
 fieldList : field (',' field)*;
 
-// TODO add floats
 literal: STRING | NUMBER | BOOLEAN;
 literalList: '(' literal (',' literal)* ')';
 
@@ -56,9 +52,39 @@ value:
 ;
 
 
+pipe:
+    'head' NUMBER  #PipeHead
+    | 'tail'  NUMBER  #PipeTail
+    | 'count' #PipeCount
+    | 'unique' fieldList #PipeUnique
+    | 'filter' condition #PipeFilter
+    | 'sort' fieldList #PipeSort
+    ;
+
 
 funcall: funcName '(' value (',' value)* ')';
-funcName: ID | ID '~';
+funcName: 
+          'add'
+        | 'between'
+        | 'cidrMatch'
+        | 'concat'
+        | 'divide'
+        | 'endsWith'
+        | 'endsWith~'
+        | 'indexOf'
+        | 'indexOf~'
+        | 'length'
+        | 'modulo'
+        | 'multiply'
+        | 'number'
+        | 'startsWith'
+        | 'startsWith~'
+        | 'string'
+        | 'stringContains'
+        | 'stringContains~'
+        | 'substring'
+        | 'subtract'
+;
 
 
 interval: INTERVAL;
@@ -69,11 +95,13 @@ MULTILINE_COMMENT: '/*' .*? '*/' -> channel(HIDDEN);
 ONELINE_COMMNET: '//' ~[\r\n]* -> channel(HIDDEN);
 BOOLEAN: 'true' | 'false';
 INTERVAL: [0-9]+[a-z];
-NUMBER: [0-9]+;
+
+NUMBER:  ('-' | ) ([0-9]+ | [0-9]* '.' [0-9]+) ([eE] [+-]? [0-9]+)?;
 
 ESC: '\\' .;
 STRING: '"' ('\\' . | '""' | ~["\\])*  '"' | '"""' .*? '"""';
 
 WS: [ \t\n\r\f]+ -> skip ;
 
+
 ID: [a-zA-Z_][.a-zA-Z0-9_-]*;
diff --git a/quesma/eql/parser/eql_base_listener.go b/quesma/eql/parser/eql_base_listener.go
diff --git a/quesma/eql/parser/eql_base_visitor.go b/quesma/eql/parser/eql_base_visitor.go