Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Introduce basic sanity test for MV used by Observability Integrations #995

Merged
merged 5 commits into from
Jan 16, 2025
+635 −0
Merged

Introduce basic sanity test for MV used by Observability Integrations #995

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ccc8c2e
Add IT for VPC flow integration
dai-chen Dec 2, 2024
b7e038e
Add IT for CloudTrail integration
dai-chen Dec 16, 2024
eeb32d6
Use UTC in all assertions
dai-chen Dec 16, 2024
e65870c
Add IT for WAF intergation
dai-chen Dec 16, 2024
9a0d213
Add more comments
dai-chen Dec 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
181 changes: 181 additions & 0 deletions integ-test/src/integration/resources/aws-logs/cloud_trail.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
CREATE TABLE {table_name} (
eventVersion STRING,
userIdentity STRUCT<
type:STRING,
principalId:STRING,
arn:STRING,
accountId:STRING,
invokedBy:STRING,
accessKeyId:STRING,
userName:STRING,
sessionContext:STRUCT<
attributes:STRUCT<
mfaAuthenticated:STRING,
creationDate:STRING
>,
sessionIssuer:STRUCT<
type:STRING,
principalId:STRING,
arn:STRING,
accountId:STRING,
userName:STRING
>,
ec2RoleDelivery:STRING,
webIdFederationData:MAP<STRING,STRING>
>
>,
eventTime STRING,
eventSource STRING,
eventName STRING,
awsRegion STRING,
sourceIPAddress STRING,
userAgent STRING,
errorCode STRING,
errorMessage STRING,
requestParameters STRING,
responseElements STRING,
additionalEventData STRING,
requestId STRING,
eventId STRING,
resources ARRAY<STRUCT<
arn:STRING,
accountId:STRING,
type:STRING
>>,
eventType STRING,
apiVersion STRING,
readOnly STRING,
recipientAccountId STRING,
serviceEventDetails STRING,
sharedEventId STRING,
vpcEndpointId STRING,
eventCategory STRING,
tlsDetails STRUCT<
tlsVersion:STRING,
cipherSuite:STRING,
clientProvidedHostHeader:STRING
>
)
USING json
OPTIONS (
recursivefilelookup='true',
multiline 'true'
);

INSERT INTO {table_name} VALUES
(
'1.08',
NAMED_STRUCT(
'type', 'IAMUser',
'principalId', 'AWS123456789012',
'arn', 'arn:aws:iam::123456789012:user/ExampleUser',
'accountId', '123456789012',
'invokedBy', null,
'accessKeyId', 'AKIA1234567890',
'userName', 'ExampleUser',
'sessionContext', NAMED_STRUCT(
'attributes', NAMED_STRUCT(
'mfaAuthenticated', 'true',
'creationDate', '2023-11-01T05:00:00Z'
),
'sessionIssuer', NAMED_STRUCT(
'type', 'Role',
'principalId', 'ARO123456789012',
'arn', 'arn:aws:iam::123456789012:role/MyRole',
'accountId', '123456789012',
'userName', 'MyRole'
),
'ec2RoleDelivery', 'true',
'webIdFederationData', MAP()
)
),
'2023-11-01T05:00:00Z',
'sts.amazonaws.com',
'AssumeRole',
'us-east-1',
'198.51.100.45',
'AWS CLI',
null,
null,
null,
null,
null,
'request-id-1',
'event-id-1',
ARRAY(NAMED_STRUCT(
'arn', 'arn:aws:iam::123456789012:role/MyRole',
'accountId', '123456789012',
'type', 'AWS::IAM::Role'
)),
'AwsApiCall',
'2015-03-31',
'true',
'123456789012',
null,
null,
null,
'Management',
NAMED_STRUCT(
'tlsVersion', 'TLSv1.2',
'cipherSuite', 'ECDHE-RSA-AES128-GCM-SHA256',
'clientProvidedHostHeader', null
)
),
(
'1.08',
NAMED_STRUCT(
'type', 'IAMUser',
'principalId', 'AWS123456789012',
'arn', 'arn:aws:iam::123456789012:user/ExampleUser',
'accountId', '123456789012',
'invokedBy', null,
'accessKeyId', 'AKIA1234567890',
'userName', 'ExampleUser',
'sessionContext', NAMED_STRUCT(
'attributes', NAMED_STRUCT(
'mfaAuthenticated', 'true',
'creationDate', '2023-11-01T05:06:00Z'
),
'sessionIssuer', NAMED_STRUCT(
'type', 'Role',
'principalId', 'ARO123456789012',
'arn', 'arn:aws:iam::123456789012:role/MyRole',
'accountId', '123456789012',
'userName', 'MyRole'
),
'ec2RoleDelivery', 'true',
'webIdFederationData', MAP()
)
),
'2023-11-01T05:06:00Z',
'sts.amazonaws.com',
'AssumeRole',
'us-east-1',
'198.51.100.45',
'AWS CLI',
null,
null,
null,
null,
null,
'request-id-2',
'event-id-2',
ARRAY(NAMED_STRUCT(
'arn', 'arn:aws:iam::123456789012:role/MyRole',
'accountId', '123456789012',
'type', 'AWS::IAM::Role'
)),
'AwsApiCall',
'2015-03-31',
'true',
'123456789012',
null,
null,
null,
'Management',
NAMED_STRUCT(
'tlsVersion', 'TLSv1.2',
'cipherSuite', 'ECDHE-RSA-AES128-GCM-SHA256',
'clientProvidedHostHeader', null
)
);
29 changes: 29 additions & 0 deletions integ-test/src/integration/resources/aws-logs/vpc_flow.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
CREATE TABLE {table_name} (
version INT,
accountId STRING,
interfaceId STRING,
srcAddr STRING,
dstAddr STRING,
srcPort INT,
dstPort INT,
protocol BIGINT,
packets BIGINT,
bytes BIGINT,
start BIGINT,
`end` BIGINT,
action STRING,
logStatus STRING
)
USING csv
OPTIONS (
sep = ' ',
recursiveFileLookup = 'true'
);

INSERT INTO {table_name} VALUES
(1, '123456789012', 'eni-abc123', '10.0.0.1', '10.0.0.2', 12345, 80, 6, 10, 200, 1698814800, 1698814860, 'ACCEPT', 'OK'), -- 05:00:00 to 05:01:00 UTC
(2, '123456789012', 'eni-def456', '10.0.0.1', '10.0.0.2', 12346, 443, 6, 5, 150, 1698814900, 1698814960, 'ACCEPT', 'OK'), -- 05:01:40 to 05:02:40 UTC
(3, '123456789013', 'eni-ghi789', '10.0.0.3', '10.0.0.4', 12347, 22, 6, 15, 300, 1698815400, 1698815460, 'ACCEPT', 'OK'), -- 05:10:00 to 05:11:00 UTC
(4, '123456789013', 'eni-jkl012', '10.0.0.5', '10.0.0.6', 12348, 21, 6, 20, 400, 1698815500, 1698815560, 'REJECT', 'OK'), -- 05:11:40 to 05:12:40 UTC
(5, '123456789014', 'eni-mno345', '10.0.0.7', '10.0.0.8', 12349, 25, 6, 25, 500, 1698816000, 1698816060, 'ACCEPT', 'OK') -- 05:20:00 to 05:21:00 UTC
;
91 changes: 91 additions & 0 deletions integ-test/src/integration/resources/aws-logs/waf.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
CREATE TABLE {table_name} (
timestamp STRING,
webaclId STRING,
action STRING,
formatVersion INT,
httpRequest STRUCT<
clientIp: STRING,
country: STRING,
headers: ARRAY<STRUCT<
name: STRING,
value: STRING
>>,
uri: STRING,
args: STRING,
httpVersion: STRING,
httpMethod: STRING,
requestId: STRING
>,
httpSourceId STRING,
httpSourceName STRING,
requestBodySize INT,
requestBodySizeInspectedByWAF INT,
terminatingRuleId STRING,
terminatingRuleType STRING,
ruleGroupList ARRAY<STRUCT<
ruleId: STRING,
ruleAction: STRING
>>,
rateBasedRuleList ARRAY<STRUCT<
ruleId: STRING
>>,
nonTerminatingMatchingRules ARRAY<STRUCT<
ruleId: STRING
>>
)
USING json
OPTIONS (
recursivefilelookup = 'true'
);

INSERT INTO {table_name} VALUES
(
1698814800000, -- 2023-11-01T05:00:00Z
'webacl-12345',
'ALLOW',
1,
NAMED_STRUCT(
'clientIp', '192.0.2.1',
'country', 'US',
'headers', ARRAY(NAMED_STRUCT('name', 'User-Agent', 'value', 'Mozilla/5.0')),
'uri', '/index.html',
'args', 'query=example',
'httpVersion', 'HTTP/1.1',
'httpMethod', 'GET',
'requestId', 'req-1'
),
'source-1',
'http-source',
500,
450,
'rule-1',
'REGULAR',
ARRAY(NAMED_STRUCT('ruleId', 'group-rule-1', 'ruleAction', 'ALLOW')),
ARRAY(),
ARRAY()
),
(
1698815400000, -- 2023-11-01T05:10:00Z
'webacl-67890',
'BLOCK',
1,
NAMED_STRUCT(
'clientIp', '192.0.2.2',
'country', 'CA',
'headers', ARRAY(NAMED_STRUCT('name', 'Referer', 'value', 'example.com')),
'uri', '/login.html',
'args', '',
'httpVersion', 'HTTP/2',
'httpMethod', 'POST',
'requestId', 'req-2'
),
'source-2',
'http-source',
750,
600,
'rule-2',
'RATE_BASED',
ARRAY(NAMED_STRUCT('ruleId', 'group-rule-2', 'ruleAction', 'BLOCK')),
ARRAY(),
ARRAY()
);
Loading
Loading