Merge branch 'main' into gil/add-error-metrics

.github/workflows/build-images-and-deploy.yaml

@@ -1,11 +1,6 @@
 name: multi-architecture docker build
 
-on:
-  push:
-    branches:
-      - main
-    tags:
-      - "v*"
+on: [push]
 
 jobs:
   build_and_deploy:

.github/workflows/e2e-test.yaml

@@ -0,0 +1,35 @@
+name: end-to-end release compatibility
+
+on:
+  merge_group:
+  push:
+  pull_request:
+
+jobs:
+  end_to_end_release_compatibility:
+    name: Check PR against end-to-end tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Generate short SHA
+        uses: benjlevesque/[email protected]
+        id: short-sha
+        with:
+          length: 9
+
+      - name: Wait for Docker image to be created
+        uses: lewagon/[email protected]
+        with:
+          ref: "${{ github.event.pull_request.head.sha || github.sha }}"
+          check-name: "build and deploy (ndc-postgres)"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Dispatch E2E tests
+        uses: aurelien-baudet/workflow-dispatch@v2
+        id: workflow_dispatch
+        with:
+          inputs: '{ "connector": "${{ steps.short-sha.outputs.sha }}" }'
+          repo: hasura/v3-e2e-testing
+          ref: main
+          token: ${{ secrets.E2E_WORKFLOW_PAT }}
+          workflow: "cargo test postgres"

ci/deploy.sh

@@ -52,6 +52,7 @@ function run {
 #
 # Additionally sets a branch tag assuming this is the latest tag for the given
 # branch. The branch tag has the form: dev-main
+# Also sets a tag with just the branch short hash
 function set_dev_tags {
     local branch="$1"
     # replace '.' and '/' in branch name with '-'
@@ -62,7 +63,7 @@ function set_dev_tags {
     local short_hash
     short_hash="$(git rev-parse --short=9 HEAD)"
     version="${branch_prefix}-${short_hash}"
-    export docker_tags=("$version" "$branch_prefix")
+    export docker_tags=("$version" "$branch_prefix" "$short_hash")
 }
 
 # The Github workflow passes a ref of the form refs/heads/<branch name> or

crates/tests/databases-tests/src/capabilities_tests.rs

@@ -1,7 +1,4 @@
-#[cfg(test)]
-mod capabilities_tests {
-    #[tokio::test]
-    async fn get_capabilities() {
-        insta::assert_json_snapshot!(ndc_postgres::capabilities::get_capabilities());
-    }
+#[tokio::test]
+async fn get_capabilities() {
+    insta::assert_json_snapshot!(ndc_postgres::capabilities::get_capabilities());
 }

crates/tests/tests-common/src/schemas.rs

@@ -1,3 +1,4 @@
+use std::fmt::Write;
 /// Checks that a given value conforms to the schema generated by `schemars`.
 ///
 /// Panics with a human-readable error if the value does not conform, or if the
@@ -17,14 +18,14 @@ pub fn check_value_conforms_to_schema<T: schemars::JsonSchema>(value: serde_json
         Err(errors) => {
             panic!(
                 "The configuration does not conform to the schema.\n{}",
-                errors
-                    .map(|error| {
-                        format!(
-                            "{}\ninstance path: {}\nschema path:   {}\n\n",
-                            error, error.instance_path, error.schema_path
-                        )
-                    })
-                    .collect::<String>()
+                errors.fold(String::new(), |mut str, error| {
+                    let _ = write!(
+                        str,
+                        "{}\ninstance path: {}\nschema path:   {}\n\n",
+                        error, error.instance_path, error.schema_path
+                    );
+                    str
+                })
             )
         }
     }

docker-compose.yaml

@@ -171,6 +171,12 @@ services:
       COLLECTOR_OTLP_ENABLED: "true"
       COLLECTOR_ZIPKIN_HOST_PORT: "9411"
 
+  auth-hook:
+    build: ../v3-engine/hasura-authn-webhook/dev-auth-webhook
+    init: true
+    ports:
+      - 3050:3050
+
 volumes:
   prom_data:
   grafana_data:

justfile

@@ -31,14 +31,14 @@ check: format-check find-unused-dependencies build lint test
 # run the connector
 run: start-dependencies
   RUST_LOG=INFO \
-  OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4317 \
+  OTLP_ENDPOINT=http://localhost:4317 \
   OTEL_SERVICE_NAME=ndc-postgres \
     cargo run --bin ndc-postgres --release -- serve --configuration {{POSTGRES_CHINOOK_DEPLOYMENT}} > /tmp/ndc-postgres.log
 
 # run the connector
 run-config: start-dependencies
   RUST_LOG=INFO \
-  OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4317 \
+  OTLP_ENDPOINT=http://localhost:4317 \
   OTEL_SERVICE_NAME=ndc-postgres-config-server \
     cargo run --bin ndc-postgres --release -- configuration serve > /tmp/ndc-postgres.log
 
@@ -83,7 +83,7 @@ run-in-docker: build-docker-with-nix start-dependencies
 # watch the code, then test and re-run on changes
 dev: start-dependencies
   RUST_LOG=INFO \
-    OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4317 \
+    OTLP_ENDPOINT=http://localhost:4317 \
     OTEL_SERVICE_NAME=ndc-postgres \
     cargo watch -i "**/snapshots/*" \
     -c \
@@ -94,7 +94,7 @@ dev: start-dependencies
 # watch the code, then run the config server
 dev-config: start-dependencies
   RUST_LOG=INFO \
-  OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4317 \
+  OTLP_ENDPOINT=http://localhost:4317 \
   OTEL_SERVICE_NAME=ndc-postgres-config-server \
     cargo watch -i "**/snapshots/*" \
     -c \
@@ -104,7 +104,7 @@ dev-config: start-dependencies
 # watch the code, then test and re-run on changes
 dev-cockroach: start-dependencies
   RUST_LOG=INFO \
-    OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4317 \
+    OTLP_ENDPOINT=http://localhost:4317 \
     OTEL_SERVICE_NAME=cockroach-ndc \
     cargo watch -i "**/snapshots/*" \
     -c \
@@ -115,7 +115,7 @@ dev-cockroach: start-dependencies
 # watch the code, then test and re-run on changes
 dev-citus: start-dependencies
   RUST_LOG=INFO \
-    OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4317 \
+    OTLP_ENDPOINT=http://localhost:4317 \
     OTEL_SERVICE_NAME=citus-ndc \
     cargo watch -i "**/snapshots/*" \
     -c \
@@ -134,7 +134,7 @@ document-openapi:
 # Run postgres, testing against external DBs like Aurora
 test-other-dbs: create-aurora-deployment start-dependencies
   RUST_LOG=INFO \
-    OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4317 \
+    OTLP_ENDPOINT=http://localhost:4317 \
     OTEL_SERVICE_NAME=ndc-postgres \
     cargo watch -i "**/snapshots/*" \
     -c \
@@ -159,7 +159,7 @@ debug: start-dependencies
 flamegraph: start-dependencies
   CARGO_PROFILE_RELEASE_DEBUG=true \
   RUST_LOG=DEBUG \
-  OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4317 \
+  OTLP_ENDPOINT=http://localhost:4317 \
   OTEL_SERVICE_NAME=postgres-ndc \
     cargo flamegraph --bin ndc-postgres -- \
     serve --configuration {{POSTGRES_CHINOOK_DEPLOYMENT}} > /tmp/ndc-postgres.log
@@ -256,16 +256,18 @@ start-metrics:
 
 # run the v3 engine binary, pointing it at our connector
 run-engine: start-dependencies
+  docker compose up --wait auth-hook
   @echo "http://localhost:3000/ for graphiql console"
   @echo "http://localhost:4002/ for jaeger console"
   # Run graphql-engine using static Chinook metadata
   # we expect the `v3-engine` repo to live next door to this one
   RUST_LOG=DEBUG \
-  OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4317 \
+  OTLP_ENDPOINT=http://localhost:4317 \
     cargo run --release \
     --manifest-path ../v3-engine/Cargo.toml \
     --bin engine -- \
-    --metadata-path ./static/chinook-metadata.json
+    --metadata-path ./static/chinook-metadata.json \
+    --authn-config-path ./static/auth_config.json
 
 # start a postgres docker image and connect to it using psql
 repl-postgres:
@@ -314,7 +316,7 @@ build-with-nix:
 massif-postgres: start-dependencies
   cargo build --bin ndc-postgres --release
   RUST_LOG=INFO \
-  OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4317 \
+  OTLP_ENDPOINT=http://localhost:4317 \
   OTEL_SERVICE_NAME=ndc-postgres \
     valgrind --tool=massif \
     target/release/ndc-postgres \
@@ -324,7 +326,7 @@ massif-postgres: start-dependencies
 heaptrack-postgres: start-dependencies
   cargo build --bin ndc-postgres --release
   RUST_LOG=INFO \
-  OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4317 \
+  OTLP_ENDPOINT=http://localhost:4317 \
   OTEL_SERVICE_NAME=ndc-postgres \
     heaptrack \
     target/release/ndc-postgres \

rust-toolchain.toml

@@ -1,4 +1,4 @@
 [toolchain]
-channel = "1.72.1"
+channel = "1.73.0"
 profile = "default"  # see https://rust-lang.github.io/rustup/concepts/profiles.html
 components = ["rust-analyzer", "rust-src"]  # see https://rust-lang.github.io/rustup/concepts/components.html

static/auth_config.json

@@ -0,0 +1,12 @@
+{
+  "version": "v1",
+  "definition": {
+    "allowRoleEmulationBy": "admin",
+    "mode": {
+      "webhook": {
+        "url": "http://localhost:3050/validate-request",
+        "method": "Post"
+      }
+    }
+  }
+}