Factor out a script to generate a new configuration. (#87)

### What

This removes some duplication, but more importantly, it allows us to get
a little more sophisticated.

Now, when a request fails, we print the response status _and_ the body.
Previously, we only printed the status.

We sometimes see a 500 Internal Server Error and don't know why.
Hopefully this will help us get to the bottom of it.

### How

Instead of using `curl -f`, which prints a simple error, we capture the
status code using `--write-out '%{http_code}\n'`, and the body using
`--output $OUTPUT_FILE`.

We can then use this to construct a more useful error message.

All other logic stays the same.

.github/workflows/benchmarks.yaml

@@ -36,11 +36,8 @@ jobs:
           mkdir -p generated
           docker compose up --detach --wait agent-configuration
           CONFIGURATION_SERVER_PORT="$(docker compose port agent-configuration 9100 | sed 's/.\+://')"
-          CONFIGURATION_SERVER_URL="http://localhost:${CONFIGURATION_SERVER_PORT}/"
-          curl -fsS "$CONFIGURATION_SERVER_URL" \
-            | jq --arg uri 'postgresql://postgres:password@postgres' '. + {"connectionUri": {"uri": $uri}}' \
-            | curl -fsS "$CONFIGURATION_SERVER_URL" -H 'Content-Type: application/json' -d @- \
-            | jq . \
+          CONFIGURATION_SERVER="localhost:${CONFIGURATION_SERVER_PORT}"
+          ../../scripts/new-configuration.sh "$CONFIGURATION_SERVER" 'postgresql://postgres:password@postgres' \
             | tee ./generated/deployment.json
           docker compose down agent-configuration
 

benchmarks/component/start.sh

@@ -36,11 +36,7 @@ if ! kill -0 "$AGENT_PID"; then
   echo >&2 'The agent stopped abruptly. Take a look at agent.log for details.'
   exit 1
 fi
-curl -fsS http://localhost:9100 \
-  | jq \
-      --arg uri "postgresql://postgres:password@${POSTGRESQL_SOCKET}" \
-      '. + {"connectionUri": {"uri": $uri}}' \
-  | curl -fsS http://localhost:9100 -H 'Content-Type: application/json' -d @- \
+../../scripts/new-configuration.sh localhost:9100 "postgresql://postgres:password@${POSTGRESQL_SOCKET}" \
   > ./generated/deployment.json
 kill "$AGENT_PID" && wait "$AGENT_PID" || :
 rm -f ./agent.pid

justfile

@@ -61,11 +61,8 @@ run-in-docker: build-docker-with-nix start-dependencies
     {{CONNECTOR_IMAGE}} \
     configuration serve
   trap 'docker stop ndc-postgres-configuration' EXIT
-  CONFIGURATION_SERVER_URL='http://localhost:9100/'
-  ./scripts/wait-until --timeout=30 --report -- nc -z localhost 9100
-  curl -fsS "$CONFIGURATION_SERVER_URL" \
-    | jq --arg uri 'postgresql://postgres:password@postgres' '. + {"connectionUri": {"uri": $uri}}' \
-    | curl -fsS "$CONFIGURATION_SERVER_URL" -H 'Content-Type: application/json' -d @- \
+  CONFIGURATION_SERVER='localhost:9100'
+  ./scripts/new-configuration.sh "$CONFIGURATION_SERVER" 'postgresql://postgres:password@postgres' \
     > "$configuration_file"
 
   echo '> Starting the server...'

scripts/generate-chinook-configuration.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -e -u -o pipefail
 
+CURRENT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null && echo "$PWD")"
+
 EXECUTABLE="$1"
 CONNECTION_STRING="$2"
 CHINOOK_DEPLOYMENT="$3"
@@ -35,19 +37,13 @@ INITIAL_DATA="$(jq '{"poolSettings": (.poolSettings // {}), "metadata": {"native
 # create a temporary file for the output so we don't overwrite data by accident
 NEW_FILE="$(mktemp)"
 
-# 1. Pass the connection string to the configuration server to generate the
-#    initial deployment from introspection
+# 1. Generate the configuration
 # 2. Splice in the preserved data from above
 # 3. Format the file
 #
 # Because we `set -o pipefail` above, this will fail if any of the steps fail,
 # and we will abort without overwriting the original file.
-curl -fsS http://localhost:9100 \
-  | jq --argjson initial_data "$INITIAL_DATA" '. * $initial_data' \
-  | jq \
-    --arg uri "$CONNECTION_STRING" \
-    '. + {"connectionUri": {"uri":$uri}}' \
-  | curl -fsS http://localhost:9100 -H 'Content-Type: application/json' -d @- \
+"${CURRENT_DIR}/new-configuration.sh" 'localhost:9100' "$CONNECTION_STRING" "$INITIAL_DATA" \
   | jq --argjson preserved_data "$PRESERVED_DATA" '. + $preserved_data' \
   | prettier --parser=json \
   > "$NEW_FILE"

scripts/new-configuration.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+set -e
+set -u
+set -o pipefail
+
+CURRENT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null && echo "$PWD")"
+
+if [[ $# -lt 2 ]]; then
+  echo >&2 "Usage: $0 CONFIGURATION-SERVER POSTGRESQL-CONNECTION-URI [EXTRA-CONFIG]"
+  echo >&2
+  echo >&2 '  The configuration server should be in the form "HOST:PORT".'
+  exit 2
+fi
+
+export CONFIGURATION_SERVER="$1"  # exported to use in the `wait-until` call
+POSTGRESQL_CONNECTION_URI="$2"
+EXTRA_CONFIG="${3:-{\}}"  # this defaults to '{}'
+
+# wait until the server is up and running
+"${CURRENT_DIR}/wait-until" --timeout=30 --report -- sh -c 'curl -fsS "http://${CONFIGURATION_SERVER}/health" > /dev/null'
+
+function get {
+  # write HTTP responses to this file
+  OUTPUT_FILE="$(mktemp)"
+  trap 'rm -f "$OUTPUT_FILE"' RETURN
+
+  # capture the status in the variable, and the body in $OUTPUT_FILE
+  response_status="$(curl --silent --output "$OUTPUT_FILE" --write-out '%{http_code}\n' "$@")"
+  if [[ "$response_status" -ge 200 && "$response_status" -lt 300 ]]; then
+    cat "$OUTPUT_FILE"
+  else
+    # on failure, log the response status and body
+    echo >&2 "Request to ${1} failed with status ${response_status}."
+    echo >&2 "Response body:"
+    cat "$OUTPUT_FILE" >&2
+    echo >&2
+    echo >&2
+    return 1
+  fi
+}
+
+# 1.  Get an empty configuration.
+# 2.  Splice in the connection URI and extra config.
+# 3.  Send that configuration back to get the real thing.
+# 4.  Reformat with `jq`.
+# 5.  Print the generated configuration to STDOUT.
+get "http://${CONFIGURATION_SERVER}/" \
+  | jq --arg uri "$POSTGRESQL_CONNECTION_URI" --argjson extra "$EXTRA_CONFIG" '. * $extra + {"connectionUri": {"uri": {"value": $uri}}}' \
+  | get "http://${CONFIGURATION_SERVER}/" -H 'Content-Type: application/json' -d @- \
+  | jq .