[OAS-10368] Grab Agenecy dump on test failures in go-driver

Makefile

@@ -78,7 +78,28 @@ else ifeq ("$(TEST_AUTH)", "jwt")
 	ARANGOARGS := --server.jwt-secret=/jwtsecret
 endif
 
-TEST_NET := --net=container:$(TESTCONTAINER)-ns
+TEST_NET := --net=host
+
+SPACE :=    
+TABV4 := $(SPACE) $(SPACE) $(SPACE) $(SPACE)
+ifdef DUMP_AGENCY_ON_FAILURE
+  	$(info Checking for jq...)
+	CHECK_JQ_INSTALLTION := $(shell jq --version 2>&1 >/dev/null | cat)
+	ifneq ($(CHECK_JQ_INSTALLTION),)
+  		$(info )
+  		$(info Error: 'jq' is not installed. This check happens because DUMP_AGENCY_ON_FAILURE is set, and 'jq' is used during the creation of the agency dump. Verified for version jq==1.6. )
+  		$(info Try installing it with:)
+  		$(info $(TABV4) sudo apt install jq)
+  		$(info )
+  		$(error $(CHECK_JQ_INSTALLTION))
+	endif
+endif
+# Installation of jq is required for processing AGENCY_DUMP
+# ifdef DUMP_AGENCY_ON_FAILURE
+# 	CHECK_JQ_INSTALLTION := $(shell command -v jq >/dev/null 2>&1 || (
+# 		
+# endif
+
 
 # By default we run tests against single endpoint to avoid problems with data propagation in Cluster mode
 # e.g. when we create a document in one endpoint, it may not be visible in another endpoint for a while
@@ -118,7 +139,6 @@ ifeq ("$(TEST_BENCHMARK)", "true")
 endif
 
 ifdef TEST_ENDPOINTS_OVERRIDE
-	TEST_NET := --net=host
 	TEST_ENDPOINTS := $(TEST_ENDPOINTS_OVERRIDE)
 endif
 
@@ -391,68 +411,57 @@ run-tests-cluster-vst-1.1-ssl:
 	@echo "Cluster server, Velocystream 1.1, SSL, with authentication"
 	@${MAKE} TEST_MODE="cluster" TEST_AUTH="rootpw" TEST_SSL="auto" TEST_CONNECTION="vst" TEST_CVERSION="1.1" __run_tests
 
+
+COMMON_DOCKER_CMD_PARAMS = \
+	--name=$(TESTCONTAINER) \
+	$(TEST_NET) \
+	-e TEST_ENDPOINTS=$(TEST_ENDPOINTS) \
+	-e TEST_NOT_WAIT_UNTIL_READY=$(TEST_NOT_WAIT_UNTIL_READY) \
+	-e TEST_AUTHENTICATION=$(TEST_AUTHENTICATION) \
+	-e TEST_JWTSECRET=$(TEST_JWTSECRET) \
+	-e TEST_MODE=$(TEST_MODE) \
+	-e TEST_BACKUP_REMOTE_REPO=$(TEST_BACKUP_REMOTE_REPO) \
+	-e TEST_BACKUP_REMOTE_CONFIG='$(TEST_BACKUP_REMOTE_CONFIG)' \
+	-e TEST_DEBUG='$(TEST_DEBUG)' \
+	-e TEST_ENABLE_SHUTDOWN=$(TEST_ENABLE_SHUTDOWN) \
+	-e ENABLE_DATABASE_EXTRA_FEATURES=$(ENABLE_DATABASE_EXTRA_FEATURES) \
+	-e GODEBUG=tls13=1 \
+	-e CGO_ENABLED=$(CGO_ENABLED)
+
+
 # Internal test tasks
 __run_tests: __test_debug__ __test_prepare __test_go_test __test_cleanup
 
+
+DOCKER_V1_CMD_PARAMS=\
+	$(COMMON_DOCKER_CMD_PARAMS) \
+	-e TEST_CONNECTION=$(TEST_CONNECTION) \
+	-e TEST_CVERSION=$(TEST_CVERSION) \
+	-e TEST_CONTENT_TYPE=$(TEST_CONTENT_TYPE) \
+	-e TEST_PPROF=$(TEST_PPROF) \
+	-e TEST_REQUEST_LOG=$(TEST_REQUEST_LOG) \
+	-e TEST_DISALLOW_UNKNOWN_FIELDS=$(TEST_DISALLOW_UNKNOWN_FIELDS) \
+	-v "${ROOTDIR}":/usr/code ${TEST_RESOURCES_VOLUME} \
+	-w /usr/code/
+
 __test_go_test:
-	$(DOCKER_CMD) \
-		--name=$(TESTCONTAINER) \
-		$(TEST_NET) \
-		-v "${ROOTDIR}":/usr/code ${TEST_RESOURCES_VOLUME} \
-		-e TEST_ENDPOINTS=$(TEST_ENDPOINTS) \
-		-e TEST_NOT_WAIT_UNTIL_READY=$(TEST_NOT_WAIT_UNTIL_READY) \
-		-e TEST_AUTHENTICATION=$(TEST_AUTHENTICATION) \
-		-e TEST_JWTSECRET=$(TEST_JWTSECRET) \
-		-e TEST_CONNECTION=$(TEST_CONNECTION) \
-		-e TEST_CVERSION=$(TEST_CVERSION) \
-		-e TEST_CONTENT_TYPE=$(TEST_CONTENT_TYPE) \
-		-e TEST_PPROF=$(TEST_PPROF) \
-		-e TEST_MODE=$(TEST_MODE) \
-		-e TEST_BACKUP_REMOTE_REPO=$(TEST_BACKUP_REMOTE_REPO) \
-		-e TEST_BACKUP_REMOTE_CONFIG='$(TEST_BACKUP_REMOTE_CONFIG)' \
-		-e TEST_DEBUG='$(TEST_DEBUG)' \
-		-e TEST_ENABLE_SHUTDOWN=$(TEST_ENABLE_SHUTDOWN) \
-		-e TEST_REQUEST_LOG=$(TEST_REQUEST_LOG) \
-		-e TEST_DISALLOW_UNKNOWN_FIELDS=$(TEST_DISALLOW_UNKNOWN_FIELDS) \
-		-e ENABLE_DATABASE_EXTRA_FEATURES=$(ENABLE_DATABASE_EXTRA_FEATURES) \
-		-e GODEBUG=tls13=1 \
-		-e CGO_ENABLED=$(CGO_ENABLED) \
-		-w /usr/code/ \
-		$(DOCKER_RUN_CMD) && echo "success!" || ( \
-			echo "failure! \n\nARANGODB-STARTER logs:"; \
-			docker logs ${TESTCONTAINER}-s; \
-			echo "\nARANGODB logs:"; \
-			docker ps -f name=${TESTCONTAINER}-s- -q | xargs -L 1 docker logs; \
-			echo "\nV1 Tests with ARGS: TEST_MODE=${TEST_MODE} TEST_AUTH=${TEST_AUTH} TEST_CONTENT_TYPE=${TEST_CONTENT_TYPE} TEST_SSL=${TEST_SSL} TEST_CONNECTION=${TEST_CONNECTION} TEST_CVERSION=${TEST_CVERSION}\n\n"; \
-			exit 1)
+	$(DOCKER_CMD) $(DOCKER_V1_CMD_PARAMS) $(DOCKER_RUN_CMD) \
+	&& echo "success!" \
+	|| ( $(foreach var,$(sort $(.VARIABLES)),$(eval export $(var)=$($(var)))) MAJOR_VERSION=1 . ./test/on_failure.sh)
+
+
 # Internal test tasks
 __run_v2_tests: __test_v2_debug__ __test_prepare __test_v2_go_test __test_cleanup
 
+DOCKER_CMD_V2_PARAMS=\
+	$(COMMON_DOCKER_CMD_PARAMS) \
+	-v "${ROOTDIR}":/usr/code:ro ${TEST_RESOURCES_VOLUME} \
+	-w /usr/code/v2/
+
 __test_v2_go_test:
-	$(DOCKER_CMD) \
-		--name=$(TESTCONTAINER) \
-		$(TEST_NET) \
-		-v "${ROOTDIR}":/usr/code:ro ${TEST_RESOURCES_VOLUME} \
-		-e TEST_ENDPOINTS=$(TEST_ENDPOINTS) \
-		-e TEST_NOT_WAIT_UNTIL_READY=$(TEST_NOT_WAIT_UNTIL_READY) \
-		-e TEST_AUTHENTICATION=$(TEST_AUTHENTICATION) \
-		-e TEST_JWTSECRET=$(TEST_JWTSECRET) \
-		-e TEST_MODE=$(TEST_MODE) \
-		-e TEST_BACKUP_REMOTE_REPO=$(TEST_BACKUP_REMOTE_REPO) \
-		-e TEST_BACKUP_REMOTE_CONFIG='$(TEST_BACKUP_REMOTE_CONFIG)' \
-		-e TEST_DEBUG='$(TEST_DEBUG)' \
-		-e TEST_ENABLE_SHUTDOWN=$(TEST_ENABLE_SHUTDOWN) \
-		-e ENABLE_DATABASE_EXTRA_FEATURES=$(ENABLE_DATABASE_EXTRA_FEATURES) \
-		-e GODEBUG=tls13=1 \
-		-e CGO_ENABLED=$(CGO_ENABLED) \
-		-w /usr/code/v2/ \
-		$(DOCKER_V2_RUN_CMD) && echo "success!" || ( \
-		    echo "failure! \n\nARANGODB-STARTER logs:"; \
-		    docker logs ${TESTCONTAINER}-s; \
-			echo "\nARANGODB logs:"; \
-			docker ps -f name=${TESTCONTAINER}-s- -q | xargs -L 1 docker logs; \
-			echo "\nV2 Tests with ARGS: TEST_MODE=${TEST_MODE} TEST_AUTH=${TEST_AUTH} TEST_CONTENT_TYPE=${TEST_CONTENT_TYPE} TEST_SSL=${TEST_SSL} TEST_CONNECTION=${TEST_CONNECTION} TEST_CVERSION=${TEST_CVERSION}\n\n"; \
-			exit 1)
+	$(DOCKER_CMD) $(DOCKER_CMD_V2_PARAMS) $(DOCKER_V2_RUN_CMD) \
+	&& echo "success!" \
+	|| ( $(foreach var,$(sort $(.VARIABLES)),$(eval export $(var)=$($(var)))) MAJOR_VERSION=2 . ./test/on_failure.sh)
 
 __test_debug__:
 ifeq ("$(DEBUG)", "true")
@@ -479,7 +488,7 @@ endif
 	@-docker rm -f -v $(TESTCONTAINER) &> /dev/null
 	@TESTCONTAINER=$(TESTCONTAINER) ARANGODB=$(ARANGODB) ALPINE_IMAGE=$(ALPINE_IMAGE) ENABLE_BACKUP=$(ENABLE_BACKUP) \
 	  ARANGO_LICENSE_KEY=$(ARANGO_LICENSE_KEY) STARTER=$(STARTER) STARTERMODE=$(TEST_MODE) TMPDIR="${TMPDIR}" \
-	  ENABLE_DATABASE_EXTRA_FEATURES=$(ENABLE_DATABASE_EXTRA_FEATURES) DEBUG_PORT=$(DEBUG_PORT) $(CLUSTERENV) "${ROOTDIR}/test/cluster.sh" start
+	  ENABLE_DATABASE_EXTRA_FEATURES=$(ENABLE_DATABASE_EXTRA_FEATURES) DEBUG_PORT=$(DEBUG_PORT) $(CLUSTERENV) DOCKER_NETWORK=${TEST_NET} "${ROOTDIR}/test/cluster.sh" start
 endif
 
 __test_cleanup:
@@ -488,7 +497,7 @@ ifdef TESTCONTAINER
 	@if [ -n "$$TESTCONTAINERS" ]; then docker rm -f -v $$(docker ps -a -q --filter="name=$(TESTCONTAINER)"); fi
 endif
 ifndef TEST_ENDPOINTS_OVERRIDE
-	@TESTCONTAINER=$(TESTCONTAINER) ARANGODB=$(ARANGODB) ALPINE_IMAGE=$(ALPINE_IMAGE) STARTER=$(STARTER) STARTERMODE=$(TEST_MODE) "${ROOTDIR}/test/cluster.sh" cleanup
+	@TESTCONTAINER=$(TESTCONTAINER) ARANGODB=$(ARANGODB) ALPINE_IMAGE=$(ALPINE_IMAGE) STARTER=$(STARTER) STARTERMODE=$(TEST_MODE) DOCKER_NETWORK=${TEST_NET} "${ROOTDIR}/test/cluster.sh" cleanup
 else
 	@-docker rm -f -v $(TESTCONTAINER) &> /dev/null
 endif

test/cluster.sh

@@ -16,6 +16,7 @@ STARTERCONTAINER=${TESTCONTAINER}-s
 CMD=$1
 DOCKERARGS=
 STARTERARGS=
+
 # Cleanup
 docker rm -f -v $(docker ps -a | grep ${TESTCONTAINER} | awk '{print $1}') &> /dev/null
 docker volume rm -f ${STARTERVOLUME} &> /dev/null
@@ -66,17 +67,20 @@ if [ "$CMD" == "start" ]; then
         DOCKER_FWD_PORTS="-p 7001:7001 -p 7002:7002 -p 7003:7003 -p 7011:7011 -p 7012:7012 -p 7013:7013 -p 7021:7021 -p 7022:7022 -p 7023:7023"
     fi
 
-    set -x
-
-    # Start network namespace
-    docker run -d --name=${NAMESPACE} $DOCKERPLATFORMARG $DOCKER_DEBUG_PORT $DOCKER_FWD_PORTS "${ALPINE_IMAGE}" sleep 365d
+    if [ -z "$DOCKER_NETWORK" ]; then
+        DOCKER_NETWORK="--net=container:${NAMESPACE}"
+        # Start network namespace
+        docker run -d --name=${NAMESPACE} $DOCKERPLATFORMARG $DOCKER_DEBUG_PORT $DOCKER_FWD_PORTS "${ALPINE_IMAGE}" sleep 365d
+    fi
 
+    set -x
+
     # pull latest version of ArangoDB image
     docker pull ${ARANGODB}
 
     # Start starters 
     # arangodb/arangodb-starter 0.7.0 or higher is needed.
-    docker run -d --name=${STARTERCONTAINER} --net=container:${NAMESPACE} \
+    docker run -d --name=${STARTERCONTAINER} ${DOCKER_NETWORK} \
         -v ${STARTERVOLUME}:/data -v /var/run/docker.sock:/var/run/docker.sock $DOCKERARGS \
         ${STARTER} \
         --starter.port=${STARTERPORT} --starter.address=127.0.0.1 \

test/jwt.sh

@@ -0,0 +1,22 @@
+#!/bin/bash 
+
+IAT=$(date -u +%s)
+EXP=$(($IAT+36000))
+
+HEADER='{"alg":"HS256","typ":"JWT"}'
+PAYLOAD1='{"exp": '
+PAYLOAD2=', "iat":'
+PAYLOAD3=', "iss":"arangodb","server_id":"arangodb"}'
+PAYLOAD="$PAYLOAD1$EXP$PAYLOAD2$IAT$PAYLOAD3"
+
+PAYLOAD='{"iss":"arangodb","server_id":"arangodb"}'
+
+jwt_header=$(echo $(echo -n '{"alg":"HS256","typ":"JWT"}' | base64) | sed 's/ /_/g' | sed 's/+/-/g' | sed -E s/=+$//)
+payload=$(echo $(echo -n "${PAYLOAD}" | base64) | sed 's/ /_/g' | sed 's/+/-/g' |  sed -E s/=+$//)
+
+hexsecret=$(echo -n "$JWTSECRET" | xxd -p | paste -sd "")
+hmac_signature=$(echo $(echo -n "${jwt_header}.${payload}" |  openssl dgst -sha256 -mac HMAC -macopt hexkey:$hexsecret -binary | base64 ) | sed 's/\//_/g' | sed 's/+/-/g' | sed -E s/=+$//)
+
+# Create the full token
+jwt="${jwt_header}.${payload}.${hmac_signature}"
+echo $jwt

test/on_failure.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+echo "failure!\n"
+
+echo "\nARANGODB-STARTER logs: "
+docker logs ${TESTCONTAINER}-s
+
+echo -n "\nARANGODB-S-* logs:"
+docker ps -f name=${TESTCONTAINER}-s- --format "{{.ID}} {{.Names}}" | xargs -L1 bash -c 'echo -e "\n\tLogs from $1:"; docker logs $0'
+
+if [ -n "${DUMP_AGENCY_ON_FAILURE}" ] && [ "${TEST_MODE}" = "cluster" ]; then
+    echo "\nAgency dump..."
+
+    if [ "${TEST_SSL}" = "auto" ]; then
+        PROTOCOL='https'
+    else
+        PROTOCOL='http'
+    fi
+
+    if [ "${TEST_AUTH}" = "jwt" ] || [ "${TEST_AUTH}" = "rootpw" ] || [ "${TEST_AUTH}" = "jwtsuper" ]; then
+        JWT_TOKEN=$(bash -c "JWTSECRET=$TEST_JWTSECRET; source ./test/jwt.sh")
+        AUTH="-H 'authorization: bearer $JWT_TOKEN'"
+    else
+        AUTH=""
+    fi
+
+    SED_DOCKER_NAME_TO_ENDPOINT="s/.*-([a-zA-Z0-9.-]+)-([0-9]+)$/${PROTOCOL}:\/\/\1:\2/"
+    ANY_ENDPOINT=$(docker ps -f name=${TESTCONTAINER}-s-agent --format '{{.Names}}'  | head -n 1 | sed -E $SED_DOCKER_NAME_TO_ENDPOINT)
+    echo "Any agent endpoint: $ANY_ENDPOINT"
+
+    # _api/agency/config returns leader endpoint with protocol that is usually not supported by curl 
+    AGENCY_CONFIG=$(bash -c "curl -k --no-progress-meter ${AUTH} ${ANY_ENDPOINT}/_api/agency/config")
+
+    LEADER_ENDPOINT_WITH_UNSUPPORTED_PROTOCOL=$(echo $AGENCY_CONFIG | jq -r '.configuration.pool[.leaderId]' | cat)
+    SED_UNSUPPORTED_PROTOCOL_ENDPOINT_TO_ENDPOINT="s/^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//${PROTOCOL}:\/\//"
+    LEADER_ENDPOINT=$(echo $LEADER_ENDPOINT_WITH_UNSUPPORTED_PROTOCOL | sed $SED_UNSUPPORTED_PROTOCOL_ENDPOINT_TO_ENDPOINT)
+    echo "Leader agent endpoint: $LEADER_ENDPOINT"
+
+    DUMP_FILE_PATH=$DUMP_AGENCY_ON_FAILURE
+    AGENCY_DUMP=$(bash -c "curl -Lk --no-progress-meter ${AUTH} ${LEADER_ENDPOINT}/_api/agency/state")
+    echo $AGENCY_DUMP > $DUMP_FILE_PATH
+    echo "Agency dump created at $(realpath $DUMP_FILE_PATH)"
+fi
+
+echo "\nV${MAJOR_VERSION} Tests with ARGS: TEST_MODE=${TEST_MODE} TEST_AUTH=${TEST_AUTH} TEST_CONTENT_TYPE=${TEST_CONTENT_TYPE} TEST_SSL=${TEST_SSL} TEST_CONNECTION=${TEST_CONNECTION} TEST_CVERSION=${TEST_CVERSION}";
+
+echo "\n"
+exit 1