diff --git a/lib/bin/s3-create-bucket.js b/lib/bin/s3-create-bucket.js index 84f16c449..f54c9ef96 100644 --- a/lib/bin/s3-create-bucket.js +++ b/lib/bin/s3-create-bucket.js @@ -9,7 +9,7 @@ const Minio = require('minio'); -const { server, bucketName, accessKey, secretKey } = require('config').get('default.external.s3blobStore'); +const { server, region, bucketName, accessKey, secretKey } = require('config').get('default.external.s3blobStore'); const minioClient = (() => { const url = new URL(server); @@ -17,7 +17,7 @@ const minioClient = (() => { const endPoint = (url.hostname + url.pathname).replace(/\/$/, ''); const port = parseInt(url.port, 10); - return new Minio.Client({ endPoint, port, useSSL, accessKey, secretKey }); + return new Minio.Client({ endPoint, port, useSSL, accessKey, secretKey, region }); })(); const log = (...args) => console.log(__filename, ...args); @@ -29,10 +29,11 @@ minioClient.bucketExists(bucketName) return; } - log('Creating bucket:', bucketName); - return minioClient.makeBucket(bucketName) - .then(() => log('Bucket created OK.')); + log('Creating bucket:', bucketName, 'in', region ?? 'default region'); + if (region) return minioClient.makeBucket(bucketName, region); + else return minioClient.makeBucket(bucketName); // eslint-disable-line no-multi-spaces }) + .then(() => log('Bucket created OK.')) .catch(err => { log('ERROR CREATING MINIO BUCKET:', err); process.exit(1); diff --git a/lib/external/s3.js b/lib/external/s3.js index ee77ba240..a51d9b165 100644 --- a/lib/external/s3.js +++ b/lib/external/s3.js @@ -12,7 +12,7 @@ const disabled = { enabled: false }; const init = (config) => { if (!config) return disabled; - const { server, accessKey, secretKey, bucketName, requestTimeout, objectPrefix } = config; + const { server, accessKey, secretKey, bucketName, region, requestTimeout, objectPrefix } = config; if (!(server && accessKey && secretKey && bucketName)) return disabled; const http = require('node:http'); @@ -83,7 +83,7 @@ const init = (config) => { return req; }; - const clientConfig = { endPoint, port, useSSL, accessKey, secretKey, transport: { request } }; + const clientConfig = { endPoint, port, useSSL, accessKey, secretKey, region, transport: { request } }; return new Minio.Client(clientConfig); })(); diff --git a/test/e2e/s3/run-tests.sh b/test/e2e/s3/run-tests.sh index 54e5a1778..ea7795703 100755 --- a/test/e2e/s3/run-tests.sh +++ b/test/e2e/s3/run-tests.sh @@ -1,7 +1,6 @@ #!/bin/bash -eu set -o pipefail -serverUrl="http://localhost:8383" userEmail="x@example.com" userPassword="secret1234" @@ -17,11 +16,6 @@ cleanup() { } trap cleanup EXIT SIGINT SIGTERM SIGHUP -if curl -s -o /dev/null $serverUrl; then - log "!!! Error: server already running at: $serverUrl" - exit 1 -fi - make base if [[ "${CI-}" = '' ]]; then @@ -52,20 +46,50 @@ EOF read -rp '' fi -NODE_CONFIG_ENV=s3-dev node lib/bin/s3-create-bucket.js -NODE_CONFIG_ENV=s3-dev make run & -serverPid=$! +run_suite() { + suite="$1" -log 'Waiting for backend to start...' -timeout 30 bash -c "while ! curl -s -o /dev/null $serverUrl; do sleep 1; done" -log 'Backend started!' + log "Running suite '$suite' ..." -cd test/e2e/s3 -npx mocha test.js + case "$suite" in + smoke) testOptions=(--fgrep @smoke-test) ;; + all) testOptions=() ;; + *) log "!!! Error: unrecongised test suite: $suite"; exit 1 ;; + esac -if ! curl -s -o /dev/null "$serverUrl"; then - log '!!! Backend died.' - exit 1 -fi + NODE_CONFIG_ENV="s3-dev" node lib/bin/s3-create-bucket.js + + serverPort="$(NODE_CONFIG_ENV="s3-dev" node -e 'console.log(require("config").default.server.port)')" + serverUrl="http://localhost:$serverPort" + if curl -s -o /dev/null "$serverUrl"; then + log "!!! Error: server already running at: $serverUrl" + exit 1 + fi + + NODE_CONFIG_ENV="s3-dev" make run & + + log 'Waiting for backend to start...' + timeout 30 bash -c "while ! curl -s -o /dev/null $serverUrl; do sleep 1; done" + log 'Backend started!' + + cd test/e2e/s3 + NODE_CONFIG_ENV="s3-dev" NODE_CONFIG_DIR=../../../config npx mocha "${testOptions[@]}" test.js + cd - + + if ! curl -s -o /dev/null "$serverUrl"; then + log '!!! Backend died.' + exit 1 + fi + + log "Suite '$suite' completed OK." +} + +NODE_CONFIG='{ "default":{ "server":{ "port":8384 }, "external":{ "s3blobStore":{ "region":"" } } } }' \ +run_suite smoke + +NODE_CONFIG='{ "default":{ "server":{ "port":8385 }, "external":{ "s3blobStore":{ "region":"eu-east-1" } } } }' \ +run_suite smoke + +run_suite all -log "Tests completed OK." +log "All tests completed OK." diff --git a/test/e2e/s3/test-forms/0.xml b/test/e2e/s3/test-forms/0.xml new file mode 100644 index 000000000..03f62859e --- /dev/null +++ b/test/e2e/s3/test-forms/0.xml @@ -0,0 +1,56 @@ + + + + Blob Test 0 + + + + + Big Bin 1 + jr://images/big-1.bin + + + + + + + + + + + + + + + + + image-big-1-bin + a + + + + + + + + + + + + + + image type with draw appearance + + + + file type with no appearance <br/> WARNING: any kind of file could be uploaded including files that contain viruses or other malware. Be sure to take proper precautions when downloading files from server. + + + diff --git a/test/e2e/s3/test.js b/test/e2e/s3/test.js index b94c2e75c..045f85a8b 100644 --- a/test/e2e/s3/test.js +++ b/test/e2e/s3/test.js @@ -17,12 +17,13 @@ const fs = require('node:fs'); const { randomBytes } = require('node:crypto'); const _ = require('lodash'); const should = require('should'); +const tmp = require('tmp'); const SUITE_NAME = 'test/e2e/s3'; const log = require('../util/logger')(SUITE_NAME); const { apiClient, mimetypeFor, Redirect } = require('../util/api'); -const serverUrl = 'http://localhost:8383'; +const serverUrl = `http://localhost:${require('config').default.server.port}`; const userEmail = 'x@example.com'; const userPassword = 'secret1234'; @@ -65,7 +66,7 @@ describe('s3 support', () => { bigFileSizeMb = 100, } = opts; // eslint-disable-line object-curly-newline - attDir = `./test-forms/${testNumber}-attachments`; + attDir = opts.attDir || `./test-forms/${testNumber}-attachments`; // given fs.mkdirSync(attDir, { recursive:true }); @@ -85,6 +86,24 @@ describe('s3 support', () => { await assertNoneRedirect(actualAttachments); } + it('should shift submission attachment to s3 @smoke-test', async function() { + this.timeout(TIMEOUT); + + // given + // randomise attDir to ensure re-runs do not use the same generated files + await setup(0, { bigFiles: 1, bigFileSizeMb: 0.01, attDir: tmp.dirSync().name }); + await assertNewStatuses({ pending: 1 }); + + // when + await cli('upload-pending'); + + // then + await assertNewStatuses({ uploaded: 1 }); + // and + await assertAllRedirect(actualAttachments); + await assertAllDownloadsMatchOriginal(actualAttachments); + }); + it('should shift submission attachments to s3', async function() { this.timeout(TIMEOUT); @@ -403,7 +422,7 @@ function cli(cmd) { cmd = `exec node lib/bin/s3 ${cmd}`; // eslint-disable-line no-param-reassign log.debug('cli()', 'calling:', cmd); - const env = { ..._.pick(process.env, 'PATH'), NODE_CONFIG_ENV:'s3-dev' }; + const env = _.omit(process.env, 'NODE_CONFIG_DIR'); const promise = new Promise((resolve, reject) => { const child = exec(cmd, { env, cwd:'../../..' }, (err, stdout, stderr) => {