ci: configure Travis CI for branch 'dev'

.travis.yml

@@ -1,31 +1,38 @@
-sudo: required
-
-dist: xenial
-
-service:
-   - docker
-
+os:
+- linux
+dist: bionic
+language: python
+services:
+- docker
+install:
+# omit automatic installation of dependencies in virtualenv
+- pip --version
 stages:
-  - if: branch = master
-    name: master build
-  - if: branch = dev
-    name: dev build
-  - name: branch build
-
+- name: test
+- name: publish
+  # do not execute if PR branch
+  if: type != pull_request
+branches:
+  only:
+  - dev
 jobs:
   include:
-  - stage: master build
+  - stage: test
+    name: Build, deploy and test application
     script:
-      - version=0.1
-      - docker build -t weselixir/elixir-wes-app:"$version" -t weselixir/elixir-wes-app:master .
-      - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-      - if [ "$TRAVIS_PULL_REQUEST" = false ]; then docker push "$DOCKER_REPO_NAME":"$version"; fi
-  - stage: dev build
+    # build and deploy application
+    - "docker-compose -f docker-compose.yaml -f docker-compose.dev.yaml up --build -d"
+    # test if endpoint 'GET /runs' is accessible
+    - "test $(curl -sL -w '%{http_code}' -X GET --header 'Accept: application/json' 'http://localhost:7777/ga4gh/wes/v1/runs' -o /dev/null) == '200'"
+  - stage: publish
+    name: Push image to registry
+    env:
+    - secure: "woVF7L1vADo4OqyKm7Kq8PC95Ej4L1q3R4Gf0Uh9p1tA81JMfD/3HDS5dG3OQvjoftcA7PqpFTrZq6GL1hR/bQqoQypHVNziH9PnDZFJPvxIYa8BUV7lknva4aSB6lcj4jmukHAnGNrusgSNXqcmFO0ziX4nyCNwcmXNGfRMKB3D25t0MAALtpSBYFovN1+7yijlklyjPCPUaEJ8Sj/HpohJ0CMye9f+dVE9RR5fDteLycHSRn2V2ftwmvu16vyI/sI9gdhYqamh18j8G9AULpeBWCdKpJ5+8nOnGwewRLCwnpClW6KEPtR95sk0qGiIs6rKigMbKuKmlgpbjh3T3vHfsvCVKpF1t+rAbFSWJy3RNv+Ru9zJ5Cl590v9DevRKfWyaJRiXdplEQRVgsMZdmfXaepZuNhYu113dw54QuVvQ5ceNS/MhR1QZs9Nkmv5LXwdvgxPn4seQM8G/uvYZS9cmyt+VkhMFkSBanRMxTafv0FAD4R+KgrHpvG8JjacwINAn7Ufy2YWMkVsolQGERH6LEo9BoxPW5SweKMoakx27K+mFKd63XwPzpwwv0zsKmPp6hBAZ7r4AnzrRVqJF7dGI9dXsFVNTTAT7+moQ7R9PI0WVNYL+ntafbd/q3/I5r2i/tuA2yr5l4oMgoOQAXB8/JuQ5uvjVudELPU1c9s="
+    - secure: "dhyNkYM5fPIcFSzrQ9k/nnOJOH3iOuWqu1Fu54IUCFDv89P7gNDkwCQAXH5hp8oipfNP0l4idOp0CmXWaAozFc67mP/tBN6A5D+zdfjjG3qhmFjlojpC1EtF7kvOvF8Wc9tCRi/J8qDiAGGqPPstxYfGD7jxUlDKeKlfgf9zhnnWQCM1kdnBqedLaEI6QIYdhpULa/8yMVLI3VOBE22FNZNc4Ftw7xDJFTMP1iW7k3DOe+gSR0bkBTwBj12rL/dCiEeYR/8UfZQJgmq4iZTI5s1EoHrqhnjp6xDxIaqmlWTkwTE6SpTm70Q7qvU0YLvwlKXmhclGmDmauxONzL+DJp0gPVT0N54pUZ6CfglxwJFAz3U8xRzi3vurcIC2hozKg67p8e6aohlJVYqook0APsK/q/z0kfvivMC9gP0HZIKUuSFvkbbwt6VAVjVmPhTbPew6pRsCmHrjntpWSfehaVAb8byKxul4BU3HnLTy+uf2RUJ9BfNjl7m7wCba+SwEU29/dow65JlLsCwUUHhEexXDRn9GO09Ci6OR+EhtCtsS7cXTS/1rOIaiUfvB7ciu5BjqdnSxJ1WkAEDaut32l4/1mfNfTg0eSmsHPs//oPsXb4NZU/f4oeXJvvVmG1PTx7mOwGnSokZT8iI+rk3ULCyf+lz0JNQy7Jx5mb+kEqI="
     script:
-      - docker build -t weselixir/elixir-wes-app:dev -t weselixir/elixir-wes-app:build-"$TRAVIS_BUILD_NUMBER" .
-      - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-      - if [ "$TRAVIS_PULL_REQUEST" = false ]; then docker push "$DOCKER_REPO_NAME":build-"$TRAVIS_BUILD_NUMBER"; fi
-  - stage: branch build
-    script:
-      - docker build -t weselixir/elixir-wes-app-branched:build-"$TRAVIS_BUILD_NUMBER" .
-      - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+    # build and tag app image
+    - docker build -t "$DOCKER_REPO_NAME":"$DOCKER_REPO_TAG" .
+    # log in
+    - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+    # push image
+    - docker push "$DOCKER_REPO_NAME":"$DOCKER_REPO_TAG"

docker-compose.dev.yaml

@@ -1,24 +1,24 @@
 version: '3.6'
 services:
 
-  celery-worker-wes-elixir:
-    environment:
-      - WES_CONFIG=/app/wes_elixir/config/override/app_config.dev.yaml
-
   wes-elixir:
     environment:
       - WES_CONFIG=/app/wes_elixir/config/override/app_config.dev.yaml
     ports:
       - "7777:8080"
 
-  rabbit-wes-elixir:
+  wes-elixir-worker:
+    environment:
+      - WES_CONFIG=/app/wes_elixir/config/override/app_config.dev.yaml
+
+  rabbit:
     ports:
       - "5672:5672"
 
-  mongo-wes-elixir:
+  mongo:
     ports:
       - "27017:27017"
 
-  flower-wes-elixir:
+  flower:
     ports:
       - "5555:5555"

docker-compose.prod.yaml

@@ -1,20 +1,20 @@
 version: '3.6'
 services:
 
-  celery-worker-wes-elixir:
-    environment:
-      - WES_CONFIG=/app/wes_elixir/config/override/app_config.prod.yaml
-
   wes-elixir:
     environment:
       - WES_CONFIG=/app/wes_elixir/config/override/app_config.prod.yaml
     ports:
       - "80:8080"
 
-  rabbit-wes-elixir:
+  wes-elixir-worker:
+    environment:
+      - WES_CONFIG=/app/wes_elixir/config/override/app_config.prod.yaml
+
+  rabbit:
     ports:
       - "5672:5672"
 
-  mongo-wes-elixir:
+  mongo:
     ports:
       - "27017:27017"

docker-compose.yaml

@@ -1,48 +1,46 @@
 version: '3.6'
 services:
 
-  celery-worker-wes-elixir:
-    image: wes-elixir:latest
+  wes-elixir:
+    image: elixircloud/wes-elixir:latest
     build:
       context: .
       dockerfile: Dockerfile
     restart: unless-stopped
     links:
-      - mongo-wes-elixir
-      - rabbit-wes-elixir
-    command: bash -c "cd /app/wes_elixir; celery worker -A celery_worker -E --loglevel=info"
+      - mongo
+    command: bash -c "cd /app/wes_elixir; gunicorn -c config.py wsgi:app"
     volumes:
-      - ../data/wes_elixir:/data
-      - /wes/vol:/tmp
+      - ../../data/wes_elixir:/data
 
-  wes-elixir:
-    image: wes-elixir:latest
-    build:
-      context: .
-      dockerfile: Dockerfile
+  wes-elixir-worker:
+    image: elixircloud/wes-elixir:latest
     restart: unless-stopped
+    depends_on:
+      - wes-elixir
     links:
-      - mongo-wes-elixir
-    command: bash -c "cd /app/wes_elixir; gunicorn -c config.py wsgi:app"
+      - mongo
+      - rabbit
+    command: bash -c "cd /app/wes_elixir; celery worker -A celery_worker -E --loglevel=info"
     volumes:
-      - ../data/wes_elixir:/data
+      - ../../data/wes_elixir:/data
 
-  rabbit-wes-elixir:
+  rabbit:
     image: "rabbitmq:3-management"
     hostname: "rabbit"
     restart: unless-stopped
     links:
-      - mongo-wes-elixir
+      - mongo
 
-  mongo-wes-elixir:
+  mongo:
     image: mongo:3.2
     restart: unless-stopped
     volumes:
-      - ../data/wes_elixir/db:/data/db
+      - ../../data/wes_elixir/db:/data/db
 
-  flower-wes-elixir:
+  flower:
     image: mher/flower:0.9
     restart: unless-stopped
     links:
-      - celery-worker-wes-elixir
-    command: flower --broker=amqp://guest:guest@rabbit-wes-elixir:5672// --port=5555
+      - wes-elixir-worker
+    command: flower --broker=amqp://guest:guest@rabbit:5672// --port=5555

wes_elixir/config/app_config.yaml

@@ -59,7 +59,7 @@ api:
           swagger_ui: True
           swagger_json: True
     endpoint_params:
-        default_page_size: 10
+        default_page_size: 5
         timeout_cancel_run: 60
         timeout_run_workflow: Null
 

wes_elixir/config/override/app_config.dev.yaml

@@ -2,11 +2,11 @@
 
 # Security settings
 security:
-    authorization_required: True
+    authorization_required: False
 
 # Database settings
 database:
-    host: 'mongo-wes-elixir'
+    host: 'mongo'
     name: wes-elixir-db-dev
 
 # Storage
@@ -16,7 +16,7 @@ storage:
 
 # Celery task queue
 celery:
-    broker_host: 'rabbit-wes-elixir'
+    broker_host: 'rabbit'
 
 # OpenAPI specs
 

wes_elixir/config/override/app_config.prod.yaml

@@ -11,7 +11,7 @@ security:
 
 # Database settings
 database:
-    host: 'mongo-wes-elixir'
+    host: 'mongo'
 
 # Storage
 storage:
@@ -20,7 +20,7 @@ storage:
 
 # Celery task queue
 celery:
-    broker_host: 'rabbit-wes-elixir'
+    broker_host: 'rabbit'
 
 # OpenAPI specs
 

wes_elixir/ga4gh/wes/endpoints/run_workflow.py

@@ -6,7 +6,6 @@
 import shutil
 import string  # noqa: F401
 import subprocess
-import string  # noqa: F401
 
 from celery import uuid
 from json import (decoder, loads)
@@ -173,7 +172,7 @@ def __create_run_environment(
     run_id_length = get_conf(config, 'database', 'run_id', 'length')
 
     # Keep on trying until a unique run id was found and inserted
-    # TODO: If no more possible IDs => inf loop; fix (raise customerror; 500
+    # TODO: If no more possible IDs => inf loop; fix (raise custom error; 500
     #       to user)
     while True:
 
@@ -193,8 +192,8 @@ def __create_run_environment(
             # TODO: Think about permissions
             # TODO: Add working dir (currently one has to run the app from
             #       outermost dir)
-            os.mkdir(current_tmp_dir)
-            os.mkdir(current_out_dir)
+            os.makedirs(current_tmp_dir)
+            os.makedirs(current_out_dir)
 
         # Try new run id if directory already exists
         except FileExistsError:
@@ -383,7 +382,7 @@ def __process_workflow_attachments(data: Dict) -> Dict:
         data['internal']['param_file_path'] = os.path.join(
             workflow_dir,
             '.'.join([
-                workflow_name_ext[0],
+                str(workflow_name_ext[0]),
                 'yml',
             ]),
         )
@@ -413,7 +412,7 @@ def __process_workflow_attachments(data: Dict) -> Dict:
                 workflow_dir,
                 'repo',
                 '.'.join([
-                    workflow_name_ext[0],
+                    str(workflow_name_ext[0]),
                     ext,
                 ]),
             )