Merge pull request #48 from ministryofjustice/fix-workflows

Fix workflows
ministryofjustice · Dec 9, 2020 · 2118186 · 2118186
2 parents 227d9d1 + b9293bb
commit 2118186
Show file tree

Hide file tree

Showing 8 changed files with 30 additions and 212 deletions.
diff --git a/.github/workflows/datascience-notebook.yml b/.github/workflows/datascience-notebook.yml
diff --git a/.github/workflows/allspark-notebook.yml → .../workflows/jupyter-lab-test-and-build.yml b/.github/workflows/allspark-notebook.yml → .../workflows/jupyter-lab-test-and-build.yml
@@ -1,17 +1,11 @@
 ---
-name: Allspark Build & Test
+name: JupyterLab - test and build
 
-"on":
+on:
   pull_request:
-    paths: ["allspark-notebook/**"]
   push:
     branches: [main]
-    paths: ["allspark-notebook/**"]
-    tags: '*'
-
-defaults:
-  run:
-    working-directory: ./allspark-notebook
+    tags: "*"
 
 jobs:
   yamllint:
@@ -31,10 +25,18 @@ jobs:
         uses: actionshub/markdownlint@master
 
   docker:
-    runs-on: [self-hosted, ecr]
+    runs-on: [self-hosted, ministryofjustice, ecr]
+    strategy:
+      fail-fast: false
+      max-parallel: 3
+      matrix:
+        flavour:
+          - "datascience-notebook"
+          - "allspark-notebook"
+          - "oracle-datascience-notebook"
     env:
-      REPOSITORY: allspark-notebook
-      ECR_REPOSITORY: allspark-notebook
+      REPOSITORY: ${{ matrix.flavour }}
+      ECR_REPOSITORY: ${{ matrix.flavour }}
     needs: [mdl, yamllint]
     steps:
       - name: Configure AWS credentials
@@ -52,7 +54,6 @@ jobs:
         id: prep
         run: |
           TAG=noop
-
           if [[ $GITHUB_REF == refs/tags/* ]]; then
             TAG=${GITHUB_REF#refs/tags/}
           elif [[ $GITHUB_REF == refs/heads/* ]]; then
@@ -66,10 +67,13 @@ jobs:
             TAG="sha-${GITHUB_SHA::8}"
           fi
 
+          echo "Docker image tag = '$TAG'"
+
           echo ::set-output name=tag::${TAG}
           echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ')
 
       - name: Build image
+        working-directory: "./${{ matrix.flavour }}"
         run: make build
         env:
           NETWORK: host
@@ -81,17 +85,21 @@ jobs:
           channel: current
           project: inspec
       - name: Test
+        working-directory: "./${{ matrix.flavour }}"
         run: make test
         env:
+          NETWORK: host
           REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           IMAGE_TAG: ${{ steps.prep.outputs.tag }}
       - name: Push image
+        working-directory: "./${{ matrix.flavour }}"
         run: make push
         env:
           REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           IMAGE_TAG: ${{ steps.prep.outputs.tag }}
       - name: Cleanup
         if: ${{ always() }}
+        working-directory: "./${{ matrix.flavour }}"
         run: make clean
         env:
           REGISTRY: ${{ steps.login-ecr.outputs.registry }}

diff --git a/.github/workflows/oracle-datascience-notebook.yml b/.github/workflows/oracle-datascience-notebook.yml
diff --git a/allspark-notebook/Dockerfile b/allspark-notebook/Dockerfile
@@ -16,9 +16,8 @@ COPY ./files/* /tmp/
 
 RUN python /tmp/pyspark-s3.py \
     && pip install --upgrade nbstripout boto3 pip\
-    && pip install git+https://github.com/moj-analytical-services/etl_manager.git#egg=etl_manager \
-    && pip install git+https://github.com/moj-analytical-services/dataengineeringutils.git \
-    && pip install git+https://github.com/moj-analytical-services/gluejobutils/#egg=gluejobutils \
+    && pip install etl-manager==7.3.0 \
+    && pip install gluejobutils==3.1.1 \
     && mv /tmp/hdfs-site.xml /usr/local/spark/conf \
     && apt-get update && apt-get install -y \
     ca-certificates-java \

diff --git a/allspark-notebook/docker-compose.yml b/allspark-notebook/docker-compose.yml
@@ -11,6 +11,7 @@ services:
       - tests:/tests
   test:
     image: ${REGISTRY}/${REPOSITORY}:${IMAGE_TAG:-latest}
+    network_mode: ${NETWORK:-default}
     command:
       - "/usr/local/bin/start-notebook.sh"
       - "--NotebookApp.token=''"
@@ -20,6 +21,7 @@ services:
       - tests:/share/tests
   inspec:
     image: chef/inspec:current
+    network_mode: ${NETWORK:-default}
     environment:
       CHEF_LICENSE: accept-no-persist
     volumes:

diff --git a/datascience-notebook/docker-compose.yml b/datascience-notebook/docker-compose.yml
@@ -11,6 +11,7 @@ services:
       - tests:/tests
   test:
     image: ${REGISTRY}/${REPOSITORY}:${IMAGE_TAG:-latest}
+    network_mode: ${NETWORK:-default}
     command:
       - "/usr/local/bin/start-notebook.sh"
       - "--NotebookApp.token=''"
@@ -20,6 +21,7 @@ services:
       - tests:/share/tests
   inspec:
     image: chef/inspec:current
+    network_mode: ${NETWORK:-default}
     environment:
       CHEF_LICENSE: accept-no-persist
     volumes:

diff --git a/datascience-notebook/tests/controls/pip_spec.rb b/datascience-notebook/tests/controls/pip_spec.rb
@@ -24,6 +24,7 @@
 
   describe command('pip install osmnx') do
     its('exit_status') { should eq 0 }
+    its('stderr') { should eq "" }
     its('stdout') { should match /Successfully installed/ }
   end
 end
diff --git a/oracle-datascience-notebook/docker-compose.yml b/oracle-datascience-notebook/docker-compose.yml
@@ -11,6 +11,7 @@ services:
       - tests:/tests
   test:
     image: ${REGISTRY}/${REPOSITORY}:${IMAGE_TAG:-latest}
+    network_mode: ${NETWORK:-default}
     command:
       - "/usr/local/bin/start-notebook.sh"
       - "--NotebookApp.token=''"
@@ -20,6 +21,7 @@ services:
       - tests:/share/tests
   inspec:
     image: chef/inspec:current
+    network_mode: ${NETWORK:-default}
     environment:
       CHEF_LICENSE: accept-no-persist
     volumes: