cloudbuild.yaml

#################################################################################
# Deployment Section
#
#  Overview:
#   1.  Test in sandbox during development
#   2.  Deploy to staging on commit to master
#   3.  Deploy to prod when a branch is tagged with prod-* or xxx-prod-*
#
#  We want to test individual components in sandbox, and avoid stepping on each
#  other, so we do NOT automate deployment to sandbox.  Each person should
#  use a branch name to trigger the single deployment that they are working on.
#
#  We want to soak all code in staging before deploying to prod.  To avoid
#  incompatible components, we deploy ALL elements to staging when we merge
#  to master branch.
#
#  Deployments to prod are done by deliberately tagging a specific commit,
#  typically in the master branch, with a tag starting with prod-*.
#  DO NOT just tag the latest version in master, as someone may have
#  pushed new code that hasn't had a chance to soak in staging.
#
#
# Deploy steps never trigger on a new Pull Request. Deploy steps will trigger
# on specific branch name patterns, after a merge to master, or on
# an explicit tag that matches "on:" conditions.
#################################################################################

timeout: 1800s

options:
  env:
  - PROJECT_ID=$PROJECT_ID
  - GIT_COMMIT=$COMMIT_SHA
  machineType: 'N1_HIGHCPU_8'

steps:
# Make all git tags available.
- name: gcr.io/cloud-builders/git
  id: "Unshallow git clone"
  args: ["fetch", "--unshallow"]

# Fetch travis submodule.
- name: gcr.io/cloud-builders/git
  id: "Update travis submodule"
  args: ["submodule", "update", "--init", "--recursive"]

# TODO: while local docker builds cache intermediate layers, CB does not.
# Combine the Dockerfile.testing with the Dockerfile using --target and
# --from-cache to speed up builds: See also:
# https://andrewlock.net/caching-docker-layers-on-serverless-build-hosts-with-multi-stage-builds---target,-and---cache-from/
- name: gcr.io/cloud-builders/docker
  id: "Build the testing docker container"
  args: [
    "build", "-t", "etl-testing", "-f", "Dockerfile.testing", "."
  ]

- name: etl-testing
  id: "Run all etl unit tests"
  args:
  - go version
  - go get -v -t ./...
  - go get -v -tags=integration -t ./...

  # Run tests.
  - go test -v -coverprofile=_unit.cov ./...
  # TODO: race detected in TestGardenerAPI_RunAll
  # - go test -v ./... -race
  # Integration testing requires additional SA credentials.
  - ./integration-testing.sh
  # Build update-schema command, with binary in cloudbuild /workspace.
  - go build ./cmd/update-schema
  env:
  - SERVICE_ACCOUNT_mlab_testing=$_SERVICE_ACCOUNT_MLAB_TESTING
  - WORKSPACE_LINK=/go/src/github.com/m-lab/etl

- name: gcr.io/cloud-builders/docker
  id: "Build the etl docker container"
  args: [
    "build",
      "--build-arg", "VERSION=${TAG_NAME}${BRANCH_NAME}",
      "-t", "gcr.io/$PROJECT_ID/etl:$_DOCKER_TAG",
      "-f", "cmd/etl_worker/Dockerfile.k8s", "."
  ]
  waitFor: ['Unshallow git clone']

- name: gcr.io/cloud-builders/docker
  id: "Push the docker container to gcr.io"
  args: [
    "push", "gcr.io/$PROJECT_ID/etl:$_DOCKER_TAG"
  ]

- name: etl-testing
  id: "Update table schemas before deploying parsers"
  entrypoint: bash
  args: [
     '-c', './update-schema -standard'
  ]
  env:
  - PROJECT=$PROJECT_ID

- name: etl-testing
  id: "Update table schemas for wehe deploying parsers"
  entrypoint: bash
  args: [
     '-c', './update-schema -experiment wehe -datatype scamper1 &&
            ./update-schema -experiment wehe -datatype annotation2 &&
            ./update-schema -experiment wehe -datatype hopannotation2'
  ]
  env:
  - PROJECT=$PROJECT_ID

- name: etl-testing
  id: "Update table schemas for msak deploying parsers"
  entrypoint: bash
  args: [
     '-c', './update-schema -experiment msak -datatype annotation2 &&
            ./update-schema -experiment msak -datatype scamper1 &&
            ./update-schema -experiment msak -datatype hopannotation2 &&
            ./update-schema -experiment msak -datatype tcpinfo &&
            ./update-schema -experiment msak -datatype pcap'
  ]
  env:
  - PROJECT=$PROJECT_ID


# UNIVERSAL PARSER: Run apply-cluster.sh
- name: gcr.io/cloud-builders/kubectl
  id: "Deploy etl parser to $_CLUSTER_NAME cluster"
  entrypoint: /bin/bash
  args: [
   '-c', '/builder/kubectl.bash version && ./apply-cluster.sh'
  ]
  env:
  - BIGQUERY_DATASET=tmp_ndt
  - CLOUDSDK_COMPUTE_REGION=$_CLUSTER_REGION
  - CLOUDSDK_CONTAINER_CLUSTER=$_CLUSTER_NAME
  - CLOUDSDK_CORE_PROJECT=$PROJECT_ID