From d96347a20232ca9571f0d87de9cc8b77c6aa7c9c Mon Sep 17 00:00:00 2001 From: Simon Liu <69875423+sliu008@users.noreply.github.com> Date: Fri, 19 Jan 2024 13:46:33 -0800 Subject: [PATCH] Feature/arm tig (#42) * revert tig.py * restore pytest * update changelog --- .github/workflows/build.yml | 126 ++++------------------------------- CHANGELOG.md | 4 ++ docker/lambdaDockerfileArm | 25 +++++++ terraform/fargate/fargate.tf | 5 ++ terraform/tig_ecr.tf | 44 ++++++++++++ terraform/tig_ecs.tf | 8 +-- terraform/tig_fargate.tf | 8 +-- terraform/tig_lambda.tf | 18 ++++- terraform/variable.tf | 15 ++++- terraform_deploy/override.py | 2 +- terraform_deploy/tig.tf | 2 +- 11 files changed, 129 insertions(+), 128 deletions(-) create mode 100644 docker/lambdaDockerfileArm create mode 100644 terraform/tig_ecr.tf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bcd65f6..a483047 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,6 +35,12 @@ jobs: with: poetry-version: 1.3.2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + ######################################################################### # Versioning (featuring weird gradle output work-arounds) ######################################################################### @@ -267,7 +273,7 @@ jobs: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release') - run: | + run: | if [ -n "$(git status --porcelain)" ]; then echo "changes=true" >> $GITHUB_ENV echo "::set-output name=changes::true" @@ -459,72 +465,16 @@ jobs: uses: docker/build-push-action@v3 with: context: . - file: ./docker/lambdaDockerfile + file: ./docker/lambdaDockerfileArm push: true pull: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + platforms: linux/arm/v7 build-args: | SOURCE=${{ env.pyproject_name }}==${{ env.the_version }} - ## Build and publish to Service ECR - - name: Upload Docker image to Service ECR - if: | - github.ref == 'refs/heads/develop' || - github.ref == 'refs/heads/main' || - startsWith(github.ref, 'refs/heads/release') - uses: vitr/actions-build-and-upload-to-ecs@master - with: - access_key_id: ${{ secrets[format('AWS_ACCESS_KEY_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }} - secret_access_key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }} - account_id: ${{ secrets[format('AWS_ACCOUNT_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }} - repo: podaac/tig - region: us-west-2 - tags: ${{ env.the_version }} - create_repo: true - dockerfile: ./docker/lambdaDockerfile - extra_build_args: --build-arg SOURCE=${{ env.pyproject_name }}==${{ env.the_version }} - - - ## Build and publish to Cumulus ECR - - name: Upload Docker image to Cumulus ECR - if: | - github.ref == 'refs/heads/develop' || - github.ref == 'refs/heads/main' || - startsWith(github.ref, 'refs/heads/release') - uses: vitr/actions-build-and-upload-to-ecs@master - with: - access_key_id: ${{ secrets[format('AWS_ACCESS_KEY_ID_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }} - secret_access_key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }} - account_id: ${{ secrets[format('AWS_ACCOUNT_ID_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }} - repo: podaac/tig - region: us-west-2 - tags: ${{ env.the_version }} - create_repo: true - dockerfile: ./docker/lambdaDockerfile - extra_build_args: --build-arg SOURCE=${{ env.pyproject_name }}==${{ env.the_version }} - - - ## Build and publish to SWOT Cumulus ECR - - name: Upload Docker image to Cumulus ECR - if: | - github.ref == 'refs/heads/develop' || - github.ref == 'refs/heads/main' || - startsWith(github.ref, 'refs/heads/release') - uses: vitr/actions-build-and-upload-to-ecs@master - with: - access_key_id: ${{ secrets[format('AWS_ACCESS_KEY_ID_CUMULUS_SWOT_{0}', env.TARGET_ENV_UPPERCASE)] }} - secret_access_key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_CUMULUS_SWOT_{0}', env.TARGET_ENV_UPPERCASE)] }} - account_id: ${{ secrets[format('AWS_ACCOUNT_ID_CUMULUS_SWOT_{0}', env.TARGET_ENV_UPPERCASE)] }} - repo: podaac/tig - region: us-west-2 - tags: ${{ env.the_version }} - create_repo: true - dockerfile: ./docker/lambdaDockerfile - extra_build_args: --build-arg SOURCE=${{ env.pyproject_name }}==${{ env.the_version }} - - ## Local tig docker builds - name: Get Local Tig Build @@ -544,68 +494,16 @@ jobs: uses: docker/build-push-action@v3 with: context: . - file: ./docker/lambdaDockerfile + file: ./docker/lambdaDockerfileArm push: true pull: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + platforms: linux/arm/v7 build-args: | DIST_PATH="dist/" SOURCE=${{ env.local_tig }} - ## Build and publish to Service ECR - - name: Upload Local TIG Docker image to Service ECR - if: | - github.event.head_commit.message == '/deploy sit' || - github.event.head_commit.message == '/deploy uat' - uses: vitr/actions-build-and-upload-to-ecs@master - with: - access_key_id: ${{ secrets[format('AWS_ACCESS_KEY_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }} - secret_access_key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }} - account_id: ${{ secrets[format('AWS_ACCOUNT_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }} - repo: podaac/tig - region: us-west-2 - tags: ${{ steps.meta.outputs.version }} - create_repo: true - dockerfile: ./docker/lambdaDockerfile - extra_build_args: --build-arg DIST_PATH="dist/" --build-arg SOURCE=${{ env.local_tig }} - - ## Build and publish to Cumulus ECR - - name: Upload Local TIG Docker image to Cumulus ECR - if: | - github.event.head_commit.message == '/deploy sit' || - github.event.head_commit.message == '/deploy uat' || - github.event.head_commit.message == '/deploy sandbox' - uses: vitr/actions-build-and-upload-to-ecs@master - with: - access_key_id: ${{ secrets[format('AWS_ACCESS_KEY_ID_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }} - secret_access_key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }} - account_id: ${{ secrets[format('AWS_ACCOUNT_ID_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }} - repo: podaac/tig - region: us-west-2 - tags: ${{ steps.meta.outputs.version }} - create_repo: true - dockerfile: ./docker/lambdaDockerfile - extra_build_args: --build-arg DIST_PATH="dist/" --build-arg SOURCE=${{ env.local_tig }} - - ## Build and publish to SWOT Cumulus ECR - - name: Upload Local TIG Docker image to Cumulus ECR - if: | - github.event.head_commit.message == '/deploy sit' || - github.event.head_commit.message == '/deploy uat' || - github.event.head_commit.message == '/deploy sandbox' - uses: vitr/actions-build-and-upload-to-ecs@master - with: - access_key_id: ${{ secrets[format('AWS_ACCESS_KEY_ID_CUMULUS_SWOT_{0}', env.TARGET_ENV_UPPERCASE)] }} - secret_access_key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_CUMULUS_SWOT_{0}', env.TARGET_ENV_UPPERCASE)] }} - account_id: ${{ secrets[format('AWS_ACCOUNT_ID_CUMULUS_SWOT_{0}', env.TARGET_ENV_UPPERCASE)] }} - repo: podaac/tig - region: us-west-2 - tags: ${{ steps.meta.outputs.version }} - create_repo: true - dockerfile: ./docker/lambdaDockerfile - extra_build_args: --build-arg DIST_PATH="dist/" --build-arg SOURCE=${{ env.local_tig }} - # ######################################################################### # # Build and Publish Documentation # ######################################################################### @@ -650,7 +548,7 @@ jobs: ls -al bin/ which python3 python3 --version - python3 override.py https://github.com/podaac/tig/releases/download/${{ env.the_version }}/tig-terraform-${{ env.the_version }}.zip ${{ secrets[format('AWS_ACCOUNT_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}.dkr.ecr.us-west-2.amazonaws.com/podaac/tig:${{ steps.meta.outputs.version }} + python3 override.py https://github.com/podaac/tig/releases/download/${{ env.the_version }}/tig-terraform-${{ env.the_version }}.zip "ghcr.io/podaac/tig:${{ env.DOCKER_METADATA_OUTPUT_VERSION }}" ls -al echo "Show override contents" cat override.tf.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 946b483..4618d50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] ### Added +- ** Add Arm Architecture ** + - update tig to be able to run with arm architecture +- ** Add image uploading ** + - update tig to upload tig image to ecr ### Changed ### Deprecated ### Removed diff --git a/docker/lambdaDockerfileArm b/docker/lambdaDockerfileArm new file mode 100644 index 0000000..02df292 --- /dev/null +++ b/docker/lambdaDockerfileArm @@ -0,0 +1,25 @@ +#ARG FUNCTION_DIR="/function" + +FROM public.ecr.aws/lambda/python:3.10-arm64 + +# Include global arg in this stage of the build +ARG SOURCE + +RUN yum -q -y install gcc + +# Create function directory +# RUN mkdir -p ${FUNCTION_DIR} +# WORKDIR ${FUNCTION_DIR} + +# Install tig from artifactory +COPY $DIST_PATH $DIST_PATH + +# install tig into working directory so we can call lambda +RUN pip3 install awslambdaric --target $LAMBDA_TASK_ROOT + +RUN pip3 install --no-cache-dir --force --index-url https://pypi.org/simple/ --extra-index-url https://test.pypi.org/simple/ --target "${LAMBDA_TASK_ROOT}" $SOURCE + +RUN rm -rf $DIST_PATH + +ENTRYPOINT [] +CMD ["podaac.lambda_handler.lambda_handler.handler"] diff --git a/terraform/fargate/fargate.tf b/terraform/fargate/fargate.tf index cea27bc..ce870e8 100644 --- a/terraform/fargate/fargate.tf +++ b/terraform/fargate/fargate.tf @@ -13,6 +13,11 @@ resource "aws_ecs_task_definition" "app" { execution_role_arn = var.iam_role task_role_arn = var.iam_role + runtime_platform { + operating_system_family = "LINUX" + cpu_architecture = "ARM64" + } + container_definitions = jsonencode([ { name = "${var.prefix}-${var.app_name}-fargate" diff --git a/terraform/tig_ecr.tf b/terraform/tig_ecr.tf new file mode 100644 index 0000000..93cd79f --- /dev/null +++ b/terraform/tig_ecr.tf @@ -0,0 +1,44 @@ +data "aws_ecr_authorization_token" "token" {} + +locals { + lambda_container_image_uri_split = split("/", var.lambda_container_image_uri) + ecr_image_name_and_tag = split(":", element(local.lambda_container_image_uri_split, length(local.lambda_container_image_uri_split) - 1)) + ecr_image_name = "${local.environment}-${element(local.ecr_image_name_and_tag, 0)}" + ecr_image_tag = element(local.ecr_image_name_and_tag, 1) +} + +resource aws_ecr_repository "lambda-image-repo" { + name = local.ecr_image_name + tags = var.tags +} + + +resource null_resource ecr_login { + triggers = { + image_uri = var.lambda_container_image_uri + } + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-e", "-c"] + command = <