Feature/arm tig (#42)

* revert tig.py * restore pytest * update changelog
podaac · Jan 19, 2024 · d96347a · d96347a
1 parent 9a7b51b
commit d96347a
Show file tree

Hide file tree

Showing 11 changed files with 129 additions and 128 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -35,6 +35,12 @@ jobs:
         with:
           poetry-version: 1.3.2
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
       #########################################################################
       # Versioning (featuring weird gradle output work-arounds)
       #########################################################################
@@ -267,7 +273,7 @@ jobs:
           github.ref == 'refs/heads/develop' ||
           github.ref == 'refs/heads/main'    ||
           startsWith(github.ref, 'refs/heads/release')
-        run: |          
+        run: |
           if [ -n "$(git status --porcelain)" ]; then
             echo "changes=true" >> $GITHUB_ENV
             echo "::set-output name=changes::true"
@@ -459,72 +465,16 @@ jobs:
         uses: docker/build-push-action@v3
         with:
           context: .
-          file: ./docker/lambdaDockerfile
+          file: ./docker/lambdaDockerfileArm
           push: true
           pull: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/arm/v7
           build-args: |
             SOURCE=${{ env.pyproject_name }}==${{ env.the_version }}
 
 
-      ## Build and publish to Service ECR
-      - name: Upload Docker image to Service ECR
-        if: |
-          github.ref == 'refs/heads/develop' ||
-          github.ref == 'refs/heads/main'    ||
-          startsWith(github.ref, 'refs/heads/release')
-        uses: vitr/actions-build-and-upload-to-ecs@master
-        with:
-          access_key_id: ${{ secrets[format('AWS_ACCESS_KEY_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
-          secret_access_key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
-          account_id: ${{ secrets[format('AWS_ACCOUNT_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
-          repo: podaac/tig
-          region: us-west-2
-          tags: ${{ env.the_version }}
-          create_repo: true
-          dockerfile: ./docker/lambdaDockerfile
-          extra_build_args: --build-arg SOURCE=${{ env.pyproject_name }}==${{ env.the_version }}
-
-
-      ## Build and publish to Cumulus ECR
-      - name: Upload Docker image to Cumulus ECR
-        if: |
-          github.ref == 'refs/heads/develop' ||
-          github.ref == 'refs/heads/main'    ||
-          startsWith(github.ref, 'refs/heads/release')
-        uses: vitr/actions-build-and-upload-to-ecs@master
-        with:
-          access_key_id: ${{ secrets[format('AWS_ACCESS_KEY_ID_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }}
-          secret_access_key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }}
-          account_id: ${{ secrets[format('AWS_ACCOUNT_ID_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }}
-          repo: podaac/tig
-          region: us-west-2
-          tags: ${{ env.the_version }}
-          create_repo: true
-          dockerfile: ./docker/lambdaDockerfile
-          extra_build_args: --build-arg SOURCE=${{ env.pyproject_name }}==${{ env.the_version }}
-
-
-      ## Build and publish to SWOT Cumulus ECR
-      - name: Upload Docker image to Cumulus ECR
-        if: |
-          github.ref == 'refs/heads/develop' ||
-          github.ref == 'refs/heads/main'    ||
-          startsWith(github.ref, 'refs/heads/release')
-        uses: vitr/actions-build-and-upload-to-ecs@master
-        with:
-          access_key_id: ${{ secrets[format('AWS_ACCESS_KEY_ID_CUMULUS_SWOT_{0}', env.TARGET_ENV_UPPERCASE)] }}
-          secret_access_key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_CUMULUS_SWOT_{0}', env.TARGET_ENV_UPPERCASE)] }}
-          account_id: ${{ secrets[format('AWS_ACCOUNT_ID_CUMULUS_SWOT_{0}', env.TARGET_ENV_UPPERCASE)] }}
-          repo: podaac/tig
-          region: us-west-2
-          tags: ${{ env.the_version }}
-          create_repo: true
-          dockerfile: ./docker/lambdaDockerfile
-          extra_build_args: --build-arg SOURCE=${{ env.pyproject_name }}==${{ env.the_version }}
-
-
     ## Local tig docker builds
 
       - name: Get Local Tig Build
@@ -544,68 +494,16 @@ jobs:
         uses: docker/build-push-action@v3
         with:
           context: .
-          file: ./docker/lambdaDockerfile
+          file: ./docker/lambdaDockerfileArm
           push: true
           pull: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/arm/v7
           build-args: |
             DIST_PATH="dist/"
             SOURCE=${{ env.local_tig }}
 
-      ## Build and publish to Service ECR
-      - name: Upload Local TIG Docker image to Service ECR
-        if: |
-          github.event.head_commit.message == '/deploy sit' ||
-          github.event.head_commit.message == '/deploy uat'
-        uses: vitr/actions-build-and-upload-to-ecs@master
-        with:
-          access_key_id: ${{ secrets[format('AWS_ACCESS_KEY_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
-          secret_access_key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
-          account_id: ${{ secrets[format('AWS_ACCOUNT_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
-          repo: podaac/tig
-          region: us-west-2
-          tags: ${{ steps.meta.outputs.version }}
-          create_repo: true
-          dockerfile: ./docker/lambdaDockerfile
-          extra_build_args: --build-arg DIST_PATH="dist/" --build-arg SOURCE=${{ env.local_tig }}
-
-      ## Build and publish to Cumulus ECR
-      - name: Upload Local TIG Docker image to Cumulus ECR
-        if: |
-          github.event.head_commit.message == '/deploy sit' ||
-          github.event.head_commit.message == '/deploy uat' ||
-          github.event.head_commit.message == '/deploy sandbox'
-        uses: vitr/actions-build-and-upload-to-ecs@master
-        with:
-          access_key_id: ${{ secrets[format('AWS_ACCESS_KEY_ID_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }}
-          secret_access_key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }}
-          account_id: ${{ secrets[format('AWS_ACCOUNT_ID_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }}
-          repo: podaac/tig
-          region: us-west-2
-          tags: ${{ steps.meta.outputs.version }}
-          create_repo: true
-          dockerfile: ./docker/lambdaDockerfile
-          extra_build_args: --build-arg DIST_PATH="dist/" --build-arg  SOURCE=${{ env.local_tig }}
-
-      ## Build and publish to SWOT Cumulus ECR
-      - name: Upload Local TIG Docker image to Cumulus ECR
-        if: |
-          github.event.head_commit.message == '/deploy sit' ||
-          github.event.head_commit.message == '/deploy uat' ||
-          github.event.head_commit.message == '/deploy sandbox'
-        uses: vitr/actions-build-and-upload-to-ecs@master
-        with:
-          access_key_id: ${{ secrets[format('AWS_ACCESS_KEY_ID_CUMULUS_SWOT_{0}', env.TARGET_ENV_UPPERCASE)] }}
-          secret_access_key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_CUMULUS_SWOT_{0}', env.TARGET_ENV_UPPERCASE)] }}
-          account_id: ${{ secrets[format('AWS_ACCOUNT_ID_CUMULUS_SWOT_{0}', env.TARGET_ENV_UPPERCASE)] }}
-          repo: podaac/tig
-          region: us-west-2
-          tags: ${{ steps.meta.outputs.version }}
-          create_repo: true
-          dockerfile: ./docker/lambdaDockerfile
-          extra_build_args: --build-arg DIST_PATH="dist/" --build-arg  SOURCE=${{ env.local_tig }}
-
       # #########################################################################
       # # Build and Publish Documentation
       # #########################################################################
@@ -650,7 +548,7 @@ jobs:
           ls -al bin/
           which python3
           python3 --version
-          python3 override.py https://github.com/podaac/tig/releases/download/${{ env.the_version }}/tig-terraform-${{ env.the_version }}.zip ${{ secrets[format('AWS_ACCOUNT_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}.dkr.ecr.us-west-2.amazonaws.com/podaac/tig:${{ steps.meta.outputs.version }}
+          python3 override.py https://github.com/podaac/tig/releases/download/${{ env.the_version }}/tig-terraform-${{ env.the_version }}.zip "ghcr.io/podaac/tig:${{ env.DOCKER_METADATA_OUTPUT_VERSION }}"
           ls -al
           echo "Show override contents"
           cat override.tf.json

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [unreleased]
 
 ### Added
+- ** Add Arm Architecture **
+  - update tig to be able to run with arm architecture
+- ** Add image uploading **
+  - update tig to upload tig image to ecr
 ### Changed
 ### Deprecated
 ### Removed

diff --git a/docker/lambdaDockerfileArm b/docker/lambdaDockerfileArm
@@ -0,0 +1,25 @@
+#ARG FUNCTION_DIR="/function"
+
+FROM public.ecr.aws/lambda/python:3.10-arm64
+
+# Include global arg in this stage of the build
+ARG SOURCE
+
+RUN yum -q -y install gcc
+
+# Create function directory
+# RUN mkdir -p ${FUNCTION_DIR}
+# WORKDIR ${FUNCTION_DIR}
+
+# Install tig from artifactory
+COPY $DIST_PATH $DIST_PATH
+
+# install tig into working directory so we can call lambda
+RUN pip3 install awslambdaric --target $LAMBDA_TASK_ROOT
+
+RUN pip3 install --no-cache-dir --force --index-url https://pypi.org/simple/ --extra-index-url https://test.pypi.org/simple/ --target "${LAMBDA_TASK_ROOT}" $SOURCE
+
+RUN rm -rf $DIST_PATH
+
+ENTRYPOINT []
+CMD ["podaac.lambda_handler.lambda_handler.handler"]
diff --git a/terraform/fargate/fargate.tf b/terraform/fargate/fargate.tf
@@ -13,6 +13,11 @@ resource "aws_ecs_task_definition" "app" {
   execution_role_arn       = var.iam_role
   task_role_arn            = var.iam_role
 
+  runtime_platform {
+    operating_system_family = "LINUX"
+    cpu_architecture        = "ARM64"
+  }
+
   container_definitions = jsonencode([
     {
       name              = "${var.prefix}-${var.app_name}-fargate"

diff --git a/terraform/tig_ecr.tf b/terraform/tig_ecr.tf
@@ -0,0 +1,44 @@
+data "aws_ecr_authorization_token" "token" {}
+
+locals {
+  lambda_container_image_uri_split = split("/", var.lambda_container_image_uri)
+  ecr_image_name_and_tag           = split(":", element(local.lambda_container_image_uri_split, length(local.lambda_container_image_uri_split) - 1))
+  ecr_image_name                   = "${local.environment}-${element(local.ecr_image_name_and_tag, 0)}"
+  ecr_image_tag                    = element(local.ecr_image_name_and_tag, 1)
+}
+
+resource aws_ecr_repository "lambda-image-repo" {
+  name = local.ecr_image_name
+  tags = var.tags
+}
+
+
+resource null_resource ecr_login {
+  triggers = {
+    image_uri = var.lambda_container_image_uri
+  }
+
+  provisioner "local-exec" {
+    interpreter = ["/bin/bash", "-e", "-c"]
+    command = <<EOF
+      docker login ${data.aws_ecr_authorization_token.token.proxy_endpoint} -u AWS -p ${data.aws_ecr_authorization_token.token.password}
+      EOF
+  }
+}
+
+
+resource null_resource upload_ecr_image {
+  depends_on = [null_resource.ecr_login]
+  triggers = {
+    image_uri = var.lambda_container_image_uri
+  }
+
+  provisioner "local-exec" {
+    interpreter = ["/bin/bash", "-e", "-c"]
+    command = <<EOF
+      docker pull ${var.lambda_container_image_uri}
+      docker tag ${var.lambda_container_image_uri} ${aws_ecr_repository.lambda-image-repo.repository_url}:${local.ecr_image_tag}
+      docker push ${aws_ecr_repository.lambda-image-repo.repository_url}:${local.ecr_image_tag}
+      EOF
+  }
+}
diff --git a/terraform/tig_ecs.tf b/terraform/tig_ecs.tf
@@ -18,7 +18,7 @@ module "tig_service" {
   tags                                  = merge(var.tags, { Project = var.prefix })
   cluster_arn                           = var.cluster_arn
   desired_count                         = var.desired_count
-  image                                 = var.image
+  image                                 = "${aws_ecr_repository.lambda-image-repo.repository_url}:${local.ecr_image_tag}"
   log_destination_arn                   = var.log_destination_arn
   cpu                                   = var.ecs_cpu
   memory_reservation                    = var.ecs_memory_reservation
@@ -30,14 +30,14 @@ module "tig_service" {
       CONFIG_DIR                  = var.config_dir
       LOGGING_LEVEL               = var.log_level  
       AWS_DEFAULT_REGION          = var.region
-      PYTHONPATH                  = ":/function"
+      PYTHONPATH                  = ":/var/task"
       CONFIG_URL                  = var.config_url
       PALETTE_URL                 = var.palette_url
   }
 
   command = [
-    "/usr/local/bin/python",
-    "/function/podaac/lambda_handler/lambda_handler.py",
+    "/var/lang/bin/python",
+    "/var/task/podaac/lambda_handler/lambda_handler.py",
     "activity",
     "--arn",
     aws_sfn_activity.tig_ecs_task.id

diff --git a/terraform/tig_fargate.tf b/terraform/tig_fargate.tf
@@ -9,22 +9,22 @@ module "tig_fargate" {
   tags = var.tags
   iam_role = var.fargate_iam_role
   command = [
-    "/usr/local/bin/python",
-    "/function/podaac/lambda_handler/lambda_handler.py",
+    "/var/lang/bin/python",
+    "/var/task//podaac/lambda_handler/lambda_handler.py",
     "activity",
     "--arn",
     aws_sfn_activity.tig_ecs_task.id
   ]
 
   environment = {
-    "PYTHONPATH": ":/function",
+    "PYTHONPATH": ":/var/task",
     "CONFIG_BUCKET": "${var.prefix}-internal",
     "CONFIG_DIR" : var.config_dir,
     "CONFIG_URL" : var.config_url,
     "PALETTE_URL" : var.palette_url
   }
 
-  image = var.image
+  image = "${aws_ecr_repository.lambda-image-repo.repository_url}:${local.ecr_image_tag}"
   ecs_cluster_arn = var.cluster_arn
   subnet_ids = var.subnet_ids
   scale_dimensions =  var.scale_dimensions != null ? var.scale_dimensions : {"ServiceName" = "${var.prefix}-${var.app_name}-fargate-service","ClusterName" = var.ecs_cluster_name}

diff --git a/terraform/tig_lambda.tf b/terraform/tig_lambda.tf
@@ -4,13 +4,20 @@ locals {
 }
 
 resource "aws_lambda_function" "tig_task" {
+
+  depends_on = [
+    null_resource.upload_ecr_image
+  ]
+
   function_name = "${local.lambda_resources_name}-lambda"
-  image_uri     = var.image
+  image_uri     = "${aws_ecr_repository.lambda-image-repo.repository_url}:${local.ecr_image_tag}"
   role          = var.role
   timeout       = var.timeout
   memory_size   = var.memory_size
   package_type  = "Image"
 
+  architectures = var.architectures
+
   image_config {
     command = var.command
     entry_point = var.entry_point
@@ -45,13 +52,20 @@ resource "aws_cloudwatch_log_group" "tig_task" {
 
 
 resource "aws_lambda_function" "tig_cleaner_task" {
+
+  depends_on = [
+    null_resource.upload_ecr_image
+  ]
+
   function_name = "${local.lambda_resources_name}-lambda-cleaner"
-  image_uri     = var.image
+  image_uri     = "${aws_ecr_repository.lambda-image-repo.repository_url}:${local.ecr_image_tag}"
   role          = var.role
   timeout       = var.timeout
   memory_size   = var.memory_size
   package_type  = "Image"
 
+  architectures = var.architectures
+
   image_config {
     command = ["podaac.lambda_handler.clean_lambda_handler.handler"]
     entry_point = var.entry_point