Skip to content

Commit

Permalink
github actions for mbed-os-env docker management
Browse files Browse the repository at this point in the history
  • Loading branch information
saheerb committed Jul 26, 2021
1 parent e2ec3cd commit 006a780
Show file tree
Hide file tree
Showing 10 changed files with 1,075 additions and 0 deletions.
124 changes: 124 additions & 0 deletions .github/workflows/ci_scripts/ghcr_utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
#!/usr/bin/env python

"""
Copyright (c) 2017-2021 ARM Limited. All rights reserved.
SPDX-License-Identifier: Apache-2.0
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations
"""
import click
import requests
import logging
import sys
import time
import json
import subprocess

"""
This file contains ghcr utlity wrapper used for:
- retrieving digest of docker image
- deleting images in ghcr
"""
@click.command()
@click.pass_context
@click.option("-r", "--repository", required=True)
@click.option("-t", "--tag", required=True)
@click.option("-p", "--platform", required=False)
def get_digest(ctx, repository, tag, platform=None):
command = f"docker run quay.io/skopeo/stable --creds={ctx.obj['username']}:{ctx.obj['passwd']} inspect docker://ghcr.io/{ctx.obj['username']}/{repository}:{tag} --raw"
output = subprocess.run(command.split(), stdout=subprocess.PIPE).stdout.decode(
"utf-8"
)
output = json.loads(output)
# logging.info(output)

images = output["manifests"]
digest = ""
if len(images) > 1 and platform == None:
logging.error(
"This tag has more than one platform associated to it, please input a platform"
)
sys.exit(1)

for image in images:
if platform != None:
if (platform.split("/")[0] == image["platform"]["os"]) and (
platform.split("/")[1] == image["platform"]["architecture"]
):
digest = image["digest"]
else:
digest = image["digest"]

print(digest)


# delete images older than provided argument (number_of_days)
@click.command()
@click.pass_context
@click.option("-r", "--repository", required=True)
@click.option(
"-n",
"--number_of_days",
default="10",
help="number of days since image was created",
required=False,
)
def delete_old_images(ctx, repository, number_of_days):
s = requests.Session()
github_api_accept = "application/vnd.github.v3+json"
s.headers.update(
{"Authorization": f'token {ctx.obj["passwd"]}', "Accept": github_api_accept}
)
r = s.get(f"https://api.github.com/user/packages/container/{repository}/versions")
versions = r.json()
version_id = None
pattern = "%d.%m.%Y %H:%M:%S"
pattern = "%Y-%m-%dT%H:%M:%SZ"
current_time = time.time()
for version in versions:
logging.info (version)
epoch = int(time.mktime(time.strptime(version["updated_at"], pattern)))

if (current_time - epoch) / (24 * 60 * 60) > int(number_of_days):
version_id = version["id"]
logging.debug(f"deleteing image with version id {version_id}")

url = f"https://api.github.com/user/packages/container/{repository}/versions/{version_id}"
resp = s.delete(url)
resp.raise_for_status()


@click.group()
@click.pass_context
@click.option("-u", "--username", required=False)
@click.option("-p", "--passwd", required=False)
@click.option("-v", "--verbose", is_flag=True, default=False)
def main(ctx, username, passwd, verbose):
ctx.obj = {"username": username, "passwd": passwd}

if verbose:
logging.basicConfig(
stream=sys.stdout,
format="%(levelname)s %(asctime)s %(message)s",
datefmt="%m/%d/%Y %I:%M:%S %p",
)
logging.getLogger().setLevel(logging.DEBUG)
else:
logging.basicConfig(
format="%(levelname)s %(asctime)s %(message)s",
datefmt="%m/%d/%Y %I:%M:%S %p",
)
logging.getLogger().setLevel(logging.INFO)


if __name__ == "__main__":
main.add_command(get_digest)
main.add_command(delete_old_images)
main()
245 changes: 245 additions & 0 deletions .github/workflows/docker_management.branch.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,245 @@
name: Publish or Update docker image for head of branch
# This work flow is disabled for forked repositories.
# If you need to enable it, find references for github.repository_owner in this file
# and replace with ARMmbed with your organisation/account name
# Read more details in https://github.com/ARMmbed/mbed-os/blob/master/docs/design-documents/docker_management



on:

# passive update once a week
schedule:
- cron: '15 4 * * 6'

# build on master branch when there is changes for active update
push:
branches:
- master

paths:
- requirements.txt
- docker_images/mbed-os-env/**
- .github/workflows/docker_management.branch.yml


# manual trigger when needed
workflow_dispatch:


jobs:
prepare-tags:
if: github.repository_owner == 'ARMmbed'
runs-on: ubuntu-latest

steps:
-
name: Extract branch name
shell: bash
run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
id: extract_branch

-
name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0

-
name: Set UUID
id: generate-uuid
uses: filipstefansson/uuid-action@v1

# set docker tags we are building, and intending to publish
# dev-tag is temporary for testing purpose. This should be considered as unstable.
# dated-tag is created for versioning purpose
# prod-tag-latest could be used by customers, CI etc for keeping up to date
-
name: Get build information
shell: bash
run: |
mkdir -p build_info
date=$(date +"%Y.%m.%dT%H.%M.%S")
echo dev-${{ steps.extract_branch.outputs.branch }}-${date}-${{ steps.generate-uuid.outputs.uuid }} > build_info/dev_tag
echo ${{ steps.extract_branch.outputs.branch }}-${date} > build_info/prod_tag_dated
echo ${{ steps.extract_branch.outputs.branch }}-latest > build_info/prod_tag_latest
echo ${{ steps.extract_branch.outputs.branch }} > build_info/mbed_os_version
-
name: Archive information
uses: actions/upload-artifact@v2
with:
name: build-info
path: build_info


build-container:
runs-on: ubuntu-latest
needs: prepare-tags
outputs:
DEV_DIGEST: ${{ steps.docker_info_dev.outputs.DIGEST }}
PROD_DIGEST: ${{ steps.docker_info_prod.outputs.DIGEST }}

steps:
-
name: unarchive artefacts
uses: actions/download-artifact@v2
with:
name: build-info

-
name: Get build info from archive
shell: bash
id: build_info
run: |
value=`cat dev_tag`
echo "DEV TAG is $value"
echo "::set-output name=DOCKER_DEV_TAG::$value"
value=`cat prod_tag_dated`
echo "PROD TAG DATED is $value"
echo "::set-output name=DOCKER_PROD_TAG_DATED::$value"
value=`cat prod_tag_latest`
echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value"
echo "PROD TAG is $value"
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1

-
name: Set up QEMU
uses: docker/setup-qemu-action@v1

-
name: Login to DockerHub
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

-
name: Checkout
uses: actions/checkout@v2

-
name: Build docker containers
uses: docker/build-push-action@v2
id: docker_build_dev
with:
context: .
platforms: linux/amd64,linux/arm64
push: true
file: ./docker_images/mbed-os-env/Dockerfile
tags: ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }}

test-container:
runs-on: ubuntu-latest
needs: build-container
strategy:
matrix:
platform: [linux/amd64, linux/arm64]

steps:
-
name: unarchive artefacts
uses: actions/download-artifact@v2
with:
name: build-info

-
name: Get build info from archive
shell: bash
id: build_info
run: |
value=`cat dev_tag`
echo "TAG is $value"
echo "::set-output name=DOCKER_DEV_TAG::$value"
value=`cat prod_tag_dated`
echo "TAG is $value"
echo "::set-output name=DOCKER_PROD_TAG_DATED::$value"
value=`cat prod_tag_latest`
echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value"
value=`cat mbed_os_version`
echo "::set-output name=MBED_OS_VERSION::$value"
-
name: Checkout
uses: actions/checkout@v2

-
name: Find DEV DOCKER DIGEST
id: docker_info_dev
run: |
DIGEST=$(python ./.github/workflows/ci_scripts/ghcr_utils.py -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} get-digest -r mbed-os-env-tmp -t ${{ steps.build_info.outputs.DOCKER_DEV_TAG }} -p ${{ matrix.platform }} )
echo "::set-output name=DIGEST::$DIGEST"
echo "Docker DIGEST: $DIGEST"
# as the dev images are created only for master branch, run test against
# development branch of blinky
-
name: Checkout
uses: actions/checkout@v2
with:
repository: ARMmbed/mbed-os-example-blinky
path: mbed-os-example-blinky
ref: development
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1

-
name: test the container
id: test
uses: addnab/docker-run-action@v3
with:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
registry: ghcr.io
options: -v ${{ github.workspace }}:/work -w=/work
image: ghcr.io/${{ github.actor }}/mbed-os-env-tmp@${{ steps.docker_info_dev.outputs.DIGEST }}
shell: bash

run: |
uname -m
ls
#./mbed-os/docker_images/mbed-os-env/test.sh DEVELOPMENT ${{ steps.build_info.outputs.MBED_OS_VERSION }}
cd mbed-os-example-blinky
mbed deploy
# build using CLI1
mbed compile -m K64F -t GCC_ARM
# build using CLI2
mbed-tools compile -m K64F -t GCC_ARM
deploy-container:
runs-on: ubuntu-latest
needs: test-container

steps:
-
name: unarchive artefacts
uses: actions/download-artifact@v2
with:
name: build-info

-
name: Get build info from archive
shell: bash
id: build_info
run: |
value=`cat dev_tag`
echo "TAG is $value"
echo "::set-output name=DOCKER_DEV_TAG::$value"
value=`cat prod_tag_dated`
echo "TAG is $value"
echo "::set-output name=DOCKER_PROD_TAG_DATED::$value"
value=`cat prod_tag_latest`
echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value"
-
name: copy dev tag to prod
run: |
docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.DOCKER_PROD_TAG_LATEST }}
docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.DOCKER_PROD_TAG_DATED }}
Loading

0 comments on commit 006a780

Please sign in to comment.