forked from ARMmbed/mbed-os
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
github actions for mbed-os-env docker management
- Loading branch information
Showing
10 changed files
with
1,075 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,124 @@ | ||
#!/usr/bin/env python | ||
|
||
""" | ||
Copyright (c) 2017-2021 ARM Limited. All rights reserved. | ||
SPDX-License-Identifier: Apache-2.0 | ||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
http://www.apache.org/licenses/LICENSE-2.0 | ||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations | ||
""" | ||
import click | ||
import requests | ||
import logging | ||
import sys | ||
import time | ||
import json | ||
import subprocess | ||
|
||
""" | ||
This file contains ghcr utlity wrapper used for: | ||
- retrieving digest of docker image | ||
- deleting images in ghcr | ||
""" | ||
@click.command() | ||
@click.pass_context | ||
@click.option("-r", "--repository", required=True) | ||
@click.option("-t", "--tag", required=True) | ||
@click.option("-p", "--platform", required=False) | ||
def get_digest(ctx, repository, tag, platform=None): | ||
command = f"docker run quay.io/skopeo/stable --creds={ctx.obj['username']}:{ctx.obj['passwd']} inspect docker://ghcr.io/{ctx.obj['username']}/{repository}:{tag} --raw" | ||
output = subprocess.run(command.split(), stdout=subprocess.PIPE).stdout.decode( | ||
"utf-8" | ||
) | ||
output = json.loads(output) | ||
# logging.info(output) | ||
|
||
images = output["manifests"] | ||
digest = "" | ||
if len(images) > 1 and platform == None: | ||
logging.error( | ||
"This tag has more than one platform associated to it, please input a platform" | ||
) | ||
sys.exit(1) | ||
|
||
for image in images: | ||
if platform != None: | ||
if (platform.split("/")[0] == image["platform"]["os"]) and ( | ||
platform.split("/")[1] == image["platform"]["architecture"] | ||
): | ||
digest = image["digest"] | ||
else: | ||
digest = image["digest"] | ||
|
||
print(digest) | ||
|
||
|
||
# delete images older than provided argument (number_of_days) | ||
@click.command() | ||
@click.pass_context | ||
@click.option("-r", "--repository", required=True) | ||
@click.option( | ||
"-n", | ||
"--number_of_days", | ||
default="10", | ||
help="number of days since image was created", | ||
required=False, | ||
) | ||
def delete_old_images(ctx, repository, number_of_days): | ||
s = requests.Session() | ||
github_api_accept = "application/vnd.github.v3+json" | ||
s.headers.update( | ||
{"Authorization": f'token {ctx.obj["passwd"]}', "Accept": github_api_accept} | ||
) | ||
r = s.get(f"https://api.github.com/user/packages/container/{repository}/versions") | ||
versions = r.json() | ||
version_id = None | ||
pattern = "%d.%m.%Y %H:%M:%S" | ||
pattern = "%Y-%m-%dT%H:%M:%SZ" | ||
current_time = time.time() | ||
for version in versions: | ||
logging.info (version) | ||
epoch = int(time.mktime(time.strptime(version["updated_at"], pattern))) | ||
|
||
if (current_time - epoch) / (24 * 60 * 60) > int(number_of_days): | ||
version_id = version["id"] | ||
logging.debug(f"deleteing image with version id {version_id}") | ||
|
||
url = f"https://api.github.com/user/packages/container/{repository}/versions/{version_id}" | ||
resp = s.delete(url) | ||
resp.raise_for_status() | ||
|
||
|
||
@click.group() | ||
@click.pass_context | ||
@click.option("-u", "--username", required=False) | ||
@click.option("-p", "--passwd", required=False) | ||
@click.option("-v", "--verbose", is_flag=True, default=False) | ||
def main(ctx, username, passwd, verbose): | ||
ctx.obj = {"username": username, "passwd": passwd} | ||
|
||
if verbose: | ||
logging.basicConfig( | ||
stream=sys.stdout, | ||
format="%(levelname)s %(asctime)s %(message)s", | ||
datefmt="%m/%d/%Y %I:%M:%S %p", | ||
) | ||
logging.getLogger().setLevel(logging.DEBUG) | ||
else: | ||
logging.basicConfig( | ||
format="%(levelname)s %(asctime)s %(message)s", | ||
datefmt="%m/%d/%Y %I:%M:%S %p", | ||
) | ||
logging.getLogger().setLevel(logging.INFO) | ||
|
||
|
||
if __name__ == "__main__": | ||
main.add_command(get_digest) | ||
main.add_command(delete_old_images) | ||
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,245 @@ | ||
name: Publish or Update docker image for head of branch | ||
# This work flow is disabled for forked repositories. | ||
# If you need to enable it, find references for github.repository_owner in this file | ||
# and replace with ARMmbed with your organisation/account name | ||
# Read more details in https://github.com/ARMmbed/mbed-os/blob/master/docs/design-documents/docker_management | ||
|
||
|
||
|
||
on: | ||
|
||
# passive update once a week | ||
schedule: | ||
- cron: '15 4 * * 6' | ||
|
||
# build on master branch when there is changes for active update | ||
push: | ||
branches: | ||
- master | ||
|
||
paths: | ||
- requirements.txt | ||
- docker_images/mbed-os-env/** | ||
- .github/workflows/docker_management.branch.yml | ||
|
||
|
||
# manual trigger when needed | ||
workflow_dispatch: | ||
|
||
|
||
jobs: | ||
prepare-tags: | ||
if: github.repository_owner == 'ARMmbed' | ||
runs-on: ubuntu-latest | ||
|
||
steps: | ||
- | ||
name: Extract branch name | ||
shell: bash | ||
run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" | ||
id: extract_branch | ||
|
||
- | ||
name: Checkout | ||
uses: actions/checkout@v2 | ||
with: | ||
fetch-depth: 0 | ||
|
||
- | ||
name: Set UUID | ||
id: generate-uuid | ||
uses: filipstefansson/uuid-action@v1 | ||
|
||
# set docker tags we are building, and intending to publish | ||
# dev-tag is temporary for testing purpose. This should be considered as unstable. | ||
# dated-tag is created for versioning purpose | ||
# prod-tag-latest could be used by customers, CI etc for keeping up to date | ||
- | ||
name: Get build information | ||
shell: bash | ||
run: | | ||
mkdir -p build_info | ||
date=$(date +"%Y.%m.%dT%H.%M.%S") | ||
echo dev-${{ steps.extract_branch.outputs.branch }}-${date}-${{ steps.generate-uuid.outputs.uuid }} > build_info/dev_tag | ||
echo ${{ steps.extract_branch.outputs.branch }}-${date} > build_info/prod_tag_dated | ||
echo ${{ steps.extract_branch.outputs.branch }}-latest > build_info/prod_tag_latest | ||
echo ${{ steps.extract_branch.outputs.branch }} > build_info/mbed_os_version | ||
- | ||
name: Archive information | ||
uses: actions/upload-artifact@v2 | ||
with: | ||
name: build-info | ||
path: build_info | ||
|
||
|
||
build-container: | ||
runs-on: ubuntu-latest | ||
needs: prepare-tags | ||
outputs: | ||
DEV_DIGEST: ${{ steps.docker_info_dev.outputs.DIGEST }} | ||
PROD_DIGEST: ${{ steps.docker_info_prod.outputs.DIGEST }} | ||
|
||
steps: | ||
- | ||
name: unarchive artefacts | ||
uses: actions/download-artifact@v2 | ||
with: | ||
name: build-info | ||
|
||
- | ||
name: Get build info from archive | ||
shell: bash | ||
id: build_info | ||
run: | | ||
value=`cat dev_tag` | ||
echo "DEV TAG is $value" | ||
echo "::set-output name=DOCKER_DEV_TAG::$value" | ||
value=`cat prod_tag_dated` | ||
echo "PROD TAG DATED is $value" | ||
echo "::set-output name=DOCKER_PROD_TAG_DATED::$value" | ||
value=`cat prod_tag_latest` | ||
echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value" | ||
echo "PROD TAG is $value" | ||
- | ||
name: Set up Docker Buildx | ||
uses: docker/setup-buildx-action@v1 | ||
|
||
- | ||
name: Set up QEMU | ||
uses: docker/setup-qemu-action@v1 | ||
|
||
- | ||
name: Login to DockerHub | ||
uses: docker/login-action@v1 | ||
with: | ||
registry: ghcr.io | ||
username: ${{ github.actor }} | ||
password: ${{ secrets.GITHUB_TOKEN }} | ||
|
||
- | ||
name: Checkout | ||
uses: actions/checkout@v2 | ||
|
||
- | ||
name: Build docker containers | ||
uses: docker/build-push-action@v2 | ||
id: docker_build_dev | ||
with: | ||
context: . | ||
platforms: linux/amd64,linux/arm64 | ||
push: true | ||
file: ./docker_images/mbed-os-env/Dockerfile | ||
tags: ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} | ||
|
||
test-container: | ||
runs-on: ubuntu-latest | ||
needs: build-container | ||
strategy: | ||
matrix: | ||
platform: [linux/amd64, linux/arm64] | ||
|
||
steps: | ||
- | ||
name: unarchive artefacts | ||
uses: actions/download-artifact@v2 | ||
with: | ||
name: build-info | ||
|
||
- | ||
name: Get build info from archive | ||
shell: bash | ||
id: build_info | ||
run: | | ||
value=`cat dev_tag` | ||
echo "TAG is $value" | ||
echo "::set-output name=DOCKER_DEV_TAG::$value" | ||
value=`cat prod_tag_dated` | ||
echo "TAG is $value" | ||
echo "::set-output name=DOCKER_PROD_TAG_DATED::$value" | ||
value=`cat prod_tag_latest` | ||
echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value" | ||
value=`cat mbed_os_version` | ||
echo "::set-output name=MBED_OS_VERSION::$value" | ||
- | ||
name: Checkout | ||
uses: actions/checkout@v2 | ||
|
||
- | ||
name: Find DEV DOCKER DIGEST | ||
id: docker_info_dev | ||
run: | | ||
DIGEST=$(python ./.github/workflows/ci_scripts/ghcr_utils.py -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} get-digest -r mbed-os-env-tmp -t ${{ steps.build_info.outputs.DOCKER_DEV_TAG }} -p ${{ matrix.platform }} ) | ||
echo "::set-output name=DIGEST::$DIGEST" | ||
echo "Docker DIGEST: $DIGEST" | ||
# as the dev images are created only for master branch, run test against | ||
# development branch of blinky | ||
- | ||
name: Checkout | ||
uses: actions/checkout@v2 | ||
with: | ||
repository: ARMmbed/mbed-os-example-blinky | ||
path: mbed-os-example-blinky | ||
ref: development | ||
- | ||
name: Set up QEMU | ||
uses: docker/setup-qemu-action@v1 | ||
|
||
- | ||
name: test the container | ||
id: test | ||
uses: addnab/docker-run-action@v3 | ||
with: | ||
username: ${{ github.actor }} | ||
password: ${{ secrets.GITHUB_TOKEN }} | ||
registry: ghcr.io | ||
options: -v ${{ github.workspace }}:/work -w=/work | ||
image: ghcr.io/${{ github.actor }}/mbed-os-env-tmp@${{ steps.docker_info_dev.outputs.DIGEST }} | ||
shell: bash | ||
|
||
run: | | ||
uname -m | ||
ls | ||
#./mbed-os/docker_images/mbed-os-env/test.sh DEVELOPMENT ${{ steps.build_info.outputs.MBED_OS_VERSION }} | ||
cd mbed-os-example-blinky | ||
mbed deploy | ||
# build using CLI1 | ||
mbed compile -m K64F -t GCC_ARM | ||
# build using CLI2 | ||
mbed-tools compile -m K64F -t GCC_ARM | ||
deploy-container: | ||
runs-on: ubuntu-latest | ||
needs: test-container | ||
|
||
steps: | ||
- | ||
name: unarchive artefacts | ||
uses: actions/download-artifact@v2 | ||
with: | ||
name: build-info | ||
|
||
- | ||
name: Get build info from archive | ||
shell: bash | ||
id: build_info | ||
run: | | ||
value=`cat dev_tag` | ||
echo "TAG is $value" | ||
echo "::set-output name=DOCKER_DEV_TAG::$value" | ||
value=`cat prod_tag_dated` | ||
echo "TAG is $value" | ||
echo "::set-output name=DOCKER_PROD_TAG_DATED::$value" | ||
value=`cat prod_tag_latest` | ||
echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value" | ||
- | ||
name: copy dev tag to prod | ||
run: | | ||
docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.DOCKER_PROD_TAG_LATEST }} | ||
docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.DOCKER_PROD_TAG_DATED }} |
Oops, something went wrong.