github actions for mbed-os-env docker management

pull/14952/head
Saheer Babu 2021-06-09 11:40:41 +01:00 committed by Saheer
parent 862a9420fe
commit 5b0c1df525
10 changed files with 1050 additions and 0 deletions

View File

@ -0,0 +1,144 @@
#!/usr/bin/env python
"""
Copyright (c) 2017-2021 ARM Limited. All rights reserved.
SPDX-License-Identifier: Apache-2.0
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations
"""
import click
import requests
import logging
import sys
import time
import json
import subprocess
"""
This file contains ghcr utlity wrapper used for:
- retrieving digest of docker image
- deleting images in ghcr
"""
@click.command()
@click.pass_context
@click.option("-r", "--repository", required=True)
@click.option("-t", "--tag", required=True)
@click.option("-p", "--platform", required=False)
def get_digest(ctx, repository, tag, platform=None):
"""
Prints docker digest of specific platform of multi architecture image
:param ctx: click context
:param repository: docker repository
:param tag: docker tag
:param platform: platform for e.g, linux/arm64
"""
command = f"docker run quay.io/skopeo/stable --creds={ctx.obj['username']}:{ctx.obj['passwd']} inspect docker://ghcr.io/{ctx.obj['username']}/{repository}:{tag} --raw"
output = subprocess.run(
command.split(), stdout=subprocess.PIPE, check=True
).stdout.decode("utf-8")
output = json.loads(output)
images = output["manifests"]
digest = ""
if len(images) and platform is None:
logging.error(
"This tag has more than one platform associated to it, please input a platform"
)
sys.exit(1)
for image in images:
if platform != None:
if (platform.split("/")[0] == image["platform"]["os"]) and (
platform.split("/")[1] == image["platform"]["architecture"]
):
digest = image["digest"]
else:
digest = image["digest"]
if digest == "":
logging.error("Digest not found. image not in repo for the given platform")
sys.exit(1)
print(digest)
@click.command()
@click.pass_context
@click.option("-r", "--repository", required=True)
@click.option(
"-n",
"--number_of_days",
default="10",
help="number of days since image was created",
required=False,
)
def delete_old_images(ctx, repository, number_of_days):
"""
delete old images from docker repository
:param ctx: click context
:param repository: docker repository
:param number_of_days: delete older than these number of days
"""
with requests.Session() as s:
github_api_accept = "application/vnd.github.v3+json"
s.headers.update(
{"Authorization": f'token {ctx.obj["passwd"]}', "Accept": github_api_accept}
)
r = s.get(
f"https://api.github.com/user/packages/container/{repository}/versions"
)
versions = r.json()
version_id = None
pattern = "%d.%m.%Y %H:%M:%S"
pattern = "%Y-%m-%dT%H:%M:%SZ"
current_time = time.time()
for version in versions:
logging.info(version)
epoch = int(time.mktime(time.strptime(version["updated_at"], pattern)))
if (current_time - epoch) / (24 * 60 * 60) > int(number_of_days):
version_id = version["id"]
logging.debug(f"deleteing image with version id {version_id}")
url = f"https://api.github.com/user/packages/container/{repository}/versions/{version_id}"
resp = s.delete(url)
resp.raise_for_status()
@click.group()
@click.pass_context
@click.option("-u", "--username", required=False)
@click.option("-p", "--passwd", required=False)
@click.option("-v", "--verbose", is_flag=True, default=False)
def main(ctx, username, passwd, verbose):
ctx.obj = {"username": username, "passwd": passwd}
if verbose:
logging.basicConfig(
stream=sys.stdout,
format="%(levelname)s %(asctime)s %(message)s",
datefmt="%m/%d/%Y %I:%M:%S %p",
)
logging.getLogger().setLevel(logging.DEBUG)
else:
logging.basicConfig(
format="%(levelname)s %(asctime)s %(message)s",
datefmt="%m/%d/%Y %I:%M:%S %p",
)
logging.getLogger().setLevel(logging.INFO)
if __name__ == "__main__":
main.add_command(get_digest)
main.add_command(delete_old_images)
main()

View File

@ -0,0 +1,237 @@
name: Publish or Update docker image for head of branch
# Design details in https://github.com/ARMmbed/mbed-os/blob/master/docs/design-documents/docker_management
on:
# passive update once a week
schedule:
- cron: '15 4 * * 6'
# build on master branch when there is changes for active update
push:
branches:
- master
paths:
- requirements.txt
- docker_images/mbed-os-env/**
- .github/workflows/docker_management.branch.yml
# manual trigger when needed
workflow_dispatch:
jobs:
prepare-tags:
runs-on: ubuntu-latest
steps:
-
name: Extract branch name
shell: bash
run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
id: extract_branch
-
name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
-
name: Set UUID
id: generate-uuid
uses: filipstefansson/uuid-action@v1
# set docker tags we are building, and intending to publish
# dev-tag is temporary for testing purpose. This should be considered as unstable.
# dated-tag is created for versioning purpose
# prod-tag-latest could be used by customers, CI etc for keeping up to date
-
name: Get build information
shell: bash
run: |
mkdir -p build_info
date=$(date +"%Y.%m.%dT%H.%M.%S")
echo dev-${{ steps.extract_branch.outputs.branch }}-${date}-${{ steps.generate-uuid.outputs.uuid }} > build_info/dev_tag
echo ${{ steps.extract_branch.outputs.branch }}-${date} > build_info/prod_tag_dated
echo ${{ steps.extract_branch.outputs.branch }}-latest > build_info/prod_tag_latest
echo ${{ steps.extract_branch.outputs.branch }} > build_info/mbed_os_version
-
name: Archive information
uses: actions/upload-artifact@v2
with:
name: build-info
path: build_info
build-container:
runs-on: ubuntu-latest
needs: prepare-tags
outputs:
DEV_DIGEST: ${{ steps.docker_info_dev.outputs.DIGEST }}
PROD_DIGEST: ${{ steps.docker_info_prod.outputs.DIGEST }}
steps:
-
name: unarchive artefacts
uses: actions/download-artifact@v2
with:
name: build-info
-
name: Get build info from archive
shell: bash
id: build_info
run: |
value=`cat dev_tag`
echo "DEV TAG is $value"
echo "::set-output name=DOCKER_DEV_TAG::$value"
value=`cat prod_tag_dated`
echo "PROD TAG DATED is $value"
echo "::set-output name=DOCKER_PROD_TAG_DATED::$value"
value=`cat prod_tag_latest`
echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value"
echo "PROD TAG is $value"
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
name: Login to DockerHub
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
-
name: Checkout
uses: actions/checkout@v2
-
name: Build docker containers
uses: docker/build-push-action@v2
id: docker_build_dev
with:
context: .
platforms: linux/amd64,linux/arm64
push: true
file: ./docker_images/mbed-os-env/Dockerfile
tags: ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }}
test-container:
runs-on: ubuntu-latest
needs: build-container
strategy:
matrix:
platform: [linux/amd64, linux/arm64]
steps:
-
name: unarchive artefacts
uses: actions/download-artifact@v2
with:
name: build-info
-
name: Get build info from archive
shell: bash
id: build_info
run: |
value=`cat dev_tag`
echo "TAG is $value"
echo "::set-output name=DOCKER_DEV_TAG::$value"
value=`cat prod_tag_dated`
echo "TAG is $value"
echo "::set-output name=DOCKER_PROD_TAG_DATED::$value"
value=`cat prod_tag_latest`
echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value"
value=`cat mbed_os_version`
echo "::set-output name=MBED_OS_VERSION::$value"
-
name: Checkout
uses: actions/checkout@v2
-
name: Find DEV DOCKER DIGEST
id: docker_info_dev
run: |
DIGEST=$(python ./.github/workflows/ci_scripts/ghcr_utils.py -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} get-digest -r mbed-os-env-tmp -t ${{ steps.build_info.outputs.DOCKER_DEV_TAG }} -p ${{ matrix.platform }} )
echo "::set-output name=DIGEST::$DIGEST"
echo "Docker DIGEST: $DIGEST"
# as the dev images are created only for master branch, run test against
# development branch of blinky
-
name: Checkout
uses: actions/checkout@v2
with:
repository: ARMmbed/mbed-os-example-blinky
path: mbed-os-example-blinky
ref: development
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
name: test the container
id: test
uses: addnab/docker-run-action@v3
with:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
registry: ghcr.io
options: -v ${{ github.workspace }}:/work -w=/work
image: ghcr.io/${{ github.actor }}/mbed-os-env-tmp@${{ steps.docker_info_dev.outputs.DIGEST }}
shell: bash
run: |
uname -m
cd mbed-os-example-blinky
mbed deploy
# build using CLI1
mbed compile -m K64F -t GCC_ARM
# build using CLI2
mbed-tools compile -m K64F -t GCC_ARM
deploy-container:
runs-on: ubuntu-latest
needs: test-container
steps:
-
name: unarchive artefacts
uses: actions/download-artifact@v2
with:
name: build-info
-
name: Get build info from archive
shell: bash
id: build_info
run: |
value=`cat dev_tag`
echo "TAG is $value"
echo "::set-output name=DOCKER_DEV_TAG::$value"
value=`cat prod_tag_dated`
echo "TAG is $value"
echo "::set-output name=DOCKER_PROD_TAG_DATED::$value"
value=`cat prod_tag_latest`
echo "::set-output name=DOCKER_PROD_TAG_LATEST::$value"
-
name: copy dev tag to prod
run: |
docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.DOCKER_PROD_TAG_LATEST }}
docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.DOCKER_PROD_TAG_DATED }}

View File

@ -0,0 +1,25 @@
name: Prune temporary docker images
on:
schedule:
- cron: '15 4 * * 6'
workflow_dispatch:
jobs:
prune-images:
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v2
-
name: Delete old temporary images
run: |
# the following command may fail because github package doesn't allow
# deletion if only one image exists or if DOCKER_MANAGEMENT_TOKEN is not
# setup. This shouldn't create any alarm as temporary image deletion is
# not a critical activity.
python ./.github/workflows/ci_scripts/ghcr_utils.py -u ${{ github.actor }} -p ${{ secrets.DOCKER_MANAGEMENT_TOKEN }} delete-old-images -r mbed-os-env-tmp || true

View File

@ -0,0 +1,285 @@
name: Release or update docker image for a released mbed-os version
# Design doc: https://github.com/ARMmbed/mbed-os/blob/master/docs/design-documents/docker_management
on:
push:
tags:
- mbed-os-6.[0-9]+.[0-9]+
schedule:
- cron: '15 4 * * 6'
workflow_dispatch:
inputs:
mbed_os_release_version:
description: 'mbed-os release version for which you want to update docker image.'
required: true
jobs:
# this job finds the necessary tags to be applied to docker image
prepare-tags:
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
-
# if manually triggered makes sure that input tag exists on the branch selected
# this is to avoid silly mistakes, for example: mbed-os-7.0.16-latest docker image
# getting released on mbed-os-6 branch
if: ${{ github.event_name == 'workflow_dispatch' }}
name: Sanity check tag of manual trigger
shell: bash
run: |
if [ -z $(git tag --merged ${GITHUB_REF} ${{ github.event.inputs.mbed_os_release_version }}) ]; then
echo "Check the tag name ${{ github.event.inputs.mbed_os_release_version }} is not found on branch ${GITHUB_REF}"
exit 1
fi
-
name: Set UUID
id: generate-uuid
uses: filipstefansson/uuid-action@v1
-
# set docker tags we are building, and intending to release
# this workflow can be executed when a tag is pushed, scheduled, or manual trigger
# depending on the trigger cache source and version of mbed-os will change
#
# when trigger is due to tag pushed (ie, a new mbed-os release is made),
# we are targeting to build docker image for the tag
# when trigger is manual
# we are targeting to build docker image for the input tag in workflow
# when trigger is scheduled
# we are targeting to build docker image for the last tag on the branch
name: Get build information
shell: bash
run: |
mkdir -p build_info
date=$(date +"%Y.%m.%dT%H.%M.%S")
if [ "push" == "${{github.event_name}}" ];then
version=${GITHUB_REF#refs/tags/}
elif [ "workflow_dispatch" == "${{github.event_name}}" ];then
version=${{ github.event.inputs.mbed_os_release_version }}
else
version=`git describe --tags --abbrev=0 --match mbed-os-[0-9]*.[0-9]*.[0-9]*`
fi
echo dev-${version}-${date}-${version} > build_info/dev_tag
echo ${version}-${date} > build_info/prod_tag_dated
echo ${version} > build_info/mbed_os_version
-
name: Archive information
uses: actions/upload-artifact@v2
with:
name: build-info
path: build_info
build-container:
runs-on: ubuntu-latest
needs: prepare-tags
outputs:
DEV_DIGEST: ${{ steps.docker_info_dev.outputs.DIGEST }}
PROD_DIGEST: ${{ steps.docker_info_prod.outputs.DIGEST }}
steps:
-
name: unarchive artefacts
uses: actions/download-artifact@v2
with:
name: build-info
# DOCKER_DEV_TAG is temporary image name.
# DOCKER_PROD_TAG_DATED is fixed tag
# prod-tag-latest could be used by customers, CI etc for keeping up to date
-
name: Get build info from archive
shell: bash
id: build_info
run: |
value=`cat dev_tag`
echo "TAG is $value"
echo "::set-output name=DOCKER_DEV_TAG::$value"
value=`cat prod_tag_dated`
echo "TAG is $value"
echo "::set-output name=DOCKER_PROD_TAG_DATED::$value"
value=`cat mbed_os_version`
echo "::set-output name=MBED_OS_VERSION::$value"
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
name: Login to DockerHub
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
-
name: Checkout
uses: actions/checkout@v2
with:
ref: refs/tags/${{ steps.build_info.outputs.MBED_OS_VERSION }}
-
name: Build docker containers
uses: docker/build-push-action@v2
id: docker_build_dev
with:
context: .
platforms: linux/amd64,linux/arm64
push: true
file: ./docker_images/mbed-os-env/Dockerfile
tags: ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }}
test-container:
runs-on: ubuntu-latest
needs: build-container
strategy:
matrix:
platform: [linux/amd64, linux/arm64]
steps:
-
name: unarchive artefacts
uses: actions/download-artifact@v2
with:
name: build-info
-
name: Get build info from archive
shell: bash
id: build_info
run: |
value=`cat dev_tag`
echo "TAG is $value"
echo "::set-output name=DOCKER_DEV_TAG::$value"
value=`cat prod_tag_dated`
echo "TAG is $value"
echo "::set-output name=DOCKER_PROD_TAG_DATED::$value"
value=`cat mbed_os_version`
echo "::set-output name=MBED_OS_VERSION::$value"
# as the dev images are created only for master branch, run test against
# development branch of blinky
-
name: Checkout example blinky
uses: actions/checkout@v2
with:
repository: ARMmbed/mbed-os-example-blinky
path: mbed-os-example-blinky
fetch-depth: 0
-
name: update the example application version to test against correct version
# When mbed-os tag is applied, and workflow is triggered to build RELEASE image, example application with same tag will be available yet.
# use release candidate branch to test the image in that case.
# When RELEASE image is passively checked, tag should be available in example application repo, then use it.
shell: bash
id: example_app_info
run: |
cd mbed-os-example-blinky
EXAMPLE_VERSION="release_candidate"
MBED_OS_VERSION=${{ steps.build_info.outputs.MBED_OS_VERSION }}
# if tag is present in example repo, use the tag
if git rev-parse "$MBED_OS_VERSION" >/dev/null 2>&1; then
EXAMPLE_VERSION=$MBED_OS_VERSION
fi
# echo "::set-output VERSION=MBED_OS_VERSION::$EXAMPLE_VERSION"
git checkout ${EXAMPLE_VERSION}
-
name: Checkout
uses: actions/checkout@v2
with:
ref: refs/tags/${{ steps.build_info.outputs.MBED_OS_VERSION }}
path: mbed-os-example-blinky/mbed-os
-
name: Find DEV DOCKER DIGEST
id: docker_info
run: |
cd mbed-os-example-blinky/mbed-os
DIGEST=$(python ./.github/workflows/ci_scripts/ghcr_utils.py -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} get-digest -r mbed-os-env-tmp -t ${{ steps.build_info.outputs.DOCKER_DEV_TAG }} -p ${{ matrix.platform }} )
echo "::set-output name=DIGEST::$DIGEST"
echo "Docker DIGEST: $DIGEST"
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
name: test the container
id: test
uses: addnab/docker-run-action@v3
with:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
registry: ghcr.io
options: -v ${{ github.workspace }}:/work -w=/work
image: ghcr.io/${{ github.actor }}/mbed-os-env-tmp@${{ steps.docker_info.outputs.DIGEST }}
shell: bash
run: |
uname -m
cd mbed-os-example-blinky
# build using CLI1
mbed compile -m K64F -t GCC_ARM
# build using CLI2
mbed-tools compile -m K64F -t GCC_ARM
deploy-container:
runs-on: ubuntu-latest
needs: test-container
steps:
-
name: unarchive artefacts
uses: actions/download-artifact@v2
with:
name: build-info
-
name: Get build info from archive
shell: bash
id: build_info
run: |
value=`cat dev_tag`
echo "DEV TAG is $value"
echo "::set-output name=DOCKER_DEV_TAG::$value"
value=`cat prod_tag_dated`
echo "DATED PROD TAG is $value"
echo "::set-output name=DOCKER_PROD_TAG_DATED::$value"
value=`cat mbed_os_version`
echo "MBED OS VERSION is $value"
echo "::set-output name=MBED_OS_VERSION::$value"
-
name: copy dev tag to prod tags
run: |
set -x
echo ${{ needs.test-container.result }}
upto_patch_version=${{ steps.build_info.outputs.MBED_OS_VERSION }}-latest
upto_min_version=${upto_patch_version%.[0-9]*}-latest
upto_major_version=${upto_patch_version%.[0-9]*.[0-9]*}-latest
docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${upto_patch_version}
docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${upto_min_version}
docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${upto_major_version}
# copy to fixed tag
docker run quay.io/skopeo/stable --src-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} --dest-creds=${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} copy --all docker://ghcr.io/${{ github.actor }}/mbed-os-env-tmp:${{ steps.build_info.outputs.DOCKER_DEV_TAG }} docker://ghcr.io/${{ github.actor }}/mbed-os-env:${{ steps.build_info.outputs.DOCKER_PROD_TAG_DATED }}

View File

@ -0,0 +1,82 @@
name: Build docker image
# This workflow is triggered when Dockerfile related or github action itself changes are made in a PR
# The workflow is quite simple - builds and test the image. Release of newer version is done only when PR is merged.
# Design doc: https://github.com/ARMmbed/mbed-os/blob/master/docs/design-documents/docker_management
on:
pull_request:
branches: [ master ]
paths:
- docker_images/mbed-os-env/**
- .github/workflows/docker_management.*
- requirements.txt
jobs:
build-container:
runs-on: ubuntu-latest
strategy:
matrix:
platform: [linux/amd64, linux/arm64]
steps:
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
# for PR tests, development branch of blinky is used
-
name: Checkout
uses: actions/checkout@v2
with:
repository: ARMmbed/mbed-os-example-blinky
path: mbed-os-example-blinky
ref: development
-
name: Remove mbed-os from example-application
shell: bash
run: |
cd mbed-os-example-blinky
rm -rf mbed-os
-
name: Checkout
uses: actions/checkout@v2
with:
path: mbed-os-example-blinky/mbed-os
-
name: Build container
uses: docker/build-push-action@v2
id: docker_build_dev
with:
context: ./mbed-os-example-blinky/mbed-os
platforms: ${{ matrix.platform }}
file: ./mbed-os-example-blinky/mbed-os/docker_images/mbed-os-env/Dockerfile
load: true
tags: mbed-os-env:a_pr_test
-
name: test the container
id: test
uses: addnab/docker-run-action@v2
with:
options: -v ${{ github.workspace }}:/work -w=/work
image: mbed-os-env:a_pr_test
shell: bash
run: |
uname -m
cd mbed-os-example-blinky
# build using CLI1
mbed compile -m K64F -t GCC_ARM
# build using CLI2
mbed-tools compile -m K64F -t GCC_ARM

View File

@ -0,0 +1,93 @@
# ------------------------------------------------------------------------------
# Pull base image
FROM ubuntu:20.04
# ------------------------------------------------------------------------------
# Arguments
ARG WORKDIR=/root
# ------------------------------------------------------------------------------
# Install tools via apt
ENV DEBIAN_FRONTEND=noninteractive
RUN set -x \
&& apt -y update \
&& apt -y install \
git \
wget \
python3 \
python3-dev \
python3-setuptools \
python3-pip \
build-essential \
astyle \
mercurial \
ninja-build \
libssl-dev \
cargo \
flex \
bison \
doxygen \
aspell \
ccache \
gcovr \
&& apt clean && rm -rf /var/lib/apt/lists \
&& update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1 \
&& : # last line
RUN set -x \
&& pip3 install -U cmake
# Set up Mbed environment
WORKDIR /tmp/
COPY requirements.txt .
RUN set -x \
&& pip3 install -r requirements.txt \
&& : # last line
COPY tools/cmake/requirements.txt .
RUN set -x \
&& pip3 install -r requirements.txt \
&& : # last line
# ------------------------------------------------------------------------------
# Install Python modules (which are not included in requirements.txt)
RUN set -x \
&& pip3 install -U \
mbed-cli \
mbed-tools \
&& : # last line
# ------------------------------------------------------------------------------
# Install arm-none-eabi-gcc
WORKDIR /opt/mbed-os-toolchain
RUN set -x \
&& [ "$(uname -m)" = "aarch64" ] && \
TARBALL="gcc-arm-none-eabi-9-2019-q4-major-aarch64-linux.tar.bz2" || \
TARBALL="gcc-arm-none-eabi-9-2019-q4-major-x86_64-linux.tar.bz2" \
&& wget -q https://developer.arm.com/-/media/Files/downloads/gnu-rm/9-2019q4/RC2.1/${TARBALL} \
&& tar -xjf ${TARBALL} \
&& rm ${TARBALL} \
&& : # last line
# ------------------------------------------------------------------------------
# Configure environment variables
ENV MBED_GCC_ARM_PATH=/opt/mbed-os-toolchain/gcc-arm-none-eabi-9-2019-q4-major/bin/
ENV PATH="${PATH}:${MBED_GCC_ARM_PATH}"
# ------------------------------------------------------------------------------
# Display, check and save environment settings
# NOTE: using bash instead of Ubuntu default bash due to unsupport for pipefail
# Pipefail is crucial here, if the tools didn't install properly, docker build should not pass because of piping
RUN /bin/bash -c \
"set -x -o pipefail \
&& arm-none-eabi-gcc --version | grep arm-none-eabi-gcc | tee env_settings \
&& cmake --version | grep version | tee -a env_settings \
&& python --version 2>&1 | tee -a env_settings \
&& (echo -n 'mbed-cli ' && mbed --version) | tee -a env_settings \
&& (echo -n 'mbed-greentea ' && mbedgt --version | grep ^[0-9]) | tee -a env_settings \
&& (echo -n 'mbed-host-tests ' && mbedhtrun --version) | tee -a env_settings \
&& (echo -n 'mbed-tools ' && mbed-tools --version) | tee -a env_settings \
&& : # LAST LINE"
WORKDIR /root

View File

@ -0,0 +1,50 @@
# Mbed OS development environment Docker image
This Docker image is the official Mbed OS development environment.
* It is based on Ubuntu 20.04
* Python and CMake are installed
* Arm-none-eabi-gcc toolchain is installed
* Latest released version of mbed-cli and mbed-greentea are installed
* All other Mbed OS dependency tools are installed.
# How to use the Docker image:
## Pull the Docker image
```bash
docker pull ghcr.io/armmbed/mbed-os-env:<label>
```
## Run Mbed OS environment without HW support (build Mbed images only)
Launch the Docker image by
```bash
docker run -it ghcr.io/armmbed/mbed-os-env:<label>
```
Then you will have a container with an Mbed OS development environment.
You should be able to compile Mbed commands/examples as recommended in the documentation
e.g.
```bash
mbed-tools import mbed-os-example-blinky
cd mbed-os-example-blinky
mbed-tools compile -m <TARGET> -t GCC_ARM
```
## Run Mbed OS environment with HW support (USB pass-through)
:warning: This currently works only on Linux host machines with [Mbed CLI 1](https://os.mbed.com/docs/mbed-os/v6.13/build-tools/mbed-cli-1.html).
If you want to use this Docker image to connect and flash your targets, you will need some extra command line option to pass-through your USB devices.
```bash
sudo docker run -it --privileged -v /dev/disk/by-id:/dev/disk/by-id -v /dev/serial/by-id:/dev/serial/by-id ghcr.io/armmbed/mbed-os-env:<label>
```
Then you will have a container with an Mbed OS development environment.
To make sure your Mbed targets have been detected, you might want to manually run the mount command and `mbedls` to check
```bash
mount /dev/sdb /mnt
mbedls
```
If `mbedls` detected your connected target, then you should be able to run Mbed tests/examples as recommended in the Mbed documentation.
``` bash
mbed import mbed-os
cd mbed-os
mbed test -t GCC_ARM -m <target>
```

View File

@ -0,0 +1,134 @@
# mbed-os-env Docker management
### Overview and background
The Docker image `mbed-os-env` bundles all the necessary tools to provide a minimal environment to build and test Mbed OS applications. This Docker image shall be used in Continuous Integration pipelines or in other use cases where Mbed OS tools and dependencies are required. This document explains versioning strategy of the Docker image and the GitHub Actions workflows that creates these Docker images.
### Types of Docker images
**Production Docker image**
These are Docker images compatible with a released version of Mbed OS. For example when `mbed-os-6.14.0` is released, a Docker image with tag `mbed-os-6.14-latest` is available.
**Development Docker image**
These are Docker images compatible with a `HEAD` of the Mbed OS master branch. These are tagged with `master-latest`.
### Types of Docker image updates
There are two types of Docker image updates.
**Active Updates**
Typically these types of changes that involve modifications in Dockerfile or changes in the requirements.txt. Examples: Mbed OS adds support for new compiler version and hence needs a Dockerfile update.
**Passive updates**
These are scheduled updates on existing mbed-os-env Docker images to provide security and bug fixes of dependent packages. In updates like this, there will be no change in Dockerfile but rebuilding the Dockerfile provides a new docker image with updated tools and packages.
### Types of Docker tags
For the purpose of mbed-os-env Docker management we will need to apply a few Docker tags on the image.
**latest tags**
These tags have the format `<name>-latest`, for example `mbed-os-6-latest`. The `-latest` at the end indicates that the same tag may be reapplied to another image when a Docker image is updated. This obviously means, a user pulling the image with same tag name may get different images at different point of time.
**fixed tags**
Using latest tags, user can retrieve an updated version of Docker image. Troubleshooting could become tricky if Docker images are updated and new images are retrieved in the middle of troubleshooting session. Also, some user could decide not to get updated version and would want to stick to a fixed version.
For reasons stated above, all the Docker images are suffixed with a fixed tag. These tags are only applied to one image and never reused. Tags are of the format `mbed-os-<version>-<date>`, for example `mbed-os-6.14.0-2021.06.19T04.43.51`, `main-2021.06.19T04.43.51` etc.
### Dockerfile versioning
The mbed-os-env image is created with the Dockerfile stored in this repository itself. This provides easy versioning of the Dockerfile as it will follows the same versioning strategy of Mbed OS.
### Docker image versioning
The picture below illustrates a typical situation where Mbed OS accepts changes to the `master` branch on daily basis and makes releases on regular basis.
![Docker Versioning](./diagrams/docker-versioning.png)
**On Day-X** There are some changes for [active updates](#Type-of-Docker-image-updates) (for example, the Dockerfile has been changed), so these are the Docker images created or updated
* `mbed-os-env:master-latest`
* `mbed-os-env:<master-day-x>` - This is a fixed tag.
:information_source: `master-latest` tag is checked for [passive updates](#Type-of-Docker-image-updates) as scheduled in the workflow.
**On Day-X+1** Though there are commits to Mbed OS source repository. These do not involve changes to Dockerfile or dependencies like `requirements.txt`. Hence, no Docker image is created at the time of merging the commit.
**On Day-X+2** A new Mbed OS release `mbed-os-6.14.0` is created. This creates an image with following Docker tag.
* `mbed-os-env:mbed-os-6-latest` - This Docker tag could be used to get a compatible image for latest `mbed-os-6` release.
* `mbed-os-env:mbed-os-6.14-latest` - This Docker tag could be used to work with updated `mbed-os-6.14` release. This image is passively updated till next mbed-os release on the branch (typically till `mbed-os-6.15.0`).
* `mbed-os-env:mbed-os-6.14-day-x` - A fixed Docker tag.
:information_source: `mbed-os-6.14-latest`, and `mbed-os-6-latest` will be passively updated as scheduled in th workflow.
**On Day X+10** Another new release `mbed-os-6.15.0` is made. This creates an image with following Docker tag.
* `mbed-os-6-latest`
* `mbed-os-6.15-latest`
* `mbed-os-6.15-day-x`
:information_source: From this point, `mbed-os-6.14-latest` will no longer be passively updated. From this point, only `mbed-os-6.15-latest` is selected for passive update. Only last release version is passively updated on a branch.
## GitHub Actions Workflows
### Goals of Docker management workflows
* Distribution of Docker image that is compatible with released version of Mbed OS.
* Providing Docker image that is compatible with the `HEAD` of the `master` branch.
* Keeping the released and development Docker image up to date.
There are 4 main workflows
**PR check**
The purpose of this workflow is to make sure build, and test of mbed-os-env Docker image works as expected. Hence this workflow is triggered when Mbed OS PR is created with changes in Dockerfile, test files, or workflow files itself has some modifications. Since most of the Mbed OS PR doesn't contain Docker image related changes, this workflow is not expected to be triggered often.
**Publish Docker image for the HEAD of Mbed OS branch**
The purpose of this workflow is to update development Docker image either when there is an active update or at nightly for passive update.
:bulb: This workflow can also be triggered manually to update the development image. For example, workflow can be triggered manually to update the Docker image during the day.
**Production Docker image Creation/Update**
This workflow will create a new Docker image with versioning strategy describe above. Also, triggered nightly for passive update.
:bulb: This workflow can also be triggered manually to update an old version of released Docker image. For example, this workflow could be manually triggered to update `mbed-os-6.14-latest` after `mbed-os-6.15.0` is released.
**Prune Docker image**
The Docker image in temporary area is pruned by this workflow. Number of dates since updated is used as criteria for pruning.
### Pipeline
The Docker image management goes through typical CI pipeline of build, test, and release stages.
There are some details worth mentioning though.
**Build**
The `docker buildx` command is used for creating multi architecture Docker image. To build Docker image using buildx, one needs to push the image to a remote repository while building it. Since, we need to test before release, the resulting images are pushed to a temporary Docker repository just after building.
The GitHub Container Registry doesn't implement yet all the Docker manifest APIs. Hence, a few features like deleting tag from an image is not available yet. Once these are implemented, we may remove temporary repository and add temporary tags to the image and delete the temporary tags after workflow.
**Test**
After the temporary Docker image is built, GitHub Actions tests the image prior to deploying the image to the production Docker repository. Please refer to the `test-container` job in the workflow for details on the test plan.
**Deploy**
In the `Deploy` job, depending on the result of test job, temporary images are moved to production repository.
### Docker repository
GitHub provides free Docker image storage for public repositories in GitHub packages. The workflows make use of `{{ secrets.GITHUB_TOKEN }}` https://docs.github.com/en/actions/reference/authentication-in-a-workflow
For deleting images from temporary repository, a new token `GITHUB_DELETE_IMAGE_TOKEN` needs to be added with package delete permissions.
There are two Docker repositories used for managing the Docker images:
**mbed-os-env** This is the public repository user can pull the images from.
**mbed-os-env-tmp** This is a private repository created for temporary management of Docker images while Docker images are being tested. This is a staging area. Unfortunately, the GitHub Container Registry does no support renaming Docker tags, hence the need for a temporary repository.
### Workflow for forks
GitHub by default will disable workflow for forked repositories. It will be up to owner of the forked repositories to enable or disable GitHub Actions as needed.
### Usage in example application
For details on branches and version, please refer to [Mbed OS and example application release workflow](https://os.mbed.com/docs/mbed-os/latest/introduction/versions-and-releases.html).
In Mbed OS example application repositories, `mbed-os-env:master-latest` can be used at the time of `release_candidate`(and `release tag` creation), as at the time of release candidate and release creation, example repositories will be pointing to the `master` branch of Mbed OS.

Binary file not shown.

After

Width:  |  Height:  |  Size: 31 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 223 KiB