diff --git a/.gitignore b/.gitignore index 4f49410..cacb9fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .DS_Store .vscode +.*.log libs/eddl libs/ecvl pylibs/pyeddl -pylibs/pyecvl \ No newline at end of file +pylibs/pyecvl diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 8b2e49b..0000000 --- a/Jenkinsfile +++ /dev/null @@ -1,231 +0,0 @@ -pipeline { - agent { - node { label 'docker && linux && !gpu' } - } - triggers{ - upstream( - upstreamProjects: 'DeepHealth/eddl/master,DeepHealth/ecvl/master,DeepHealth/pyeddl/master,DeepHealth/pyecvl/master', - threshold: hudson.model.Result.SUCCESS) - } - environment { - BASE_SRC = "/usr/local/src" - ECVL_SRC = "${BASE_SRC}/ecvl" - EDDL_SRC = "${BASE_SRC}/eddl" - PYECVL_SRC = "${BASE_SRC}/pyecvl" - PYEDDL_SRC = "${BASE_SRC}/pyeddl" - // ECVL Settings - ECVL_REPOSITORY = "https://github.com/deephealthproject/ecvl.git" - ECVL_BRANCH = "master" - ECVL_REVISION = sh(returnStdout: true, script: "git ls-remote ${ECVL_REPOSITORY} ${ECVL_BRANCH} | awk '{print \$1}'").trim() - // PyECVL Settings - PYECVL_REPOSITORY = "https://github.com/deephealthproject/pyecvl.git" - PYECVL_BRANCH = "master" - PYECVL_REVISION = sh(returnStdout: true, script: "git ls-remote ${PYECVL_REPOSITORY} ${PYECVL_BRANCH} | awk '{print \$1}'").trim() - // EDDL Settings - EDDL_REPOSITORY = "https://github.com/deephealthproject/eddl.git" - EDDL_BRANCH = "master" - EDDL_REVISION = sh(returnStdout: true, script: "git ls-remote ${EDDL_REPOSITORY} ${EDDL_BRANCH} | awk '{print \$1}'").trim() - // PyEDDL Settings - PYEDDL_REPOSITORY = "https://github.com/deephealthproject/pyeddl.git" - PYEDDL_BRANCH = "master" - PYEDDL_REVISION = sh(returnStdout: true, script: "git ls-remote ${PYEDDL_REPOSITORY} ${PYEDDL_BRANCH} | awk '{print \$1}'").trim() - // Extract additional info - NORMALIZED_BRANCH_NAME = sh(returnStdout: true, script: "echo ${BRANCH_NAME} | sed 's+/+-+g'").trim() - REPO_TAG = sh(returnStdout: true, script: "tag=\$(git tag -l --points-at HEAD); if [[ -n \${tag} ]]; then echo \${tag}; else git rev-parse --short HEAD --short; fi").trim() - // Docker Settings - DOCKER_IMAGE_LATEST = sh(returnStdout: true, script: "if [ '${GIT_BRANCH}' = 'master' ]; then echo 'true'; else echo 'false'; fi").trim() - DOCKER_IMAGE_TAG = "${NORMALIZED_BRANCH_NAME}_build${BUILD_NUMBER}" - DOCKER_IMAGE_TAG_EXTRA = "${REPO_TAG} ${REPO_TAG}_build${BUILD_NUMBER}" - DOCKER_REPOSITORY_OWNER = "dhealth" - // Docker credentials - registryCredential = 'dockerhub-deephealthproject' - // Skip DockerHub - DOCKER_LOGIN_DONE = true - } - stages { - - stage('Configure') { - steps { - sh 'git fetch --tags' - sh 'printenv' - } - } - - stage('Build') { - - parallel { - - stage('Master Build') { - when { - allOf { - branch 'master' ; - not { triggeredBy 'UpstreamCause' } - } - } - steps { - script { - sh 'make build' - docker.withRegistry( '', registryCredential ) { - sh 'CONFIG_FILE="" DOCKER_IMAGE_TAG_EXTRA="" make push_libs_toolkit' - sh 'CONFIG_FILE="" DOCKER_IMAGE_TAG_EXTRA="" make push_pylibs_toolkit' - } - } - } - } - - stage('Development Build') { - when { - allOf { - not { branch "master" } ; - triggeredBy 'UpstreamCause' - } - } - steps { - script { - sh 'CONFIG_FILE="" make build' - docker.withRegistry( '', registryCredential ) { - sh 'CONFIG_FILE="" DOCKER_IMAGE_TAG_EXTRA="" make push_libs_toolkit' - sh 'CONFIG_FILE="" DOCKER_IMAGE_TAG_EXTRA="" make push_pylibs_toolkit' - } - } - } - } - } - } - - stage('Test') { - - parallel { - - stage('Test EDDL') { - agent { - docker { image '${DOCKER_REPOSITORY_OWNER}/libs-toolkit:${DOCKER_IMAGE_TAG}' } - } - steps { - sh 'cd ${EDDL_SRC}/build && ctest -C Debug -VV' - } - } - - stage('Test ECVL') { - agent { - docker { image '${DOCKER_REPOSITORY_OWNER}/libs-toolkit:${DOCKER_IMAGE_TAG}' } - } - steps { - sh 'cd ${ECVL_SRC}/build && ctest -C Debug -VV' - } - } - - stage('Test PyEDDL') { - agent { - docker { image '${DOCKER_REPOSITORY_OWNER}/pylibs-toolkit:${DOCKER_IMAGE_TAG}' } - } - steps { - sh 'cd ${PYEDDL_SRC} && pytest tests' - sh 'cd ${PYEDDL_SRC}/examples && python3 Tensor/eddl_tensor.py' - sh 'cd ${PYEDDL_SRC}/examples && python3 NN/other/eddl_ae.py --epochs 1' - } - } - - stage('Test PyECVL') { - agent { - docker { image '${DOCKER_REPOSITORY_OWNER}/pylibs-toolkit:${DOCKER_IMAGE_TAG}' } - } - steps { - sh 'cd ${PYECVL_SRC} && pytest tests' - sh 'cd ${PYECVL_SRC}/examples && python3 dataset.py "${ECVL_SRC}/examples/data/mnist/mnist.yml"' - sh 'cd ${PYECVL_SRC}/examples && python3 ecvl_eddl.py "${ECVL_SRC}/examples/data/test.jpg" "${ECVL_SRC}/examples/data/mnist/mnist.yml"' - sh 'cd ${PYECVL_SRC}/examples && python3 img_format.py "${ECVL_SRC}/examples/data/nifti/LR_nifti.nii" "${ECVL_SRC}/data/isic_dicom/ISIC_0000008.dcm"' - sh 'cd ${PYECVL_SRC}/examples && python3 imgproc.py "${ECVL_SRC}/examples/data/test.jpg"' - sh 'cd ${PYECVL_SRC}/examples && python3 openslide.py "${ECVL_SRC}/examples/data/hamamatsu/test3-DAPI 2 (387).ndpi"' - sh 'cd ${PYECVL_SRC}/examples && python3 read_write.py "${ECVL_SRC}/examples/data/test.jpg test_mod.jpg"' - } - } - } - } - - stage('Publish') { - - parallel { - - stage('Publish Master Build') { - when { - allOf { - branch 'master'; - not { triggeredBy 'UpstreamCause' } - } - } - steps { - script { - docker.withRegistry( '', registryCredential ) { - sh ''' - tag=$(git tag -l --points-at HEAD); - if [ -n "${tag}" ]; then - REPO_TAG="${tag}" - else - REPO_TAG=$(git rev-parse --short HEAD --short) - fi - DOCKER_IMAGE_TAG_EXTRA="${DOCKER_IMAGE_TAG_EXTRA} ${REPO_TAG} ${REPO_TAG}_build${BUILD_NUMBER}" - echo "Pushing tags: ${DOCKER_IMAGE_TAG_EXTRA}" - make push - ''' - } - } - } - } - - stage('Publish Development Build') { - when { - allOf { - not { branch "master" }; - triggeredBy 'UpstreamCause' - } - } - steps { - script { - docker.withRegistry( '', registryCredential ) { - sh ''' - tag=$(git tag -l --points-at HEAD); - if [ -n "${tag}" ]; then - REPO_TAG="${tag}" - else - REPO_TAG=$(git rev-parse --short HEAD --short) - fi - DOCKER_IMAGE_TAG_EXTRA="${DOCKER_IMAGE_TAG_EXTRA} ${REPO_TAG} ${REPO_TAG}_build${BUILD_NUMBER}" - echo "Pushing tags: ${DOCKER_IMAGE_TAG_EXTRA}" - CONFIG_FILE="" make push - ''' - } - } - } - } - } - } - } - - post { - always { - echo 'One way or another, I have finished' - } - success { - echo "Docker images successfully build and published with tags: ${DOCKER_IMAGE_TAG} ${DOCKER_IMAGE_TAG_EXTRA}" - echo "Library revisions..." - echo "* ECVL revision: ${ECVL_REVISION}" - echo "* EDDL revision: ${EDDL_REVISION}" - echo "* PyECVL revision: ${PYECVL_REVISION}" - echo "* PyEDDL revision: ${PYEDDL_REVISION}" - } - unstable { - echo 'I am unstable :/' - } - failure { - echo 'I failed :(' - } - cleanup { - // sh 'make clean' - deleteDir() /* clean up our workspace */ - sh 'docker images' - sh 'docker image prune -f' - sh 'if [ "$(docker images | grep -E \"(l|pyl)ibs([[:space:]]|-toolkit)\")" ]; then docker images | grep -E "(l|pyl)ibs([[:space:]]|-toolkit)" | awk \'{print $3}\' | uniq | xargs docker rmi -f; fi;' - } - } -} \ No newline at end of file diff --git a/LICENCE b/LICENCE new file mode 100644 index 0000000..8feab66 --- /dev/null +++ b/LICENCE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019-2020 CRS4 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile index 2d307f0..5e9afe0 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,40 @@ -# version -VERSION := 0.1 - # set bash as default interpreter SHELL := /bin/bash +# detect OS +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Linux) + XARGS_OPT = --no-run-if-empty +endif +ifeq ($(UNAME_S),Darwin) + XARGS_OPT = +endif + +# config file +CONFIG_FILE ?= settings.conf +ifneq ($(wildcard $(CONFIG_FILE)),) +include $(CONFIG_FILE) +endif + # date.time as build number BUILD_NUMBER := $(or ${BUILD_NUMBER},$(shell date '+%Y%m%d.%H%M%S')) +# set build target: (CPU, GPU) +BUILD_TARGET := $(or $(BUILD_TARGET),CPU) +build_target_opts := --build-arg BUILD_TARGET=$(BUILD_TARGET) + +# log file with dependencies +IMAGES_LOG := .images.log +LIBRARIES_LOG := .libraries.log +DEPENDENCIES_LOG := .dependencies.log +DEPENDENCY_GRAPH_FILE := graph.dot + # set docker user credentials DOCKER_USER := $(or ${DOCKER_USER},${USER}) DOCKER_PASSWORD := ${DOCKER_PASSWORD} # use DockerHub as default registry -DOCKER_REGISTRY := $(or ${DOCKER_REGISTRY},registry.hub.docker.com) +DOCKER_REGISTRY := $(or ${DOCKER_REGISTRY},) # set Docker repository DOCKER_REPOSITORY_OWNER := $(or ${DOCKER_REPOSITORY_OWNER},${DOCKER_USER}) @@ -20,15 +42,54 @@ DOCKER_REPOSITORY_OWNER := $(or ${DOCKER_REPOSITORY_OWNER},${DOCKER_USER}) runtime_suffix = develop_suffix = -toolkit +# extract info about repository revision +LIBS_TAG := $(shell git tag -l --points-at HEAD | tail -n 1) +LIBS_REVISION := $(shell git rev-parse --short HEAD | sed -E 's/-//; s/ .*//') +LIBS_BRANCH := $(shell git name-rev --name-only HEAD | sed -E 's+(remotes/|origin/)++g; s+/+-+g; s/ .*//') +LIBS_VERSION := $(shell if [[ -n "${LIBS_TAG}" ]]; then echo ${LIBS_TAG}; else echo ${LIBS_REVISION}; fi) + +# set container version equal to the repository version +CONTAINER_VERSION := $(LIBS_VERSION) + +# set base image version +DOCKER_BASE_IMAGE_VERSION := ${DOCKER_BASE_IMAGE_VERSION} + # latest tag settings DOCKER_IMAGE_LATEST := $(or ${DOCKER_IMAGE_LATEST},false) # extra tags DOCKER_IMAGE_TAG_EXTRA := ${DOCKER_IMAGE_TAG_EXTRA} +# set tag suffix +DOCKER_IMAGE_TAG_SUFFIX := $(shell [[ "$(ENABLE_TARGET_SUFFIX_IMAGE_TAG)" == "true" ]] && echo -$(BUILD_TARGET) | tr '[:upper:]' '[:lower:]') + # set default Docker image TAG DOCKER_IMAGE_TAG := $(or ${DOCKER_IMAGE_TAG},${BUILD_NUMBER}) +# set docker-libs version tag +DOCKER_LIBS_IMAGE_VERSION_TAG := $(or ${LIBS_IMAGE_VERSION_TAG},${LIBS_TAG},${LIBS_VERSION})$(DOCKER_IMAGE_TAG_SUFFIX) +DOCKER_LIBS_EXTRA_TAGS := $(LIBS_VERSION)$(DOCKER_IMAGE_TAG_SUFFIX) $(LIBS_REVISION)$(DOCKER_IMAGE_TAG_SUFFIX) + +# set default Base images +DOCKER_BASE_IMAGE_SKIP_PULL := $(or ${DOCKER_BASE_IMAGE_SKIP_PULL},true) +DOCKER_UBUNTU_IMAGE := $(or ${DOCKER_UBUNTU_IMAGE},ubuntu:18.04) +DOCKER_NVIDIA_DEVELOP_IMAGE := $(or ${DOCKER_NVIDIA_DEVELOP_IMAGE},nvidia/cuda:10.1-devel-ubuntu18.04) +DOCKER_NVIDIA_RUNTIME_IMAGE := $(or ${DOCKER_NVIDIA_RUNTIME_IMAGE},nvidia/cuda:10.1-runtime-ubuntu18.04) + +# extract name and tag of nvidia images +DOCKER_UBUNTU_IMAGE_NAME := $(shell echo ${DOCKER_UBUNTU_IMAGE} | sed -e 's+:.*++') +DOCKER_UBUNTU_IMAGE_TAG := $(shell echo ${DOCKER_UBUNTU_IMAGE} | sed -E 's@.+:(.+)@\1@') +DOCKER_NVIDIA_DEVELOP_IMAGE_NAME := $(shell echo ${DOCKER_NVIDIA_DEVELOP_IMAGE} | sed -e 's+:.*++') +DOCKER_NVIDIA_DEVELOP_IMAGE_TAG := $(shell echo ${DOCKER_NVIDIA_DEVELOP_IMAGE} | sed -E 's@.+:(.+)@\1@') +DOCKER_NVIDIA_RUNTIME_IMAGE_NAME := $(shell echo ${DOCKER_NVIDIA_RUNTIME_IMAGE} | sed -e 's+:.*++') +DOCKER_NVIDIA_RUNTIME_IMAGE_TAG := $(shell echo ${DOCKER_NVIDIA_RUNTIME_IMAGE} | sed -E 's@.+:(.+)@\1@') + +DOCKER_BASE_IMAGE_VERSION_TAG := $(CONTAINER_VERSION)$(DOCKER_IMAGE_TAG_SUFFIX) +EDDL_IMAGE_VERSION_TAG := $(or ${EDDL_IMAGE_VERSION_TAG},${EDDL_REVISION}) +ECVL_IMAGE_VERSION_TAG := $(or ${ECVL_IMAGE_VERSION_TAG},${ECVL_REVISION}) +PYEDDL_IMAGE_VERSION_TAG := $(or ${PYEDDL_IMAGE_VERSION_TAG},${PYEDDL_REVISION}) +PYECVL_IMAGE_VERSION_TAG := $(or ${PYECVL_IMAGE_VERSION_TAG},${PYECVL_REVISION}) + # current path CURRENT_PATH := $(PWD) @@ -44,75 +105,116 @@ PYEDDL_LIB_PATH = ${LOCAL_PYLIBS_PATH}/pyeddl ECVL_REPOSITORY := $(or ${ECVL_REPOSITORY},https://github.com/deephealthproject/ecvl.git) ECVL_BRANCH := $(or ${ECVL_BRANCH},master) ECVL_REVISION := ${ECVL_REVISION} +ECVL_TAG := # PyECVL repository PYECVL_REPOSITORY := $(or ${PYECVL_REPOSITORY},https://github.com/deephealthproject/pyecvl.git) PYECVL_BRANCH := $(or ${PYECVL_BRANCH},master) PYECVL_REVISION := ${PYECVL_REVISION} +PYECVL_TAG := # EDDL repository EDDL_REPOSITORY := $(or ${EDDL_REPOSITORY},https://github.com/deephealthproject/eddl.git) EDDL_BRANCH := $(or ${EDDL_BRANCH},master) EDDL_REVISION := ${EDDL_REVISION} +EDDL_TAG := # PyEDDL repository PYEDDL_REPOSITORY := $(or ${PYEDDL_REPOSITORY},https://github.com/deephealthproject/pyeddl.git) PYEDDL_BRANCH := $(or ${PYEDDL_BRANCH},master) -PYEDDL_REVISION := ${PYEDDL_REVISION} - -# config file -CONFIG_FILE ?= settings.sh -ifneq ($(wildcard $(CONFIG_FILE)),) -include $(CONFIG_FILE) +PYEDDL_REVISION := ${PYEDDL_REVISION} +PYEDDL_TAG := + +# disable image pull +DISABLE_PULL ?= 0 +_DO_NOT_PULL_DOCKER_IMAGES = 0 +ifeq ($(DISABLE_PULL),$(filter $(DISABLE_PULL),1 true TRUE)) +$(info Docker image pull disabled) +_DO_NOT_PULL_DOCKER_IMAGES = 1 endif # set no cache option -DISABLE_CACHE ?= -BUILD_CACHE_OPT ?= -ifneq ("$(DISABLE_CACHE)", "") +DISABLE_CACHE ?= 0 +BUILD_CACHE_OPT ?= +ifeq ($(DISABLE_CACHE),$(filter $(DISABLE_CACHE),1 true TRUE)) +$(info Docker cache disabled) BUILD_CACHE_OPT = --no-cache +_DO_NOT_USE_DOCKER_CACHE = 1 endif # enable latest tags push_latest_tags = false -ifeq ("${DOCKER_IMAGE_LATEST}", "true") +ifeq ($(DOCKER_IMAGE_LATEST),$(filter $(DOCKER_IMAGE_LATEST),1 true TRUE)) push_latest_tags = true endif # auxiliary flag DOCKER_LOGIN_DONE := $(or ${DOCKER_LOGIN_DONE},false) +# Arguments to execute tests with Docker +DOCKER_RUN := docker run -i --rm #-u 1000:1000 +ifneq (${GPU_RUNTIME},) + DOCKER_RUN := ${DOCKER_RUN} ${GPU_RUNTIME} +endif + +define build_new_image + echo "Building Docker image '${image_name}' ( tags: ${tag} ${extra_tags})..." ; \ + $(eval tags := $(filter-out undefined,$(foreach tag,$(extra_tags),-t $(image_name):$(tag)))) + $(eval _DO_NOT_PULL_DOCKER_IMAGES := 1) + cd ${image} \ + && docker build ${BUILD_CACHE_OPT} \ + -f ${target}.Dockerfile \ + ${base} ${toolkit} ${extra_args} \ + -t ${image_name}:${tag} ${tags} ${latest_tags} ${labels} . +endef + define build_image $(eval image := $(1)) $(eval target := $(2)) - $(eval labels := $(3)) - $(eval base := $(if $(4), --build-arg BASE_IMAGE=$(4))) - $(eval toolkit := $(if $(5), --build-arg TOOLKIT_IMAGE=$(5))) - $(eval image_name := ${DOCKER_IMAGE_PREFIX}${image}${${target}_suffix}) + $(eval tag := $(3)) + $(eval labels := $(4)) + $(eval base := $(if $(5), --build-arg BASE_IMAGE=$(5))) + $(eval toolkit := $(if $(6), --build-arg TOOLKIT_IMAGE=$(6))) + $(eval extra_tags := $(7)) + $(eval extra_args := $(8)) + $(eval image_name := ${DOCKER_IMAGE_PREFIX}${target}${${target}_suffix}) + $(eval full_image_name := $(shell prefix=""; if [ -n "${DOCKER_REGISTRY}" ]; then prefix="${DOCKER_REGISTRY}/"; fi; echo "${prefix}${DOCKER_REPOSITORY_OWNER}/${image_name}")) $(eval latest_tags := $(shell if [ "${push_latest_tags}" == "true" ]; then echo "-t ${image_name}:latest"; fi)) - @echo "Building Docker image '${image_name}'..." - cd ${image} \ - && docker build ${BUILD_CACHE_OPT} \ - -f ${target}.Dockerfile \ - ${base} ${toolkit} \ - -t ${image_name}:${DOCKER_IMAGE_TAG} ${latest_tags} ${labels} . + $(eval tagged_image := ${image_name}:${tag}) + $(eval images := $(shell docker images -q ${tagged_image})) + $(eval exists := $(shell curl --silent -f -lSL https://index.docker.io/v1/repositories/${full_image_name}/tags/${tag} 2>/dev/null)) + @printf "\n\n" ; \ + $(if $(or $(findstring ${_DO_NOT_USE_DOCKER_CACHE},1),$(findstring ${_DO_NOT_PULL_DOCKER_IMAGES},1)),\ + $(call build_new_image), + $(if ${images},\ + @echo "Docker image '${tagged_image}' exists (id: ${images})", \ + $(if $(and ${exists},$(findstring ${_DO_NOT_PULL_DOCKER_IMAGES},0)), \ + @echo "Pulling image '${full_image_name}:${tag}'..."; + docker pull ${full_image_name}:${tag} && docker tag ${full_image_name}:${tag} ${tagged_image}, \ + @echo "Docker image '${full_image_name}:${tag}' doesn't exist", \ + $(call build_new_image) + ) + ) + ) + $(call log_image_revision,$(target),$(tag),extend,$(shell echo $(5))) + $(if $(6),$(call log_image_revision,$(target),$(tag),use,$(shell echo $(6)))) endef define push_image $(eval image := $(1)) - $(eval target := $(2)) - $(eval image_name := ${DOCKER_IMAGE_PREFIX}${image}${${target}_suffix}) + $(eval tag := $(or $(2),${DOCKER_IMAGE_TAG})) + $(eval extra_tags := $(filter-out $(tag),$(foreach t,$(3) ${DOCKER_IMAGE_TAG_EXTRA},$(t)))) + $(eval image_name := ${DOCKER_IMAGE_PREFIX}${image}) $(eval full_image_name := $(shell prefix=""; if [ -n "${DOCKER_REGISTRY}" ]; then prefix="${DOCKER_REGISTRY}/"; fi; echo "${prefix}${DOCKER_REPOSITORY_OWNER}/${image_name}")) - $(eval full_tag := ${full_image_name}:$(DOCKER_IMAGE_TAG)) + $(eval full_tag := ${full_image_name}:$(tag)) $(eval latest_tag := ${full_image_name}:latest) - $(eval tags := ${DOCKER_IMAGE_TAG_EXTRA}) @echo "Tagging images... " - docker tag ${image_name}:$(DOCKER_IMAGE_TAG) ${full_tag} - @if [ ${push_latest_tags} == true ]; then docker tag ${image_name}:$(DOCKER_IMAGE_TAG) ${latest_tag}; fi + docker tag ${image_name}:$(tag) ${full_tag} + @if [ ${push_latest_tags} == true ]; then docker tag ${image_name}:$(tag) ${latest_tag}; fi @echo "Pushing Docker image '${image_name}'..." docker push ${full_tag} @if [ ${push_latest_tags} == true ]; then docker push ${latest_tag}; fi - @for tag in $(tags); \ + @for tag in $(extra_tags); \ do \ img_tag=${full_image_name}:$$tag ; \ docker tag ${full_tag} $$img_tag ; \ @@ -126,197 +228,795 @@ endef # 4 --> REVISION # 5 --> RECURSIVE SUBMODULE CLONE (true|false) define clone_repository - @if [ ! -d ${1} ]; then \ + if [ ! -d ${1} ]; then \ git clone --branch "${3}" ${2} ${1} \ && cd "${1}" \ && if [ -n "${4}" ]; then git reset --hard ${4} -- ; fi \ && if [ ${5} == true ]; then git submodule update --init --recursive ; fi \ + && cd - ; \ else \ echo "Using existing ${1} repository..." ; \ fi endef -define clean_build - $(eval lib := $(1)) # libs or pylibs - @echo "Removing $(lib)/{eddl,ecvl}..." - @rm -rf $(lib)/{*eddl,*ecvl} - @echo "Removing sources... DONE" - @echo "Stopping docker containers... " - # We'd use xargs --no-run-if-empty if we only wanted to work on Linux, but we also care - # about Mac OSX so we ignore an error in the following line (which will happen if xargs - # is called without any input on stdin) - @docker ps -a | grep -E "(${DOCKER_IMAGE_PREFIX})?$(lib)-(runtime|develop)" | awk '{print $$1}' | xargs docker rm -f || true - @echo "Stopping docker containers... DONE" - @echo "Removing docker images... " - @docker images | grep -E "(${DOCKER_IMAGE_PREFIX})?$(lib)-(runtime|develop)" | awk '{print $$3}' | xargs docker rmi -f || true - @echo "Removing docker images... DONE" +define clean_sources + $(eval path := $(1)) + @printf "Removing sources '$(path)'... " + @rm -rf $(path) + @printf "DONE\n" +endef + + +define clean_image + $(eval image := $(1)) + @printf "Stopping docker containers instances of image '$(image)'... " + @docker ps -a | grep -E "^$(image)\s" | awk '{print $$1}' | xargs ${XARGS_OPT} docker rm -f || true + @printf "DONE\n" + @printf "Removing docker image '$(image)'... " + @docker images | grep -E "^$(image)\s" | awk '{print $$1 ":" $$2}' | xargs ${XARGS_OPT} docker rmi -f || true + @docker images | grep -E "^${DOCKER_REPOSITORY_OWNER}/$(image)\s" | awk '{print $$1 ":" $$2}' | xargs ${XARGS_OPT} docker rmi -f || true + @printf "DONE\n" + @printf "Removing unused docker image... " + @docker image prune -f + @printf "DONE\n" endef # 1: library path -# 2: actual revision -define get_revision -$(shell if [[ -z "${2}" ]]; then cd ${1} && git rev-parse HEAD; else echo "${2}" ; fi) +define get_tag + $(eval tag := $(shell cd ${1} && git tag -l --points-at HEAD | tail -n 1)) \ + $(strip $(shell echo ${tag};)) +endef + +# 1: library path +define get_revision + $(eval head := $(shell cd ${1} && git rev-parse --short HEAD | sed -E 's/-//; s/ .*//')) \ + $(strip $(shell echo ${head};)) +endef + +# 1: submodule path +# 2: library name +# 3: revision +define submodule_revision + $(eval rev = $(shell cd $(1) && git submodule status -- $(2) | sed -E 's/-//; s/ .*//' | cut -c1-7;)) \ + $(shell if [[ -n "$(3)" ]]; then echo $(3); else echo $(rev); fi) +endef + +# 1: library path +# 2: library name +define set_library_revision + $(eval lib := $(shell echo $(2) | tr a-z A-Z)) + $(eval ${lib}_REVISION := $(call get_revision,$(1)/$(2))) + $(eval $(lib)_TAG = $(call get_tag,$(1)/$(2))) + $(eval ${lib}_IMAGE_VERSION_TAG = $(or $(filter %-cpu %-gpu,$(${lib}_IMAGE_VERSION_TAG)),$(or ${${lib}_IMAGE_VERSION_TAG},${${lib}_TAG},${${lib}_REVISION})$(DOCKER_IMAGE_TAG_SUFFIX))) + @echo "${lib} rev: ${${lib}_REVISION} ${${lib}_TAG} (image-tag: ${${lib}_IMAGE_VERSION_TAG})" +endef + +define log_library_revision + $(file >>${LIBRARIES_LOG},$(1) [label="$(1) (rev. ${$(1)_REVISION} ${$(1)_TAG})"];) \ + $(if $(2),$(file >>${DEPENDENCIES_LOG},$(2) -> $(1) [label=" << depends on >> ",color="black:invis:black"];)) +endef + +define log_image_revision + $(eval A := $(shell echo $(1):$(2) | sed -e 's+[-:/\.]+_+g')) + $(eval B := $(shell echo $(4) | sed -e 's+[-:/\.]+_+g')) + $(eval relation_style := $(shell \ + if [[ "$(3)" == "install" ]]; then echo 'style=dashed,color="black"' ; \ + elif [[ "$(3)" == "extend" ]]; then echo 'color="black"' ; \ + elif [[ "$(3)" == "use" ]]; then echo 'style=dotted,color="black"' ; \ + else echo "style=dotted,color=gray" ; fi \ + )) + $(file >>${IMAGES_LOG},${A} [label="$(1) (tag. $(2))"];) \ + $(if $(4),$(file >>${DEPENDENCIES_LOG},$(A) -> $(B) [label=" << $(3) >> ",$(relation_style)];)) endef .DEFAULT_GOAL := help +# version +version: ## Output the current version of this Makefile + @echo $(LIBS_VERSION) + help: ## Show help @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) +libraries_list: + @sort -u ${LIBRARIES_LOG} + +images_list: + @sort -u ${IMAGES_LOG} + +dependencies_list: + @sort -u ${DEPENCIES_LOG} + +dependency_graph: ## make a dependency graph of the involved libraries + @echo "digraph {" > ${DEPENDENCY_GRAPH_FILE} \ + && sort -u ${LIBRARIES_LOG} >> ${DEPENDENCY_GRAPH_FILE} \ + && sort -u ${IMAGES_LOG} >> ${DEPENDENCY_GRAPH_FILE} \ + && sort -u ${DEPENDENCIES_LOG} >> ${DEPENDENCY_GRAPH_FILE} \ + && echo "}" >> ${DEPENDENCY_GRAPH_FILE} \ + && if [[ $$(command -v dot) ]]; then \ + dot -Tpdf ${DEPENDENCY_GRAPH_FILE} -o ${DEPENDENCY_GRAPH_FILE}.pdf; \ + fi + +##################################################################################################################################### +############# Clone sources ############# +##################################################################################################################################### + libs_folder: - $(info Creating ${LOCAL_LIBS_PATH} folder...) - @mkdir -p ${LOCAL_LIBS_PATH} + $(if $(wildcard ${LOCAL_LIBS_PATH}),, \ + $(info Creating ${LOCAL_LIBS_PATH} folder...) ; \ + @mkdir -p ${LOCAL_LIBS_PATH} ; \ + ) + +_eddl_folder: libs_folder + $(if $(wildcard ${EDDL_LIB_PATH}),$(info Using existing '${EDDL_LIB_PATH}' repository), \ + $(call clone_repository,${EDDL_LIB_PATH},${EDDL_REPOSITORY},${EDDL_BRANCH},${EDDL_REVISION},true) ; \ + ) + +eddl_folder: _eddl_folder + $(call set_library_revision,libs,eddl) \ + $(call log_library_revision,EDDL) + + +define clone_ecvl + $(if $(wildcard ${ECVL_LIB_PATH}),$(info Using existing '${ECVL_LIB_PATH}' repository), \ + $(call clone_repository,${ECVL_LIB_PATH},${ECVL_REPOSITORY},${ECVL_BRANCH},${ECVL_REVISION},true) ; \ + ) +endef -ecvl_folder: libs_folder - $(call clone_repository,${ECVL_LIB_PATH},${ECVL_REPOSITORY},${ECVL_BRANCH},${ECVL_REVISION},true) +_ecvl_folder: + $(call clone_ecvl) -eddl_folder: libs_folder - $(call clone_repository,${EDDL_LIB_PATH},${EDDL_REPOSITORY},${EDDL_BRANCH},${EDDL_REVISION},true) +ecvl_folder: _ecvl_folder pyecvl_folder + $(call set_library_revision,libs,ecvl) \ + $(call log_library_revision,ECVL) pylibs_folder: @mkdir -p ${LOCAL_PYLIBS_PATH} -pyecvl_folder: pylibs_folder - $(call clone_repository,${PYECVL_LIB_PATH},${PYECVL_REPOSITORY},${PYECVL_BRANCH},${PYECVL_REVISION},false) - @echo "Copying revision '${ECVL_REVISION}' of ECVL library..." - @rm -rf ${PYECVL_LIB_PATH}/third_party/ecvl - @cp -a ${CURRENT_PATH}/${ECVL_LIB_PATH} ${CURRENT_PATH}/${PYECVL_LIB_PATH}/third_party/ecvl +define pyeddl_shallow_clone + $(if $(wildcard ${PYEDDL_LIB_PATH}),$(info Using existing '${PYEDDL_LIB_PATH}' repository), \ + $(call clone_repository,${PYEDDL_LIB_PATH},${PYEDDL_REPOSITORY},${PYEDDL_BRANCH},${PYEDDL_REVISION},false) ; \ + ) +endef + +define pyeddl_clone_dependencies + $(eval EDDL_REVISION = $(call submodule_revision,${PYEDDL_LIB_PATH}/third_party,eddl,${EDDL_REVISION})) + if [[ -d ${EDDL_LIB_PATH} ]]; then \ + echo "Using existing '${EDDL_LIB_PATH}' repository" ; \ + else \ + $(call clone_repository,${EDDL_LIB_PATH},${EDDL_REPOSITORY},${EDDL_BRANCH},${EDDL_REVISION},true) ; \ + printf "Copying revision '${EDDL_REVISION}' of EDDL library... " ; \ + rm -rf ${PYEDDL_LIB_PATH}/third_party/eddl && cp -a ${EDDL_LIB_PATH} ${PYEDDL_LIB_PATH}/third_party/eddl ; \ + printf "DONE\n" ; \ + $(call log_library_revision,EDDL,PYEDDL) \ + fi +endef + +_pyeddl_shallow_clone: pylibs_folder + $(call pyeddl_shallow_clone) + +_pyeddl_dependencies: _pyeddl_shallow_clone + $(call pyeddl_clone_dependencies) + +pyeddl_folder: _pyeddl_dependencies + $(call set_library_revision,libs,eddl) \ + $(call set_library_revision,pylibs,pyeddl) \ + $(call log_library_revision,PYEDDL) + +define pyecvl_shallow_clone + $(if $(wildcard ${PYECVL_LIB_PATH}),$(info Using existing '${PYECVL_LIB_PATH}' repository), \ + $(call clone_repository,${PYECVL_LIB_PATH},${PYECVL_REPOSITORY},${PYECVL_BRANCH},${PYECVL_REVISION},false) ; \ + ) +endef + +define pyecvl_resolve_dependencies + $(eval PYEDDL_REVISION = $(call submodule_revision,${PYECVL_LIB_PATH}/third_party,pyeddl,${PYEDDL_REVISION})) + $(eval ECVL_REVISION = $(call submodule_revision,${PYECVL_LIB_PATH}/third_party,ecvl,${ECVL_REVISION})) + $(call log_library_revision,PYEDDL,PYECVL) \ + $(call log_library_revision,ECVL,PYECVL) \ + if [[ -d ${PYEDDL_LIB_PATH} ]]; then \ + echo "Using existing '${PYEDDL_LIB_PATH}' repository" ; \ + else \ + $(call pyeddl_shallow_clone) \ + printf "Copying revision '${PYEDDL_REVISION}' of PYEDDL library... " ; \ + rm -rf ${PYECVL_LIB_PATH}/third_party/pyeddl && cp -a ${PYEDDL_LIB_PATH} ${PYECVL_LIB_PATH}/third_party/pyeddl ; \ + printf "DONE\n" ; \ + $(call log_library_revision,PYEDDL,PYECVL) \ + fi + if [[ -d ${ECVL_LIB_PATH} ]]; then \ + echo "Using existing '${ECVL_LIB_PATH}' repository" ; \ + else \ + echo "Using ECVL revision '${ECVL_REVISION}'" ; \ + $(call clone_ecvl) \ + printf "Copying revision '${ECVL_REVISION}' of ECVL library... " ; \ + rm -rf ${PYECVL_LIB_PATH}/third_party/ecvl && cp -a ${ECVL_LIB_PATH} ${PYECVL_LIB_PATH}/third_party/ecvl ; \ + printf "DONE\n" ; \ + $(call log_library_revision,ECVL,PYECVL) \ + fi +endef + +_pyecvl_shallow_clone: pylibs_folder + $(call pyecvl_shallow_clone) + +_pyecvl_first_level_dependencies: _pyecvl_shallow_clone + $(call pyecvl_resolve_dependencies) + +_pyecvl_second_level_dependencies: _pyecvl_first_level_dependencies + $(call set_library_revision,libs,ecvl) \ + $(call set_library_revision,pylibs,pyeddl) \ + $(call pyeddl_clone_dependencies) -pyeddl_folder: pylibs_folder - $(call clone_repository,${PYEDDL_LIB_PATH},${PYEDDL_REPOSITORY},${PYEDDL_BRANCH},${PYEDDL_REVISION},false) - @echo "Copying revision '${EDDL_REVISION}' of EDDL library..." - @rm -rf ${PYEDDL_LIB_PATH}/third_party/eddl - @cp -a ${EDDL_LIB_PATH} ${PYEDDL_LIB_PATH}/third_party/eddl +pyecvl_folder: _pyecvl_second_level_dependencies + $(call set_library_revision,pylibs,pyecvl) + $(call log_library_revision,PYECVL) -apply_pyeddl_patches: - # TODO: remove this patch when not required +# TODO: remove this patch when not required +apply_pyeddl_patches: pyeddl_folder @echo "Applying patches to the EDDL repository..." - $(call clone_repository,${PYEDDL_LIB_PATH},${PYEDDL_REPOSITORY},${PYEDDL_BRANCH},${PYEDDL_REVISION},false) cd ${EDDL_LIB_PATH} && git apply ../../${PYEDDL_LIB_PATH}/eddl_0.3.patch || true - # @echo "Copying revision '${EDDL_REVISION}' of EDDL library..." - # @rm -rf ${PYEDDL_LIB_PATH}/third_party/eddl - # @cp -a ${EDDL_LIB_PATH} ${PYEDDL_LIB_PATH}/third_party/eddl +# # TODO: remove this patch when not required +apply_pyecvl_patches: + + +##################################################################################################################################### +############# Build Docker images ############# +##################################################################################################################################### # Targets to build container images -build: _build ## Build and tag all Docker images +build: _build ## Build all Docker images _build: \ - build_libs \ - build_pylibs + build_eddl build_ecvl build_libs \ + build_eddl_toolkit build_ecvl_toolkit build_libs_toolkit \ + build_pyeddl build_pyecvl build_pylibs \ + build_pyeddl_toolkit build_pyecvl_toolkit build_pylibs_toolkit + +############# libs-toolkit ############# + +_build_libs_base_toolkit: + $(if $(findstring $(BUILD_TARGET), GPU),\ + echo "Building for GPU"; \ + $(call build_image,libs,libs-base-toolkit,${DOCKER_BASE_IMAGE_VERSION_TAG}, --label CONTAINER_VERSION=$(CONTAINER_VERSION),$(DOCKER_NVIDIA_DEVELOP_IMAGE),,,${build_target_opts}) \ + $(call log_image_revision,$(DOCKER_NVIDIA_DEVELOP_IMAGE_NAME),$(DOCKER_NVIDIA_DEVELOP_IMAGE_TAG)),\ + echo "Building for CPU"; \ + $(call build_image,libs,libs-base-toolkit,${DOCKER_BASE_IMAGE_VERSION_TAG}, --label CONTAINER_VERSION=$(CONTAINER_VERSION),$(DOCKER_UBUNTU_IMAGE),,,${build_target_opts}) \ + $(call log_image_revision,$(DOCKER_UBUNTU_IMAGE_NAME),$(DOCKER_UBUNTU_IMAGE_TAG)) \ + ) + +build_eddl_toolkit: eddl_folder _build_libs_base_toolkit apply_pyeddl_patches ## Build 'eddl-toolkit' image + $(call build_image,libs,eddl-toolkit,${EDDL_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION) \ + --label EDDL_REPOSITORY=${EDDL_REPOSITORY} \ + --label EDDL_BRANCH=${EDDL_BRANCH} \ + --label EDDL_REVISION=${EDDL_REVISION},libs-base-toolkit:$(DOCKER_BASE_IMAGE_VERSION_TAG)) + $(call log_image_revision,eddl-toolkit,${EDDL_IMAGE_VERSION_TAG},install,EDDL) -build_libs: build_libs_toolkit ## Build and tag 'libs' image - $(call build_image,libs,runtime,\ - --label CONTAINER_VERSION=${DOCKER_IMAGE_TAG} \ +build_ecvl_toolkit: ecvl_folder build_eddl_toolkit ## Build 'ecvl-toolkit' image + $(call build_image,libs,ecvl-toolkit,${ECVL_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION) \ + --label ECVL_REPOSITORY=${ECVL_REPOSITORY} \ + --label ECVL_BRANCH=${ECVL_BRANCH} \ + --label ECVL_REVISION=${ECVL_REVISION},eddl-toolkit:$(EDDL_IMAGE_VERSION_TAG)) + $(call log_image_revision,ecvl-toolkit,${ECVL_IMAGE_VERSION_TAG},install,ECVL) + +build_libs_toolkit: build_ecvl_toolkit ## Build 'libs-toolkit' image + $(call build_image,libs,libs-toolkit,${DOCKER_LIBS_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION) \ --label EDDL_REPOSITORY=${EDDL_REPOSITORY} \ --label EDDL_BRANCH=${EDDL_BRANCH} \ - --label EDDL_REVISION=$(call get_revision,${EDDL_LIB_PATH},${EDDL_REVISION}) \ + --label EDDL_REVISION=${EDDL_REVISION} \ --label ECVL_REPOSITORY=${ECVL_REPOSITORY} \ --label ECVL_BRANCH=${ECVL_BRANCH} \ - --label ECVL_REVISION=$(call get_revision,${ECVL_LIB_PATH},${ECVL_REVISION}),libs-toolkit:$(DOCKER_IMAGE_TAG)) + --label ECVL_REVISION=${ECVL_REVISION},ecvl-toolkit:$(ECVL_IMAGE_VERSION_TAG),,${DOCKER_LIBS_EXTRA_TAGS}) + + +############# libs ############# + +_build_libs_base: _build_libs_base_toolkit + $(if $(findstring $(BUILD_TARGET), GPU),\ + $(call build_image,libs,libs-base,${DOCKER_BASE_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION),$(DOCKER_NVIDIA_RUNTIME_IMAGE),libs-base-toolkit:$(DOCKER_BASE_IMAGE_VERSION_TAG),,${build_target_opts})\ + $(call log_image_revision,$(DOCKER_NVIDIA_RUNTIME_IMAGE_NAME),$(DOCKER_NVIDIA_RUNTIME_IMAGE_TAG)),\ + $(call build_image,libs,libs-base,${DOCKER_BASE_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION),$(DOCKER_UBUNTU_IMAGE),libs-base-toolkit:$(DOCKER_BASE_IMAGE_VERSION_TAG),,${build_target_opts})\ + $(call log_image_revision,$(DOCKER_UBUNTU_IMAGE_NAME),$(DOCKER_UBUNTU_IMAGE_TAG))\ + ) + +build_eddl: _build_libs_base build_eddl_toolkit ## Build 'eddl' image + $(call build_image,libs,eddl,${EDDL_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION) \ + --label EDDL_REPOSITORY=${EDDL_REPOSITORY} \ + --label EDDL_BRANCH=${EDDL_BRANCH} \ + --label EDDL_REVISION=${EDDL_REVISION},libs-base:$(DOCKER_BASE_IMAGE_VERSION_TAG),eddl-toolkit:$(EDDL_IMAGE_VERSION_TAG)) + $(call log_image_revision,eddl,${EDDL_IMAGE_VERSION_TAG},install,EDDL) -build_libs_toolkit: ecvl_folder eddl_folder apply_pyeddl_patches ## Build and tag 'libs-toolkit' image - $(call build_image,libs,develop,\ - --label CONTAINER_VERSION=${DOCKER_IMAGE_TAG} \ +build_ecvl: _build_libs_base build_ecvl_toolkit build_eddl ## Build 'ecvl' image + $(call build_image,libs,ecvl,${ECVL_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION) \ --label EDDL_REPOSITORY=${EDDL_REPOSITORY} \ --label EDDL_BRANCH=${EDDL_BRANCH} \ - --label EDDL_REVISION=$(call get_revision,${EDDL_LIB_PATH},${EDDL_REVISION}) \ + --label EDDL_REVISION=${EDDL_REVISION} \ --label ECVL_REPOSITORY=${ECVL_REPOSITORY} \ --label ECVL_BRANCH=${ECVL_BRANCH} \ - --label ECVL_REVISION=$(call get_revision,${ECVL_LIB_PATH},${ECVL_REVISION}) \ - ) + --label ECVL_REVISION=${ECVL_REVISION},eddl:$(EDDL_IMAGE_VERSION_TAG),ecvl-toolkit:$(ECVL_IMAGE_VERSION_TAG)) + $(call log_image_revision,ecvl,${ECVL_IMAGE_VERSION_TAG},install,ECVL) + +build_libs: build_ecvl ## Build 'libs' image + $(call build_image,libs,libs,${DOCKER_LIBS_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION) \ + --label EDDL_REPOSITORY=${EDDL_REPOSITORY} \ + --label EDDL_BRANCH=${EDDL_BRANCH} \ + --label EDDL_REVISION=${EDDL_REVISION} \ + --label ECVL_REPOSITORY=${ECVL_REPOSITORY} \ + --label ECVL_BRANCH=${ECVL_BRANCH} \ + --label ECVL_REVISION=${ECVL_REVISION},ecvl:$(ECVL_IMAGE_VERSION_TAG),,${DOCKER_LIBS_EXTRA_TAGS}) + + + +############# pylibs-toolkit ############# + +_build_pyeddl_base_toolkit: build_eddl_toolkit + $(eval PYLIBS_BASE_IMAGE_VERSION_TAG := base_${DOCKER_BASE_IMAGE_VERSION_TAG}-eddl_${EDDL_IMAGE_VERSION_TAG}) + $(call build_image,pylibs,pylibs-base-toolkit,${PYLIBS_BASE_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION),eddl-toolkit:$(EDDL_IMAGE_VERSION_TAG)) + +_build_pyecvl_base_toolkit: build_ecvl_toolkit pyeddl_folder apply_pyeddl_patches + $(eval PYLIBS_BASE_IMAGE_VERSION_TAG := base_${DOCKER_BASE_IMAGE_VERSION_TAG}-pyeddl_${PYEDDL_IMAGE_VERSION_TAG}-eddl_${EDDL_IMAGE_VERSION_TAG}-ecvl_${ECVL_IMAGE_VERSION_TAG}) + $(call build_image,pylibs,pylibs-base-toolkit,${PYLIBS_BASE_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION),ecvl-toolkit:$(ECVL_IMAGE_VERSION_TAG)) + $(call build_image,pylibs,pyeddl-toolkit,${PYLIBS_BASE_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION) \ + --label EDDL_REPOSITORY=${EDDL_REPOSITORY} \ + --label EDDL_BRANCH=${EDDL_BRANCH} \ + --label EDDL_REVISION=${EDDL_REVISION} \ + --label ECVL_REPOSITORY=${ECVL_REPOSITORY} \ + --label ECVL_BRANCH=${ECVL_BRANCH} \ + --label ECVL_REVISION=${ECVL_REVISION} \ + --label PYEDDL_REPOSITORY=${PYEDDL_REPOSITORY} \ + --label PYEDDL_BRANCH=${PYEDDL_BRANCH} \ + --label PYEDDL_REVISION=${PYEDDL_REVISION},pylibs-base-toolkit:$(PYLIBS_BASE_IMAGE_VERSION_TAG)) + +build_pyeddl_toolkit: pyeddl_folder _build_pyeddl_base_toolkit apply_pyeddl_patches ## Build 'pyeddl-toolkit' image + $(call build_image,pylibs,pyeddl-toolkit,${PYEDDL_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION) \ + --label EDDL_REPOSITORY=${EDDL_REPOSITORY} \ + --label EDDL_BRANCH=${EDDL_BRANCH} \ + --label EDDL_REVISION=${EDDL_REVISION} \ + --label ECVL_REPOSITORY=${ECVL_REPOSITORY} \ + --label ECVL_BRANCH=${ECVL_BRANCH} \ + --label ECVL_REVISION=${ECVL_REVISION} \ + --label PYEDDL_REPOSITORY=${PYEDDL_REPOSITORY} \ + --label PYEDDL_BRANCH=${PYEDDL_BRANCH} \ + --label PYEDDL_REVISION=${PYEDDL_REVISION},pylibs-base-toolkit:$(PYLIBS_BASE_IMAGE_VERSION_TAG)) + $(call log_image_revision,pyeddl-toolkit,${PYEDDL_IMAGE_VERSION_TAG},install,PYEDDL) -build_pylibs: build_pylibs_toolkit ## Build and tag 'pylibs' image - $(call build_image,pylibs,runtime,\ - --label CONTAINER_VERSION=${DOCKER_IMAGE_TAG} \ +build_pyecvl_toolkit: pyecvl_folder _build_pyecvl_base_toolkit ## Build 'pyecvl-toolkit' image + $(call build_image,pylibs,pyecvl-toolkit,${PYECVL_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION) \ --label EDDL_REPOSITORY=${EDDL_REPOSITORY} \ --label EDDL_BRANCH=${EDDL_BRANCH} \ - --label EDDL_REVISION=$(call get_revision,${EDDL_LIB_PATH},${EDDL_REVISION}) \ + --label EDDL_REVISION=${EDDL_REVISION} \ --label ECVL_REPOSITORY=${ECVL_REPOSITORY} \ --label ECVL_BRANCH=${ECVL_BRANCH} \ - --label ECVL_REVISION=$(call get_revision,${ECVL_LIB_PATH},${ECVL_REVISION}) \ + --label ECVL_REVISION=${ECVL_REVISION} \ --label PYECVL_REPOSITORY=${PYECVL_REPOSITORY} \ --label PYECVL_BRANCH=${PYECVL_BRANCH} \ - --label PYECVL_REVISION=$(call get_revision,${PYECVL_LIB_PATH},${PYECVL_REVISION}) \ + --label PYECVL_REVISION=${PYECVL_REVISION} \ --label PYEDDL_REPOSITORY=${PYEDDL_REPOSITORY} \ --label PYEDDL_BRANCH=${PYEDDL_BRANCH} \ - --label PYEDDL_REVISION=$(call get_revision,${PYEDDL_LIB_PATH},${PYEDDL_REVISION}),libs:$(DOCKER_IMAGE_TAG),pylibs-toolkit:$(DOCKER_IMAGE_TAG)) + --label PYEDDL_REVISION=${PYEDDL_REVISION},pyeddl-toolkit:$(PYLIBS_BASE_IMAGE_VERSION_TAG)) + $(call log_image_revision,pyecvl-toolkit,${PYECVL_IMAGE_VERSION_TAG},install,PYECVL) -build_pylibs_toolkit: pyeddl_folder pyecvl_folder ## Build and tag 'pylibs-toolkit' image - $(call build_image,pylibs,develop,\ - --label CONTAINER_VERSION=${DOCKER_IMAGE_TAG} \ +build_pylibs_toolkit: build_pyecvl_toolkit ## Build 'pylibs-toolkit' image + $(call build_image,pylibs,pylibs-toolkit,${DOCKER_LIBS_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION) \ --label EDDL_REPOSITORY=${EDDL_REPOSITORY} \ --label EDDL_BRANCH=${EDDL_BRANCH} \ - --label EDDL_REVISION=$(call get_revision,${EDDL_LIB_PATH},${EDDL_REVISION}) \ + --label EDDL_REVISION=${EDDL_REVISION} \ --label ECVL_REPOSITORY=${ECVL_REPOSITORY} \ --label ECVL_BRANCH=${ECVL_BRANCH} \ - --label ECVL_REVISION=$(call get_revision,${ECVL_LIB_PATH},${ECVL_REVISION}) \ + --label ECVL_REVISION=${ECVL_REVISION} \ --label PYECVL_REPOSITORY=${PYECVL_REPOSITORY} \ --label PYECVL_BRANCH=${PYECVL_BRANCH} \ - --label PYECVL_REVISION=$(call get_revision,${PYECVL_LIB_PATH},${PYECVL_REVISION}) \ + --label PYECVL_REVISION=${PYECVL_REVISION} \ --label PYEDDL_REPOSITORY=${PYEDDL_REPOSITORY} \ --label PYEDDL_BRANCH=${PYEDDL_BRANCH} \ - --label PYEDDL_REVISION=$(call get_revision,${PYEDDL_LIB_PATH},${PYEDDL_REVISION}),libs-toolkit:$(DOCKER_IMAGE_TAG)) - -# Docker push -push: _push ## Push all built images + --label PYEDDL_REVISION=${PYEDDL_REVISION},pyecvl-toolkit:$(PYECVL_IMAGE_VERSION_TAG),,${DOCKER_LIBS_EXTRA_TAGS}) + + + +############# pylibs ############# + +_build_pyeddl_base: build_eddl + $(eval PYLIBS_BASE_IMAGE_VERSION_TAG := base_${DOCKER_BASE_IMAGE_VERSION_TAG}-eddl_${EDDL_IMAGE_VERSION_TAG}) + $(call build_image,pylibs,pylibs-base,${PYLIBS_BASE_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION),eddl:$(EDDL_IMAGE_VERSION_TAG)) + +_build_pyecvl_base: build_ecvl build_pyeddl_toolkit + $(eval PYLIBS_BASE_IMAGE_VERSION_TAG := base_${DOCKER_BASE_IMAGE_VERSION_TAG}-pyeddl_${PYEDDL_IMAGE_VERSION_TAG}-eddl_${EDDL_IMAGE_VERSION_TAG}-ecvl_${ECVL_IMAGE_VERSION_TAG}) + $(call build_image,pylibs,pylibs-base,${PYLIBS_BASE_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION),ecvl:$(ECVL_IMAGE_VERSION_TAG)) + $(call build_image,pylibs,pyeddl,${PYLIBS_BASE_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION) \ + --label EDDL_REPOSITORY=${EDDL_REPOSITORY} \ + --label EDDL_BRANCH=${EDDL_BRANCH} \ + --label EDDL_REVISION=${EDDL_REVISION} \ + --label ECVL_REPOSITORY=${ECVL_REPOSITORY} \ + --label ECVL_BRANCH=${ECVL_BRANCH} \ + --label ECVL_REVISION=${ECVL_REVISION} \ + --label PYEDDL_REPOSITORY=${PYEDDL_REPOSITORY} \ + --label PYEDDL_BRANCH=${PYEDDL_BRANCH} \ + --label PYEDDL_REVISION=${PYEDDL_REVISION},pylibs-base:$(PYLIBS_BASE_IMAGE_VERSION_TAG),pyeddl-toolkit:$(PYEDDL_IMAGE_VERSION_TAG)) + +build_pyeddl: build_pyeddl_toolkit _build_pyeddl_base ## Build 'pyeddl' image + $(call build_image,pylibs,pyeddl,${PYEDDL_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION) \ + --label EDDL_REPOSITORY=${EDDL_REPOSITORY} \ + --label EDDL_BRANCH=${EDDL_BRANCH} \ + --label EDDL_REVISION=${EDDL_REVISION} \ + --label ECVL_REPOSITORY=${ECVL_REPOSITORY} \ + --label ECVL_BRANCH=${ECVL_BRANCH} \ + --label ECVL_REVISION=${ECVL_REVISION} \ + --label PYEDDL_REPOSITORY=${PYEDDL_REPOSITORY} \ + --label PYEDDL_BRANCH=${PYEDDL_BRANCH} \ + --label PYEDDL_REVISION=${PYEDDL_REVISION},pylibs-base:$(PYLIBS_BASE_IMAGE_VERSION_TAG),pyeddl-toolkit:$(PYEDDL_IMAGE_VERSION_TAG)) + $(call log_image_revision,pyeddl,${PYEDDL_IMAGE_VERSION_TAG},install,PYEDDL) + +build_pyecvl: build_pyecvl_toolkit _build_pyecvl_base ## Build 'pyecvl' image + $(call build_image,pylibs,pyecvl,${PYECVL_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION) \ + --label EDDL_REPOSITORY=${EDDL_REPOSITORY} \ + --label EDDL_BRANCH=${EDDL_BRANCH} \ + --label EDDL_REVISION=${EDDL_REVISION} \ + --label ECVL_REPOSITORY=${ECVL_REPOSITORY} \ + --label ECVL_BRANCH=${ECVL_BRANCH} \ + --label ECVL_REVISION=${ECVL_REVISION} \ + --label PYECVL_REPOSITORY=${PYECVL_REPOSITORY} \ + --label PYECVL_BRANCH=${PYECVL_BRANCH} \ + --label PYECVL_REVISION=${PYECVL_REVISION} \ + --label PYEDDL_REPOSITORY=${PYEDDL_REPOSITORY} \ + --label PYEDDL_BRANCH=${PYEDDL_BRANCH} \ + --label PYEDDL_REVISION=${PYEDDL_REVISION},pyeddl:$(PYLIBS_BASE_IMAGE_VERSION_TAG),pyecvl-toolkit:$(PYECVL_IMAGE_VERSION_TAG)) + $(call log_image_revision,pyecvl,${PYECVL_IMAGE_VERSION_TAG},install,PYECVL) + +build_pylibs: build_pyecvl ## Build 'pylibs' image + $(call build_image,pylibs,pylibs,${DOCKER_LIBS_IMAGE_VERSION_TAG},\ + --label CONTAINER_VERSION=$(CONTAINER_VERSION) \ + --label EDDL_REPOSITORY=${EDDL_REPOSITORY} \ + --label EDDL_BRANCH=${EDDL_BRANCH} \ + --label EDDL_REVISION=${EDDL_REVISION} \ + --label ECVL_REPOSITORY=${ECVL_REPOSITORY} \ + --label ECVL_BRANCH=${ECVL_BRANCH} \ + --label ECVL_REVISION=${ECVL_REVISION} \ + --label PYECVL_REPOSITORY=${PYECVL_REPOSITORY} \ + --label PYECVL_BRANCH=${PYECVL_BRANCH} \ + --label PYECVL_REVISION=${PYECVL_REVISION} \ + --label PYEDDL_REPOSITORY=${PYEDDL_REPOSITORY} \ + --label PYEDDL_BRANCH=${PYEDDL_BRANCH} \ + --label PYEDDL_REVISION=${PYEDDL_REVISION},pyecvl:$(PYECVL_IMAGE_VERSION_TAG),,${DOCKER_LIBS_EXTRA_TAGS}) + + +############################################################################################################################ +### Tests +############################################################################################################################ +define check_image + printf "\nSearching image $(1)... " ; \ + images=$(docker images -q ${1}) 2> /dev/null ; \ + if [ -z "$${images}" ]; then \ + [ "${_DO_NOT_PULL_DOCKER_IMAGES}" == "0" ] && docker pull ${DOCKER_REPOSITORY_OWNER}/${1} 2> /dev/null ; \ + docker tag ${DOCKER_REPOSITORY_OWNER}/${1} ${1} 2> /dev/null ; \ + fi ; \ + printf "\n" +endef + +define test_image + $(eval image := $(1)) + $(eval test_script := $(2)) + $(eval container_paths := $(3)) + $(eval rname := $(shell cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)) + echo -e "\n\n\n*************************************************************************************************************" >&2 ; \ + echo -e "*** Test: '${test_script}' @ '${image}' image ***" >&2 ; \ + echo -e "*************************************************************************************************************\n" >&2 ; \ + cnames="" ; \ + for cpath in ${container_paths}; do \ + xcpath=($$(echo $${cpath} | tr "=" " ")) ; \ + cname=$$(echo $${xcpath[0]} | sed -e +s+:.*++)-${rname} ; \ + cnames="$${cnames} $${cname}" ; \ + volumes="$${volumes} --volumes-from $${cname}" ; \ + printf "\nCreating temp container instance of '$${xcpath[0]}' (name: $${cname})... " >&2; \ + $(call check_image,$${xcpath[0]}) ; \ + docker create --name $${cname} -v "$${xcpath[1]}" $${xcpath[0]} > /dev/null ; \ + printf "DONE\n" >&2 ; \ + done ; \ + printf "\n\n" ; \ + $(call check_image,${image}) ; \ + cat ${test_script} | ${DOCKER_RUN} -e GPU_RUNTIME="${GPU_RUNTIME}" $${volumes} ${image} /bin/bash ; \ + exit_code=$$? ; \ + for cname in $${cnames}; do \ + printf "\nRemoving temp container instance '$${cname}'... " >&2; \ + docker rm -f $${cname} > /dev/null ; printf "DONE" >&2 ; done ; \ + echo -e "\n*************************************************************************************************************\n\n\n" >&2 ; \ + exit $${exit_code} +endef + +test: _test ## Test all docker images + +_test: \ + test_eddl test_eddl_toolkit \ + test_ecvl test_ecvl_toolkit \ + test_pyeddl test_pyeddl_toolkit \ + test_pyecvl test_pyecvl_toolkit + +test_eddl: eddl_folder ## Test 'eddl' images + $(call set_library_revision,libs,eddl) + @$(call test_image,\ + eddl:${EDDL_IMAGE_VERSION_TAG},\ + $(shell python3 tests/inventory.py ${EDDL_LIB_PATH} tests/eddl),\ + eddl-toolkit:${EDDL_IMAGE_VERSION_TAG}=/usr/local/src/eddl \ + ) + +test_eddl_toolkit: eddl_folder ## Test 'eddl' images + $(call set_library_revision,libs,eddl) + @$(call test_image,eddl-toolkit:${EDDL_IMAGE_VERSION_TAG},\ + $(shell python3 tests/inventory.py ${EDDL_LIB_PATH} tests/eddl)) + +test_ecvl: ecvl_folder ## Test 'ecvl' images + $(call set_library_revision,libs,ecvl) + @$(call test_image,\ + ecvl:${ECVL_IMAGE_VERSION_TAG},\ + $(shell python3 tests/inventory.py ${ECVL_LIB_PATH} tests/ecvl),\ + ecvl-toolkit:${ECVL_IMAGE_VERSION_TAG}=/usr/local/src/ecvl \ + ) + +test_ecvl_toolkit: ecvl_folder ## Test 'ecvl' images + $(call set_library_revision,libs,ecvl) + @$(call test_image,ecvl-toolkit:${ECVL_IMAGE_VERSION_TAG},\ + $(shell python3 tests/inventory.py ${ECVL_LIB_PATH} tests/ecvl)) + +test_pyeddl: pyeddl_folder ## Test 'ecvl' images + $(call set_library_revision,pylibs,pyeddl) + @$(call test_image,\ + pyeddl:${PYEDDL_IMAGE_VERSION_TAG},\ + $(shell python3 tests/inventory.py ${PYEDDL_LIB_PATH} tests/pyeddl),\ + pyeddl-toolkit:${PYEDDL_IMAGE_VERSION_TAG}=/usr/local/src/pyeddl\ + ) + +test_pyeddl_toolkit: pyeddl_folder ## Test 'ecvl' images + $(call set_library_revision,pylibs,pyeddl) + @$(call test_image,pyeddl-toolkit:${PYEDDL_IMAGE_VERSION_TAG},\ + $(shell python3 tests/inventory.py ${PYEDDL_LIB_PATH} tests/pyeddl)) + +test_pyecvl: pyecvl_folder ## Test 'ecvl' images + $(call set_library_revision,libs,ecvl) + $(call set_library_revision,pylibs,pyecvl) + @$(call test_image,\ + pyecvl:${PYECVL_IMAGE_VERSION_TAG},\ + $(shell python3 tests/inventory.py ${PYECVL_LIB_PATH} tests/pyecvl),\ + 'ecvl-toolkit:${ECVL_IMAGE_VERSION_TAG}=/usr/local/src/ecvl' \ + 'pyecvl-toolkit:${PYECVL_IMAGE_VERSION_TAG}=/usr/local/src/pyecvl' \ + ) + +test_pyecvl_toolkit: #pyecvl_folder ## Test 'ecvl' images + $(call set_library_revision,pylibs,pyecvl) + @$(call test_image,pyecvl-toolkit:${PYECVL_IMAGE_VERSION_TAG},\ + $(shell python3 tests/inventory.py ${PYECVL_LIB_PATH} tests/pyecvl)) + +############################################################################################################################ +### Push Docker images +############################################################################################################################ +push: _push ## Push all images + _push: \ - push_libs_toolkit push_libs \ - push_pylibs_toolkit push_pylibs + push_libs_base push_libs_base_toolkit \ + push_libs push_libs_toolkit\ + push_eddl push_eddl_toolkit \ + push_ecvl push_ecvl_toolkit \ + push_pylibs push_pylibs_toolkit \ + push_pyeddl push_pyeddl_toolkit \ + push_pyecvl push_pyecvl_toolkit + +push_libs: docker_login ## Push 'libs' image + $(call push_image,libs,${DOCKER_LIBS_IMAGE_VERSION_TAG},${DOCKER_LIBS_EXTRA_TAGS}) + +push_libs_base: docker_login ## Push 'lib-base' image + $(call push_image,libs-base,${DOCKER_BASE_IMAGE_VERSION_TAG}) + +push_eddl: docker_login eddl_folder ## Push 'eddl' image + $(call push_image,eddl,${EDDL_IMAGE_VERSION_TAG},${EDDL_REVISION} ${EDDL_TAG}) + +push_ecvl: docker_login ecvl_folder ## Push 'ecvl' image + $(call push_image,ecvl,${ECVL_IMAGE_VERSION_TAG},${ECVL_REVISION} ${ECVL_TAG}) + +push_libs_toolkit: docker_login ## Push 'libs-toolkit' image + $(call push_image,libs-toolkit,${DOCKER_LIBS_IMAGE_VERSION_TAG},${DOCKER_LIBS_EXTRA_TAGS}) + +push_libs_base_toolkit: docker_login ## Push 'libs-base-toolkit' image + $(call push_image,libs-base-toolkit,${DOCKER_BASE_IMAGE_VERSION_TAG}) + +push_eddl_toolkit: docker_login eddl_folder ## Push 'eddl-toolkit' images + $(call push_image,eddl-toolkit,${EDDL_IMAGE_VERSION_TAG},${EDDL_REVISION} ${EDDL_TAG}) + +push_ecvl_toolkit: docker_login ecvl_folder ## Push 'ecvl-toolkit' images + $(call push_image,ecvl-toolkit,${ECVL_IMAGE_VERSION_TAG},${ECVL_REVISION} ${ECVL_TAG}) + +push_pylibs: docker_login ## Push 'pylibs' images + $(call push_image,pylibs,${DOCKER_LIBS_IMAGE_VERSION_TAG},${DOCKER_LIBS_EXTRA_TAGS}) + +push_pyeddl: docker_login pyeddl_folder ## Push 'pyeddl' images + $(call push_image,pyeddl,${PYEDDL_IMAGE_VERSION_TAG},${PYEDDL_REVISION} ${PYEDDL_TAG}) + +push_pyecvl: docker_login pyecvl_folder ## Push 'pyecvl' images + $(call push_image,pyecvl,${PYECVL_IMAGE_VERSION_TAG},${PYECVL_REVISION} ${PYECVL_TAG}) + +push_pylibs_toolkit: docker_login ## Push 'pylibs-toolkit' images + $(call push_image,pylibs-toolkit,${DOCKER_LIBS_IMAGE_VERSION_TAG},${DOCKER_LIBS_EXTRA_TAGS}) + +push_pyeddl_toolkit: docker_login pyeddl_folder ## Push 'pyeddl-toolkit' images + $(call push_image,pyeddl-toolkit,${PYEDDL_IMAGE_VERSION_TAG},${PYEDDL_REVISION} ${PYEDDL_TAG}) + +push_pyecvl_toolkit: docker_login pyecvl_folder ## Push 'pyeddl-toolkit' images + $(call push_image,pyecvl-toolkit,${PYECVL_IMAGE_VERSION_TAG},${PYECVL_REVISION} ${PYECVL_TAG}) + +############################################################################################################################ +### Piblish Docker images +############################################################################################################################ +publish: build push ## Publish all images to a Docker Registry (e.g., DockerHub) -push_libs: repo-login ## Push 'libs' images - $(call push_image,libs,runtime) +publish_libs: build_libs push_libs ## Publish 'libs' image -push_libs_toolkit: repo-login ## Push 'libs-toolkit' images - $(call push_image,libs,develop) +publish_eddl: build_eddl push_eddl ## Publish 'eddl' image -push_pylibs: repo-login ## Push 'pylibs' images - $(call push_image,pylibs,runtime) +publish_ecvl: build_ecvl push_ecvl ## Publish 'ecvl' image -push_pylibs_toolkit: repo-login ## Push 'pylibs-toolkit' images - $(call push_image,pylibs,develop) +publish_libs_toolkit: build_libs_toolkit push_libs_toolkit ## Publish 'libs-toolkit' image -# Docker publish -publish: build push ## Publish all built images to a Docker Registry (e.g., DockerHub) +publish_eddl_toolkit: build_eddl_toolkit push_eddl_toolkit ## Publish 'eddl-toolkit' image -publish_libs: build_libs push_libs ## Publish 'libs' images +publish_ecvl_toolkit: build_ecvl_toolkit push_ecvl_toolkit ## Publish 'ecvl-toolkit' image -publish_libs_toolkit: build_libs_toolkit push_libs_toolkit ## Publish 'libs-toolkit' images +publish_pylibs: build_pylibs push_pylibs ## Publish 'pylibs' image -publish_pylibs: build_pylibs push_pylibs ## Publish 'pylibs' images +publish_pyeddl: build_pyeddl push_pyeddl ## Publish 'pyeddl' image -publish_pylibs_toolkit: build_pylibs_toolkit push_pylibs_toolkit ## Publish 'pylibs-toolkit' images +publish_pyecvl: build_pyecvl push_pyecvl ## Publish 'pyecvl' image + +publish_pylibs_toolkit: build_pylibs_toolkit push_pylibs_toolkit ## Publish 'pylibs-toolkit' image + +publish_pyeddl_toolkit: build_pyeddl_toolkit push_pyeddl_toolkit ## Publish 'pyeddl-toolkit' image + +publish_pyecvl_toolkit: build_pyecvl_toolkit push_pyecvl_toolkit ## Publish 'pyecvl-toolkit' image # login to the Docker HUB repository -repo-login: ## Login to the Docker Registry +_docker_login: @if [[ ${DOCKER_LOGIN_DONE} == false ]]; then \ echo "Logging into Docker registry ${DOCKER_REGISTRY}..." ; \ - echo ${DOCKER_PASSWORD} | docker login ${DOCKER_REGISTRY} -u ${DOCKER_USER} --password-stdin ; \ - DOCKER_LOGIN_DONE=true ;\ + echo ${DOCKER_PASSWORD} | docker login ${DOCKER_REGISTRY} --username ${DOCKER_USER} --password-stdin \ else \ echo "Logging into Docker registry already done" ; \ fi -version: ## Output the current version of this Makefile - @echo $(VERSION) +docker-login: _docker-login ## Login to the Docker Registry + $(eval DOCKER_LOGIN_DONE := true) + + +############################################################################################################################ +### Clean sources +############################################################################################################################ +clean_eddl_sources: ## clean repository containing EDDL source code + $(call clean_sources,libs/eddl) + +clean_ecvl_sources: ## clean repository containing ECVL source code + $(call clean_sources,libs/ecvl) + +clean_pyeddl_sources: ## clean repository containing PyEDDL source code + $(call clean_sources,pylibs/pyeddl) + +clean_pyecvl_sources: ## clean repository containing PyECVL source code + $(call clean_sources,pylibs/pyecvl) + +clean_libs_sources: clean_eddl_sources clean_ecvl_sources ## clean repository containing libs source code + +clean_pylibs_sources: clean_pyeddl_sources clean_pyecvl_sources ## clean repository containing pylibs source code + +clean_sources: clean_pylibs_sources clean_libs_sources _clean_libraries_logs ## clean repository containing source code + + +############################################################################################################################ +### Clean Docker images +############################################################################################################################ +clean_base_images: + $(call clean_image,libs-base) + $(call clean_image,pylibs-base) + $(call clean_image,libs-base-toolkit) + $(call clean_image,pylibs-base-toolkit) + +clean_eddl_images: + $(call clean_image,eddl) + $(call clean_image,eddl-toolkit) + +clean_ecvl_images: + $(call clean_image,ecvl) + $(call clean_image,ecvl-toolkit) + +clean_libs_images: clean_ecvl_images clean_eddl_images + $(call clean_image,libs) + $(call clean_image,libs-toolkit) + +clean_pyeddl_images: + $(call clean_image,pyeddl) + $(call clean_image,pyeddl-toolkit) + +clean_pyecvl_images: + $(call clean_image,pyecvl) + $(call clean_image,pyecvl-toolkit) + +clean_pylibs_images: clean_pyecvl_images clean_pyeddl_images + $(call clean_image,pylibs) + $(call clean_image,pylibs-toolkit) + +clean_images: \ + clean_pylibs_images clean_libs_images clean_base_images \ + clean_ecvl_images clean_eddl_images \ + clean_pyecvl_images clean_pylibs_images \ + _clean_images_logs + +#################################################################### + +_clean_dependencies_logs: + $(file >${DEPENDENCIES_LOG},) + @echo "Logs of dependencies deleted" + +_clean_images_logs: _clean_dependencies_logs + $(file >${IMAGES_LOG},) + @echo "Logs of images deleted" + +_clean_libraries_logs: _clean_dependencies_logs + $(file >${LIBRARIES_LOG},) + @echo "Logs of libraries deleted" -clean: clean_libs clean_pylibs +clean_logs: _clean_images_logs _clean_libraries_logs ## clean logs -clean_libs: - $(call clean_build,libs) -clean_pylibs: - $(call clean_build,pylibs) +############################################################################################################################ +### Clean Sources and Docker images +############################################################################################################################ +clean: clean_images clean_sources clean_logs -.PHONY: help clean clean_libs clean_pylibs apply_libs_patches \ - build _build build_libs_toolkit build_libs \ - build_pylibs_toolkit build_pylibs \ - ecvl_folder eddl_folder pyecvl_folder pyeddl_folder \ - repo-login \ - push \ - _push push_libs_toolkit push_libs \ - push_pylibs_toolkit push_pylibs \ +.PHONY: help \ + clean_logs _clean_dependencies_logs _clean_libraries_logs _clean_images_logs \ + libraries_list images_list dependencies_list dependency_graph \ + libs_folder eddl_folder ecvl_folder pylibs_folder \ + pyeddl_folder _pyeddl_shallow_clone _pyecvl_second_level_dependencies \ + pyecvl_folder _pyeddl_shallow_clone _pyecvl_first_level_dependencies _pyecvl_second_level_dependencies \ + apply_pyeddl_patches apply_pyecvl_patches \ + clean clean_libs clean_pylibs apply_libs_patches \ + build _build \ + _build_libs_base_toolkit \ + build_eddl_toolkit build_ecvl_toolkit build_libs_toolkit \ + _build_libs_base build_eddl build_ecvl build_libs \ + _build_pyeddl_base_toolkit _build_pyecvl_base_toolkit _build_pylibs_base \ + build_pyeddl_toolkit build_pyecvl_toolkit build_pylibs_toolkit\ + _build_pyeddl_base _build_pyecvl_base build_pyeddl build_pyecvl build_pylibs \ + _docker_login docker_login \ + push _push \ + push_libs push_eddl push_ecvl \ + push_libs_toolkit push_eddl_toolkit push_ecvl_toolkit \ + push_pylibs push_pyeddl push_pyecvl \ + push_pylibs_toolkit push_pyeddl_toolkit push_pyecvl_toolkit \ publish \ - publish_libs_toolkit publish_libs \ - publish_pylibs_toolkit publish_pylibs \ No newline at end of file + publish_libs publish_eddl publish_ecvl \ + publish_libs_toolkit publish_eddl_toolkit publish_ecvl_toolkit \ + publish_pylibs publish_pyeddl publish_pyecvl \ + publish_pylibs_toolkit publish_pyeddl_toolkit publish_pyecvl_toolkit \ + tests _tests test_eddl test_eddl_toolkit test_ecvl test_ecvl_toolkit test_pyeddl \ + test_pyeddl_toolkit test_pyecvl test_pyecvl_toolkit \ + clean_sources \ + clean_eddl_sources clean_ecvl_sources \ + clean_pyeddl_sources clean_pyecvl_sources \ + clean \ + clean_images \ + clean_base_images \ + clean_eddl_images clean_ecvl_images clean_libs_images \ + clean_pyeddl_images clean_pyecvl_images clean_pylibs_images diff --git a/README.md b/README.md index 8fd0ca3..0296747 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,68 @@ +![GitHub release (latest by date)](https://img.shields.io/github/v/release/deephealthproject/docker-libs)![GitHub](https://img.shields.io/github/license/deephealthproject/docker-libs) + + # docker-libs -Docker images to develop and run software based on the [EDDL](https://github.com/deephealthproject/eddl) and [ECVL](https://github.com/deephealthproject/ecvl) libraries and their Python wrappers ([PyEDDL](https://github.com/deephealthproject/pyeddl) and [PyECVL](https://github.com/deephealthproject/pycvl)). +Docker images to develop and run software based on the [EDDL](https://github.com/deephealthproject/eddl) and [ECVL](https://github.com/deephealthproject/ecvl) libraries and their respective Python interfaces ([PyEDDL](https://github.com/deephealthproject/pyeddl) and [PyECVL](https://github.com/deephealthproject/pycvl)). + +## TL;DR + +E.g., `docker pull dhealth/pyecvl:0.1.0` + + +* Every library has a corresponding image repository: + - `dhealth/eddl` + - `dhealth/ecvl` + - `dhealth/pyeddl` + - `dhealth/pyecvl` + +* Every tag and commit id you see in the git repository has a corresponding image tag + - e.g., PyECVL [version 0.1.0](https://github.com/deephealthproject/pyecvl/tree/0.1.0) corresponds to the image tag [dhealth/pyecvl:0.1.0](https://hub.docker.com/layers/dhealth/pyecvl/0.1.0/) + - e.g., PyECVL at [commit id 23a79c5](https://github.com/deephealthproject/pyecvl/tree/23a79c5b6ba39a5049901933edff2ca372713df7) corresponds to the image tag [dhealth/pyecvl:23a79c5](https://hub.docker.com/layers/dhealth/pyecvl/23a79c5/images/sha256-bea02aa37dbb4f0f987b56d5c33d319e4018c809b562bca09bd1df0b4c755425?context=explore) (use the first 7 characters of the commit id) + +### Dependencies + +When you use the DeepHealth image for a library, the image also contains the libraries on which it depends: +* PyECVL -> also contains PyEDDL, EDDL, ECVL +* PyEDDL -> also contains EDDL +* ECVL -> also contains EDDL +* EDDL -> on its own + +If you want everything, **use the PyECVL image**. + +### Toolkit + +For all images, a toolkit version is also built that contains build requirements for software (compiler, headers, etc.). You can use these to build your own software or rebuild the DeepHealth libraries from source. Keep reading the section below for details. + -## Description +## Detailed description -The `docker-libs` repository allows users to build and publish on a registry (e.g., DockerHub) Docker images containing the [EDDL](https://github.com/deephealthproject/eddl) and [ECVL](https://github.com/deephealthproject/ecvl) libraries and their Python wrappers, [PyEDDL](https://github.com/deephealthproject/pyeddl) and [PyECVL](https://github.com/deephealthproject/pycvl). All the images are based on the [NVIDIA Container Toolkit](https://github.com/NVIDIA/nvidia-docker) and the EDDL and ECVL libraries are configured to leverage NVIDIA GPUs. +The `docker-libs` repository allows users to build and publish on a registry (e.g., DockerHub) Docker images containing the DeepHealth libraries: [EDDL](https://github.com/deephealthproject/eddl), [ECVL](https://github.com/deephealthproject/ecvl) and their Python wrappers, [PyEDDL](https://github.com/deephealthproject/pyeddl) and [PyECVL](https://github.com/deephealthproject/pycvl). All the images are based on the [NVIDIA Container Toolkit](https://github.com/NVIDIA/nvidia-docker) and the EDDL and ECVL libraries are configured to leverage NVIDIA GPUs. -Precompiled images are published on [DockerHub](https://hub.docker.com/u/dhealth). +Precompiled images for each DeepHealth library are published on [DockerHub](https://hub.docker.com/u/dhealth): -The available images are: +* **[`dhealth/eddl`](https://hub.docker.com/r/dhealth/eddl)** contains an installation of the EDDL library +* **[`dhealth/ecvl`](https://hub.docker.com/r/dhealth/ecvl)** contains an installation of the ECVL library with support for EDDL +* **[`dhealth/pyeddl`](https://hub.docker.com/r/dhealth/pyeddl)** contains an installation of the PyEDDL and EDDL library +* **[`dhealth/pyecvl`](https://hub.docker.com/r/dhealth/ecvl-toolkit)** contains an installation of the PyECVL library with support for PyEDDL + +They are available with the `-toolkit` variant, which provides optional development tools (sources, compilers, etc.). + +In addition, to provide, under a common Docker tag, library revisions compatible with each other, the following images are periodically released on [DockerHub](https://hub.docker.com/u/dhealth) (see GitHub **[releases](https://github.com/deephealthproject/docker-libs/releases)**): * **[`dhealth/libs`](https://hub.docker.com/r/dhealth/libs)** contains an installation of the EDDL and ECVL libraries * **[`dhealth/libs-toolkit`](https://hub.docker.com/r/dhealth/libs-toolkit)** contains an installation of the EDDL and ECVL libraries, the source code of two libraries and all the development tools (compilers, libraries, etc.) you need to compile them - * **[`dhealth/pylibs`](https://hub.docker.com/r/dhealth/pylibs)** extends the `libs` image with the PyEDDL and PyECVL libraries * **[`dhealth/pylibs-toolkit`](https://hub.docker.com/r/dhealth/pylibs-toolkit)** extends the `libs-toolkit` image with the PyEDDL and PyECVL libraries + + +##### Image TAGs + +All the images are tagged accordingly with the revision of the corresponding libraries on GitHub, identified by commit ID or TAG: e.g., `dhealth/ecvl:1512be8` and`dhealth/ecvl-toolkit:1512be8` are built on the `1512be8` revision (commit ID) of the ECVL library. If that revision is also associated with a Git TAG, it will be used as a Docker tag as well. + + + ## Example usage Open a shell in a container with access to the DeepHealth libraries: @@ -27,34 +74,78 @@ docker run -it --rm dhealth/libs /bin/bash Open a shell to compile your local project: ```bash -docker run -it -u $(id -u) -v $(pwd):/tests --rm dhealth/libs-toolkit:0.1.2 /bin/bash +docker run -it -u $(id -u) -v $(pwd):/tests --rm dhealth/libs-toolkit /bin/bash ``` -## How to build +## How to build, test and publish -A `Makefile` allows to easily compile and publish the Docker images. Type `make help` to see the available `Makefile` targets, i.e.: +A `Makefile` allows to easily compile, test and publish the Docker images. Type `make help` to see the available `Makefile` targets, i.e.: ```bash +version Output the current version of this Makefile help Show help -build Build and tag all Docker images -build_libs Build and tag 'libs' image -build_libs_toolkit Build and tag 'libs-toolkit' image -build_pylibs Build and tag 'pylibs' image -build_pylibs_toolkit Build and tag 'pylibs-toolkit' image -push Push all built images -push_libs Push 'libs' images -push_libs_toolkit Push 'libs-toolkit' images +dependency_graph make a dependency graph of the involved libraries +build Build all Docker images +build_eddl_toolkit Build 'eddl-toolkit' image +build_ecvl_toolkit Build 'ecvl-toolkit' image +build_libs_toolkit Build 'libs-toolkit' image +build_eddl Build 'eddl' image +build_ecvl Build 'ecvl' image +build_libs Build 'libs' image +build_pyeddl_toolkit Build 'pyeddl-toolkit' image +build_pyecvl_toolkit Build 'pyecvl-toolkit' image +build_pylibs_toolkit Build 'pylibs-toolkit' image +build_pyeddl Build 'pyeddl' image +build_pyecvl Build 'pyecvl' image +build_pylibs Build 'pylibs' image +test Test all docker images +test_eddl Test 'eddl' images +test_eddl_toolkit Test 'eddl' images +test_ecvl Test 'ecvl' images +test_ecvl_toolkit Test 'ecvl' images +test_pyeddl Test 'ecvl' images +test_pyeddl_toolkit Test 'ecvl' images +test_pyecvl Test 'ecvl' images +test_pyecvl_toolkit Test 'ecvl' images +push Push all images +push_libs Push 'libs' image +push_libs_base Push 'lib-base' image +push_eddl Push 'eddl' image +push_ecvl Push 'ecvl' image +push_libs_toolkit Push 'libs-toolkit' image +push_libs_base_toolkit Push 'libs-base-toolkit' image +push_eddl_toolkit Push 'eddl-toolkit' images +push_ecvl_toolkit Push 'ecvl-toolkit' images push_pylibs Push 'pylibs' images +push_pyeddl Push 'pyeddl' images +push_pyecvl Push 'pyecvl' images push_pylibs_toolkit Push 'pylibs-toolkit' images -publish Publish all built images to a Docker Registry (e.g., DockerHub) -publish_libs Publish 'libs' images -publish_libs_toolkit Publish 'libs-toolkit' images -publish_pylibs Publish 'pylibs' images -publish_pylibs_toolkit Publish 'pylibs-toolkit' images -repo-login Login to the Docker Registry -version Output the current version of this Makefile +push_pyeddl_toolkit Push 'pyeddl-toolkit' images +push_pyecvl_toolkit Push 'pyeddl-toolkit' images +publish Publish all images to a Docker Registry (e.g., DockerHub) +publish_libs Publish 'libs' image +publish_eddl Publish 'eddl' image +publish_ecvl Publish 'ecvl' image +publish_libs_toolkit Publish 'libs-toolkit' image +publish_eddl_toolkit Publish 'eddl-toolkit' image +publish_ecvl_toolkit Publish 'ecvl-toolkit' image +publish_pylibs Publish 'pylibs' image +publish_pyeddl Publish 'pyeddl' image +publish_pyecvl Publish 'pyecvl' image +publish_pylibs_toolkit Publish 'pylibs-toolkit' image +publish_pyeddl_toolkit Publish 'pyeddl-toolkit' image +publish_pyecvl_toolkit Publish 'pyecvl-toolkit' image +docker-login Login to the Docker Registry +clean_eddl_sources clean repository containing EDDL source code +clean_ecvl_sources clean repository containing ECVL source code +clean_pyeddl_sources clean repository containing PyEDDL source code +clean_pyecvl_sources clean repository containing PyECVL source code +clean_libs_sources clean repository containing libs source code +clean_pylibs_sources clean repository containing pylibs source code +clean_sources clean repository containing source code ``` -Edit the file `settings.sh` to customize your images (name, software revision, Docker registry, etc.) +Edit the file `settings.conf` to customize your images (e.g., software revision, Docker registry, etc.) + diff --git a/jenkins.sh b/jenkins.sh new file mode 100755 index 0000000..b7a63ec --- /dev/null +++ b/jenkins.sh @@ -0,0 +1,316 @@ +#!/bin/bash + +#set -x +set -o nounset +set -o errexit +set -o pipefail +# without errtrace functions don't inherit the ERR trap +set -o errtrace + +# default libs version +export DOCKER_LIBS_REPO="${DOCKER_LIBS_REPO:-https://github.com/deephealthproject/docker-libs.git}" +export DOCKER_LIBS_BRANCH=${DOCKER_LIBS_BRANCH:-develop} +export DOCKER_LIBS_VERSION="" + +# set Docker repository +export DOCKER_REPOSITORY_OWNER="${DOCKER_REPOSITORY_OWNER:-dhealth}" + +# env requirements +DOCKER_USER=${DOCKER_USER:-} +DOCKER_PASSWORD=${DOCKER_PASSWORD:-} + +GIT_URL=${GIT_URL:-} + +# set script version +VERSION=0.3.0 + +function abspath() { + local path="${*}" + + if [[ -d "${path}" ]]; then + echo "$( cd "${path}" >/dev/null && pwd )" + else + echo "$( cd "$( dirname "${path}" )" >/dev/null && pwd )/$(basename "${path}")" + fi +} + +function log() { + echo -e "${@}" >&2 +} + +function debug_log() { + if [[ -n "${DEBUG:-}" ]]; then + echo -e "DEBUG: ${@}" >&2 + fi +} + +function error_log() { + echo -e "ERROR: ${@}" >&2 +} + +function error_trap() { + error_log "Error at line ${BASH_LINENO[1]} running the following command:\n\n\t${BASH_COMMAND}\n\n" + error_log "Stack trace:" + for (( i=1; i < ${#BASH_SOURCE[@]}; ++i)); do + error_log "$(printf "%$((4*$i))s %s:%s\n" " " "${BASH_SOURCE[$i]}" "${BASH_LINENO[$i]}")" + done + exit 2 +} + +trap error_trap ERR + +function usage_error() { + if [[ $# > 0 ]]; then + echo -e "ERROR: ${@}" >&2 + fi + help + exit 2 +} + +function print_version() { + echo ${VERSION} +} + +function help() { + local script_name=$(basename "$0") + echo -e "\nUsage of '${script_name}' + + ${script_name} [options] + ${script_name} -h prints this help message + ${script_name} -v prints the '${script_name}' version + + OPTIONS: + --clean-sources Remove library sources + --clean-images Remove local Docker images + --disable-cache Disable Docker cache + --disable-build Disable build of Docker images + --disable-pull Disable pull of existing Docker images + --disable-push Disable push of Docker images + --disable-tests Disable tests of Docker images + --disable-docker-login Disable login on Docker registry + + ENVIRONMENT requirements: + * DOCKER_USER + * DOCKER_PASSWORD + + ENVIRONMENT defaults: + * DOCKER_BASE_IMAGE_VERSION_TAG => 0.2.0 + * DOCKER_LIBS_REPO => https://github.com/deephealthproject/docker-libs.git + * DOCKER_LIBS_BRANCH => develop + * DOCKER_REPOSITORY_OWNER => dhealth + " >&2 +} + +# sw version +LIBS_TAG=$(git tag -l --points-at HEAD | tail -n 1) +LIBS_REVISION=$(git rev-parse --short HEAD | sed -E 's/-//; s/ .*//') +LIBS_BRANCH=$(git rev-parse --abbrev-ref HEAD | sed -E 's+(remotes/|origin/)++g; s+/+-+g; s/ .*//') +LIBS_VERSION=$(if [[ -n "${LIBS_TAG}" ]]; then echo ${LIBS_TAG}; else echo ${LIBS_BRANCH}-${LIBS_REVISION}; fi) + +# various settings +CLEAN_IMAGES=0 +CLEAN_SOURCES=0 +DISABLE_CACHE=0 +DISABLE_PULL=0 +DISABLE_BUILD=0 +DISABLE_PUSH=0 +DISABLE_TESTS=0 +DISABLE_DOCKER_LOGIN=0 + +# get docker-libs repository +function clone_docker_libs() { + if [[ ! -d "docker-libs" ]]; then + log "Cloning docker-libs (branch=${DOCKER_LIBS_BRANCH}, rev=${DOCKER_LIBS_VERSION})..." + if [[ -n "${DOCKER_LIBS_BRANCH}" ]]; then + branch="-b ${DOCKER_LIBS_BRANCH}" + fi + git clone ${branch} ${DOCKER_LIBS_REPO} + fi + if [[ -n "${DOCKER_LIBS_VERSION}" ]]; then + log "Cloning docker-libs (rev.${DOCKER_LIBS_VERSION})..." + cd docker-libs && git fetch && git reset --hard ${DOCKER_LIBS_VERSION} + fi + log "DONE" +} + +# Docker login +function docker_login() { + echo ${DOCKER_PASSWORD} | docker login --username ${DOCKER_USER} --password-stdin + export DOCKER_LOGIN_DONE="true" +} + +function run() { + local REPOSITORY="" + local LIB_NAME="" + # set repository + if [[ -n "${GIT_URL}" ]]; then \ + REPOSITORY=$(echo "${GIT_URL}" | sed -E 's+(.*)/([^/]*)\.git+\2+') ; \ + else + REPOSITORY=$(basename $(git rev-parse --show-toplevel)) ; + fi + # set library name + LIB_NAME=$(echo "${REPOSITORY}" | tr a-z A-Z | sed 's+DOCKER-LIBS+LIBS+; s+-+_+') + + # set git tag & branch & image prefix + # and define whether to push lates tag + GIT_BRANCH=${GIT_BRANCH:-$(git rev-parse --abbrev-ref HEAD | sed -E 's+(remotes/|origin/|tags/)++g; s+/+-+g; s/ .*//')} + BRANCH_NAME=$(echo "${GIT_BRANCH}" | sed 's+origin/++g; s+refs/tags/++g') + NORMALIZED_BRANCH_NAME=$(echo "${BRANCH_NAME}" | sed 's+/+-+g; s+[[:space:]]++g') + TAG=$(git tag -l --points-at HEAD | tail -n 1 | sed 's+[[:space:]]++g') + REVISION="$(git rev-parse --short HEAD --short | sed 's+[[:space:]]++g')" + if [ -n "${TAG}" ]; then + DOCKER_IMAGE_PREFIX="${TAG}" + # update latest if a tag exists and the branch is master + if [ "${BRANCH_NAME}" == "master" ]; then + export DOCKER_IMAGE_LATEST=true + fi + else + DOCKER_IMAGE_PREFIX="${REVISION}" + fi + + # define Docker image tags + export BUILD_NUMBER=${BUILD_NUMBER:-$(date '+%Y%m%d%H%M%S')} + #export DOCKER_IMAGE_TAG_EXTRA="${DOCKER_IMAGE_PREFIX}_build${BUILD_NUMBER}" + #export DOCKER_IMAGE_TAG_EXTRA="${DOCKER_IMAGE_TAG_EXTRA} ${NORMALIZED_BRANCH_NAME}_build${BUILD_NUMBER}" + + # set branch + export ${LIB_NAME}_BRANCH=${BRANCH_NAME} + + # log environment + printenv + + # detect repository + lib_suffix="" + if [[ "${LIB_NAME}" != "LIBS" ]]; then + lib_suffix="_${REPOSITORY}" + export CONFIG_FILE="" + clone_docker_libs + cd docker-libs + fi + + # cleanup sources + if [[ "${CLEAN_SOURCES}" == "true" ]]; then + make clean_sources + fi + + # cleanup images + if [[ "${CLEAN_IMAGES}" == "true" ]]; then + make clean_images + fi + + # build images + if [[ ${DISABLE_BUILD} == 0 ]]; then + log "Docker images before..." + docker images + make build${lib_suffix} ; + log "Docker images after..." + docker images + fi + + # make tests + if [[ ${DISABLE_TESTS} == 0 ]]; then + if [[ -n ${lib_suffix} ]]; then + make test${lib_suffix} ; + make test${lib_suffix}_toolkit ; + else + make test ; + fi + fi + + # push images + if [[ ${DISABLE_PUSH} == 0 ]]; then + # login to DockerHub + if [[ ${DISABLE_DOCKER_LOGIN} == 0 ]]; then + if [[ -z ${DOCKER_USER} || -z ${DOCKER_PASSWORD} ]]; then + usage_error "You need to set DOCKER_USER and DOCKER_PASSWORD on your environment" + fi + docker_login + export DOCKER_LOGIN_DONE="true" + fi + if [[ -n ${lib_suffix} ]]; then + make push${lib_suffix} ; + make push${lib_suffix}_toolkit ; + else + make push ; + fi + fi + + # print images + log "\n\nList of Docker images..." + log "*************************************************************************************************************" + make images_list + + # print libraries + log "\n\nList of Libraries..." + log "*************************************************************************************************************" + make libraries_list +} + +# parse arguments +POSITIONAL=() +while [[ $# -gt 0 ]] +do + opt="${1}" + value="${2:-}" + case $opt in + + -h) help; exit 0 ;; + -v) print_version; exit 0 ;; + + --clean-sources) + CLEAN_SOURCES=true + shift ;; + + --clean-images) + CLEAN_IMAGES=true + shift ;; + + --disable-cache) + DISABLE_CACHE=1 + shift ;; + + --disable-build) + DISABLE_BUILD=1 + shift ;; + + --disable-pull) + DISABLE_PULL=1 + shift ;; + + --disable-push) + DISABLE_PUSH=1 + shift ;; + + --disable-tests) + DISABLE_TESTS=1 + shift ;; + + --disable-docker-login) + DISABLE_DOCKER_LOGIN=1 + shift ;; + + *) # unknown option + POSITIONAL+=("$opt") # save it in an array for later + shift ;; + esac +done +set -- "${POSITIONAL[@]}" # restore positional parameters + +# get the absolute path of CONFIG_DIR +CONFIG_DIR=$(abspath $(pwd)) + +debug_log "docker-libs branch: ${DOCKER_LIBS_BRANCH}" +debug_log "docker-libs revision: ${DOCKER_LIBS_VERSION}" +debug_log "disable cache: ${DISABLE_CACHE}" +debug_log "disable pull: ${DISABLE_PULL}" + +# +export DOCKER_LIBS_BRANCH +export DOCKER_LIBS_VERSION +export CLEAN_IMAGES +export CLEAN_SOURCES +export DISABLE_CACHE +export DISABLE_PULL + +# exec +run diff --git a/libs/develop.Dockerfile b/libs/develop.Dockerfile deleted file mode 100644 index 83dff95..0000000 --- a/libs/develop.Dockerfile +++ /dev/null @@ -1,81 +0,0 @@ -FROM nvidia/cuda:10.1-devel - -# set metadata -LABEL website="https://github.com/deephealthproject/" -LABEL description="DeepHealth European Distributed Deep Learning Library" -LABEL software="deephealth-eddl,deephealth-ecvl" - - -ARG cmake_release="3.14.6" -# set arguments -ARG eddl_src_origin="eddl" -ARG ecvl_src_origin="ecvl" -ARG eddl_src_target="/usr/local/src/eddl" -ARG ecvl_src_target="/usr/local/src/ecvl" - -# expose lib source paths on environment -ENV EDDL_SRC ${eddl_src_target} -ENV ECVL_SRC ${ecvl_src_target} - -# Install software requirements -RUN \ - echo "\nInstalling software requirements..." >&2 \ - && export DEBIAN_FRONTEND=noninteractive \ - && apt-get update -y -q \ - && apt-get install -y --no-install-recommends \ - build-essential git gcc-8 g++-8 wget rsync graphviz \ - libopencv-dev libwxgtk3.0-dev libopenslide-dev \ - && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 70 \ - --slave /usr/bin/g++ g++ /usr/bin/g++-7 \ - --slave /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/x86_64-linux-gnu-gcc-7 \ - --slave /usr/bin/x86_64-linux-gnu-g++ x86_64-linux-gnu-g++ /usr/bin/x86_64-linux-gnu-g++-7 \ - && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 80 \ - --slave /usr/bin/g++ g++ /usr/bin/g++-8 \ - --slave /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/x86_64-linux-gnu-gcc-8 \ - --slave /usr/bin/x86_64-linux-gnu-g++ x86_64-linux-gnu-g++ /usr/bin/x86_64-linux-gnu-g++-8 \ - && apt-get clean \ - && cd /tmp/ \ - && wget --quiet https://github.com/Kitware/CMake/releases/download/v3.14.6/cmake-${cmake_release}-Linux-x86_64.tar.gz \ - && tar xzf cmake-${cmake_release}-Linux-x86_64.tar.gz \ - && rm cmake*.tar.gz \ - && mv cmake*/bin/* /usr/local/bin/ \ - && mv cmake*/share/* /usr/local/share/ \ - && chown root:root /usr/local/bin/* /usr/local/share/* \ - && chmod a+rx /usr/local/bin/* \ - && rm -rf /tmp/cmake* - -# copy libraries -COPY ${ecvl_src_origin} ${ECVL_SRC} -COPY ${eddl_src_origin} ${EDDL_SRC} - -# Build and install EDDL library -RUN echo "\nBuilding EDDL library..." >&2 \ - && cd ${EDDL_SRC} \ - && mkdir build \ - && cd build \ - && cmake \ - -D BUILD_TARGET=GPU \ - -D BUILD_TESTS=ON \ - -D EDDL_SHARED=ON \ - .. \ - && make -j$(grep -c ^processor /proc/cpuinfo) \ - && echo "\n Installing EDDL library..." >&2 \ - && make install - -# Build and install ECVL library -RUN echo "\nBuilding ECVL library..." >&2 \ - && cd ${ECVL_SRC} \ - && mkdir build \ - && cd build \ - && cmake \ - -D ECVL_BUILD_GUI=OFF \ - -D ECVL_WITH_OPENSLIDE=ON \ - -D ECVL_DATASET_PARSER=ON \ - -D ECVL_WITH_DICOM=ON \ - -D ECVL_BUILD_EDDL=ON \ - -D ECVL_BUILD_EXAMPLES=ON \ - -D EDDL_DIR=${EDDL_SRC}/build/install \ - .. \ - && make -j$(grep -c ^processor /proc/cpuinfo) \ - && echo "\n Installing ECVL library..." >&2 \ - && make install diff --git a/libs/ecvl-toolkit.Dockerfile b/libs/ecvl-toolkit.Dockerfile new file mode 100644 index 0000000..239a2be --- /dev/null +++ b/libs/ecvl-toolkit.Dockerfile @@ -0,0 +1,35 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} AS libs.ecvl-toolkit + +# set metadata +LABEL website="https://github.com/deephealthproject/" +LABEL description="DeepHealth European Distributed Deep Learning Library" +LABEL software="deephealth-ecvl" + +# set arguments +ARG ecvl_src_origin="ecvl" +ARG ecvl_src_target="/usr/local/src/ecvl" + +# expose lib source paths on environment +ENV ECVL_SRC ${ecvl_src_target} + +# copy libraries +COPY ${ecvl_src_origin} ${ECVL_SRC} + +# Build and install ECVL library +RUN echo "\nBuilding ECVL library..." >&2 \ + && cd ${ECVL_SRC} \ + && mkdir build \ + && cd build \ + && cmake \ + -D ECVL_BUILD_EXAMPLES=ON \ + -D ECVL_BUILD_GUI=OFF \ + -D ECVL_WITH_OPENSLIDE=ON \ + -D ECVL_DATASET_PARSER=ON \ + -D ECVL_WITH_DICOM=ON \ + -D ECVL_BUILD_EDDL=ON \ + .. \ + && make -j$(grep -c ^processor /proc/cpuinfo) \ + && echo "\n Installing ECVL library..." >&2 \ + && make install \ + && ldconfig diff --git a/libs/ecvl.Dockerfile b/libs/ecvl.Dockerfile new file mode 100644 index 0000000..4e52880 --- /dev/null +++ b/libs/ecvl.Dockerfile @@ -0,0 +1,61 @@ +ARG BASE_IMAGE +ARG TOOLKIT_IMAGE + +# Declare Toolkit image +FROM ${TOOLKIT_IMAGE} AS toolkit + +############################ +#### INTERMEDIATE Stage #### +############################ +FROM ${BASE_IMAGE} AS prepare_install + +# set arguments +ARG ecvl_src_origin="ecvl" +ARG ecvl_src_target="/usr/local/src/ecvl" + +# make a temporary copy of libraries +COPY --from=toolkit /usr/local/etc /tmp/local/etc +COPY --from=toolkit /usr/local/include /tmp/local/include +COPY --from=toolkit /usr/local/lib /tmp/local/lib +COPY --from=toolkit /usr/local/share /tmp/local/share +COPY --from=toolkit /usr/local/src/ecvl/build/install_manifest.txt /tmp/local/install_manifest.txt + + +# merge existing system directories with those containing libraries +RUN cd /tmp/local && sed -e 's+/usr/local/++g' /tmp/local/install_manifest.txt | \ + while IFS= read -r line; do echo ">>> $line" ; rsync --relative "${line}" "/usr/local/" || exit ; done + +###################### +#### TARGET Stage #### +###################### +FROM scratch AS libs.ecvl + +# set metadata +LABEL website="https://github.com/deephealthproject/" \ + description="DeepHealth European Distributed Deep Learning Library" \ + software="deephealth-ecvl" + +COPY --from=prepare_install /bin /bin +COPY --from=prepare_install /boot /boot +COPY --from=prepare_install /dev /dev +COPY --from=prepare_install /etc /etc +COPY --from=prepare_install /home /home +COPY --from=prepare_install /lib /lib +COPY --from=prepare_install /lib64 /lib64 +COPY --from=prepare_install /media /media +COPY --from=prepare_install /mnt /mnt +COPY --from=prepare_install /opt /opt +COPY --from=prepare_install /proc /proc +COPY --from=prepare_install /root /root +COPY --from=prepare_install /run /run +COPY --from=prepare_install /sbin /sbin +COPY --from=prepare_install /srv /srv +COPY --from=prepare_install /sys /sys +COPY --from=prepare_install /usr /usr +COPY --from=prepare_install /var /var + +# create the /tmp folder with right permissions +RUN mkdir -p /tmp && chmod 1777 /tmp + +# default cmd +CMD ["/bin/bash"] \ No newline at end of file diff --git a/libs/eddl-toolkit.Dockerfile b/libs/eddl-toolkit.Dockerfile new file mode 100644 index 0000000..681a914 --- /dev/null +++ b/libs/eddl-toolkit.Dockerfile @@ -0,0 +1,34 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} AS libs.eddl-toolkit + +# set metadata +LABEL website="https://github.com/deephealthproject/" \ + description="DeepHealth European Distributed Deep Learning Library" \ + software="deephealth-eddl" + +# set arguments +ARG eddl_src_origin="eddl" +ARG eddl_src_target="/usr/local/src/eddl" + +# expose lib source paths on environment +ENV EDDL_SRC ${eddl_src_target} + +# copy libraries +COPY ${eddl_src_origin} ${EDDL_SRC} + +# Build and install EDDL library +RUN echo "\nBuilding EDDL library..." >&2 \ + && cd ${EDDL_SRC} \ + && mkdir build \ + && cd build \ + && cmake \ + -D BUILD_TARGET=${BUILD_TARGET} \ + -D DBUILD_EXAMPLES=ON \ + -D BUILD_TESTS=ON \ + -D BUILD_SHARED_LIB=ON \ + -D BUILD_PROTOBUF=ON \ + .. \ + && make -j$(grep -c ^processor /proc/cpuinfo) \ + && echo "\n Installing EDDL library..." >&2 \ + && make install \ + && ldconfig \ No newline at end of file diff --git a/libs/eddl.Dockerfile b/libs/eddl.Dockerfile new file mode 100644 index 0000000..76488b0 --- /dev/null +++ b/libs/eddl.Dockerfile @@ -0,0 +1,60 @@ +ARG BASE_IMAGE +ARG TOOLKIT_IMAGE + +# Declare Toolkit image +FROM ${TOOLKIT_IMAGE} AS toolkit + +############################ +#### INTERMEDIATE Stage #### +############################ +FROM ${BASE_IMAGE} AS prepare_install + +# set arguments +ARG eddl_src_origin="eddl" +ARG eddl_src_target="/usr/local/src/eddl" + +# make a temporary copy of libraries +COPY --from=toolkit /usr/local/etc /tmp/local/etc +COPY --from=toolkit /usr/local/include /tmp/local/include +COPY --from=toolkit /usr/local/lib /tmp/local/lib +COPY --from=toolkit /usr/local/share /tmp/local/share +COPY --from=toolkit /usr/local/src/eddl/build/install_manifest.txt /tmp/local/install_manifest.txt + +# merge existing system directories with those containing libraries +RUN cd /tmp/local && sed -e 's+/usr/local/++g' /tmp/local/install_manifest.txt | \ + while IFS= read -r line; do echo ">>> $line" ; rsync --relative "${line}" "/usr/local/" || exit ; done + +###################### +#### TARGET Stage #### +###################### +FROM scratch AS libs.eddl + +# Set metadata +LABEL website="https://github.com/deephealthproject" \ + description="DeepHealth European Distributed Deep Learning Library" \ + software="deephealth-eddl" + +COPY --from=prepare_install /bin /bin +COPY --from=prepare_install /boot /boot +COPY --from=prepare_install /dev /dev +COPY --from=prepare_install /etc /etc +COPY --from=prepare_install /home /home +COPY --from=prepare_install /lib /lib +COPY --from=prepare_install /lib64 /lib64 +COPY --from=prepare_install /media /media +COPY --from=prepare_install /mnt /mnt +COPY --from=prepare_install /opt /opt +COPY --from=prepare_install /proc /proc +COPY --from=prepare_install /root /root +COPY --from=prepare_install /run /run +COPY --from=prepare_install /sbin /sbin +COPY --from=prepare_install /srv /srv +COPY --from=prepare_install /sys /sys +COPY --from=prepare_install /usr /usr +COPY --from=prepare_install /var /var + +# create the /tmp folder with right permissions +RUN mkdir -p /tmp && chmod 1777 /tmp + +# default cmd +CMD ["/bin/bash"] \ No newline at end of file diff --git a/libs/libs-base-toolkit.Dockerfile b/libs/libs-base-toolkit.Dockerfile new file mode 100644 index 0000000..0eeeaf6 --- /dev/null +++ b/libs/libs-base-toolkit.Dockerfile @@ -0,0 +1,114 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} as libs.base-toolkit + +# set metadata +LABEL website="https://github.com/deephealthproject/" \ + description="DeepHealth European Distributed Deep Learning Library" \ + software="deephealth-eddl,deephealth-ecvl" \ + maintainer="marcoenrico.piras@crs4.it" + +# set cmake version +ARG cmake_release="3.14.6" + +# set OpenCV version +ARG opencv_release="3.4.6" +ENV OPENCV_RELEASE ${opencv_release} +ENV OPENCV_INSTALL_MANIFEST "/usr/local/opencv/install_manifest.txt" + +# set Eigen version +ARG eigen_release="3.3.7" +ENV EIGEN_RELEASE ${eigen_release} +ENV EIGEN_INSTALL_MANIFEST "/usr/local/eigen/install_manifest.txt" +ENV CPATH="/usr/local/include/eigen3:${CPATH}" + +# set ProtoBuf version +ARG protobuf_release="3.11.4" +ENV PROTOBUF_RELEASE ${protobuf_release} +ENV PROTOBUF_INSTALL_MANIFEST "/usr/local/protobuf/install_manifest.txt" + +# set build target +ARG BUILD_TARGET="CPU" +ENV BUILD_TARGET=${BUILD_TARGET} + +# Install software requirements +RUN \ + echo "\nInstalling base software requirements..." >&2 \ + && export DEBIAN_FRONTEND=noninteractive \ + && apt-get update -y -q \ + && apt-get install -y --no-install-recommends \ + build-essential git gcc-8 g++-8 wget rsync graphviz \ + libwxgtk3.0-dev libopenslide-dev zlib1g-dev libblas-dev \ + libavcodec-dev libavformat-dev libswscale-dev \ + libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev \ + && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 70 \ + --slave /usr/bin/g++ g++ /usr/bin/g++-7 \ + --slave /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/x86_64-linux-gnu-gcc-7 \ + --slave /usr/bin/x86_64-linux-gnu-g++ x86_64-linux-gnu-g++ /usr/bin/x86_64-linux-gnu-g++-7 \ + && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 80 \ + --slave /usr/bin/g++ g++ /usr/bin/g++-8 \ + --slave /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/x86_64-linux-gnu-gcc-8 \ + --slave /usr/bin/x86_64-linux-gnu-g++ x86_64-linux-gnu-g++ /usr/bin/x86_64-linux-gnu-g++-8 \ + && echo "\n > Installing cmake (version '${cmake_release}')..." >&2 \ + && cd /tmp/ \ + && wget --quiet --no-check-certificate \ + https://github.com/Kitware/CMake/releases/download/v${cmake_release}/cmake-${cmake_release}-Linux-x86_64.tar.gz \ + && tar xzf cmake-${cmake_release}-Linux-x86_64.tar.gz \ + && rm cmake*.tar.gz \ + && mv cmake*/bin/* /usr/local/bin/ \ + && mv cmake*/share/* /usr/local/share/ \ + && chown root:root /usr/local/bin/* /usr/local/share/* \ + && chmod a+rx /usr/local/bin/* \ + && rm -rf /tmp/cmake* \ + && echo "\n > Installing OpenCV (version '${opencv_release}')..." >&2 \ + && cd /tmp/ \ + && wget --quiet --no-check-certificate \ + https://github.com/opencv/opencv/archive/${opencv_release}.tar.gz \ + && tar xzf ${opencv_release}.tar.gz \ + && rm ${opencv_release}.tar.gz \ + && cd opencv-${opencv_release} \ + && mkdir build \ + && cd build \ + && cmake -D CMAKE_BUILD_TYPE=RELEASE \ + -D OPENCV_GENERATE_PKGCONFIG=ON .. \ + && make -j$(nproc) \ + && make install \ + && mkdir -p $(dirname ${OPENCV_INSTALL_MANIFEST}) \ + && cp $(basename ${OPENCV_INSTALL_MANIFEST}) $(dirname ${OPENCV_INSTALL_MANIFEST})/ \ + && rm -rf /tmp/opencv-${opencv_release} \ + # Eigen version installed by APT is too old to work properly with CUDA + # https://devtalk.nvidia.com/default/topic/1026622/nvcc-can-t-compile-code-that-uses-eigen/ + && echo "\n > Installing Eigen (version '${eigen_release}')..." >&2 \ + && cd /tmp \ + && wget --quiet --no-check-certificate \ + https://gitlab.com/libeigen/eigen/-/archive/${eigen_release}/eigen-${eigen_release}.tar.gz \ + && tar xzf eigen-${eigen_release}.tar.gz \ + && rm eigen-${eigen_release}.tar.gz \ + && cd eigen-${eigen_release} \ + && mkdir build \ + && cd build \ + && cmake -D OpenGL_GL_PREFERENCE=GLVND .. \ + && make install \ + && mkdir -p $(dirname ${EIGEN_INSTALL_MANIFEST}) \ + && cp $(basename ${EIGEN_INSTALL_MANIFEST}) $(dirname ${EIGEN_INSTALL_MANIFEST})/ \ + && rm -rf /tmp/eigen-${eigen_release} \ + && echo "\n > Installing ProtoBuf (version '${protobuf_release}')..." >&2 \ + && cd /tmp \ + && wget --quiet --no-check-certificate \ + https://github.com/protocolbuffers/protobuf/releases/download/v${protobuf_release}/protobuf-all-${protobuf_release}.tar.gz \ + && tar xf protobuf-all-${protobuf_release}.tar.gz \ + && rm protobuf-all-${protobuf_release}.tar.gz \ + && cd protobuf-${protobuf_release}/ \ + && ./configure \ + && make -j$(nproc) \ + && make install \ + && rm -rf /tmp/protobuf-${protobuf_release} \ + && echo "\n > Installing GTest library..." >&2 \ + && apt-get install -y --no-install-recommends libgtest-dev \ + && cd /usr/src/gtest \ + && mkdir build \ + && cd build \ + && cmake .. \ + && make install \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && ldconfig diff --git a/libs/libs-base.Dockerfile b/libs/libs-base.Dockerfile new file mode 100644 index 0000000..521e8ce --- /dev/null +++ b/libs/libs-base.Dockerfile @@ -0,0 +1,67 @@ +ARG BASE_IMAGE +ARG TOOLKIT_IMAGE +ARG BUILD_TARGET="CPU" +ARG OPENV_INSTALL_MANIFEST="/usr/local/opencv/install_manifest.txt" + +# set toolkit image +FROM ${TOOLKIT_IMAGE} as toolkit + +############################ +#### INTERMEDIATE Stage #### +############################ +FROM ${BASE_IMAGE} AS prepare_install + +RUN apt-get update -y -q \ + && apt-get install -y --no-install-recommends wget rsync \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# make a temporary copy of libraries +COPY --from=toolkit /usr/local/bin /tmp/local/bin +COPY --from=toolkit /usr/local/etc /tmp/local/etc +COPY --from=toolkit /usr/local/include /tmp/local/include +COPY --from=toolkit /usr/local/lib /tmp/local/lib +COPY --from=toolkit /usr/local/share /tmp/local/share +COPY --from=toolkit /usr/local/opencv/install_manifest.txt /tmp/local/opencv_manifest.txt +COPY --from=toolkit /usr/local/eigen/install_manifest.txt /tmp/local/eigen_manifest.txt + +# merge existing system directories with those containing libraries +RUN cd /tmp/local && sed -e 's+/usr/local/++g' *_manifest.txt | \ + while IFS= read -r line; do echo ">>> $line" ; rsync --relative "${line}" "/usr/local/" || exit ; done \ + && find /tmp/local/lib -not -type d -execdir cp "{}" /usr/local/lib ";" + +#################### +#### BASE image #### +#################### +FROM ${BASE_IMAGE} AS base + +LABEL website="https://github.com/deephealthproject" \ + description="DeepHealth European Distributed Deep Learning Library" \ + software="deephealth-eddl,deephealth-ecvl" \ + maintainer="marcoenrico.piras@crs4.it" + +# set build target +ARG BUILD_TARGET +ENV BUILD_TARGET ${BUILD_TARGET} + +# Install software requirements +RUN \ + echo "\nInstalling software requirements..." >&2 \ + && export DEBIAN_FRONTEND=noninteractive \ + && apt-get update -y -q \ + && apt-get install -y --no-install-recommends \ + wget \ + rsync \ + libavcodec-dev libavformat-dev libswscale-dev \ + libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev \ + libopenslide-dev \ + libgomp1 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && ldconfig + +# copy libraries to the target paths +COPY --from=prepare_install /usr/local/etc /usr/local/etc +COPY --from=prepare_install /usr/local/include /usr/local/include +COPY --from=prepare_install /usr/local/lib /usr/local/lib +COPY --from=prepare_install /usr/local/share /usr/local/share \ No newline at end of file diff --git a/libs/libs-toolkit.Dockerfile b/libs/libs-toolkit.Dockerfile new file mode 100644 index 0000000..d090eb1 --- /dev/null +++ b/libs/libs-toolkit.Dockerfile @@ -0,0 +1,7 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} AS libs-toolkit + +# set metadata +LABEL website="https://github.com/deephealthproject/" \ + description="DeepHealth European Distributed Deep Learning Library" \ + software="deephealth-eddl,deephealth-ecvl" \ No newline at end of file diff --git a/libs/libs.Dockerfile b/libs/libs.Dockerfile new file mode 100644 index 0000000..2e7d2d5 --- /dev/null +++ b/libs/libs.Dockerfile @@ -0,0 +1,7 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} AS libs + +# Set metadata +LABEL website="https://github.com/deephealthproject" \ + description="DeepHealth European Distributed Deep Learning Library" \ + software="deephealth-eddl,deephealth-ecvl" diff --git a/libs/runtime.Dockerfile b/libs/runtime.Dockerfile deleted file mode 100644 index 6d45a34..0000000 --- a/libs/runtime.Dockerfile +++ /dev/null @@ -1,73 +0,0 @@ -ARG BASE_IMAGE -FROM ${BASE_IMAGE} AS toolkit - -#################### -#### BASE image #### -#################### - -FROM nvidia/cuda:10.1-runtime AS base - -LABEL website="https://github.com/deephealthproject" -LABEL description="DeepHealth European Distributed Deep Learning Library" -LABEL software="deephealth-eddl,deephealth-ecvl" - - -RUN \ - echo "\nInstalling software requirements..." >&2 \ - && export DEBIAN_FRONTEND=noninteractive \ - && apt-get update -y -q \ - && apt-get install -y --no-install-recommends \ - wget \ - rsync \ - libopencv-core-dev \ - libopencv-imgproc-dev \ - libopencv-imgcodecs-dev \ - libopenslide-dev \ - libgomp1 \ - && apt-get clean - -# set arguments -ARG eddl_src_origin="eddl" -ARG ecvl_src_origin="ecvl" -ARG eddl_src_target="/usr/local/src/eddl" -ARG ecvl_src_target="/usr/local/src/ecvl" - -######################### -#### INTERMEDIATE Stage #### -######################### -FROM base as prepare_install - -# install missing rsync utility -RUN export DEBIAN_FRONTEND=noninteractive \ - && apt-get update -y -q \ - && apt-get install -y --no-install-recommends rsync \ - && apt-get clean - -# create a temp directory -RUN mkdir /tmp/local - -# make a temporary copy of libraries -COPY --from=toolkit /usr/local/etc /tmp/local/etc -COPY --from=toolkit /usr/local/include /tmp/local/include -COPY --from=toolkit /usr/local/lib /tmp/local/lib -COPY --from=toolkit /usr/local/share /tmp/local/share -COPY --from=toolkit /usr/local/src/ecvl/build/install_manifest.txt /tmp/local/ecvl_manifest.txt -COPY --from=toolkit /usr/local/src/eddl/build/install_manifest.txt /tmp/local/eddl_manifest.txt - -# change working directory -WORKDIR /tmp/local - -# merge existing system directories with those containing libraries -RUN sed -e 's+/usr/local/++' *_manifest.txt | \ - while IFS= read -r line; do echo ">>> $line" ; rsync --relative "${line}" "/usr/local/" || exit ; done - -###################### -#### TARGET Stage #### -###################### -FROM base - -# copy libraries to the target paths -COPY --from=prepare_install /usr/local/etc /usr/local/etc -COPY --from=prepare_install /usr/local/include /usr/local/include -COPY --from=prepare_install /usr/local/lib /usr/local/lib -COPY --from=prepare_install /usr/local/share /usr/local/share \ No newline at end of file diff --git a/pylibs/develop.Dockerfile b/pylibs/develop.Dockerfile deleted file mode 100644 index 4f5b567..0000000 --- a/pylibs/develop.Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -ARG BASE_IMAGE -FROM ${BASE_IMAGE} as base - - -# set metadata -LABEL website="https://github.com/deephealthproject/" -LABEL description="DeepHealth European Distributed Deep Learning Library" -LABEL software="deephealth-eddl,deephealth-ecvl,deephealth-pyecvl,deephealth-pyeddl" - - -ARG eddl_src="/usr/local/src/eddl" -ARG ecvl_src="/usr/local/src/ecvl" - -ARG pyeddl_src_origin="pyeddl" -ARG pyecvl_src_origin="pyecvl" -ARG pyeddl_src_target="/usr/local/src/pyeddl" -ARG pyecvl_src_target="/usr/local/src/pyecvl" - -# Run git submodule update [--init] --recursive first -COPY ${pyeddl_src_origin} ${pyeddl_src_target} -COPY ${pyecvl_src_origin} ${pyecvl_src_target} - -RUN \ - echo "\nInstalling software requirements..." >&2 \ - && apt-get -y update && apt-get -y install --no-install-recommends \ - python3-dev python3-pip \ - && apt-get clean \ - && python3 -m pip install --upgrade --no-cache-dir \ - setuptools pip numpy pybind11 pytest - -RUN \ - cd ${pyeddl_src_target} \ - && echo "\nLinking eddl library..." >&2 \ - && rm -r third_party/eddl \ - && ln -s ${eddl_src} third_party/ \ - && echo "\nInstalling pyeddl module..." >&2 \ - && python3 setup.py install \ - && rm -rf build/temp.* - -RUN \ - cd ${pyecvl_src_target} \ - && echo "\nLinking ecvl library..." >&2 \ - && rm -r third_party/ecvl \ - && rm -r third_party/pyeddl \ - && ln -s ${ecvl_src} third_party/ecvl \ - && ln -s ${pyeddl_src_target} third_party/pyeddl \ - && echo "\nInstalling pyecvl module..." >&2 \ - && python3 setup.py install \ - && rm -rf build/temp.* \ No newline at end of file diff --git a/pylibs/pyecvl-toolkit.Dockerfile b/pylibs/pyecvl-toolkit.Dockerfile new file mode 100644 index 0000000..c6cf063 --- /dev/null +++ b/pylibs/pyecvl-toolkit.Dockerfile @@ -0,0 +1,28 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} as base + +# set metadata +LABEL website="https://github.com/deephealthproject/" \ + description="DeepHealth European Distributed Deep Learning Library" \ + software="deephealth-eddl,deephealth-ecvl,deephealth-pyecvl,deephealth-pyeddl" + +# set paths +ARG ecvl_src="/usr/local/src/ecvl" +ARG pyecvl_src_origin="pyecvl" +ARG pyeddl_src_target="/usr/local/src/pyeddl" +ARG pyecvl_src_target="/usr/local/src/pyecvl" + +# Run git submodule update [--init] --recursive first +COPY ${pyecvl_src_origin} ${pyecvl_src_target} + +# build & install +RUN \ + cd ${pyecvl_src_target} \ + && echo "\nLinking ecvl library..." >&2 \ + && rm -rf third_party/ecvl \ + && rm -rf third_party/pyeddl \ + && ln -s ${ecvl_src} third_party/ecvl \ + && ln -s ${pyeddl_src_target} third_party/pyeddl \ + && echo "\nInstalling pyecvl module..." >&2 \ + && python3 setup.py install --record install.log \ + && rm -rf build/temp.* \ No newline at end of file diff --git a/pylibs/pyecvl.Dockerfile b/pylibs/pyecvl.Dockerfile new file mode 100644 index 0000000..5216ffe --- /dev/null +++ b/pylibs/pyecvl.Dockerfile @@ -0,0 +1,27 @@ +# base image to start from +ARG BASE_IMAGE +ARG TOOLKIT_IMAGE + +# set toolkit as intermediate stage +FROM ${TOOLKIT_IMAGE} as intermediate_stage + +ARG pyecvl_src="/usr/local/src/pyecvl" + +WORKDIR ${pyecvl_src} + +# merge existing system directories with those containing libraries +RUN sed -e 's+/usr/local/++g' install.log | \ + while IFS= read -r line; do echo ">>> $line" ; rsync --relative "/usr/local/./${line}/" "/intermediate_path/" || exit ; done + +# prepare target image +FROM ${BASE_IMAGE} as base + +# set metadata +LABEL website="https://github.com/deephealthproject/" \ + description="DeepHealth European Distributed Deep Learning Library" \ + software="deephealth-eddl,deephealth-ecvl,deephealth-pyecvl,deephealth-pyeddl" + +# Run git submodule update [--init] --recursive first +COPY --from=intermediate_stage /intermediate_path/bin/* /usr/local/bin/ +COPY --from=intermediate_stage /intermediate_path/lib/python3.6/dist-packages/* /usr/local/lib/python3.6/dist-packages/ + diff --git a/pylibs/pyeddl-toolkit.Dockerfile b/pylibs/pyeddl-toolkit.Dockerfile new file mode 100644 index 0000000..a709c34 --- /dev/null +++ b/pylibs/pyeddl-toolkit.Dockerfile @@ -0,0 +1,32 @@ +ARG BASE_IMAGE + +# base image +FROM ${BASE_IMAGE} as base + +# set metadata +LABEL website="https://github.com/deephealthproject/" \ + description="DeepHealth European Distributed Deep Learning Library" \ + software="deephealth-eddl,deephealth-ecvl,deephealth-pyeddl" + +ARG eddl_src="/usr/local/src/eddl" + +ARG pyeddl_src_origin="pyeddl" +ARG pyeddl_src_target="/usr/local/src/pyeddl" + +# Run git submodule update [--init] --recursive first +COPY ${pyeddl_src_origin} ${pyeddl_src_target} + +# link the cudart, cublas and curand libraries on "standard" system locations +RUN /bin/bash -c "if [[ \"${BUILD_TARGET}\" == \"GPU\" ]]; then \ + ln -s /usr/local/cuda-10.1/targets/x86_64-linux/lib/libcudart.so /usr/lib/ \ + && ln -s /usr/local/cuda-10.1/targets/x86_64-linux/lib/libcurand.so /usr/lib/ \ + && ln -s /usr/local/cuda-10.1/targets/x86_64-linux/lib/libcublas.so /usr/lib/ \ + && export EDDL_WITH_CUDA=\"true\" ; \ + fi" \ + && cd ${pyeddl_src_target} \ + && echo "\nLinking eddl library..." >&2 \ + && rm -rf third_party/eddl \ + && ln -s ${eddl_src} third_party/ \ + && echo "\nInstalling pyeddl module..." >&2 \ + && python3 setup.py install --record install.log \ + && rm -rf build/temp.* diff --git a/pylibs/pyeddl.Dockerfile b/pylibs/pyeddl.Dockerfile new file mode 100644 index 0000000..d42a421 --- /dev/null +++ b/pylibs/pyeddl.Dockerfile @@ -0,0 +1,34 @@ +# base image to start from +ARG BASE_IMAGE +ARG TOOLKIT_IMAGE + +# set toolkit as intermediate stage +FROM ${TOOLKIT_IMAGE} as intermediate_stage + +ARG pyeddl_src="/usr/local/src/pyeddl" + +WORKDIR ${pyeddl_src} + +# merge existing system directories with those containing libraries +RUN sed -e 's+/usr/local/++g' install.log | \ + while IFS= read -r line; do echo ">>> $line" ; rsync --relative "/usr/local/./${line}/" "/intermediate_path/" || exit ; done + +# prepare target image +FROM ${BASE_IMAGE} as base + +# set metadata +LABEL website="https://github.com/deephealthproject/" \ + description="DeepHealth European Distributed Deep Learning Library" \ + software="deephealth-eddl,deephealth-ecvl,deephealth-pyeddl" + +# link the cudart, cublas and curand libraries on "standard" system locations +RUN /bin/bash -c "if [[ \"${BUILD_TARGET}\" == \"GPU\" ]]; then \ + ln -s /usr/local/cuda-10.1/targets/x86_64-linux/lib/libcudart.so /usr/lib/ \ + && ln -s /usr/local/cuda-10.1/targets/x86_64-linux/lib/libcurand.so /usr/lib/ \ + && ln -s /usr/local/cuda-10.1/targets/x86_64-linux/lib/libcublas.so /usr/lib/ ; \ + fi" + +# Run git submodule update [--init] --recursive first +COPY --from=intermediate_stage /intermediate_path/bin/* /usr/local/bin/ +COPY --from=intermediate_stage /intermediate_path/lib/python3.6/dist-packages/* /usr/local/lib/python3.6/dist-packages/ + diff --git a/pylibs/pylibs-base-toolkit.Dockerfile b/pylibs/pylibs-base-toolkit.Dockerfile new file mode 100644 index 0000000..1e125bb --- /dev/null +++ b/pylibs/pylibs-base-toolkit.Dockerfile @@ -0,0 +1,16 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} as base + +# set metadata +LABEL website="https://github.com/deephealthproject/" \ + description="DeepHealth European Distributed Deep Learning Library" + +# software requirements +RUN \ + echo "\nInstalling software requirements..." >&2 \ + && apt-get -y update && apt-get -y install --no-install-recommends \ + python3-dev python3-pip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && python3 -m pip install --upgrade --no-cache-dir \ + setuptools pip numpy pybind11 pytest diff --git a/pylibs/pylibs-base.Dockerfile b/pylibs/pylibs-base.Dockerfile new file mode 100644 index 0000000..42bf6fd --- /dev/null +++ b/pylibs/pylibs-base.Dockerfile @@ -0,0 +1,17 @@ +# base image to start from +ARG BASE_IMAGE +FROM ${BASE_IMAGE} as base + +# set metadata +LABEL website="https://github.com/deephealthproject/" \ + description="DeepHealth European Distributed Deep Learning Library" + +# Install software requirements +RUN \ + echo "\nInstalling software requirements..." >&2 \ + && apt-get -y update && apt-get -y install --no-install-recommends \ + python3-dev python3-pip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && python3 -m pip install --upgrade --no-cache-dir \ + setuptools pip numpy pybind11 pytest \ No newline at end of file diff --git a/pylibs/pylibs-toolkit.Dockerfile b/pylibs/pylibs-toolkit.Dockerfile new file mode 100644 index 0000000..04de904 --- /dev/null +++ b/pylibs/pylibs-toolkit.Dockerfile @@ -0,0 +1,7 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} as base + +# set metadata +LABEL website="https://github.com/deephealthproject/" \ + description="DeepHealth European Distributed Deep Learning Library" \ + software="deephealth-eddl,deephealth-ecvl,deephealth-pyecvl,deephealth-pyeddl" diff --git a/pylibs/pylibs.Dockerfile b/pylibs/pylibs.Dockerfile new file mode 100644 index 0000000..67096a3 --- /dev/null +++ b/pylibs/pylibs.Dockerfile @@ -0,0 +1,8 @@ +# base image to start from +ARG BASE_IMAGE +FROM ${BASE_IMAGE} as base + +# set metadata +LABEL website="https://github.com/deephealthproject/" \ + description="DeepHealth European Distributed Deep Learning Library" \ + software="deephealth-eddl,deephealth-ecvl,deephealth-pyecvl,deephealth-pyeddl" diff --git a/pylibs/runtime.Dockerfile b/pylibs/runtime.Dockerfile deleted file mode 100644 index 8ebe492..0000000 --- a/pylibs/runtime.Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -# base image to start from -ARG BASE_IMAGE -# pylibs-toolkit image -ARG TOOLKIT_IMAGE - -FROM ${TOOLKIT_IMAGE} as toolkit_image - -FROM ${BASE_IMAGE} as base - -# set metadata -LABEL website="https://github.com/deephealthproject/" -LABEL description="DeepHealth European Distributed Deep Learning Library" -LABEL software="deephealth-eddl,deephealth-ecvl,deephealth-pyecvl,deephealth-pyeddl" - - -ARG eddl_src="/usr/local/src/eddl" -ARG ecvl_src="/usr/local/src/ecvl" - -ARG pyeddl_src_origin="pyeddl" -ARG pyecvl_src_origin="pyecvl" -ARG pyeddl_src_target="/usr/local/src/pyeddl" -ARG pyecvl_src_target="/usr/local/src/pyecvl" - -# Run git submodule update [--init] --recursive first -COPY --from=toolkit_image ${pyeddl_src_target} ${pyeddl_src_target} -COPY --from=toolkit_image ${pyecvl_src_target} ${pyecvl_src_target} - -RUN \ - echo "\nInstalling software requirements..." >&2 \ - && apt-get -y update && apt-get -y install --no-install-recommends \ - python3-dev python3-pip \ - && apt-get clean \ - && python3 -m pip install --upgrade --no-cache-dir \ - setuptools pip numpy pybind11 pytest \ - && cd ${pyeddl_src_target} \ - && echo "\nInstalling pyeddl module..." >&2 \ - && python3 setup.py install \ - && cd ${pyecvl_src_target} \ - && echo "\nInstalling pyecvl module..." >&2 \ - && python3 setup.py install \ No newline at end of file diff --git a/settings.sh b/settings.conf similarity index 65% rename from settings.sh rename to settings.conf index f0e8929..ae6a937 100644 --- a/settings.sh +++ b/settings.conf @@ -1,39 +1,46 @@ +# set build target (CPU or GPU) +BUILD_TARGET=GPU + +# enable/disable suffix `-cpu` or `-gpu` +ENABLE_TARGET_SUFFIX_IMAGE_TAG=false + # set docker user credentials -DOCKER_USER=deephealth -DOCKER_PASSWORD="" +#DOCKER_USER=deephealth +#DOCKER_PASSWORD="" +DOCKER_LOGIN_DONE=true # use DockerHub as default registry -DOCKER_REGISTRY=registry.hub.docker.com +#DOCKER_REGISTRY=registry.hub.docker.com # set Docker repository #DOCKER_REPOSITORY_OWNER=dhealth -DOCKER_IMAGE_PREFIX= +#DOCKER_IMAGE_PREFIX= # latest tag settings #DOCKER_IMAGE_LATEST=false +# date.time as build number +#DOCKER_IMAGE_TAG=0.1.2 + # additional tags #DOCKER_IMAGE_TAG_EXTRA= # ECVL repository ECVL_REPOSITORY=https://github.com/deephealthproject/ecvl.git ECVL_BRANCH=master -ECVL_REVISION=v0.1.0 +ECVL_REVISION=e873315 # PyECVL PYECVL_REPOSITORY=https://github.com/deephealthproject/pyecvl.git PYECVL_BRANCH=master -PYECVL_REVISION=0.1.0 +PYECVL_REVISION=8effe45 # EDDL repository EDDL_REPOSITORY=https://github.com/deephealthproject/eddl.git EDDL_BRANCH=master -EDDL_REVISION=0.3 +EDDL_REVISION=7467cb6 # PyEDDL repository PYEDDL_REPOSITORY=https://github.com/deephealthproject/pyeddl.git PYEDDL_BRANCH=master -PYEDDL_REVISION=0.2.0 - -# date.time as build number -#DOCKER_IMAGE_TAG=0.1.2 +PYEDDL_REVISION=a55c3c4 diff --git a/tests/ecvl/3e865965bc4abe2e040deb7ce44b1f386f35a5c1.sh b/tests/ecvl/3e865965bc4abe2e040deb7ce44b1f386f35a5c1.sh new file mode 100644 index 0000000..15b1183 --- /dev/null +++ b/tests/ecvl/3e865965bc4abe2e040deb7ce44b1f386f35a5c1.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# exit on error +set -e + +# set the path containing tests +ECVL_SRC=${ECVL_SRC:-"/usr/local/src/ecvl"} + +# install cmake just to run ctest +if [ ! $(command -v cmake) ]; then + apt-get update -y -q && apt-get install -y -q cmake +fi + +# run all tests +cd ${ECVL_SRC}/build && ctest -C Debug -VV diff --git a/tests/eddl/ddb079b8dd624204bad3d902761e6b291e3f74fa.sh b/tests/eddl/ddb079b8dd624204bad3d902761e6b291e3f74fa.sh new file mode 100644 index 0000000..4be61f2 --- /dev/null +++ b/tests/eddl/ddb079b8dd624204bad3d902761e6b291e3f74fa.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# exit on error +set -e + +# set the path containing sources and tests +EDDL_SRC=${EDDL_SRC:-"/usr/local/src/eddl"} + +# install cmake just to run ctest +if [ ! $(command -v cmake) ]; then + apt-get update -y -q && apt-get install -y -q cmake +fi + +# run all tests +cd ${EDDL_SRC}/build && ctest -C Debug -VV diff --git a/tests/inventory.py b/tests/inventory.py new file mode 100644 index 0000000..c44b9e5 --- /dev/null +++ b/tests/inventory.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +import os +import logging +import argparse +import subprocess +from glob import glob + +# Set script logger +_logger = logging.getLogger() + + +def get_repo_commits(repo_path): + if not os.path.exists(repo_path): + raise FileNotFoundError("The repository %s doesn't exist", repo_path) + cmd_result = subprocess.Popen("cd {} && git rev-list --all --reverse".format(repo_path), shell=True, + stdout=subprocess.PIPE) + subprocess_return = cmd_result.stdout.read() + repo_list = subprocess_return.decode("utf-8").split('\n') + _logger.debug(repo_list) + return repo_list + + +def get_current_repo_revision(repo_path): + if not os.path.exists(repo_path): + raise FileNotFoundError("The repository %s doesn't exist", repo_path) + cmd_result = subprocess.Popen("cd {} && git rev-parse HEAD".format(repo_path), shell=True, stdout=subprocess.PIPE) + subprocess_return = cmd_result.stdout.read() + repo_list = subprocess_return.decode("utf-8").strip() + _logger.debug(repo_list) + return repo_list + + +def get_test_revisions(tests_path=".", commits=None, reverse=False): + revisions = sorted([(os.path.basename(v).split(".")[0],v) \ + for v in glob(os.path.join(tests_path, "*.sh"))], \ + key=lambda x: commits.index(x[0]), reverse=reverse) + _logger.debug("revisions %r", revisions) + return revisions + + +def find_test_revision(repo_path=".", tests_path="."): + found_revision = None + commits = get_repo_commits(repo_path) + current_revision = get_current_repo_revision(repo_path) + test_revisions = get_test_revisions(tests_path, commits, reverse=True) + if len(test_revisions) == 0: + return None + current_candidate = test_revisions.pop() + while (len(test_revisions)>0): + next_candidate = test_revisions.pop() + if commits.index(current_revision) < commits.index(current_candidate[0]) \ + or commits.index(current_revision) >= commits.index(current_candidate[0]) \ + and commits.index(current_revision) < commits.index(next_candidate[0]): + break + else: + current_candidate = next_candidate + return current_candidate + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('repo_path', metavar='REPO_PATH', type=str, help='Path of the git repository') + parser.add_argument('test_revisions_path', metavar='TEST_REVISIONS_PATH', type=str, + help='Path containing revisions of a test') + parser.add_argument('--debug', action='store_true', help='Enable debug') + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO) + found_test_revision = find_test_revision(args.repo_path, args.test_revisions_path) + _logger.debug(found_test_revision) + if found_test_revision: + print(found_test_revision[1]) + + +if __name__ == "__main__": + main() diff --git a/tests/pyecvl/36da4abde1a8d6633310ea9f39b828c50369ecba.sh b/tests/pyecvl/36da4abde1a8d6633310ea9f39b828c50369ecba.sh new file mode 100644 index 0000000..fe448c1 --- /dev/null +++ b/tests/pyecvl/36da4abde1a8d6633310ea9f39b828c50369ecba.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# exit on error +set -e + +# set the path containing sources and tests +ECVL_SRC=${ECVL_SRC:-"/usr/local/src/ecvl"} +PYECVL_SRC=${PYECVL_SRC:-"/usr/local/src/pyecvl"} + +# run tests +cd ${PYECVL_SRC} && pytest tests + +# run examples +cd ${PYECVL_SRC}/examples +python3 dataset.py "${ECVL_SRC}/examples/data/mnist/mnist.yml" +python3 ecvl_eddl.py "${ECVL_SRC}/examples/data/test.jpg" "${ECVL_SRC}/examples/data/mnist/mnist.yml" +python3 img_format.py "${ECVL_SRC}/examples/data/nifti/LR_nifti.nii" "${ECVL_SRC}/examples/data/dicom/ISIC_0000008.dcm" +python3 imgproc.py "${ECVL_SRC}/examples/data/test.jpg" +python3 openslide.py "${ECVL_SRC}/examples/data/hamamatsu/test3-DAPI 2 (387).ndpi" +python3 read_write.py "${ECVL_SRC}/examples/data/test.jpg" "test_mod.jpg" \ No newline at end of file diff --git a/tests/pyecvl/8effe452abc060e8a748c1cde54d8e03c2b7580b.sh b/tests/pyecvl/8effe452abc060e8a748c1cde54d8e03c2b7580b.sh new file mode 100644 index 0000000..6daff19 --- /dev/null +++ b/tests/pyecvl/8effe452abc060e8a748c1cde54d8e03c2b7580b.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# exit on error +set -e + +# set the path containing sources and tests +ECVL_SRC=${ECVL_SRC:-"/usr/local/src/ecvl"} +PYECVL_SRC=${PYECVL_SRC:-"/usr/local/src/pyecvl"} + +# run tests +cd ${PYECVL_SRC} && pytest tests + +# run examples +cd ${PYECVL_SRC}/examples +bash run_all.sh "${ECVL_SRC}/examples/data" \ No newline at end of file diff --git a/tests/pyeddl/a55c3c40414a2e751c3e2d392b7ae1ae201c5168.sh b/tests/pyeddl/a55c3c40414a2e751c3e2d392b7ae1ae201c5168.sh new file mode 100644 index 0000000..dc76468 --- /dev/null +++ b/tests/pyeddl/a55c3c40414a2e751c3e2d392b7ae1ae201c5168.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# exit on error +set -e + +# set the path containing sources and tests +PYEDDL_SRC=${PYEDDL_SRC:-"/usr/local/src/pyeddl"} + +# run tests +cd ${PYEDDL_SRC} + +echo 'Downloading test dataset' +wget -q https://www.dropbox.com/s/khrb3th2z6owd9t/trX.bin +wget -q https://www.dropbox.com/s/m82hmmrg46kcugp/trY.bin +wget -q https://www.dropbox.com/s/7psutd4m4wna2d5/tsX.bin +wget -q https://www.dropbox.com/s/q0tnbjvaenb4tjs/tsY.bin + +echo 'Running tests....' +pytest tests + +# run examples +if [ -z "${GPU_RUNTIME}" ]; then + echo 'Running CPU examples...' + python3 examples/Tensor/array_tensor_save.py + python3 examples/NN/1_MNIST/mnist_auto_encoder.py --epochs 1 +else + echo "Running GPU examples..." + python3 examples/Tensor/array_tensor_save.py + bash examples/NN/1_MNIST/run_all_fast.sh + bash examples/NN/py_loss_metric/run_all_fast.sh + bash examples/onnx/run_all_fast.sh +fi