Skip to content

Commit

Permalink
added buildscript for testcontainers and manual; dind not working yet
Browse files Browse the repository at this point in the history
  • Loading branch information
L6REKcodw committed Aug 16, 2024
1 parent 990d6e4 commit c8e728a
Show file tree
Hide file tree
Showing 3 changed files with 184 additions and 50 deletions.
76 changes: 26 additions & 50 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,22 +33,6 @@ variables:
cache:
key: "$CI_JOB_NAME"

before_script:
- apt update
- apt -y install git mesa-utils python3-dev libblas-dev liblapack-dev libsndfile1-dev libsox-dev cmake ninja-build curl build-essential python-is-python3


# Install poetry using recommended method
- curl -sSL https://install.python-poetry.org/ | python3
- export PATH=${POETRY_HOME}/bin:${PATH}

# install and manage dependencies with poetry
- poetry config installer.max-workers 10
- poetry install --all-extras

# Show limits
- ulimit -a

test_prebuilt_docker:
stage: test
image: cgerum/hannah:latest
Expand All @@ -69,21 +53,36 @@ test_prebuilt_docker:
dependencies: []


build_testcontainers:
stage: build
image: gitlab/gitlab-runner:latest
tags:
- dind
before_script:
- apk add --no-cache bash
script:
- bash ./build_testcontainer.sh all
allow_failure: false
when: manual

test_python_39:
stage: test
image: python:3.9
image: es-git-registry.cs.uni-tuebingen.de/es/ai/hannah/hannah/testing/python:3.9
script:
- set -e
- poetry config installer.max-workers 10
- poetry install -E vision
- poetry run python3 -m pytest -v --cov=hannah test hannah --integration
tags:
- docker
interruptible: true
rules:
- if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
dependencies: []
cache:
paths:
- .cache/
rules:
- if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- when: manual

#test_tvm:
# stage: test
Expand All @@ -107,9 +106,11 @@ test_python_39:

test_python_310:
stage: test
image: python:3.10
image: es-git-registry.cs.uni-tuebingen.de/es/ai/hannah/hannah/testing/python:3.10
script:
- set -e
- poetry config installer.max-workers 10
- poetry install -E vision
- "echo 'import coverage; coverage.process_startup()' > sitecustomize.py"
- export PYTHONPATH=$PWD
- export COVERAGE_PROCESS_START=$PWD/.coveragerc
Expand All @@ -132,12 +133,15 @@ test_python_310:
- .cache/
rules:
- if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- when: manual

test_python_311:
stage: test
image: python:3.11
image: es-git-registry.cs.uni-tuebingen.de/es/ai/hannah/hannah/es/ai/hannah/hannah/testing/python:3.11
script:
- set -e
- poetry config installer.max-workers 10
- poetry install -E vision
- "echo 'import coverage; coverage.process_startup()' > sitecustomize.py"
- export PYTHONPATH=$PWD
- export COVERAGE_PROCESS_START=$PWD/.coveragerc
Expand All @@ -160,35 +164,7 @@ test_python_311:
- .cache/
rules:
- if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

test_python_312:
stage: test
image: python:3.12
script:
- set -e
- "echo 'import coverage; coverage.process_startup()' > sitecustomize.py"
- export PYTHONPATH=$PWD
- export COVERAGE_PROCESS_START=$PWD/.coveragerc
- poetry run coverage run --source hannah -m pytest test
- unset PYTHONPATH
- unset COVERAGE_PROCESS_START
- poetry run coverage report
- poetry run coverage xml
tags:
- gpu
interruptible: true
artifacts:
reports:
coverage_report:
coverage_format: cobertura
path: coverage.xml
dependencies: []
cache:
paths:
- .cache/
rules:
- if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

- when: manual

build_docs:
stage: test
Expand Down
44 changes: 44 additions & 0 deletions Dockerfile.python3
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# ARG for Python version, defaulting to 3.12
ARG PYTHON_VERSION=3.12

# Use specified Python version as the base image
FROM python:${PYTHON_VERSION}

# Set working directory
WORKDIR /data

# Set environment variables
ENV PIP_CACHE_DIR="/data/.cache/pip" \
POETRY_HOME="/data/.poetry" \
POETRY_CACHE_DIR="/data/.cache/pypoetry" \
GIT_SUBMODULE_STRATEGY="recursive" \
DEBIAN_FRONTEND="noninteractive"

# Install system dependencies
RUN apt-get update && apt-get install -y \
git \
mesa-utils \
python3-dev \
libblas-dev \
liblapack-dev \
libsndfile1-dev \
libsox-dev \
cmake \
ninja-build \
&& rm -rf /var/lib/apt/lists/*

# Install Poetry
RUN curl -sSL https://install.python-poetry.org/ | python3 -
ENV PATH="${POETRY_HOME}/bin:${PATH}"

# Copy project files
# COPY hannah/pyproject.toml hannah/poetry.lock* ./

COPY hannah/ ./

# Configure Poetry and install dependencies
RUN poetry config installer.max-workers 10 && \
poetry install --with dev

# Show limits (this will be executed when the container runs)
CMD ["sh", "-c", "ulimit -a && /bin/bash"]
114 changes: 114 additions & 0 deletions build_testcontainer.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
#!/bin/bash
##
## Copyright (c) 2024 Hannah contributors.
##
## This file is part of hannah.
## See https://github.com/ekut-es/hannah for further info.
##
## Licensed under the Apache License, Version 2.0 (the "License");
## you may not use this file except in compliance with the License.
## You may obtain a copy of the License at
##
## http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## See the License for the specific language governing permissions and
## limitations under the License.
##


# Define the Python versions to build
VERSIONS=(3.9 3.10 3.11 3.12)
# VERSIONS=(3.12)

HANNAH_FOLDER=$(pwd)/hannah

# Set the registry and namespace
REGISTRY="es-git-registry.cs.uni-tuebingen.de"
NAMESPACE="es/ai/hannah"
PROJECT="hannah"
IMAGE_NAME="testing/python"


get_tag() {
local VERSION="$1"
echo "${REGISTRY}/${NAMESPACE}/${PROJECT}/${IMAGE_NAME}:${VERSION}"
}

build_image() {
local VERSION="$1"
local TAG=$(get_tag "$VERSION")
docker build -f Dockerfile.python3 --build-arg PYTHON_VERSION="${VERSION}" -t "${TAG}" .
}
# Function to push an image
push_image() {
local VERSION="$1"
local TAG=$(get_tag "$VERSION")
echo "Pushing image ${TAG}..."
docker push "${TAG}"
}

# Function to test an image
test_image() {
local VERSION="$1"
local VERSION_WITHOUT_DOT=${VERSION/.}
cd $HANNAH_FOLDER
# --docker-pull-policy "never"
gitlab-runner exec docker --cicd-config-file ".gitlab-ci.yml" --clone-url "." "test_python_${VERSION_WITHOUT_DOT}"
}

# Function to echo an image tag
echo_image() {
local VERSION="$1"
local TAG=$(get_tag "$VERSION")
echo "${TAG}"
}

# Check if a second argument is provided to override VERSIONS
if [ -n "$2" ]; then
VERSIONS=("$2")
fi


# Default action is build
action=${1:-build}

case "$action" in
push)
for VERSION in "${VERSIONS[@]}"; do
push_image "$VERSION"
done
;;

build)
for VERSION in "${VERSIONS[@]}"; do
build_image "$VERSION"
done
;;

test)
for VERSION in "${VERSIONS[@]}"; do
test_image "$VERSION"
done
;;

echo)
for VERSION in "${VERSIONS[@]}"; do
echo_image "$VERSION"
done
;;

all)
for VERSION in "${VERSIONS[@]}"; do
build_image "$VERSION"
test_image "$VERSION"
push_image "$VERSION"
done
;;

*)
echo "Unknown action: $action. Options are push build test"
;;
esac

0 comments on commit c8e728a

Please sign in to comment.