Merge branch 'cpp17'

Changes: - Implement weather simulations - Change the way max intensity is set & passed on - Change `intensity_range` from float to enum - Move `inensity_range` into `point_cloud_data` namespace - Use OpenMP to speed up simulations - Add the possibility to use docker to build/use the library - Fix `google-test` workflow
ekut-es · Oct 28, 2024 · db506d2 · db506d2
2 parents cb3ecc5 + c0b4203
commit db506d2
Show file tree

Hide file tree

Showing 38 changed files with 2,176 additions and 180 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,8 @@
+
+LidarAug.egg-info
+build
+build_files
+**/__pycache__
+.cache
+**/compile_commands.json
+.idea/
diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml
@@ -0,0 +1,17 @@
+name: Build Docker Image
+on:
+  push:
+    branches: [ "docker" ]
+  pull_request:
+    branches: [ "docker" ]
+
+jobs:
+
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Build the Docker image
+      run: docker build . --file Dockerfile --tag lidaraug:0.0.1
diff --git a/.github/workflows/build-module.yml b/.github/workflows/build-module.yml
@@ -18,10 +18,22 @@ jobs:
         uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install C++ build tools
-        run: sudo apt-get update && sudo apt-get install -y build-essential
+      - name: Install C++ build tools & git
+        run: sudo apt-get update && sudo apt-get install -y build-essential git
       - name: Install Boost
         run: sudo apt-get update && sudo apt-get install -y libboost-all-dev
+      - name: Install OpenMP
+        run: sudo apt-get update && sudo apt-get install -y libomp-dev
+      - name: Install cnpy
+        run: |
+          git clone https://github.com/TomSchammo/cnpy/
+          cd cnpy
+          git checkout cpp17
+          mkdir build
+          cd build
+          cmake .. -DCMAKE_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=0"
+          make VERBOSE=1
+          sudo make install
       - name: Install setuptools
         run: python -m pip install setuptools==69.5.1
       - name: Install dependencies
@@ -36,7 +48,9 @@ jobs:
           echo "TORCH_PATH=$(python -c 'import torch; import os; print(os.path.join(torch.__path__[0], "share", "cmake"))')" >> $GITHUB_ENV
       - name: Build and install C++ module
         run: |
-          python -m pip install .
+          export CXXFLAGS="-fopenmp ${CXXFLAGS}"
+          export CFLAGS="-fopenmp ${CFLAGS}"
+          python -m pip install . -v --no-cache-dir
       - name: Test with pytest
         run: |
           python -m pytest ./pytest/test.py -v
diff --git a/.github/workflows/google-test.yml b/.github/workflows/google-test.yml
@@ -4,42 +4,52 @@ on: [ push, pull_request ]
 
 jobs:
   build:
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
 
     strategy:
       matrix:
         compiler: [ gcc ]
+        python-version: [ "3.11" ]
 
     steps:
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: setup google test
         uses: Bacondish2023/setup-googletest@v1
 
+      - name: Install C++ build tools & git
+        run: sudo apt-get update && sudo apt-get install -y build-essential git
       - name: Install CMake
         run: sudo apt-get update && sudo apt-get install -y cmake
-
       - name: Install Boost
         run: sudo apt-get update && sudo apt-get install -y libboost-all-dev
+      - name: Install OpenMP
+        run: sudo apt-get update && sudo apt-get install -y libomp-dev
 
+      - name: Install cnpy
+        run: |
+          git clone https://github.com/TomSchammo/cnpy/
+          cd cnpy
+          git checkout cpp17
+          mkdir build
+          cd build
+          cmake .. -DCMAKE_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=0"
+          make VERBOSE=1
+          sudo make install
       - name: Install libtorch
         run: |
-          wget https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.3.0%2Bcpu.zip
-          unzip libtorch-cxx11-abi-shared-with-deps-2.3.0+cpu.zip -d /opt/
-
-      - name: Create build directory
-        run: mkdir build
-        working-directory: ./cpp
-
-      - name: Configure and build
+          python3.11 -m pip install --upgrade pip
+          python3.11 -m pip install setuptools==69.5.1 numpy==1.24.0
+          python3.11 -m pip install torch==2.1.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
+      - name: Find PyTorch cmake path
+        id: find-torch-cmake-path
         run: |
-          cd build
-          cmake -DBOOST_ROOT=/usr/include/boost -DCMAKE_PREFIX_PATH=/opt/libtorch ..
-          make
-        working-directory: ./cpp
+          echo "TORCH_PATH=$(python3.11 -c 'import torch; import os; print(os.path.join(torch.__path__[0], "share", "cmake"))')" >> $GITHUB_ENV
 
-      - name: Run Google Test
+      - name: Build and run Google Test
         run: |
-          cd build
-          ctest --output-on-failure
-        working-directory: ./cpp
+          make ctest
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,179 @@
+
+FROM ubuntu:22.04 AS base
+
+ARG RUN_CTEST="true"
+ARG RUN_PYTEST="true"
+ARG OPENSSL_VERSION="3.3.2"
+ARG PYTHON_VERSION="3.11.0"
+
+
+RUN apt-get update && apt-get dist-upgrade -y
+RUN DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata
+
+FROM base AS python_builder
+RUN apt-get install -y \
+  build-essential \
+  libbz2-dev \
+  libffi-dev \
+  libgdbm-dev \
+  liblzma-dev \
+  libncurses5-dev \
+  libnss3-dev \
+  libsqlite3-dev \
+  wget \
+  zlib1g-dev
+
+RUN echo "Downloading sources..."
+
+RUN \
+  wget -cq https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -O - | tar -xz && \
+  wget -cq https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz -O - | tar -xz
+
+RUN echo "Building OpenSSL ${OPENSSL_VERSION}..."
+WORKDIR /openssl-${OPENSSL_VERSION}
+
+# RPATH from https://wiki.openssl.org/index.php/Compilation_and_Installation#Using_RPATHs
+RUN ./config -Wl,-rpath=/usr/local/ssl/lib64:/usr/local/lib -Wl,--enable-new-dtags --prefix=/usr/local/ssl --openssldir=/usr/local/ssl
+RUN make
+RUN make install
+
+RUN echo "Building Python ${PYTHON_VERSION}"
+WORKDIR /Python-${PYTHON_VERSION}
+
+# Setting env variables for python compilation
+ENV LDFLAGS="-L/usr/local/ssl/lib64/ -Wl,-rpath=/usr/local/ssl/lib64:/usr/local/lib"
+ENV LD_LIBRARY_PATH="/usr/local/ssl/lib/:/usr/local/ssl/lib64/"
+ENV CPPFLAGS="-I/usr/local/ssl/include -I/usr/local/ssl/include/openssl"
+
+RUN \
+  ./configure \
+  --with-openssl=/usr/local/ssl \
+  --enable-loadable-sqlite-extensions \
+  --enable-shared \
+  --with-openssl-rpath=auto \
+  --enable-optimizations --with-lto
+
+RUN make
+RUN make altinstall
+
+# taken from the official image https://github.com/docker-library/python/
+RUN find /usr/local -depth \
+  \( \
+  \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
+  -o \
+  \( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
+  \) -exec rm -rf '{}' +;
+
+# Cleaning up openSSL headers and doc
+RUN rm -rf /usr/local/ssl/share /usr/local/ssl/include
+
+FROM base AS python_base
+RUN apt-get install -y --no-install-recommends \
+  bzip2 \
+  ca-certificates \
+  curl \
+  libffi8 \
+  libgdbm6 \
+  liblzma5 \
+  libncurses6 \
+  libnss3 \
+  sqlite3 \
+  wget \
+  zlib1g \
+  && apt-get autoclean \
+  && apt-get clean \
+  && apt-get autoremove -y \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY --from=python_builder /usr/local/ /usr/local/
+
+ARG PYTHON_VERSION
+ENV LD_LIBRARY_PATH="/usr/local/ssl/lib/:/usr/local/ssl/lib64/"
+RUN echo "Python ${PYTHON_VERSION} has been successfully installed!"
+
+# OpenSSL looks into $OPENSSLDIR/certs as CA trust store. By default this is
+# empty, and installing ca-certificates with apt-get populates it in the system
+# openssl at /usr/lib/ssl/certs/. Our compiled openssl looks into
+# /usr/local/ssl/certs, we create a symlink between the two to let Python access
+# the OS trust store.
+RUN rm -rf /usr/local/ssl/certs && ln -s /usr/lib/ssl/certs/ /usr/local/ssl/certs
+
+WORKDIR /usr/local/bin/
+
+RUN ln -sf python${PYTHON_VERSION%.*} python && ln -sf python${PYTHON_VERSION%.*} python3
+
+# Verify pip installation
+RUN python3.11 -m pip --version
+RUN python3.11 -m pip install --upgrade pip
+
+FROM python_base AS module_builder
+
+RUN apt-get update
+
+# Install C++ build tools & git
+RUN apt-get install -y build-essential git cmake
+RUN apt-get install -y libgl1
+
+# Install Boost
+RUN apt-get install -y libboost-all-dev
+
+# Install OpenMP
+RUN apt-get install -y libomp-dev
+
+# Install cnpy
+RUN git clone https://github.com/TomSchammo/cnpy /opt/cnpy
+WORKDIR /opt/cnpy
+RUN git checkout cpp17
+RUN mkdir /opt/cnpy/build
+WORKDIR /opt/cnpy/build
+RUN cmake .. -DCMAKE_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=0"
+RUN make VERBOSE=1
+RUN make install
+
+# Install setuptools
+RUN python3.11 -m pip install setuptools==69.5.1
+
+# Install dependencies
+RUN python3.11 -m pip install pytest pybind11 numpy==1.24.0 scipy
+# RUN python3.11 -m pip install torch==2.1.0 -f https://download.pytorch.org/whl/torch_stable.html
+RUN python3.11 -m pip install torch==2.1.0 --index-url https://download.pytorch.org/whl/cpu
+
+
+# Set CXX flags
+ENV CXXFLAGS="-fopenmp"
+ENV CFLAGS="-fopenmp"
+
+# Copy the source/project files
+COPY . /opt/LidarAug
+
+# Set the working directory
+WORKDIR /opt/LidarAug
+
+RUN if [ -d ./cpp/build_files ]; then make clean; fi
+
+# Build the library using cmake and run the c++ tests
+RUN if [ "${RUN_CTEST}" = "true" ]; then export TORCH_PATH=$(python3.11 -c 'import torch; import os; print(os.path.join(torch.__path__[0], "share", "cmake"))'); make ctest; fi
+
+# Build and install the python module
+RUN make install
+
+# Test the python module
+RUN if [ "${RUN_PYTEST}" = "true" ]; then make testpy; fi
+
+# Clean up
+RUN python3.11 -m pip uninstall --yes pybind11 pytest
+
+FROM python_base AS final
+
+RUN apt-get update
+RUN apt-get install --only-upgrade -y libstdc++6
+RUN apt-get install -y \
+  libgomp1 \
+  libx11-6 \
+  libgl1
+
+RUN python3.11 --version
+
+COPY --from=module_builder /usr/local/lib/python3.11/site-packages/ /usr/local/lib/python3.11/site-packages/
+
+RUN python3.11 -c 'import torch; import LidarAug'
diff --git a/Makefile b/Makefile
@@ -1,3 +1,15 @@
+ifeq ($(OS),Windows_NT)
+    CXXFLAGS += openmp
+    CFLAGS+= openmp
+else
+	UNAME_S := $(shell uname -s)
+	ifeq ($(UNAME_S),Darwin)
+		CXXFLAGS += -Xpreprocessor
+		CFLAGS += -Xpreprocessor
+	endif
+	CXXFLAGS += -fopenmp
+	CFLAGS += -fopenmp
+endif
 
 all: install testpy build ctest
 
@@ -8,10 +20,10 @@ configure_test:
 	cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH="$(TORCH_PATH)" -S ./cpp/ -B ./cpp/build_files
 
 build: configure_test
-	cmake --build ./cpp/build_files -j 4
+	cmake --build ./cpp/build_files -j 8
 
 release: configure
-	cmake --build ./cpp/build_files -j 4 --config
+	cmake --build ./cpp/build_files -j 8
 
 ctest: build
 	cd ./cpp/build_files && ctest
@@ -25,8 +37,16 @@ testpy: ./pytest/test.py
 rerun: ./cpp/build_files
 	cd ./cpp/build_files && ctest --rerun-failed --output-on-failure
 
+sim: release
+	cd ./cpp/build_files && ctest --output-on-failure -R 'Simulation.*'
+
 install:
-	rm -rf ./build ./src/LidarAug.egg-info && mkdir -p ./tmp && TMPDIR=./tmp python3.11 -m pip install . && rm -rf ./tmp
+	@echo "CXXFLAGS: $(CXXFLAGS)"
+	@echo "CFLAGS: $(CFLAGS)"
+	rm -rf ./build ./src/LidarAug.egg-info && mkdir -p ./tmp && TMPDIR=./tmp python3.11 -m pip install -v . && rm -rf ./tmp
+
+docker:
+	docker build -t lidaraug:0.0.1 .
 
 clean: ./cpp/build_files
 	rm -rfv ./cpp/build_files