updated compressed spilo image with cartodb

Dockerfile

@@ -1,88 +1,114 @@
 # Spilo base image for postgis + timescale
-FROM registry.opensource.zalan.do/acid/spilo-11:1.6-p1
+ARG PGVERSION=11
+ARG DEMO=false
+
+FROM registry.opensource.zalan.do/acid/spilo-11:1.6-p1 as builder
 
 # Configuring locales
-ENV CRANKSHAFT_VERSION=0.8.1
-ENV CARTODB_POSTGRESQL_VERSION=0.3.0
-ENV DATASERVICES_VERSION=master
-ENV DATAERVICESAPI_VERSION=0.35.1-server
-ENV OBSERVATORY_VERSION=develop
-ENV CARTODB_VERSION=v4.29.0
 ENV DEBIAN_FRONTEND=noninteractive
+ENV POSTGRES_USER=postgres
 
-WORKDIR /home/postgres
-
+## UNPACK
 RUN tar xpJf /a.tar.xz -C / > /dev/null
 
-# Define build packages
+WORKDIR /home/postgres
 
-RUN BUILD_PACKAGES="software-properties-common python2.7 python-setuptools gcc-6 g++-6 git make pkg-config libpq-dev python2.7-dev build-essential libssl-dev libffi-dev libxml2-dev libxslt1-dev zlib1g-dev"
+# Define build packages
 
 # Add libraries
-RUN apt-get -y update \
-  && apt-get -y --no-install-recommends install apt-utils
-RUN apt-get -y --no-install-recommends install $BUILD_PACKAGES \
-  && apt-get -y --no-install-recommends install gdal-bin libgdal-dev
-
-RUN apt-get -y --no-install-recommends install libpython2.7-minimal libpython2.7-stdlib python2.7 python2.7-minimal git-man git liberror-perl binutils binutils-common binutils-x86-64-linux-gnu cpp cpp-6 cpp-7 dpkg-dev g++ g++-7 gcc gcc-6-base gcc-7 gcc-7-base gir1.2-glib-2.0 gir1.2-harfbuzz-0.0 git-man gpg gpgconf icu-devtools iso-codes libapt-inst2.0 libasan3 libasan4 libassuan0 libatomic1 libbinutils libc-dev-bin libc6-dev libcc1-0 libcilkrts5 libdbus-1-3 libdpkg-perl libelf1 liberror-perl libexpat1 libexpat1-dev libfreetype6 libgcc-6-dev libgcc-7-dev libgirepository-1.0-1 libglib2.0-0 libglib2.0-bin libglib2.0-data libglib2.0-dev libglib2.0-dev-bin libgomp1 libgraphite2-3 libgraphite2-dev libharfbuzz-dev libharfbuzz-gobject0 libharfbuzz-icu0 libharfbuzz0b libicu-dev libicu-le-hb-dev libicu-le-hb0 libiculx60 libisl19 libitm1 liblsan0 libmpc3 libmpx2 libpcre16-3 libpcre3-dev libpcre32-3 libpcrecpp0v5 libpng16-16 libpython-stdlib libpython2.7 libpython2.7-dev libquadmath0 libstdc++-6-dev libstdc++-7-dev libtsan0 libubsan0 linux-libc-dev patch python python-apt-common python-minimal python-pkg-resources python3-apt python3-dbus python3-gi python3-software-properties
-RUN apt-get -y --no-install-recommends install postgresql-server-dev-11 postgresql-common
-RUN apt-get -y --no-install-recommends install ruby2.5-dev 
-
-# Add crankshaft
+RUN apt-get -y update
+
+RUN apt-get -y --no-install-recommends install libpython2.7-minimal libpython2.7-stdlib python2.7 python2.7-minimal git-man git liberror-perl binutils binutils-common binutils-x86-64-linux-gnu cpp cpp-6 cpp-7 dpkg-dev g++ g++-7 gcc gcc-6-base gcc-7 gcc-7-base gir1.2-glib-2.0 gir1.2-harfbuzz-0.0 git-man gpg gpgconf icu-devtools iso-codes libapt-inst2.0 libasan3 libasan4 libassuan0 libatomic1 libbinutils libc-dev-bin libc6-dev libcc1-0 libcilkrts5 libdbus-1-3 libdpkg-perl libelf1 liberror-perl libexpat1 libexpat1-dev libfreetype6 libgcc-6-dev libgcc-7-dev libgirepository-1.0-1 libglib2.0-0 libglib2.0-bin libglib2.0-data libglib2.0-dev libglib2.0-dev-bin libgomp1 libgraphite2-3 libgraphite2-dev libharfbuzz-dev libharfbuzz-gobject0 libharfbuzz-icu0 libharfbuzz0b libicu-dev libicu-le-hb-dev libicu-le-hb0 libiculx60 libisl19 libitm1 liblsan0 libmpc3 libmpx2 libpcre16-3 libpcre3-dev libpcre32-3 libpcrecpp0v5 libpng16-16 libpython-stdlib libpython2.7 libpython2.7-dev libquadmath0 libstdc++-6-dev libstdc++-7-dev libtsan0 libubsan0 linux-libc-dev patch python python-apt-common python-minimal python-pkg-resources python3-apt python3-dbus python3-gi python3-software-properties software-properties-common python2.7 python-setuptools gcc-6 g++-6 git make pkg-config libpq-dev python2.7-dev build-essential libssl-dev libffi-dev libxml2-dev libxslt1-dev zlib1g-dev postgresql-server-dev-11 postgresql-common gdal-bin libgdal-dev
+RUN apt-get -y --no-install-recommends install ruby2.5-dev postgresql-plpython-11
+
+#carto version
+
+# https://github.com/CartoDB/cartodb-postgresql version
+ENV CARTODB_POSTGRESQL_VERSION=0.28.1
+# https://github.com/CartoDB/crankshaft version
+ENV CRANKSHAFT_VERSION=master
+# https://github.com/CartoDB/data-services
+ENV DATASERVICES_VERSION=0.0.2
+# https://github.com/CartoDB/dataservices-api
+ENV DATASERVICESAPI_VERSION=0.35.1-server
+# https://github.com/CartoDB/observatory-extension
+ENV OBSERVATORY_VERSION=1.9.0
+
+# CartoDB main postgresql extensions
+RUN git clone \
+  --branch ${CARTODB_POSTGRESQL_VERSION} \
+  --depth 1 git://github.com/CartoDB/cartodb-postgresql.git \
+  && cd cartodb-postgresql \
+  && make all install \
+  && cd /home/postgres  \
+  && rm -rf cartodb-postgresql
+
+# Crankshaft extensions
 RUN curl https://bootstrap.pypa.io/get-pip.py | python && \
   pip install --user numpy && \
-  git clone https://github.com/CartoDB/crankshaft.git && \
-  cd crankshaft && \
-  git checkout $CRANKSHAFT_VERSION && \
-  PGUSER=postgres make install && \
-  cd .. && \
-  rm -rf crankshaft
-
-# Add cartodb-postgresql
-RUN git clone https://github.com/CartoDB/cartodb-postgresql.git && \
-  cd cartodb-postgresql/ && \
-  git checkout $CARTODB_POSTGRESQL_VERSION && \
-  PGUSER=postgres make all install && \
-  cd .. && \
-  rm -rf cartodb-postgresql
-
-# Add data-services
-RUN git clone https://github.com/CartoDB/data-services.git && \
-  cd data-services/geocoder/extension && \
-  git checkout $DATASERVICES_VERSION && \
-  PGUSER=postgres make all install && \
-  cd .. && \
-  rm -rf data-services
+  pip install --user scipy && \
+  git clone \
+    --branch ${CRANKSHAFT_VERSION} \
+    --depth 1 git://github.com/CartoDB/crankshaft.git \
+  && cd crankshaft \
+  && make install \
+  # Numpy gets upgraded after scikit-learn is installed
+  # make sure scikit-learn is compatible with currently installed numpy, by reinstalling
+  && pip install --force-reinstall --no-cache-dir scikit-learn==0.14.1 \
+  && cd /home/postgres \
+  && rm -rf crankshaft
+
+# Add Dataservices / geocoder
+RUN git clone \
+  --branch ${DATASERVICES_VERSION} \
+  --depth 1 git://github.com/CartoDB/data-services.git \
+  && cd data-services/geocoder/extension \
+  && PGUSER=$POSTGRES_USER make all install \
+  && cd /home/postgres \
+  && rm -rf data-services
 
 # Add data-services-api
-RUN git clone https://github.com/CartoDB/dataservices-api.git && \
-  cd dataservices-api/server/extension && \
-  PGUSER=postgres make install && \
-  cd ../../client && \
-  PGUSER=postgres make install && \
-  cd ../..  && \
-  rm -rf dataservices-api
+RUN git clone \
+  --branch ${DATASERVICESAPI_VERSION} \
+  --depth 1 git://github.com/CartoDB/dataservices-api.git \
+  && cd dataservices-api/server/extension \
+  && PGUSER=$POSTGRES_USER make install \
+  && cd ../lib/python/cartodb_services \
+  && pip install -r requirements.txt && pip install . \
+  && cd ../../../../client \
+  && PGUSER=$POSTGRES_USER make install \
+  && cd /home/postgres \
+  && rm -rf dataservices-api
 
 # Add observatory-extension
-RUN git clone --recursive https://github.com/CartoDB/observatory-extension.git && \
-  cd observatory-extension && \
-  git checkout $OBSERVATORY_VERSION && \
-  PGUSER=postgres make deploy && \
-  cd .. && \
-  rm -rf observatory-extension
-
-# Add cartodb
-RUN git clone --recursive git://github.com/CartoDB/cartodb.git && \
-  cd cartodb && \
-  git checkout $CARTODB_VERSION && \
-  cd lib/sql && \
-  PGUSER=postgres make install && \
-  cd ../../.. && \
-  rm -rf cartodb
+RUN git clone \
+  --branch ${OBSERVATORY_VERSION} \
+  --depth 1 git://github.com/CartoDB/observatory-extension.git \
+  && cd observatory-extension \
+  && PGUSER=$POSTGRES_USER make deploy \
+  && cd /home/postgres \
+  && rm -rf observatory-extension
+
+
+ENV BUILD_PACKAGES="git-man git liberror-perl binutils binutils-common binutils-x86-64-linux-gnu"
+
+RUN apt-get purge -y ${BUILD_PACKAGES}
+
+ENV BUILD_PACKAGES="cpp cpp-6 cpp-7 dpkg-dev g++ g++-7 gcc gcc-6-base gcc-7 gcc-7-base gir1.2-glib-2.0 gir1.2-harfbuzz-0.0 git-man gpg gpgconf libbinutils libc-dev-bin libc6-dev libexpat1-dev"
+
+RUN apt-get purge -y ${BUILD_PACKAGES}
+
+ENV BUILD_PACKAGES="libfreetype6 libgcc-6-dev libgcc-7-dev libglib2.0-dev libglib2.0-dev-bin libgraphite2-dev libharfbuzz-dev libicu-dev libicu-le-hb-dev libpcre3-dev libpython2.7-dev libstdc++-6-dev"
 
+RUN apt-get purge -y ${BUILD_PACKAGES}
+
+ENV BUILD_PACKAGES="libstdc++-7-dev linux-libc-dev patch software-properties-common python2.7 gcc-6 g++-6 git make pkg-config libpq-dev python2.7-dev build-essential libssl-dev libffi-dev libxml2-dev libxslt1-dev zlib1g-dev postgresql-server-dev-11 libgdal-dev ruby2.5-dev"
+
+RUN apt-get purge -y ${BUILD_PACKAGES}
+
+#RUN apt-get -y --no-install-recommends install python3 postgresql-plpython-11
 # Clean up
-RUN apt-get purge -y ${BUILD_PACKAGES} \
-    && apt-get autoremove -y \
+RUN apt-get autoremove -y \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* \
             /var/cache/debconf/* \
@@ -106,7 +132,51 @@ RUN apt-get purge -y ${BUILD_PACKAGES} \
     && rm -rf /a.tar.xz \
     && find /var/log -type f -exec truncate --size 0 {} \;
 
-# Fix access from localhost
-# RUN sed -i 's+host    all             all                127.0.0.1/32       md5+host    all             all                127.0.0.1/32       trust+g' /home/postgres/postgres.yml \
-#  && sed -i 's+host    all             all                ::1/128            md5+host    all             all                ::1/128            trust+g' /home/postgres/postgres.yml
+ARG COMPRESS=false
+
+WORKDIR /
+
+RUN set -ex \
+    && if [ "$COMPRESS" = "true" ]; then \
+        apt-get update \
+        && apt-get install -y busybox xz-utils \
+        && apt-get clean \
+        && rm -rf /var/lib/apt/lists/* /var/cache/debconf/* /usr/share/doc /usr/share/man /etc/rc?.d /etc/systemd \
+        && ln -snf busybox /bin/sh \
+        && files="/bin/sh" \
+        && libs="$(ldd $files | awk '{print $3;}' | grep '^/' | sort -u) /lib/x86_64-linux-gnu/ld-linux-x86-64.so.* /lib/x86_64-linux-gnu/libnsl.so.* /lib/x86_64-linux-gnu/libnss_compat.so.*" \
+        && (echo /var/run $files $libs | tr ' ' '\n' && realpath $files $libs) | sort -u | sed 's/^\///' > /exclude \
+        && find /etc/alternatives -xtype l -delete \
+        && save_dirs="usr lib var bin sbin etc/ssl etc/init.d etc/alternatives etc/apt" \
+        && XZ_OPT=-e9v tar -X /exclude -cpJf a.tar.xz $save_dirs \
+        && rm -fr /usr/local/lib/python* \
+        && /bin/busybox sh -c "(find $save_dirs -not -type d && cat /exclude /exclude && echo exclude) | sort | uniq -u | xargs /bin/busybox rm" \
+        && /bin/busybox --install -s \
+        && /bin/busybox sh -c "find $save_dirs -type d -depth -exec rmdir -p {} \; 2> /dev/null"; \
+    fi
+
+FROM scratch
+COPY --from=builder / /
+
+LABEL maintainer="Martel Innovate"
+
+ARG PGVERSION
+ARG DEMO
+
+EXPOSE 5432 8008 8080
+
+ENV LC_ALL=en_US.utf-8 \
+    PATH=$PATH:/usr/lib/postgresql/$PGVERSION/bin \
+    PGHOME=/home/postgres \
+    DEMO=$DEMO
+
+ENV WALE_ENV_DIR=$PGHOME/etc/wal-e.d/env \
+    PGROOT=$PGHOME/pgdata/pgroot \
+    LOG_ENV_DIR=$PGHOME/etc/log.d/env
+
+ENV PGDATA=$PGROOT/data \
+    PGLOG=$PGROOT/pg_log
+
+WORKDIR $PGHOME
 
+CMD ["/bin/sh", "/launch.sh"]

build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker build --build-arg COMPRESS=true . -t orchestracities/spilo

docker-compose.yaml

@@ -0,0 +1,28 @@
+version: '3.5'
+
+services:
+  postgres:
+    container_name: postgres_container
+    image: orchestracities/spilo
+    ports:
+      - "5432:5432"
+    restart: unless-stopped
+
+  pgadmin:
+    container_name: pgadmin_container
+    image: dpage/pgadmin4
+    environment:
+      PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:[email protected]}
+      PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin}
+    volumes:
+       - pgadmin:/root/.pgadmin
+    ports:
+      - "5050:80"
+    restart: unless-stopped
+
+networks:
+  default:
+
+volumes:
+    postgres:
+    pgadmin: