From bfcc52072e1d4468c6e235b2a8a2913932102238 Mon Sep 17 00:00:00 2001 From: Nicola Jordan Date: Tue, 13 Feb 2018 10:52:03 +0100 Subject: [PATCH 01/19] prepare config to use environment variables without sacrificing backwards compatibility --- config.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/config.py b/config.py index 35f5e4d..fe1a3d7 100644 --- a/config.py +++ b/config.py @@ -1,18 +1,21 @@ import os BASE_DIR = os.path.abspath(os.path.dirname(__file__)) -DEBUG = True +DEBUG = os.environ.get('DEBUG', True) -DATABASE_URI = 'sqlite:///' + os.path.join(BASE_DIR, 'audit.db') +DATABASE_URI = os.environ.get('DATABASE_URI', 'sqlite:///' + os.path.join(BASE_DIR, 'audit.db')) # DATABASE_URI = 'postgresql://localhost/cf_audit' -MAX_CONTENT_LENGTH = 16*1024*1024 +MAX_CONTENT_LENGTH = int(os.environ.get('MAX_CONTENT_LENGTH', 16*2014*1024)) -ADMINS = set([290271]) # Zverik +ADMINS = set(os.environ.get('ADMINS', '').split(',')) +if not ADMINS: + ADMINS = set([290271]) # Zverik -# Override these (and anything else) in config_local.py -OAUTH_KEY = '' -OAUTH_SECRET = '' -SECRET_KEY = 'sdkjfhsfljhsadf' +# Override these (and anything else) in config_local.py or +# set environment variable accordingly. +OAUTH_KEY = os.environ.get('OAUTH_KEY', '') +OAUTH_SECRET = os.environ.get('OAUTH_SECRET', '') +SECRET_KEY = os.environ.get('SECRET_KEY', 'sdkjfhsfljhsadf') try: from config_local import * From 2ee6f1b70e487c79754cb1a147e4e5b2d81a4789 Mon Sep 17 00:00:00 2001 From: Nicola Jordan Date: Tue, 13 Feb 2018 10:59:25 +0100 Subject: [PATCH 02/19] let docker ignore some files --- .dockerignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..38f5e04 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.git/ +.gitignore +Dockerfile +docker-compose.yml From ab917ffb30a05b6ea8966eb1da3e7ba4695c9efd Mon Sep 17 00:00:00 2001 From: Nicola Jordan Date: Tue, 13 Feb 2018 11:00:05 +0100 Subject: [PATCH 03/19] add Dockerfile --- Dockerfile | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..959b854 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3 + +LABEL maintainer="Geometalab " + +RUN apt-get update && apt-get install -y wget && rm -rf /var/lib/apt/lists/* + +ENV DOCKERIZE_VERSION v0.6.0 +RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ + && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ + && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz + +RUN pip install uwsgi psycopg2 + +COPY requirements.txt /tmp/requirements.txt +RUN pip install -r /tmp/requirements.txt + +EXPOSE 8080 + +COPY . /opt/cf_audit +WORKDIR /opt/cf_audit +ENTRYPOINT [ "dockerize", "-wait", "tcp://database:5432" ] +CMD [ "uwsgi", "--chdir", "/opt/cf_audit/", "--http", ":8080", "--wsgi-file", "app.wsgi" ] From a805e72ac388487e99dffc05e18361e217f40121 Mon Sep 17 00:00:00 2001 From: Nicola Jordan Date: Tue, 13 Feb 2018 11:00:42 +0100 Subject: [PATCH 04/19] add a simplified uwsgi compatible running file --- app.wsgi | 1 + 1 file changed, 1 insertion(+) create mode 100644 app.wsgi diff --git a/app.wsgi b/app.wsgi new file mode 100644 index 0000000..7590029 --- /dev/null +++ b/app.wsgi @@ -0,0 +1 @@ +from www import app as application \ No newline at end of file From d94df11a05b756b4938660b68bcf8127abccb3c3 Mon Sep 17 00:00:00 2001 From: Nicola Jordan Date: Tue, 13 Feb 2018 11:02:04 +0100 Subject: [PATCH 05/19] add migration file --- migrate.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 migrate.py diff --git a/migrate.py b/migrate.py new file mode 100644 index 0000000..68a1c5a --- /dev/null +++ b/migrate.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python3 +from www.db import migrate +migrate() From 430abb069e5b90b08c0a86a5b421fe7957823031 Mon Sep 17 00:00:00 2001 From: Nicola Jordan Date: Tue, 13 Feb 2018 11:04:27 +0100 Subject: [PATCH 06/19] use a docker-compose file for easier running and building docker image/container --- docker-compose.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..7b8edbe --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,28 @@ +version: '3.3' +services: + cf-audit: &auditor + build: . + environment: + # uncomment following line to enable Debugging + # DEBUG: yes + DATABASE_URI: 'postgresql://cf_audit:pw@database:5432/cf_audit' + OAUTH_KEY: 'your-key' + OAUTH_SECRET: 'your-secret' + ports: + - "8080:8080" + depends_on: + - database + migrate: + <<: *auditor + command: python3 migrate.py + ports: [] + database: + image: postgres:10 + ports: + - "5432:5432" + environment: + POSTGRES_USER: cf_audit + POSTGRES_PASSWORD: pw + POSTGRES_DB: cf_audit + ADMINS: '290271' + From 2579c0f18f096886e6578027a8537c7d8d99f16d Mon Sep 17 00:00:00 2001 From: Nicola Jordan Date: Tue, 13 Feb 2018 12:18:19 +0100 Subject: [PATCH 07/19] do not expose the db ports --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7b8edbe..d6c24bb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,8 +18,6 @@ services: ports: [] database: image: postgres:10 - ports: - - "5432:5432" environment: POSTGRES_USER: cf_audit POSTGRES_PASSWORD: pw From c3eed1cff55f22c613cb61eabaa2728cd5b0355b Mon Sep 17 00:00:00 2001 From: Nicola Jordan Date: Tue, 13 Feb 2018 12:27:30 +0100 Subject: [PATCH 08/19] remove security settings and use .env file --- .gitignore | 1 + docker-compose.yml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 5530c37..8ceb755 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ config_local.py *.backup *.sql *.gz +.env diff --git a/docker-compose.yml b/docker-compose.yml index d6c24bb..a00b655 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.3' +version: '3.4' services: cf-audit: &auditor build: . @@ -6,8 +6,8 @@ services: # uncomment following line to enable Debugging # DEBUG: yes DATABASE_URI: 'postgresql://cf_audit:pw@database:5432/cf_audit' - OAUTH_KEY: 'your-key' - OAUTH_SECRET: 'your-secret' + env_file: + - .env ports: - "8080:8080" depends_on: From 8586656876b24f7e10bbf9af774c630f6c9793a8 Mon Sep 17 00:00:00 2001 From: Nicola Jordan Date: Tue, 13 Feb 2018 12:28:04 +0100 Subject: [PATCH 09/19] add basic documentation on how to use this project with docker and compose --- README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/README.md b/README.md index f90c0fa..3606add 100644 --- a/README.md +++ b/README.md @@ -7,3 +7,28 @@ It records any changes and produces a file that can be later feeded back to the ## Author and License All this was written by Ilya Zverev for MAPS.ME. Published under Apache License 2.0. + +## Running using docker-compose + +You can run this project locally (or using an adapted version +on your server as well) using the provided docker-compose file. + +1. Create an application on https://www.openstreetmap.org/user//oauth_clients, +and use `localhost:8080` as your main application URL. +2. Copy the "Consumer Key" respectively the "Consumer Secret" to a new file called `.env` as follows: + +``` +OAUTH_KEY='your-key' +OAUTH_SECRET='your-secret' +``` + +3. Then start it on your machine using docker-compose: `docker-compose up --build`. + +Open your browser at `localhost:8080` and start using it. + +In case you don't have admin-rights, login first, then execute a database query as follows +(replace with your user osm-id): + +``` +docker-compose exec database psql -U postgres cf_audit -c 'UPDATE public."user" SET admin = TRUE WHERE uid = ;' +``` From e7953f06e4188cf2d7cd5a2240ae4c3a2b93cc3a Mon Sep 17 00:00:00 2001 From: Nicola Jordan Date: Tue, 13 Feb 2018 12:36:31 +0100 Subject: [PATCH 10/19] spelling in comment --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index a00b655..4de00de 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ services: cf-audit: &auditor build: . environment: - # uncomment following line to enable Debugging + # uncomment following line to enable debugging # DEBUG: yes DATABASE_URI: 'postgresql://cf_audit:pw@database:5432/cf_audit' env_file: From 5f521312253e0a732ca86fa83ec708acf43a0cc2 Mon Sep 17 00:00:00 2001 From: Nicola Jordan Date: Thu, 15 Feb 2018 17:04:09 +0100 Subject: [PATCH 11/19] fix syntax highlighting --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3606add..f9780ec 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,10 @@ on your server as well) using the provided docker-compose file. 1. Create an application on https://www.openstreetmap.org/user//oauth_clients, and use `localhost:8080` as your main application URL. + 2. Copy the "Consumer Key" respectively the "Consumer Secret" to a new file called `.env` as follows: -``` +```bash OAUTH_KEY='your-key' OAUTH_SECRET='your-secret' ``` @@ -29,6 +30,6 @@ Open your browser at `localhost:8080` and start using it. In case you don't have admin-rights, login first, then execute a database query as follows (replace with your user osm-id): -``` +```bash docker-compose exec database psql -U postgres cf_audit -c 'UPDATE public."user" SET admin = TRUE WHERE uid = ;' ``` From 38f899e7551158fa4cb379ed18a3072f3b6b4e60 Mon Sep 17 00:00:00 2001 From: Nicola Jordan Date: Thu, 15 Feb 2018 17:10:32 +0100 Subject: [PATCH 12/19] make mardown linter happy --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index f9780ec..3dbe059 100644 --- a/README.md +++ b/README.md @@ -13,22 +13,22 @@ All this was written by Ilya Zverev for MAPS.ME. Published under Apache License You can run this project locally (or using an adapted version on your server as well) using the provided docker-compose file. -1. Create an application on https://www.openstreetmap.org/user//oauth_clients, -and use `localhost:8080` as your main application URL. +1. Create an application on https://www.openstreetmap.org/user//oauth_clients, + and use `localhost:8080` as your main application URL. -2. Copy the "Consumer Key" respectively the "Consumer Secret" to a new file called `.env` as follows: +1. Copy the "Consumer Key" respectively the "Consumer Secret" to a new file called `.env` as follows: -```bash -OAUTH_KEY='your-key' -OAUTH_SECRET='your-secret' -``` + ```bash + OAUTH_KEY='your-key' + OAUTH_SECRET='your-secret' + ``` -3. Then start it on your machine using docker-compose: `docker-compose up --build`. +1. Then start it on your machine using docker-compose: `docker-compose up --build`. Open your browser at `localhost:8080` and start using it. In case you don't have admin-rights, login first, then execute a database query as follows -(replace with your user osm-id): +(replace `` with your user osm-id): ```bash docker-compose exec database psql -U postgres cf_audit -c 'UPDATE public."user" SET admin = TRUE WHERE uid = ;' From d976f30ad93c93686fa8dca15ad9d9ad3b0b5953 Mon Sep 17 00:00:00 2001 From: Nicola Jordan Date: Thu, 15 Feb 2018 17:14:35 +0100 Subject: [PATCH 13/19] correct escaping --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3dbe059..b0edbfd 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ All this was written by Ilya Zverev for MAPS.ME. Published under Apache License You can run this project locally (or using an adapted version on your server as well) using the provided docker-compose file. -1. Create an application on https://www.openstreetmap.org/user//oauth_clients, +1. Create an application on `https://www.openstreetmap.org/user//oauth_clients` and use `localhost:8080` as your main application URL. 1. Copy the "Consumer Key" respectively the "Consumer Secret" to a new file called `.env` as follows: From c9deb3fd05b66683aed6f520da6ade15405f7474 Mon Sep 17 00:00:00 2001 From: Nicola Jordan Date: Fri, 16 Feb 2018 11:01:58 +0100 Subject: [PATCH 14/19] use python 2 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 959b854..8d4d164 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3 +FROM python:2 LABEL maintainer="Geometalab " From 488944165eb37524a34b5e11ad05d353d987fe49 Mon Sep 17 00:00:00 2001 From: Nicola Jordan Date: Fri, 16 Feb 2018 11:03:40 +0100 Subject: [PATCH 15/19] reuse existing wsgi file instead of adding a new one --- Dockerfile | 2 +- app.wsgi | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 app.wsgi diff --git a/Dockerfile b/Dockerfile index 8d4d164..59aa2b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,4 +19,4 @@ EXPOSE 8080 COPY . /opt/cf_audit WORKDIR /opt/cf_audit ENTRYPOINT [ "dockerize", "-wait", "tcp://database:5432" ] -CMD [ "uwsgi", "--chdir", "/opt/cf_audit/", "--http", ":8080", "--wsgi-file", "app.wsgi" ] +CMD [ "uwsgi", "--chdir", "/opt/cf_audit/", "--http", ":8080", "--wsgi-file", "cf_audit.wsgi" ] diff --git a/app.wsgi b/app.wsgi deleted file mode 100644 index 7590029..0000000 --- a/app.wsgi +++ /dev/null @@ -1 +0,0 @@ -from www import app as application \ No newline at end of file From e091bc7c86f0e71dff69ae3d395380a208a858c0 Mon Sep 17 00:00:00 2001 From: Nicola Jordan Date: Fri, 16 Feb 2018 11:03:54 +0100 Subject: [PATCH 16/19] default migrate to python 2 --- migrate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrate.py b/migrate.py index 68a1c5a..471d85d 100644 --- a/migrate.py +++ b/migrate.py @@ -1,3 +1,3 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python2 from www.db import migrate migrate() From 95a56fe1e8caee4e2a234c022ddfe6eee2ac62f0 Mon Sep 17 00:00:00 2001 From: Nicola Jordan Date: Fri, 16 Feb 2018 12:52:24 +0100 Subject: [PATCH 17/19] update settings and adapt readme accordingly --- README.md | 13 +++++-------- config.py | 20 ++++++++++++-------- docker-compose.yml | 5 +++-- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index b0edbfd..42df5df 100644 --- a/README.md +++ b/README.md @@ -19,17 +19,14 @@ on your server as well) using the provided docker-compose file. 1. Copy the "Consumer Key" respectively the "Consumer Secret" to a new file called `.env` as follows: ```bash - OAUTH_KEY='your-key' - OAUTH_SECRET='your-secret' + OAUTH_KEY= + OAUTH_SECRET= + SECRET_KEY= ``` 1. Then start it on your machine using docker-compose: `docker-compose up --build`. Open your browser at `localhost:8080` and start using it. -In case you don't have admin-rights, login first, then execute a database query as follows -(replace `` with your user osm-id): - -```bash -docker-compose exec database psql -U postgres cf_audit -c 'UPDATE public."user" SET admin = TRUE WHERE uid = ;' -``` +In case you don't have admin-rights, uncomment the line in the `docker-compose.yml` with `# ADMINS: ''`. +You can add multiple admins by separating the values with a comma (`,`), ie. `ADMINS: '1234,98765'` diff --git a/config.py b/config.py index fe1a3d7..9b14b3a 100644 --- a/config.py +++ b/config.py @@ -4,20 +4,24 @@ DEBUG = os.environ.get('DEBUG', True) DATABASE_URI = os.environ.get('DATABASE_URI', 'sqlite:///' + os.path.join(BASE_DIR, 'audit.db')) -# DATABASE_URI = 'postgresql://localhost/cf_audit' -MAX_CONTENT_LENGTH = int(os.environ.get('MAX_CONTENT_LENGTH', 16*2014*1024)) +MAX_CONTENT_LENGTH = int(os.environ.get('MAX_CONTENT_LENGTH', 16*1024*1024)) -ADMINS = set(os.environ.get('ADMINS', '').split(',')) -if not ADMINS: - ADMINS = set([290271]) # Zverik +ADMINS = os.environ.get('ADMINS') +if ADMINS: + ADMINS = set(ADMINS.split(',')) # Override these (and anything else) in config_local.py or # set environment variable accordingly. -OAUTH_KEY = os.environ.get('OAUTH_KEY', '') -OAUTH_SECRET = os.environ.get('OAUTH_SECRET', '') -SECRET_KEY = os.environ.get('SECRET_KEY', 'sdkjfhsfljhsadf') +OAUTH_KEY = os.environ.get('OAUTH_KEY') +OAUTH_SECRET = os.environ.get('OAUTH_SECRET') +SECRET_KEY = os.environ.get('SECRET_KEY') try: from config_local import * except ImportError: pass + +assert OAUTH_KEY +assert OAUTH_SECRET +assert SECRET_KEY +assert ADMINS diff --git a/docker-compose.yml b/docker-compose.yml index 4de00de..8382001 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,8 @@ services: # uncomment following line to enable debugging # DEBUG: yes DATABASE_URI: 'postgresql://cf_audit:pw@database:5432/cf_audit' + # enter the osm-ids here, which should get admin-rights. + ADMINS: '' env_file: - .env ports: @@ -14,7 +16,7 @@ services: - database migrate: <<: *auditor - command: python3 migrate.py + command: python migrate.py ports: [] database: image: postgres:10 @@ -22,5 +24,4 @@ services: POSTGRES_USER: cf_audit POSTGRES_PASSWORD: pw POSTGRES_DB: cf_audit - ADMINS: '290271' From d3e82d141643bdb59076c99c1ba6245531a1e64f Mon Sep 17 00:00:00 2001 From: Nicola Jordan Date: Fri, 16 Feb 2018 14:54:51 +0100 Subject: [PATCH 18/19] intify strings --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 9b14b3a..db90c8c 100644 --- a/config.py +++ b/config.py @@ -8,7 +8,7 @@ ADMINS = os.environ.get('ADMINS') if ADMINS: - ADMINS = set(ADMINS.split(',')) + ADMINS = set([int(adm_id) for adm_id in ADMINS.split(',')]) # Override these (and anything else) in config_local.py or # set environment variable accordingly. From 67b743d1c7699c41fbd8f6243bd45537aa6b6c48 Mon Sep 17 00:00:00 2001 From: Nicola Jordan Date: Mon, 11 Jun 2018 11:21:41 +0200 Subject: [PATCH 19/19] added mapillary setting; remove asserts --- Dockerfile | 4 ++-- README.md | 1 + config.py | 5 ----- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 59aa2b5..d155c3f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,12 +4,12 @@ LABEL maintainer="Geometalab " RUN apt-get update && apt-get install -y wget && rm -rf /var/lib/apt/lists/* -ENV DOCKERIZE_VERSION v0.6.0 +ENV DOCKERIZE_VERSION v0.6.1 RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz -RUN pip install uwsgi psycopg2 +RUN pip install uwsgi psycopg2-binary COPY requirements.txt /tmp/requirements.txt RUN pip install -r /tmp/requirements.txt diff --git a/README.md b/README.md index 42df5df..c835ad6 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ on your server as well) using the provided docker-compose file. OAUTH_KEY= OAUTH_SECRET= SECRET_KEY= + MAPILLARY_CLIENT_ID= ``` 1. Then start it on your machine using docker-compose: `docker-compose up --build`. diff --git a/config.py b/config.py index 669f092..2238bcd 100644 --- a/config.py +++ b/config.py @@ -21,8 +21,3 @@ from config_local import * except ImportError: pass - -assert OAUTH_KEY -assert OAUTH_SECRET -assert SECRET_KEY -assert ADMINS