From 5b0782c36c0458dca33588ecd5b46e6a3ea901cc Mon Sep 17 00:00:00 2001 From: David Greenwood Date: Tue, 5 Nov 2024 12:59:34 +0000 Subject: [PATCH] making variables more clear (#8) --- .env.example | 56 ++++----- .env.markdown | 116 ++++++++++++++++++ .env.obstracts-web | 3 - .github/workflows/deploy-image-production.yml | 2 - .github/workflows/deploy-image-staging.yml | 2 - Dockerfile.deploy | 4 - README.md | 27 +--- 7 files changed, 144 insertions(+), 66 deletions(-) create mode 100644 .env.markdown diff --git a/.env.example b/.env.example index 561f1a2..70c6729 100644 --- a/.env.example +++ b/.env.example @@ -1,43 +1,41 @@ #django settings -DJANGO_SECRET=insecure_django_secret -DJANGO_DEBUG=True -DJANGO_ALLOWED_HOSTS=#MODIFY IT RUNNING ON NON LOCAL SERVERS -DJANGO_CORS_ALLOW_ALL_ORIGINS=True #MODIFY TO FALSE IF RUNNING IN PROD -DJANGO_CORS_ALLOWED_ORIGINS=#MODIFY IF RUNNING IN PROD +DJANGO_SECRET= +DJANGO_DEBUG= +DJANGO_ALLOWED_HOSTS= +DJANGO_CORS_ALLOW_ALL_ORIGINS= +DJANGO_CORS_ALLOWED_ORIGINS= #postgres settings POSTGRES_HOST= POSTGRES_PORT= -POSTGRES_DB=postgres -POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres +POSTGRES_DB= +POSTGRES_USER= +POSTGRES_PASSWORD= #celery settings -CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP=1 +CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP= # obstracts settings -MAX_PAGE_SIZE=50 # max size of api response payload -DEFAULT_PAGE_SIZE=50 # default size of api response payload +MAX_PAGE_SIZE= +DEFAULT_PAGE_SIZE= # stix2arango settings -ARANGODB_HOST_URL='http://host.docker.internal:8529' -ARANGODB_USERNAME=root +ARANGODB_HOST_URL= +ARANGODB_USERNAME= ARANGODB_PASSWORD= # history4feed settings -HISTORY4FEED_URL='http://host.docker.internal:8002/' +HISTORY4FEED_URL= # txt2stix settings -BIN_LIST_API_KEY= #[OPTIONAL -- for enriching credit card extractions] needed for extracting credit card information -OPENAI_API_KEY= # [REQUIRED IF USING AI MODES] needed if using AI relationship mode or AI extractions -OPENAI_MODEL=gpt-4 # [REQUIRED IF USING AI MODES] choose an OpenAI model of your choice. Ensure the input/output token count meets requirements (and adjust INPUT_TOKEN_LIMIT accordingly). List of models here: https://platform.openai.com/docs/models -INPUT_TOKEN_LIMIT=50 +BIN_LIST_API_KEY= +OPENAI_API_KEY= +OPENAI_MODEL= +INPUT_TOKEN_LIMIT= ## CTIBUTLER FOR ATT&CK, CAPEC, CWE, ATLAS, AND LOCATION LOOKUPS -CTIBUTLER_HOST='http://host.docker.internal:8006' -CTIBUTLER_APIKEY= #[OPTIONAL] if using https://app.ctibutler.com +CTIBUTLER_HOST= ## VULMATCH FOR CVE AND CPE LOOKUPS -VULMATCH_HOST='http://host.docker.internal:8005' -VULMATCH_APIKEY= #[OPTIONAL] if using https://app.vulmatch.com +VULMATCH_HOST= # file2txt settings -GOOGLE_VISION_API_KEY= # [REQUIRED -- to extract text from blog images] +GOOGLE_VISION_API_KEY= # R2 storage configuration -USE_S3_STORAGE=0 # Need to set to 1 to enable -R2_ENDPOINT_URL= # Looks like https://ID.r2.cloudflarestorage.com -R2_BUCKET_NAME= # the bucket name -R2_ACCESS_KEY= # generated when creating an R2 API token. Make sure has read+write to R2_BUCKET_NAME specified -R2_SECRET_KEY= # generated when creating an R2 API token -R2_CUSTOM_DOMAIN= # this value is optional, but if you don't set your bucket to public, your images will hit 403s as they will hit the raw endpoint (e.g. https://ID.r2.cloudflarestorage.com/BUCKET/IMAGE/PATH.jpg) which will be inaccessible. The easiest way to do this is to enable R2.dev subdomain for the bucket. Looks like pub-ID.r2.dev . Do not include the https:// part \ No newline at end of file +USE_S3_STORAGE= +R2_ENDPOINT_URL= +R2_BUCKET_NAME= +R2_ACCESS_KEY= +R2_SECRET_KEY= +R2_CUSTOM_DOMAIN= \ No newline at end of file diff --git a/.env.markdown b/.env.markdown new file mode 100644 index 0000000..68db697 --- /dev/null +++ b/.env.markdown @@ -0,0 +1,116 @@ +# Environmental file info + +If you're running in production, you should set these securely. + +However, if you just want to experiment, set the following values + +## Django Settings + +These are all Django settings, defined in `obstracts/settings.py` + +* `DJANGO_SECRET`: `insecure_django_secret` +* `DJANGO_DEBUG`: `True` +* `DJANGO_ALLOWED_HOSTS`: BLANK +* `DJANGO_CORS_ALLOW_ALL_ORIGINS`: `True` +* `DJANGO_CORS_ALLOWED_ORIGINS`: LEAVE EMPTY + +## Postgres Settings + +These are all Django settings, defined in `obstracts/settings.py` + +* `POSTGRES_HOST`: `localhost` +* `POSTGRES_PORT`: BLANK +* `POSTGRES_DB`: `postgres` +* `POSTGRES_USER`: `postgres` +* `POSTGRES_PASSWORD`: `postgres` + +## Celery settings + +* `CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP`: `1` + +## Obstracts API settings + +These define how the API behaves. + +* `MAX_PAGE_SIZE`: `50` + * This is the maximum number of results the API will ever return before pagination +* `DEFAULT_PAGE_SIZE`: `50` + * The default page size of result returned by the API + +## ArangoDB settings + +Note, this code will not install an ArangoDB instance. + +If you're new to ArangoDB, [you can install the community edition quickly by following the instructions here](https://arangodb.com/community-server/). + +The script will automatically create a database called `obstracts_database` when the container is spun up (if it does not exist). + +For each blog added, two new collections will be created in the format + +`_-_collection` + +e.g. + +* `graham_cluley_9288374-0298740-94875-vertex_collection` +* `graham_cluley_9288374-0298740-94875-edge_collection` + + +* `ARANGODB_HOST_URL`: `'http://host.docker.internal:8529'` + * If you are running ArangoDB locally, be sure to set `ARANGODB_HOST_URL='http://host.docker.internal:8529'` in the `.env` file otherwise you will run into networking errors. +* `ARANGODB_USERNAME`: `root` + * Change this if neeed +* `ARANGODB_PASSWORD`: USE PASSWORD OF ARANGODB_USERNAME + +## history4feed settings + +Obstracts requires [history4feed](https://github.com/muchdogesec/history4feed) to download and store blog posts. + +* `HISTORY4FEED_URL`: `'http://host.docker.internal:8002/'` + * If you are running history4feed locally, be sure to set `'http://host.docker.internal:8002/'` in the `.env` file otherwise you will run into networking errors. + +## txt2stix settings + +* `BIN_LIST_API_KEY`: BLANK + * for enriching credit card extractions] needed for extracting credit card information +* `OPENAI_API_KEY`: YOUR_API_KEY + * (REQUIRED IF USING AI MODES) get it from https://platform.openai.com/api-keys +* `OPENAI_MODEL`: `gpt-4` + * (REQUIRED IF USING AI MODES) List of models here: https://platform.openai.com/docs/models +* `INPUT_TOKEN_LIMIT`: 15000 + * (REQUIRED IF USING AI MODES) Ensure the input/output token count meets requirements and is supported by the model selected + +## CTIBUTLER + +Obstracts requires [ctibutler](https://github.com/muchdogesec/ctibutler) to lookup ATT&CK, CAPEC, CWE, ATLAS, and locations in blogs + +* `CTIBUTLER_HOST`: `'http://host.docker.internal:8006'` + * If you are running CTI Butler locally, be sure to set `'http://host.docker.internal:8006'` in the `.env` file otherwise you will run into networking errors. + +## VULMATCH FOR CVE AND CPE LOOKUPS + +Obstracts requires [vulmatch](https://github.com/muchdogesec/vulmatch) to lookup CVEs and CPEs in blogs + +* `VULMATCH_HOST`: `'http://host.docker.internal:8005'` + * If you are running vulmatch locally, be sure to set `'http://host.docker.internal:8005'` in the `.env` file otherwise you will run into networking errors. + +## file2txt settings + +* `GOOGLE_VISION_API_KEY`: YOUR_API_KEY + * Instructions here: https://github.com/muchdogesec/file2txt?tab=readme-ov-file#optional-add-googles-cloud-vision-api-key + +## R2 storage configuration + +You can choose to store static assets on Cloudflare on R2. Default is local. + +* `USE_S3_STORAGE`: `0` + * Set to `1` to enable +* `R2_ENDPOINT_URL`: BLANK + * Will be something like `https://ID.r2.cloudflarestorage.com` +* `R2_BUCKET_NAME`: BLANK + * The bucket name you want to use. +* `R2_ACCESS_KEY`: BLANK + * generated when creating an R2 API token. Make sure has read+write to R2_`BUCKET_NAME` specified +* `R2_SECRET_KEY`: BLANK + * generated when creating an R2 API token +* `R2_CUSTOM_DOMAIN`: BLANK + * this value is optional when using R2, but if you don't set your bucket to public, your images will hit 403s as they will hit the raw endpoint (e.g. https://ID.r2.cloudflarestorage.com/BUCKET/IMAGE/PATH.jpg) which will be inaccessible. The easiest way to do this is to enable R2.dev subdomain for the bucket. Looks like `pub-ID.r2.dev` . Do not include the `https://` part \ No newline at end of file diff --git a/.env.obstracts-web b/.env.obstracts-web index 12141f0..2b13ec1 100644 --- a/.env.obstracts-web +++ b/.env.obstracts-web @@ -16,13 +16,10 @@ OPENAI_MODEL= INPUT_TOKEN_LIMIT= # CTIBUTLER FOR ATT&CK, CAPEC, CWE, ATLAS, AND LOCATION LOOKUPS - CTIBUTLER_HOST= -CTIBUTLER_APIKEY= # VULMATCH FOR CVE AND CPE LOOKUPS VULMATCH_HOST= -VULMATCH_APIKEY= # file2txt settings GOOGLE_VISION_API_KEY= diff --git a/.github/workflows/deploy-image-production.yml b/.github/workflows/deploy-image-production.yml index 1cd162a..5a0ab1e 100644 --- a/.github/workflows/deploy-image-production.yml +++ b/.github/workflows/deploy-image-production.yml @@ -55,9 +55,7 @@ jobs: OPENAI_MODEL=${{ secrets.OPENAI_MODEL }} INPUT_TOKEN_LIMIT=${{ secrets.INPUT_TOKEN_LIMIT }} CTIBUTLER_HOST=${{ secrets.CTIBUTLER_HOST }} - CTIBUTLER_APIKEY=${{ secrets.CTIBUTLER_APIKEY }} VULMATCH_HOST=${{ secrets.VULMATCH_HOST }} - VULMATCH_APIKEY=${{ secrets.VULMATCH_APIKEY }} GOOGLE_VISION_API_KEY=${{ secrets.GOOGLE_VISION_API_KEY }} USE_S3_STORAGE=${{ secrets.USE_S3_STORAGE }}1 R2_ENDPOINT_URL=${{ secrets.R2_ENDPOINT_URL }} diff --git a/.github/workflows/deploy-image-staging.yml b/.github/workflows/deploy-image-staging.yml index 0f22d68..f3b30b3 100644 --- a/.github/workflows/deploy-image-staging.yml +++ b/.github/workflows/deploy-image-staging.yml @@ -55,9 +55,7 @@ jobs: OPENAI_MODEL=${{ secrets.OPENAI_MODEL }} INPUT_TOKEN_LIMIT=${{ secrets.INPUT_TOKEN_LIMIT }} CTIBUTLER_HOST=${{ secrets.CTIBUTLER_HOST }} - CTIBUTLER_APIKEY=${{ secrets.CTIBUTLER_APIKEY }} VULMATCH_HOST=${{ secrets.VULMATCH_HOST }} - VULMATCH_APIKEY=${{ secrets.VULMATCH_APIKEY }} GOOGLE_VISION_API_KEY=${{ secrets.GOOGLE_VISION_API_KEY }} USE_S3_STORAGE=${{ secrets.USE_S3_STORAGE }}1 R2_ENDPOINT_URL=${{ secrets.R2_ENDPOINT_URL }} diff --git a/Dockerfile.deploy b/Dockerfile.deploy index 40d990d..af45cb1 100644 --- a/Dockerfile.deploy +++ b/Dockerfile.deploy @@ -11,9 +11,7 @@ ARG OPENAI_API_KEY= ARG OPENAI_MODEL= ARG INPUT_TOKEN_LIMIT= ARG CTIBUTLER_HOST= -ARG CTIBUTLER_APIKEY= ARG VULMATCH_HOST= -ARG VULMATCH_APIKEY= ARG GOOGLE_VISION_API_KEY= ARG USE_S3_STORAGE= ARG R2_ENDPOINT_URL= @@ -32,9 +30,7 @@ ENV OPENAI_API_KEY=${OPENAI_API_KEY} ENV OPENAI_MODEL=${OPENAI_MODEL} ENV INPUT_TOKEN_LIMIT=${INPUT_TOKEN_LIMIT} ENV CTIBUTLER_HOST=${CTIBUTLER_HOST} -ENV CTIBUTLER_APIKEY=${CTIBUTLER_APIKEY} ENV VULMATCH_HOST=${VULMATCH_HOST} -ENV VULMATCH_APIKEY=${VULMATCH_APIKEY} ENV GOOGLE_VISION_API_KEY=${GOOGLE_VISION_API_KEY} ENV USE_S3_STORAGE=${USE_S3_STORAGE} ENV R2_ENDPOINT_URL=${R2_ENDPOINT_URL} diff --git a/README.md b/README.md index e5b250d..0de32e5 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,6 @@ It works at a high level like so: ## Install -### Download and run history4feed - -Obstracts requires [history4feed](https://github.com/muchdogesec/history4feed) to download and store blog posts. - -You'll need to set the location of history4feed later in the Obstracts `.env` file. - -If you are running history4feed locally, be sure to set `HISTORY4FEED_URL='http://host.docker.internal:8002/'` in the `.env` file otherwise you will run into networking errors. - ### Download and configure ```shell @@ -57,24 +49,7 @@ To create one using the default settings: cp .env.example .env ``` -#### A note on ArangoDB secrets - -Note, this script will not install an ArangoDB instance. - -If you're new to ArangoDB, [you can install the community edition quickly by following the instructions here](https://arangodb.com/community-server/). - -If you are running ArangoDB locally, be sure to set `ARANGODB_HOST_URL='http://host.docker.internal:8529'` in the `.env` file otherwise you will run into networking errors. - -The script will automatically create a database called `obstracts_database` when the container is spun up (if it does not exist). - -For each blog added, two new collections will be created in the format - -`_-_collection` - -e.g. - -* `graham_cluley_9288374-0298740-94875-vertex_collection` -* `graham_cluley_9288374-0298740-94875-edge_collection` +To see more information about how to set the variables, and what they do, read the `.env.markdown` file. #### A note on Django and Postgres secrets