Cloud run support and tracing (#457)

* cloud run changes

* deploy

* heroku supported runtime

* add args to context manager

* testfix

* dockerfile

* fix dockerfile and documentation

* rm gunicorn from dev dockerfile

* improve dev dockerfile and makefile

* remove /everything call

* fix e2e tests

* tmp git ignore

* Revert "tmp git ignore"

This reverts commit f8a15e2.

* speed up docker dev build

* fix makefile installation of dependencies

* disable explorer until we manage to paginate the /everything call

.dockerignore

@@ -4,4 +4,73 @@ docs
 tmp
 .vscode
 .github
-.git
+.git
+
+# This file specifies files that are *not* uploaded to Google Cloud
+# using gcloud. It follows the same syntax as .gitignore, with the addition of
+# "#!include" directives (which insert the entries of the given .gitignore-style
+# file at that point).
+#
+# For more information, run:
+#   $ gcloud topic gcloudignore
+#
+.gcloudignore
+# If you would like to upload your .git directory, .gitignore file or files
+# from your .gitignore file, remove the corresponding line
+# below:
+.git
+.gitignore
+
+# Python pycache:
+__pycache__/
+# Ignored by the build system
+/setup.cfg
+
+gcp_secret.json
+/Gemfile
+/Gemfile.lock
+_site/
+__pycache__
+.coverage
+.aws-sam/
+*.orig
+.vagrant/*
+*.patch
+Vagrantfile
+
+### vscode ###
+.vscode/
+*.code-workspace
+
+## act secrets
+.secrets/
+.env
+
+### conventions ###
+venv/
+venv-*/
+v/
+
+### Frontend 
+node_modules/
+www/
+.DS_Store
+yarn-error.log
+coverage/
+
+### Dev db
+*.sqlite
+
+### Docs
+*.md
+
+### Dev DBDumps
+*.sql
+*.dump
+
+### Neo4j
+neo4j/
+.neo4j/
+
+.mypy_cache
+tmp/

.github/workflows/e2e.yml

@@ -27,6 +27,7 @@ jobs:
           [ -d "./venv" ] && . ./venv/bin/activate
           export FLASK_APP=./cre.py
           export FLASK_CONFIG=development
+          export INSECURE_REQUESTS=1
           FLASK_CONFIG=development flask run &
           sleep 20s
           yarn test:e2e

.gitignore

@@ -1,3 +1,23 @@
+# This file specifies files that are *not* uploaded to Google Cloud
+# using gcloud. It follows the same syntax as .gitignore, with the addition of
+# "#!include" directives (which insert the entries of the given .gitignore-style
+# file at that point).
+#
+# For more information, run:
+#   $ gcloud topic gcloudignore
+#
+.gcloudignore
+# If you would like to upload your .git directory, .gitignore file or files
+# from your .gitignore file, remove the corresponding line
+# below:
+.git
+.gitignore
+
+# Python pycache:
+__pycache__/
+# Ignored by the build system
+/setup.cfg
+
 gcp_secret.json
 /Gemfile
 /Gemfile.lock
@@ -31,7 +51,14 @@ yarn-error.log
 coverage/
 
 ### Dev db
-standards_cache.sqlite
+*.sqlite
+
+### Docs
+*.md
+
+### Dev DBDumps
+*.sql
+*.dump
 
 ### Neo4j
 neo4j/

Dockerfile

@@ -0,0 +1,14 @@
+FROM node:lts as build
+
+WORKDIR /code
+COPY . /code
+RUN yarn install && yarn build
+
+FROM python:3.11.0 as run
+
+COPY --from=build /code /code
+WORKDIR /code
+RUN pip install -r requirements.txt gunicorn
+
+ENTRYPOINT gunicorn
+CMD ["--timeout","800","--workers","8","cre:app"]

Dockerfile-dev

@@ -1,22 +1,17 @@
-FROM python:alpine
+FROM node:lts as build
 
-# For pete's sake do not put this on the internet
-
-RUN apk add  --no-cache --update --upgrade linux-headers py3-numpy &&\
- apk add --no-cache --virtual .build-deps gcc g++ zlib-dev make python3-dev py3-numpy-dev jpeg-dev
-
-RUN addgroup -S credev && adduser -S credev -G credev
-
-COPY --chown=credev:credev . /home/credev/cre_sync
-USER credev
-
-WORKDIR /home/credev/cre_sync
-
-RUN export PATH=$PATH:/home/credev/.local/bin &&\
-    pip install -r requirements.txt &&\
-    chmod +x docker-entrypoint.sh
-
-ENTRYPOINT ["/home/credev/cre_sync/docker-entrypoint.sh"]
+WORKDIR /code
+COPY . /code
+RUN make install-deps-typescript && make install-typescript
 
+FROM python:3.11.0 as run
 
+COPY --from=build /code /code
+WORKDIR /code
+RUN apt update &&\
+    apt install -y python3-numpy &&\
+    pip install virtualenv &&\
+    make install-deps-python &&\
+    make install-python
 
+ENTRYPOINT make dev-flask

Makefile

@@ -20,13 +20,14 @@ start-worker:
 
 dev-flask:
 	. ./venv/bin/activate
-	FLASK_APP=`pwd`/cre.py  FLASK_CONFIG=development flask run
+	INSECURE_REQUESTS=1 FLASK_APP=`pwd`/cre.py  FLASK_CONFIG=development flask run
 
 e2e:
 	yarn build
 	[ -d "./venv" ] && . ./venv/bin/activate
 	export FLASK_APP=$(CURDIR)/cre.py
 	export FLASK_CONFIG=development
+	export INSECURE_REQUESTS=1
 	flask run &
 	sleep 5
 	yarn test:e2e
@@ -42,22 +43,36 @@ test:
 cover:
 	. ./venv/bin/activate && FLASK_APP=cre.py FLASK_CONFIG=testing flask test --cover
 
-install-deps:
+install-deps-python:
 	[ -d "./venv" ] && . ./venv/bin/activate 
 	pip install -r requirements.txt
-	cd application/frontend
-	yarn install
 
-install:
+install-deps-typescript:
+	(cd application/frontend && yarn install)
+
+install-deps: install-deps-python install-deps-typescript
+
+install-python:
 	virtualenv -p python3 venv
 	. ./venv/bin/activate
-	make install-deps
-	(cd application/frontend && yarn build)
+	make install-deps-python
+	playwright install
+
+install-typescript:
+	cd application/frontend && yarn build
+
+install: install-typescript install-python
+
+docker-dev:
+	docker build -f Dockerfile-dev -t opencre-dev:$(shell git rev-parse HEAD) .
 
-docker:
-	docker build -f Dockerfile-dev -t opencre:$(shell git rev-parse HEAD) .
+docker-prod:
+	docker build -f Dockerfile -t opencre:$(shell git rev-parse HEAD) .
 
-docker-run:
+docker-dev-run:
+	 docker run -it -p 5000:5000 opencre-dev:$(shell git rev-parse HEAD)
+
+docker-prod-run:
 	 docker run -it -p 5000:5000 opencre:$(shell git rev-parse HEAD)
 
 lint:
@@ -104,12 +119,21 @@ import-neo4j:
 	export FLASK_APP=$(CURDIR)/cre.py && python cre.py --populate_neo4j_db
 
 preload-map-analysis: 
-	make start-worker& make start-worker& make  start-worker& make  start-worker& make  start-worker& make  start-worker& make  start-worker& make  start-worker& make  start-worker& make  start-worker& make dev-flask&
+	make docker-redis&\
+	make start-worker&\
+	make start-worker&\
+	make  start-worker&\
+	make  start-worker&\
+	make  start-worker&\
+	make start-worker&\
+	make  start-worker&\
+	make  start-worker&\
+	make  start-worker&\
+	make  start-worker&\
+	make dev-flask&
+	sleep 5
 	[ -d "./venv" ] && . ./venv/bin/activate
 	export FLASK_APP=$(CURDIR)/cre.py 
-
 	python cre.py --preload_map_analysis_target_url 'http://127.0.0.1:5000'	
 	killall python flask
-
-
 all: clean lint test dev dev-run

README.md

@@ -13,28 +13,27 @@
 
 [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://github.com/codespaces/new?hide_repo_select=true&ref=main&repo=400297709&machine=standardLinux32gb&devcontainer_path=.devcontainer%2Fdevcontainer.json&location=WestEurope)
 
-Common Requirements Enumeration Application
-===============================
+# Common Requirements Enumeration Application
+
 See the application working and more explanation at https://www.opencre.org
 CRE is an interactive content linking platform for uniting security standards and guidelines. It offers easy and robust access to relevant information when designing, developing, testing and procuring secure software.
 This python web and cli application handles adding and presenting CREs.
 
-WHY?
-==========
+# WHY?
 
 Independent software security professionals got together to find a solution for the complexity and fragmentation in today’s landscape of security standards and guidelines. These people are Spyros Gasteratos, Rob van der Veer and friends, in close collaboration with the SKF, OpenSSF and Owasp Top 10 project.
 
-HOW?
-======
+# HOW?
+
 The CRE links each section of a standard to a shared topic (a Common Requirement), causing that section to also link with all other resources that map to the same topic. This 1) enables users to find all combined information from relevant sources, 2) it facilitates a shared and better understanding of cyber security, and 3) it allows standard makers to have links that keep working and offer all the information that readers need, so they don’t have to cover it all themselves. The CRE maintains itself: topic links in the standard text are scanned automatically. Furthermore, topics are linked with related other topics, creating a semantic web for security.
 
 Example: the session time-out topic will take the user to relevant criteria in several standards, and to testing guides, development tips, more technical detail, threat descriptions, articles etc. From there, the user can navigate to resources about session management in general.
 WHEN?
 
 Some of the data has been kindly contributed by the SKF and ASVS projects
 
-Installing
----
+# Installing
+
 
 To install this application you need python3, yarn and virtualenv.
 Clone the repository:
@@ -47,8 +46,9 @@ Install dependencies
 <pre> make install </pre>
 
 
-Running
--------
+# Running
+
+### Locally
 
 To run the CLI application, you can run
 <pre>python cre.py --help</pre>
@@ -79,16 +79,37 @@ enviroment varaibles for app to connect to neo4jDB (default):
 To run the web application for production you need gunicorn and you can run from within the cre_sync dir
 <pre>make prod-run</pre>
 
-Developing
----
+# Docker
+You can build the production or the development docker images with 
+`make docker-prod` and `make docker-dev` respectively
+The environment variables used by OpenCRE are:
+```
+        - name: NEO4J_URL
+        - name: NO_GEN_EMBEDDINGS
+        - name: FLASK_CONFIG
+        - name: DEV_DATABASE_URL
+        - name: INSECURE_REQUESTS # development or TLS terminated environments only
+        - name: REDIS_HOST
+        - name: REDIS_PORT
+        - name: REDIS_NO_SSL
+        - name: REDIS_URL # in case REDIS_HOST and REDIS_PORT are unavailable
+        - name: GCP_NATIVE # if there are ambient GCP credentials, only useful for VERTEX chatbot
+        - name: GOOGLE_SECRET_JSON # if not running on GCP
+        - name: GOOGLE_CLIENT_ID # useful for login only
+        - name: GOOGLE_CLIENT_SECRET # useful for login only
+        - name: LOGIN_ALLOWED_DOMAINS # useful for login only
+        - name: ENABLE_TRACING
+```
+You can run the containers with `make docker-prod-run` and `make-docker-dev-run`
+
+# Developing
+
 You can run backend tests with
 <pre>make test</pre>
 You can run get a coverage report with 
 <pre>make cover</pre>
 Try to keep the coverage above 70%
 
-Repo Moved here from https://github.com/northdpole/www-project-integration-standards
-
 Contributing
 ---
 Please see [Contributing](CONTRIBUTING.md) for contributing instructions