Cloud native security controls (#450)

* slugignore

* add cnsc

* cleanup

* lint

* nit

* lint

* add makefile target

.slugignore

@@ -2,4 +2,12 @@ cres/
 .devcontainer/
 docs/
 application/tests
-application/frontend/src/test/basic-e2e.test.ts
+application/frontend/src/test/basic-e2e.test.ts
+.github
+README.md
+LICENSE
+CONTRIBUTING.md
+.gitignore
+.dockerignore
+Dockerfile-dev
+docker-entrypoint.sh

Makefile

@@ -92,7 +92,12 @@ migrate-downgrade:
 
 import-all:
 	[ -d "./venv" ] && . ./venv/bin/activate
-	rm -rf standards_cache.sqlite && make migrate-upgrade && export FLASK_APP=$(CURDIR)/cre.py && python cre.py --add --from_spreadsheet https://docs.google.com/spreadsheets/d/1eZOEYgts7d_-Dr-1oAbogPfzBLh6511b58pX3b59kvg && python cre.py --generate_embeddings && python cre.py --zap_in --cheatsheets_in --github_tools_in  --capec_in --owasp_secure_headers_in --pci_dss_4_in --juiceshop_in &&	python cre.py --generate_embeddings
+	rm -rf standards_cache.sqlite &&\
+	make migrate-upgrade && export FLASK_APP=$(CURDIR)/cre.py &&\
+	python cre.py --add --from_spreadsheet https://docs.google.com/spreadsheets/d/1eZOEYgts7d_-Dr-1oAbogPfzBLh6511b58pX3b59kvg &&\
+	python cre.py --generate_embeddings && \
+	python cre.py --zap_in --cheatsheets_in --github_tools_in  --capec_in --owasp_secure_headers_in --pci_dss_4_in --juiceshop_in --cloud_native_security_controls_in &&\
+	python cre.py --generate_embeddings
 
 import-neo4j:
 	[ -d "./venv" ] && . ./venv/bin/activate

application/cmd/cre_main.py

@@ -25,6 +25,7 @@
     secure_headers,
     pci_dss,
     juiceshop,
+    cloud_native_security_controls,
 )
 from application.prompt_client import prompt_client as prompt_client
 from application.utils import gap_analysis
@@ -414,6 +415,10 @@ def run(args: argparse.Namespace) -> None:  # pragma: no cover
         juiceshop.parse(
             cache=db_connect(args.cache_file),
         )
+    if args.cloud_native_security_controls_in:
+        cloud_native_security_controls.parse(
+            cache=db_connect(args.cache_file),
+        )
     if args.generate_embeddings:
         generate_embeddings(args.cache_file)
     if args.owasp_proj_meta:

application/frontend/src/test/basic-e2e.test.ts

@@ -14,7 +14,7 @@ describe('App.js', () => {
     jest.setTimeout(1000000);
     browser = await puppeteer.launch(debug);
     page = await browser.newPage();
-    page.setDefaultTimeout(15000)
+    page.setDefaultTimeout(15000);
   });
 
   it('contains the welcome text', async () => {
@@ -42,7 +42,7 @@ describe('App.js', () => {
     await page.waitForSelector('.content');
     const text = await page.$eval('.content', (e) => e.textContent);
     expect(text).not.toContain('No results match your search term');
-    
+
     await page.waitForSelector('.standard-page__links-container');
     const results = await page.$$('.standard-page__links-container');
     expect(results.length).toBeGreaterThan(1);
@@ -61,7 +61,7 @@ describe('App.js', () => {
     expect(text).not.toContain('No results match your search term');
 
     await page.waitForSelector('.standard-page__links-container');
-    
+
     // title match
     const page_title = await page.$eval('.standard-page__heading', (e) => e.textContent);
     expect(page_title).toContain('ASVS');
@@ -109,7 +109,7 @@ describe('App.js', () => {
     await page.waitForSelector('.content');
     const text = await page.$$('.content', (e) => e.textContent);
     expect(text).not.toContain('No results match your search term');
-    
+
     await page.waitForSelector('.standard-page__links-container');
 
     // title match

application/utils/external_project_parsers/cloud_native_security_controls.py

@@ -0,0 +1,85 @@
+from io import StringIO
+import csv
+import urllib
+from pprint import pprint
+import logging
+import os
+from typing import Dict, Any
+from application.database import db
+from application.defs import cre_defs as defs
+import re
+from application.utils import spreadsheet as sheet_utils
+from application.prompt_client import prompt_client as prompt_client
+import requests
+
+logging.basicConfig()
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+def parse(
+    cache: db.Node_collection,
+):
+    prompt = prompt_client.PromptHandler(cache)
+    resp = requests.get(
+        "https://raw.githubusercontent.com/cloud-native-security-controls/controls-catalog/main/controls/controls_catalog.csv"
+    )
+
+    if resp.status_code != 200:
+        logger.fatal(
+            f"could not retrieve cnsclenges yaml, status code {resp.status_code}"
+        )
+        return
+    entries = csv.DictReader(StringIO(resp.text), delimiter=",")
+    for entry in entries:
+        cnsc = defs.Standard(
+            description=entry.get("Control Implementation"),
+            name="Cloud Native Security Controls",
+            section=entry.get("Section"),
+            sectionID=entry.get("ID"),
+            subsection=entry.get("Control Title"),
+            hyperlink="https://github.com/cloud-native-security-controls/controls-catalog/blob/main/controls/controls_catalog.csv#L"
+            + str(entry.get("ID") + 1),
+            version=entry.get("Originating Document"),
+        )
+        existing = cache.get_nodes(
+            name=cnsc.name, section=cnsc.section, sectionID=cnsc.sectionID
+        )
+        if existing:
+            embeddings = cache.get_embeddings_for_doc(existing[0])
+            if embeddings:
+                logger.info(
+                    f"Node {cnsc.todict()} already exists and has embeddings, skipping"
+                )
+                continue
+        cnsc_embeddings = prompt.get_text_embeddings(cnsc.subsection)
+        cre_id = prompt.get_id_of_most_similar_cre(cnsc_embeddings)
+        if not cre_id:
+            logger.info(
+                f"could not find an appropriate CRE for Clound Native Security Control {cnsc.section}, findings similarities with standards instead"
+            )
+            standard_id = prompt.get_id_of_most_similar_node(cnsc_embeddings)
+            dbstandard = cache.get_node_by_db_id(standard_id)
+            logger.info(
+                f"found an appropriate standard for Cloud Native Security Control {cnsc.section}:{cnsc.subsection}, it is: {dbstandard.name}:{dbstandard.section}"
+            )
+            cres = cache.find_cres_of_node(dbstandard)
+            if cres:
+                cre_id = cres[0].id
+        cre = cache.get_cre_by_db_id(cre_id)
+        cnsc_copy = cnsc.shallow_copy()
+        cnsc_copy.description = ""
+        dbnode = cache.add_node(cnsc_copy)
+        if not dbnode:
+            logger.error(f"could not store database node {cnsc_copy.__repr__()}")
+            continue
+        cache.add_embedding(
+            dbnode, cnsc_copy.doctype, cnsc_embeddings, cnsc_copy.__repr__()
+        )
+        if cre:
+            cache.add_link(db.dbCREfromCRE(cre), dbnode)
+            logger.info(f"successfully stored {cnsc_copy.__repr__()}")
+        else:
+            logger.info(
+                f"stored {cnsc_copy.__repr__()} but could not link it to any CRE reliably"
+            )

cre.py

@@ -181,7 +181,11 @@ def main() -> None:
         action="store_true",
         help="import juiceshop challenges from their repo",
     )
-
+    parser.add_argument(
+        "--cloud_native_security_controls_in",
+        action="store_true",
+        help="import cloud native security controls challenges from their repo (https://raw.githubusercontent.com/cloud-native-security-controls/controls-catalog/main/controls/controls_catalog.csv)",
+    )
     parser.add_argument(
         "--generate_embeddings",
         action="store_true",