From b99079a2f81c5378a5fd9aa3b365ffa00544fead Mon Sep 17 00:00:00 2001 From: Spyros Date: Sun, 19 Nov 2023 13:52:37 +0000 Subject: [PATCH] Cloud native security controls (#450) * slugignore * add cnsc * cleanup * lint * nit * lint * add makefile target --- .slugignore | 10 ++- Makefile | 7 +- application/cmd/cre_main.py | 5 ++ .../frontend/src/test/basic-e2e.test.ts | 8 +- .../cloud_native_security_controls.py | 85 +++++++++++++++++++ cre.py | 6 +- 6 files changed, 114 insertions(+), 7 deletions(-) create mode 100644 application/utils/external_project_parsers/cloud_native_security_controls.py diff --git a/.slugignore b/.slugignore index 5082ea21..6a295170 100644 --- a/.slugignore +++ b/.slugignore @@ -2,4 +2,12 @@ cres/ .devcontainer/ docs/ application/tests -application/frontend/src/test/basic-e2e.test.ts \ No newline at end of file +application/frontend/src/test/basic-e2e.test.ts +.github +README.md +LICENSE +CONTRIBUTING.md +.gitignore +.dockerignore +Dockerfile-dev +docker-entrypoint.sh \ No newline at end of file diff --git a/Makefile b/Makefile index e2a094ae..e9280093 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,12 @@ migrate-downgrade: import-all: [ -d "./venv" ] && . ./venv/bin/activate - rm -rf standards_cache.sqlite && make migrate-upgrade && export FLASK_APP=$(CURDIR)/cre.py && python cre.py --add --from_spreadsheet https://docs.google.com/spreadsheets/d/1eZOEYgts7d_-Dr-1oAbogPfzBLh6511b58pX3b59kvg && python cre.py --generate_embeddings && python cre.py --zap_in --cheatsheets_in --github_tools_in --capec_in --owasp_secure_headers_in --pci_dss_4_in --juiceshop_in && python cre.py --generate_embeddings + rm -rf standards_cache.sqlite &&\ + make migrate-upgrade && export FLASK_APP=$(CURDIR)/cre.py &&\ + python cre.py --add --from_spreadsheet https://docs.google.com/spreadsheets/d/1eZOEYgts7d_-Dr-1oAbogPfzBLh6511b58pX3b59kvg &&\ + python cre.py --generate_embeddings && \ + python cre.py --zap_in --cheatsheets_in --github_tools_in --capec_in --owasp_secure_headers_in --pci_dss_4_in --juiceshop_in --cloud_native_security_controls_in &&\ + python cre.py --generate_embeddings import-neo4j: [ -d "./venv" ] && . ./venv/bin/activate diff --git a/application/cmd/cre_main.py b/application/cmd/cre_main.py index 93d387cd..b908f0e2 100644 --- a/application/cmd/cre_main.py +++ b/application/cmd/cre_main.py @@ -25,6 +25,7 @@ secure_headers, pci_dss, juiceshop, + cloud_native_security_controls, ) from application.prompt_client import prompt_client as prompt_client from application.utils import gap_analysis @@ -414,6 +415,10 @@ def run(args: argparse.Namespace) -> None: # pragma: no cover juiceshop.parse( cache=db_connect(args.cache_file), ) + if args.cloud_native_security_controls_in: + cloud_native_security_controls.parse( + cache=db_connect(args.cache_file), + ) if args.generate_embeddings: generate_embeddings(args.cache_file) if args.owasp_proj_meta: diff --git a/application/frontend/src/test/basic-e2e.test.ts b/application/frontend/src/test/basic-e2e.test.ts index 2a982bff..92486146 100644 --- a/application/frontend/src/test/basic-e2e.test.ts +++ b/application/frontend/src/test/basic-e2e.test.ts @@ -14,7 +14,7 @@ describe('App.js', () => { jest.setTimeout(1000000); browser = await puppeteer.launch(debug); page = await browser.newPage(); - page.setDefaultTimeout(15000) + page.setDefaultTimeout(15000); }); it('contains the welcome text', async () => { @@ -42,7 +42,7 @@ describe('App.js', () => { await page.waitForSelector('.content'); const text = await page.$eval('.content', (e) => e.textContent); expect(text).not.toContain('No results match your search term'); - + await page.waitForSelector('.standard-page__links-container'); const results = await page.$$('.standard-page__links-container'); expect(results.length).toBeGreaterThan(1); @@ -61,7 +61,7 @@ describe('App.js', () => { expect(text).not.toContain('No results match your search term'); await page.waitForSelector('.standard-page__links-container'); - + // title match const page_title = await page.$eval('.standard-page__heading', (e) => e.textContent); expect(page_title).toContain('ASVS'); @@ -109,7 +109,7 @@ describe('App.js', () => { await page.waitForSelector('.content'); const text = await page.$$('.content', (e) => e.textContent); expect(text).not.toContain('No results match your search term'); - + await page.waitForSelector('.standard-page__links-container'); // title match diff --git a/application/utils/external_project_parsers/cloud_native_security_controls.py b/application/utils/external_project_parsers/cloud_native_security_controls.py new file mode 100644 index 00000000..3605f12e --- /dev/null +++ b/application/utils/external_project_parsers/cloud_native_security_controls.py @@ -0,0 +1,85 @@ +from io import StringIO +import csv +import urllib +from pprint import pprint +import logging +import os +from typing import Dict, Any +from application.database import db +from application.defs import cre_defs as defs +import re +from application.utils import spreadsheet as sheet_utils +from application.prompt_client import prompt_client as prompt_client +import requests + +logging.basicConfig() +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +def parse( + cache: db.Node_collection, +): + prompt = prompt_client.PromptHandler(cache) + resp = requests.get( + "https://raw.githubusercontent.com/cloud-native-security-controls/controls-catalog/main/controls/controls_catalog.csv" + ) + + if resp.status_code != 200: + logger.fatal( + f"could not retrieve cnsclenges yaml, status code {resp.status_code}" + ) + return + entries = csv.DictReader(StringIO(resp.text), delimiter=",") + for entry in entries: + cnsc = defs.Standard( + description=entry.get("Control Implementation"), + name="Cloud Native Security Controls", + section=entry.get("Section"), + sectionID=entry.get("ID"), + subsection=entry.get("Control Title"), + hyperlink="https://github.com/cloud-native-security-controls/controls-catalog/blob/main/controls/controls_catalog.csv#L" + + str(entry.get("ID") + 1), + version=entry.get("Originating Document"), + ) + existing = cache.get_nodes( + name=cnsc.name, section=cnsc.section, sectionID=cnsc.sectionID + ) + if existing: + embeddings = cache.get_embeddings_for_doc(existing[0]) + if embeddings: + logger.info( + f"Node {cnsc.todict()} already exists and has embeddings, skipping" + ) + continue + cnsc_embeddings = prompt.get_text_embeddings(cnsc.subsection) + cre_id = prompt.get_id_of_most_similar_cre(cnsc_embeddings) + if not cre_id: + logger.info( + f"could not find an appropriate CRE for Clound Native Security Control {cnsc.section}, findings similarities with standards instead" + ) + standard_id = prompt.get_id_of_most_similar_node(cnsc_embeddings) + dbstandard = cache.get_node_by_db_id(standard_id) + logger.info( + f"found an appropriate standard for Cloud Native Security Control {cnsc.section}:{cnsc.subsection}, it is: {dbstandard.name}:{dbstandard.section}" + ) + cres = cache.find_cres_of_node(dbstandard) + if cres: + cre_id = cres[0].id + cre = cache.get_cre_by_db_id(cre_id) + cnsc_copy = cnsc.shallow_copy() + cnsc_copy.description = "" + dbnode = cache.add_node(cnsc_copy) + if not dbnode: + logger.error(f"could not store database node {cnsc_copy.__repr__()}") + continue + cache.add_embedding( + dbnode, cnsc_copy.doctype, cnsc_embeddings, cnsc_copy.__repr__() + ) + if cre: + cache.add_link(db.dbCREfromCRE(cre), dbnode) + logger.info(f"successfully stored {cnsc_copy.__repr__()}") + else: + logger.info( + f"stored {cnsc_copy.__repr__()} but could not link it to any CRE reliably" + ) diff --git a/cre.py b/cre.py index 967183a3..c6fa9df9 100644 --- a/cre.py +++ b/cre.py @@ -181,7 +181,11 @@ def main() -> None: action="store_true", help="import juiceshop challenges from their repo", ) - + parser.add_argument( + "--cloud_native_security_controls_in", + action="store_true", + help="import cloud native security controls challenges from their repo (https://raw.githubusercontent.com/cloud-native-security-controls/controls-catalog/main/controls/controls_catalog.csv)", + ) parser.add_argument( "--generate_embeddings", action="store_true",