diff --git a/.gitignore b/.gitignore index ea287de2..8413f156 100644 --- a/.gitignore +++ b/.gitignore @@ -303,3 +303,4 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ /venv_test/ +/development/docker-compose.yml diff --git a/CveXplore/.schema_version b/CveXplore/.schema_version index c11efbde..78215650 100644 --- a/CveXplore/.schema_version +++ b/CveXplore/.schema_version @@ -1,4 +1,4 @@ { - "version": "1.6", + "version": "1.7", "rebuild_needed": true } diff --git a/CveXplore/VERSION b/CveXplore/VERSION index a0a1ce09..a6b633f4 100644 --- a/CveXplore/VERSION +++ b/CveXplore/VERSION @@ -1 +1 @@ -0.3.20.dev11 \ No newline at end of file +0.3.20.dev19 \ No newline at end of file diff --git a/CveXplore/alembic.ini b/CveXplore/alembic.ini new file mode 100644 index 00000000..073d468b --- /dev/null +++ b/CveXplore/alembic.ini @@ -0,0 +1,116 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python>=3.9 or backports.zoneinfo library. +# Any required deps can installed by adding `alembic[tz]` to the pip requirements +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +;sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/CveXplore/alembic/README b/CveXplore/alembic/README new file mode 100644 index 00000000..98e4f9c4 --- /dev/null +++ b/CveXplore/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/CveXplore/alembic/env.py b/CveXplore/alembic/env.py new file mode 100644 index 00000000..afbfe462 --- /dev/null +++ b/CveXplore/alembic/env.py @@ -0,0 +1,86 @@ +import os +import sys +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../")) +sys.path.insert(0, folder) + +from CveXplore.common.config import Configuration +from CveXplore.core.database_models.models import CveXploreBase + +app_config = Configuration() +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +config.set_main_option("sqlalchemy.url", app_config.SQLALCHEMY_DATABASE_URI) + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = CveXploreBase.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/CveXplore/alembic/script.py.mako b/CveXplore/alembic/script.py.mako new file mode 100644 index 00000000..fbc4b07d --- /dev/null +++ b/CveXplore/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/CveXplore/alembic/versions/53df0e286532_first_2_models.py b/CveXplore/alembic/versions/53df0e286532_first_2_models.py new file mode 100644 index 00000000..f9a8730f --- /dev/null +++ b/CveXplore/alembic/versions/53df0e286532_first_2_models.py @@ -0,0 +1,43 @@ +"""First 2 models + +Revision ID: 53df0e286532 +Revises: +Create Date: 2023-12-20 16:18:04.244419 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "53df0e286532" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "info", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("db", sa.String(length=25), nullable=True), + sa.Column("lastModified", sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "schema", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("rebuild_needed", sa.Boolean(), nullable=True), + sa.Column("version", sa.Float(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("schema") + op.drop_table("info") + # ### end Alembic commands ### diff --git a/CveXplore/alembic/versions/ecb1788b7e08_initial_full_model_setup.py b/CveXplore/alembic/versions/ecb1788b7e08_initial_full_model_setup.py new file mode 100644 index 00000000..d88530a1 --- /dev/null +++ b/CveXplore/alembic/versions/ecb1788b7e08_initial_full_model_setup.py @@ -0,0 +1,175 @@ +"""Initial full model setup + +Revision ID: ecb1788b7e08 +Revises: 53df0e286532 +Create Date: 2023-12-21 06:39:09.346006 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'ecb1788b7e08' +down_revision: Union[str, None] = '53df0e286532' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('cpe', + sa.Column('_id', sa.BigInteger(), nullable=False), + sa.Column('id', sa.String(length=50), nullable=True), + sa.Column('cpeName', sa.String(length=50), nullable=True), + sa.Column('cpeNameId', sa.String(length=50), nullable=True), + sa.Column('created', sa.DateTime(), nullable=True), + sa.Column('deprecated', sa.Boolean(), nullable=True), + sa.Column('deprecatedBy', sa.String(length=50), nullable=True), + sa.Column('lastModified', sa.DateTime(), nullable=True), + sa.Column('padded_version', sa.String(length=50), nullable=True), + sa.Column('product', sa.String(length=50), nullable=True), + sa.Column('stem', sa.String(length=50), nullable=True), + sa.Column('title', sa.String(length=150), nullable=True), + sa.Column('vendor', sa.String(length=50), nullable=True), + sa.Column('version', sa.String(length=50), nullable=True), + sa.PrimaryKeyConstraint('_id') + ) + op.create_index(op.f('ix_cpe__id'), 'cpe', ['_id'], unique=True) + op.create_index(op.f('ix_cpe_cpeName'), 'cpe', ['cpeName'], unique=False) + op.create_index(op.f('ix_cpe_deprecated'), 'cpe', ['deprecated'], unique=False) + op.create_index(op.f('ix_cpe_id'), 'cpe', ['id'], unique=True) + op.create_index(op.f('ix_cpe_lastModified'), 'cpe', ['lastModified'], unique=False) + op.create_index(op.f('ix_cpe_padded_version'), 'cpe', ['padded_version'], unique=False) + op.create_index(op.f('ix_cpe_product'), 'cpe', ['product'], unique=False) + op.create_index(op.f('ix_cpe_stem'), 'cpe', ['stem'], unique=False) + op.create_index(op.f('ix_cpe_title'), 'cpe', ['title'], unique=False) + op.create_index(op.f('ix_cpe_vendor'), 'cpe', ['vendor'], unique=False) + op.create_table('cpeother', + sa.Column('id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_cpeother_id'), 'cpeother', ['id'], unique=True) + op.create_table('cves', + sa.Column('_id', sa.BigInteger(), nullable=False), + sa.Column('id', sa.String(length=50), nullable=True), + sa.Column('access', sa.JSON(), nullable=True), + sa.Column('assigner', sa.String(length=50), nullable=True), + sa.Column('cvss', sa.Float(), nullable=True), + sa.Column('cvss3', sa.Float(), nullable=True), + sa.Column('cvssSource', sa.String(length=50), nullable=True), + sa.Column('cvssTime', sa.DateTime(), nullable=True), + sa.Column('cvssVector', sa.String(length=100), nullable=True), + sa.Column('cwe', sa.String(length=50), nullable=True), + sa.Column('epss', sa.Float(), nullable=True), + sa.Column('epssMetric', sa.JSON(), nullable=True), + sa.Column('exploitabilityScore', sa.Float(), nullable=True), + sa.Column('impact', sa.JSON(), nullable=True), + sa.Column('impactScore', sa.Float(), nullable=True), + sa.Column('lastModified', sa.DateTime(), nullable=True), + sa.Column('modified', sa.DateTime(), nullable=True), + sa.Column('products', sa.JSON(), nullable=True), + sa.Column('published', sa.DateTime(), nullable=True), + sa.Column('references', sa.JSON(), nullable=True), + sa.Column('status', sa.String(length=25), nullable=True), + sa.Column('summary', sa.Text(), nullable=True), + sa.Column('vendors', sa.JSON(), nullable=True), + sa.Column('vulnerable_configuration', sa.JSON(), nullable=True), + sa.Column('vulnerable_configuration_cpe_2_2', sa.JSON(), nullable=True), + sa.Column('vulnerable_configuration_stems', sa.JSON(), nullable=True), + sa.Column('vulnerable_product', sa.JSON(), nullable=True), + sa.Column('vulnerable_product_stems', sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint('_id') + ) + op.create_index(op.f('ix_cves__id'), 'cves', ['_id'], unique=True) + op.create_index(op.f('ix_cves_assigner'), 'cves', ['assigner'], unique=False) + op.create_index(op.f('ix_cves_cvss'), 'cves', ['cvss'], unique=False) + op.create_index(op.f('ix_cves_cvss3'), 'cves', ['cvss3'], unique=False) + op.create_index(op.f('ix_cves_cwe'), 'cves', ['cwe'], unique=False) + op.create_index(op.f('ix_cves_epss'), 'cves', ['epss'], unique=False) + op.create_index(op.f('ix_cves_id'), 'cves', ['id'], unique=True) + op.create_index(op.f('ix_cves_lastModified'), 'cves', ['lastModified'], unique=False) + op.create_index(op.f('ix_cves_modified'), 'cves', ['modified'], unique=False) + op.create_index(op.f('ix_cves_published'), 'cves', ['published'], unique=False) + op.create_index(op.f('ix_cves_status'), 'cves', ['status'], unique=False) + op.create_table('cwe', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('name', sa.String(length=250), nullable=True), + sa.Column('status', sa.String(length=25), nullable=True), + sa.Column('weaknessabs', sa.String(length=25), nullable=True), + sa.Column('related_weaknesses', sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_cwe_id'), 'cwe', ['id'], unique=True) + op.create_index(op.f('ix_cwe_name'), 'cwe', ['name'], unique=False) + op.create_index(op.f('ix_cwe_status'), 'cwe', ['status'], unique=False) + op.create_table('mgmt_blacklist', + sa.Column('id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_mgmt_blacklist_id'), 'mgmt_blacklist', ['id'], unique=True) + op.create_table('mgmt_whitelist', + sa.Column('id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_mgmt_whitelist_id'), 'mgmt_whitelist', ['id'], unique=True) + op.create_table('via4', + sa.Column('_id', sa.Integer(), nullable=False), + sa.Column('id', sa.String(length=50), nullable=True), + sa.Column('db', sa.String(length=25), nullable=True), + sa.Column('searchables', sa.JSON(), nullable=True), + sa.Column('sources', sa.JSON(), nullable=True), + sa.Column('msbulletin', sa.JSON(), nullable=True), + sa.Column('redhat', sa.JSON(), nullable=True), + sa.Column('refmap', sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint('_id') + ) + op.create_index(op.f('ix_via4__id'), 'via4', ['_id'], unique=True) + op.create_index(op.f('ix_via4_id'), 'via4', ['id'], unique=False) + op.create_index(op.f('ix_capec_typical_severity'), 'capec', ['typical_severity'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_capec_typical_severity'), table_name='capec') + op.drop_index(op.f('ix_via4_id'), table_name='via4') + op.drop_index(op.f('ix_via4__id'), table_name='via4') + op.drop_table('via4') + op.drop_index(op.f('ix_mgmt_whitelist_id'), table_name='mgmt_whitelist') + op.drop_table('mgmt_whitelist') + op.drop_index(op.f('ix_mgmt_blacklist_id'), table_name='mgmt_blacklist') + op.drop_table('mgmt_blacklist') + op.drop_index(op.f('ix_cwe_status'), table_name='cwe') + op.drop_index(op.f('ix_cwe_name'), table_name='cwe') + op.drop_index(op.f('ix_cwe_id'), table_name='cwe') + op.drop_table('cwe') + op.drop_index(op.f('ix_cves_status'), table_name='cves') + op.drop_index(op.f('ix_cves_published'), table_name='cves') + op.drop_index(op.f('ix_cves_modified'), table_name='cves') + op.drop_index(op.f('ix_cves_lastModified'), table_name='cves') + op.drop_index(op.f('ix_cves_id'), table_name='cves') + op.drop_index(op.f('ix_cves_epss'), table_name='cves') + op.drop_index(op.f('ix_cves_cwe'), table_name='cves') + op.drop_index(op.f('ix_cves_cvss3'), table_name='cves') + op.drop_index(op.f('ix_cves_cvss'), table_name='cves') + op.drop_index(op.f('ix_cves_assigner'), table_name='cves') + op.drop_index(op.f('ix_cves__id'), table_name='cves') + op.drop_table('cves') + op.drop_index(op.f('ix_cpeother_id'), table_name='cpeother') + op.drop_table('cpeother') + op.drop_index(op.f('ix_cpe_vendor'), table_name='cpe') + op.drop_index(op.f('ix_cpe_title'), table_name='cpe') + op.drop_index(op.f('ix_cpe_stem'), table_name='cpe') + op.drop_index(op.f('ix_cpe_product'), table_name='cpe') + op.drop_index(op.f('ix_cpe_padded_version'), table_name='cpe') + op.drop_index(op.f('ix_cpe_lastModified'), table_name='cpe') + op.drop_index(op.f('ix_cpe_id'), table_name='cpe') + op.drop_index(op.f('ix_cpe_deprecated'), table_name='cpe') + op.drop_index(op.f('ix_cpe_cpeName'), table_name='cpe') + op.drop_index(op.f('ix_cpe__id'), table_name='cpe') + op.drop_table('cpe') + # ### end Alembic commands ### diff --git a/CveXplore/common/config.py b/CveXplore/common/config.py index c53bb3b8..99854e59 100644 --- a/CveXplore/common/config.py +++ b/CveXplore/common/config.py @@ -92,6 +92,7 @@ class Configuration(object): DATASOURCE = os.getenv("DATASOURCE", "mongodb") DATASOURCE_PROTOCOL = os.getenv("DATASOURCE_PROTOCOL", "mongodb") + DATASOURCE_DBAPI = os.getenv("DATASOURCE_DBAPI", None) DATASOURCE_HOST = os.getenv( "DATASOURCE_HOST", os.getenv("MONGODB_HOST", "127.0.0.1") ) @@ -99,6 +100,23 @@ class Configuration(object): os.getenv("DATASOURCE_PORT", int(os.getenv("MONGODB_PORT", 27017))) ) + DATASOURCE_USER = os.getenv("DATASOURCE_USER", "cvexplore") + DATASOURCE_PASSWORD = os.getenv("DATASOURCE_PASSWORD", "cvexplore") + DATASOURCE_DBNAME = os.getenv("DATASOURCE_DBNAME", "cvexplore") + + SQLALCHEMY_DATABASE_URI = os.getenv( + "SQLALCHEMY_DATABASE_URI", + f"{DATASOURCE_PROTOCOL}://{DATASOURCE_USER}:{DATASOURCE_PASSWORD}@{DATASOURCE_HOST}:{DATASOURCE_PORT}/{DATASOURCE_DBNAME}" + if DATASOURCE_DBAPI is None + else f"{DATASOURCE_PROTOCOL}+{DATASOURCE_DBAPI}://{DATASOURCE_USER}:{DATASOURCE_PASSWORD}@{DATASOURCE_HOST}:{DATASOURCE_PORT}/{DATASOURCE_DBNAME}", + ) + SQLALCHEMY_TRACK_MODIFICATIONS = getenv_bool( + "SQLALCHEMY_TRACK_MODIFICATIONS", "False" + ) + SQLALCHEMY_ENGINE_OPTIONS = getenv_dict( + "SQLALCHEMY_ENGINE_OPTIONS", {"pool_recycle": 299, "pool_timeout": 20} + ) + # keep these for now to maintain backwards compatibility MONGODB_HOST = os.getenv("MONGODB_HOST", "127.0.0.1") MONGODB_PORT = int(os.getenv("MONGODB_PORT", 27017)) diff --git a/CveXplore/core/database_migration/database_migrator.py b/CveXplore/core/database_migration/database_migrator.py index 10439e7d..bd1cf8c5 100644 --- a/CveXplore/core/database_migration/database_migrator.py +++ b/CveXplore/core/database_migration/database_migrator.py @@ -18,9 +18,7 @@ def __init__(self, cwd: str = None): cwd if cwd is not None else os.path.dirname( - os.path.dirname( - os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - ) + os.path.dirname(os.path.dirname(os.path.realpath(__file__))) ) ) @@ -62,7 +60,7 @@ def db_down(self, count: int) -> None: def __parse_command_output(self, cmd_output: CompletedProcess) -> None: if cmd_output.returncode != 0: - self.logger.error(cmd_output.stdout.split("\n")[0]) + self.logger.error(cmd_output.stdout) else: output_list = cmd_output.stdout.split("\n") @@ -84,7 +82,7 @@ def __cli_runner(self, command: int, message: str | int = None) -> CompletedProc command_mapping = { 1: f"alembic init alembic", - 2: f"alembic revision -m {message}", + 2: f'alembic revision --autogenerate -m "{message}"', 3: f"alembic upgrade head", 4: f"alembic current", 5: f"alembic history --verbose", diff --git a/CveXplore/core/database_models/models.py b/CveXplore/core/database_models/models.py index e69de29b..497c00ab 100644 --- a/CveXplore/core/database_models/models.py +++ b/CveXplore/core/database_models/models.py @@ -0,0 +1,151 @@ +from sqlalchemy import Column, Integer, String, DateTime, Boolean, Float, JSON, Text, BigInteger + +from sqlalchemy.orm import declarative_base + +CveXploreBase = declarative_base() + + +class Info(CveXploreBase): + __tablename__ = "info" + id = Column(Integer, primary_key=True) + db = Column(String(25)) + lastModified = Column(DateTime) + + def __repr__(self): + return f"<< Info: {self.db} >>" + + +class Schema(CveXploreBase): + __tablename__ = "schema" + id = Column(Integer, primary_key=True) + rebuild_needed = Column(Boolean, default=False) + version = Column(Float) + + def __repr__(self): + return f"<< Schema: {self.id} >>" + + +class Capec(CveXploreBase): + __tablename__ = "capec" + id = Column(Integer, primary_key=True, unique=True, index=True) + loa = Column(String(25), index=True) + name = Column(String(250), index=True) + prerequisites = Column(Text) + solutions = Column(Text) + summary = Column(Text) + typical_severity = Column(String(25), index=True) + execution_flow = Column(JSON, default={}) + related_capecs = Column(JSON, default=[]) + related_weakness = Column(JSON, default=[]) + taxonomy = Column(JSON, default=[]) + + def __repr__(self): + return f"<< Capec: {self.id} >>" + + +class Cpe(CveXploreBase): + __tablename__ = "cpe" + _id = Column(BigInteger, primary_key=True, unique=True, index=True) + id = Column(String(50), unique=True, index=True) + cpeName = Column(String(50), index=True) + cpeNameId = Column(String(50)) + created = Column(DateTime) + deprecated = Column(Boolean, default=False, index=True) + deprecatedBy = Column(String(50)) + lastModified = Column(DateTime, index=True) + padded_version = Column(String(50), index=True) + product = Column(String(50), index=True) + stem = Column(String(50), index=True) + title = Column(String(150), index=True) + vendor = Column(String(50), index=True) + version = Column(String(50)) + + def __repr__(self): + return f"<< Cpe: {self.id} >>" + + +class Cves(CveXploreBase): + __tablename__ = "cves" + _id = Column(BigInteger, primary_key=True, unique=True, index=True) + id = Column(String(50), unique=True, index=True) + access = Column(JSON, default={}) + assigner = Column(String(50), index=True) + cvss = Column(Float, index=True) + cvss3 = Column(Float, index=True) + cvssSource = Column(String(50)) + cvssTime = Column(DateTime) + cvssVector = Column(String(100)) + cwe = Column(String(50), index=True) + epss = Column(Float, index=True) + epssMetric = Column(JSON) + exploitabilityScore = Column(Float) + impact = Column(JSON) + impactScore = Column(Float) + lastModified = Column(DateTime, index=True) + modified = Column(DateTime, index=True) + products = Column(JSON, default=[]) + published = Column(DateTime, index=True) + references = Column(JSON) + status = Column(String(25), index=True) + summary = Column(Text) + vendors = Column(JSON, default=[]) + vulnerable_configuration = Column(JSON, default=[]) + vulnerable_configuration_cpe_2_2 = Column(JSON, default=[]) + vulnerable_configuration_stems = Column(JSON, default=[]) + vulnerable_product = Column(JSON, default=[]) + vulnerable_product_stems = Column(JSON, default=[]) + + def __repr__(self): + return f"<< Cves: {self.id} >>" + + +class Cwe(CveXploreBase): + __tablename__ = "cwe" + id = Column(Integer, primary_key=True, unique=True, index=True) + description = Column(Text) + name = Column(String(250), index=True) + status = Column(String(25), index=True) + weaknessabs = Column(String(25)) + related_weaknesses = Column(JSON, default=[]) + + def __repr__(self): + return f"<< Cwe: {self.id} >>" + + +class Via4(CveXploreBase): + __tablename__ = "via4" + _id = Column(Integer, primary_key=True, unique=True, index=True) + id = Column(String(50), index=True) + db = Column(String(25)) + searchables = Column(JSON, default=[]) + sources = Column(JSON, default=[]) + msbulletin = Column(JSON, default=[]) + redhat = Column(JSON, default={}) + refmap = Column(JSON, default={}) + + def __repr__(self): + return f"<< Via4: {self.db} >>" + + +class Cpeother(CveXploreBase): + __tablename__ = "cpeother" + id = Column(Integer, primary_key=True, unique=True, index=True) + + def __repr__(self): + return f"<< Cpeother: {self.id} >>" + + +class MgmtBlacklist(CveXploreBase): + __tablename__ = "mgmt_blacklist" + id = Column(Integer, primary_key=True, unique=True, index=True) + + def __repr__(self): + return f"<< MgmtBlacklist: {self.id} >>" + + +class MgmtWhitelist(CveXploreBase): + __tablename__ = "mgmt_whitelist" + id = Column(Integer, primary_key=True, unique=True, index=True) + + def __repr__(self): + return f"<< MgmtWhitelist: {self.id} >>" diff --git a/CveXplore/core/general/datasources.py b/CveXplore/core/general/datasources.py index 98a8f34e..1e57ff9a 100644 --- a/CveXplore/core/general/datasources.py +++ b/CveXplore/core/general/datasources.py @@ -1 +1 @@ -supported_datasources = {"mongodb", "api"} +supported_datasources = {"mongodb", "api", "mysql"} diff --git a/CveXplore/database/connection/database_connection.py b/CveXplore/database/connection/database_connection.py index f4b8003c..e6772c43 100644 --- a/CveXplore/database/connection/database_connection.py +++ b/CveXplore/database/connection/database_connection.py @@ -1,6 +1,7 @@ from CveXplore.api.connection.api_db import ApiDatabaseSource from CveXplore.database.connection.base.db_connection_base import DatabaseConnectionBase from CveXplore.database.connection.mongodb.mongo_db import MongoDBConnection +from CveXplore.database.connection.sqlbase.sql_base import SQLBase class DatabaseConnection(object): @@ -11,6 +12,7 @@ def __init__(self, database_type: str, database_init_parameters: dict): self._database_connnections = { "mongodb": MongoDBConnection, "api": ApiDatabaseSource, + "mysql": SQLBase, } self._database_connection = self._database_connnections[self.database_type]( diff --git a/CveXplore/database/connection/mongodb/mongo_db.py b/CveXplore/database/connection/mongodb/mongo_db.py index d8956b9f..677e36c7 100644 --- a/CveXplore/database/connection/mongodb/mongo_db.py +++ b/CveXplore/database/connection/mongodb/mongo_db.py @@ -40,7 +40,7 @@ def __init__( self._dbclient = self.client[database] try: - collections = self.db_client.list_collection_names() + collections = self.dbclient.list_collection_names() except ServerSelectionTimeoutError as err: raise DatabaseConnectionException( f"Connection to the database failed: {err}" @@ -50,7 +50,7 @@ def __init__( for each in collections: self.__setattr__( f"store_{each}", - CveSearchCollection(database=self.db_client, name=each), + CveSearchCollection(database=self.dbclient, name=each), ) atexit.register(self.disconnect) @@ -60,12 +60,12 @@ def dbclient(self): return self._dbclient def set_handlers_for_collections(self): - for each in self.db_client.list_collection_names(): + for each in self.dbclient.list_collection_names(): if not hasattr(self, each): setattr( self, f"store_{each}", - CveSearchCollection(database=self.db_client, name=each), + CveSearchCollection(database=self.dbclient, name=each), ) def disconnect(self): diff --git a/CveXplore/database/connection/sqlbase/connection.py b/CveXplore/database/connection/sqlbase/connection.py new file mode 100644 index 00000000..acbfb552 --- /dev/null +++ b/CveXplore/database/connection/sqlbase/connection.py @@ -0,0 +1,10 @@ +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from CveXplore.common.config import Configuration + +config = Configuration() + +engine = create_engine(config.SQLALCHEMY_DATABASE_URI, echo=True) + +Session = sessionmaker(bind=engine) diff --git a/CveXplore/database/connection/sqlbase/sql_base.py b/CveXplore/database/connection/sqlbase/sql_base.py index 44047dc0..57ca39a0 100644 --- a/CveXplore/database/connection/sqlbase/sql_base.py +++ b/CveXplore/database/connection/sqlbase/sql_base.py @@ -2,10 +2,10 @@ class SQLBase(DatabaseConnectionBase): - def __init__(self): + def __init__(self, **kwargs): super().__init__(logger_name=__name__) - self._dbclient = None + self._dbclient = {"schema": "test"} @property def dbclient(self): diff --git a/development/.docker/images/kafka/dockerfile-kafka b/development/.docker/images/kafka/dockerfile-kafka new file mode 100644 index 00000000..8646b529 --- /dev/null +++ b/development/.docker/images/kafka/dockerfile-kafka @@ -0,0 +1,3 @@ +FROM confluentinc/cp-kafka + +EXPOSE 9092 \ No newline at end of file diff --git a/development/.docker/images/kafka_connect/dockerfile b/development/.docker/images/kafka_connect/dockerfile new file mode 100644 index 00000000..b9412a1c --- /dev/null +++ b/development/.docker/images/kafka_connect/dockerfile @@ -0,0 +1,3 @@ +FROM confluentinc/cp-kafka-connect + +RUN confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:10.7.4 \ No newline at end of file diff --git a/development/.docker/images/kafka_registry/dockerfile b/development/.docker/images/kafka_registry/dockerfile new file mode 100644 index 00000000..c642ebd8 --- /dev/null +++ b/development/.docker/images/kafka_registry/dockerfile @@ -0,0 +1,4 @@ +FROM ubuntu:latest +LABEL authors="paul" + +ENTRYPOINT ["top", "-b"] \ No newline at end of file diff --git a/development/.docker/images/mysql/dockerfile-mysql b/development/.docker/images/mysql/dockerfile-mysql new file mode 100644 index 00000000..38e2288f --- /dev/null +++ b/development/.docker/images/mysql/dockerfile-mysql @@ -0,0 +1,3 @@ +FROM mysql:latest + +EXPOSE 3306 diff --git a/development/.docker/images/redis/dockerfile-redis b/development/.docker/images/redis/dockerfile-redis new file mode 100644 index 00000000..6973ae38 --- /dev/null +++ b/development/.docker/images/redis/dockerfile-redis @@ -0,0 +1,4 @@ +FROM redis:latest + +EXPOSE 6379 + diff --git a/development/.env.example b/development/.env.example new file mode 100644 index 00000000..3fc70e8b --- /dev/null +++ b/development/.env.example @@ -0,0 +1,4 @@ +MYSQL_DATABASE=cvexplore +MYSQL_USER=cvexplore +MYSQL_PASSWORD=<< MYSQL USER PASSWORD >> +MYSQL_ROOT_PASSWORD=<< MYSQL ROOT PASSWORD TO BE CONFIGURED >> diff --git a/development/README.md b/development/README.md new file mode 100644 index 00000000..c3cdb10c --- /dev/null +++ b/development/README.md @@ -0,0 +1,4 @@ +1. rename .env.example to .env; +2. rename docker-compose.yml.example to docker-compose.yml and alter IP addresses if needed; +3. fill out values for MYSQL_PASSWORD and MYSQL_ROOT_PASSWORD; +4. start docker-compose (docker-compose up)! \ No newline at end of file diff --git a/development/docker-compose.yml.example b/development/docker-compose.yml.example new file mode 100644 index 00000000..7ebf2ffc --- /dev/null +++ b/development/docker-compose.yml.example @@ -0,0 +1,32 @@ +version: '3.5' + +services: + mysql: + image: cvexplore-mysql + build: + context: . + dockerfile: .docker/images/mysql/dockerfile-mysql + hostname: mysql + restart: always + env_file: + - .env + expose: + - 3306 + security_opt: + - seccomp:unconfined + volumes: + - mysql_data:/var/lib/mysql + networks: + backend: + ipv4_address: 172.16.50.5 + +networks: + backend: + driver_opts: + com.docker.network.bridge.host_binding_ipv4: "172.16.50.1" + ipam: + config: + - subnet: 172.16.50.0/24 + +volumes: + mysql_data: \ No newline at end of file diff --git a/requirements/modules/mysql.txt b/requirements/modules/mysql.txt index 6e24b009..1d6afde0 100644 --- a/requirements/modules/mysql.txt +++ b/requirements/modules/mysql.txt @@ -1 +1,3 @@ --r ./sqlalchemy.txt \ No newline at end of file +-r ./sqlalchemy.txt + +PyMySQL>=1.1.0 \ No newline at end of file diff --git a/requirements/modules/sqlalchemy.txt b/requirements/modules/sqlalchemy.txt index 7c4226e3..86e975c9 100644 --- a/requirements/modules/sqlalchemy.txt +++ b/requirements/modules/sqlalchemy.txt @@ -1,2 +1,2 @@ sqlalchemy>=2.0.23 -alembic>=1.13.0 +alembic>=1.13.0 \ No newline at end of file