diff --git a/CveXplore/.schema_version b/CveXplore/.schema_version index d38695d7..15d4ae93 100644 --- a/CveXplore/.schema_version +++ b/CveXplore/.schema_version @@ -1,4 +1,4 @@ { - "version": "1.8", + "version": "2.0", "rebuild_needed": true } diff --git a/CveXplore/alembic/versions/9aea9133b2b8_first_setup.py b/CveXplore/alembic/versions/9aea9133b2b8_first_setup.py deleted file mode 100644 index fcfb6198..00000000 --- a/CveXplore/alembic/versions/9aea9133b2b8_first_setup.py +++ /dev/null @@ -1,228 +0,0 @@ -"""First Setup - -Revision ID: 9aea9133b2b8 -Revises: -Create Date: 2024-04-24 05:17:08.297675 - -""" - -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision: str = "9aea9133b2b8" -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "capec", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("loa", sa.String(length=25), nullable=True), - sa.Column("name", sa.String(length=250), nullable=True), - sa.Column("prerequisites", sa.Text(), nullable=True), - sa.Column("solutions", sa.Text(), nullable=True), - sa.Column("summary", sa.Text(), nullable=True), - sa.Column("typical_severity", sa.String(length=25), nullable=True), - sa.Column("execution_flow", sa.JSON(), nullable=True), - sa.Column("related_capecs", sa.JSON(), nullable=True), - sa.Column("related_weakness", sa.JSON(), nullable=True), - sa.Column("taxonomy", sa.JSON(), nullable=True), - sa.PrimaryKeyConstraint("id"), - ) - op.create_index(op.f("ix_capec_id"), "capec", ["id"], unique=True) - op.create_index(op.f("ix_capec_loa"), "capec", ["loa"], unique=False) - op.create_index(op.f("ix_capec_name"), "capec", ["name"], unique=False) - op.create_index( - op.f("ix_capec_typical_severity"), "capec", ["typical_severity"], unique=False - ) - op.create_table( - "cpe", - sa.Column("_id", sa.BigInteger(), nullable=False), - sa.Column("id", sa.String(length=50), nullable=True), - sa.Column("cpeName", sa.String(length=50), nullable=True), - sa.Column("cpeNameId", sa.String(length=50), nullable=True), - sa.Column("created", sa.DateTime(), nullable=True), - sa.Column("deprecated", sa.Boolean(), nullable=True), - sa.Column("deprecatedBy", sa.String(length=50), nullable=True), - sa.Column("lastModified", sa.DateTime(), nullable=True), - sa.Column("padded_version", sa.String(length=50), nullable=True), - sa.Column("product", sa.String(length=50), nullable=True), - sa.Column("stem", sa.String(length=50), nullable=True), - sa.Column("title", sa.String(length=150), nullable=True), - sa.Column("vendor", sa.String(length=50), nullable=True), - sa.Column("version", sa.String(length=50), nullable=True), - sa.PrimaryKeyConstraint("_id"), - ) - op.create_index(op.f("ix_cpe__id"), "cpe", ["_id"], unique=True) - op.create_index(op.f("ix_cpe_cpeName"), "cpe", ["cpeName"], unique=False) - op.create_index(op.f("ix_cpe_deprecated"), "cpe", ["deprecated"], unique=False) - op.create_index(op.f("ix_cpe_id"), "cpe", ["id"], unique=True) - op.create_index(op.f("ix_cpe_lastModified"), "cpe", ["lastModified"], unique=False) - op.create_index( - op.f("ix_cpe_padded_version"), "cpe", ["padded_version"], unique=False - ) - op.create_index(op.f("ix_cpe_product"), "cpe", ["product"], unique=False) - op.create_index(op.f("ix_cpe_stem"), "cpe", ["stem"], unique=False) - op.create_index(op.f("ix_cpe_title"), "cpe", ["title"], unique=False) - op.create_index(op.f("ix_cpe_vendor"), "cpe", ["vendor"], unique=False) - op.create_table( - "cpeother", - sa.Column("id", sa.Integer(), nullable=False), - sa.PrimaryKeyConstraint("id"), - ) - op.create_index(op.f("ix_cpeother_id"), "cpeother", ["id"], unique=True) - op.create_table( - "cves", - sa.Column("_id", sa.BigInteger(), nullable=False), - sa.Column("id", sa.String(length=50), nullable=True), - sa.Column("access", sa.JSON(), nullable=True), - sa.Column("assigner", sa.String(length=50), nullable=True), - sa.Column("cvss", sa.Float(), nullable=True), - sa.Column("cvss3", sa.Float(), nullable=True), - sa.Column("cvssSource", sa.String(length=50), nullable=True), - sa.Column("cvssTime", sa.DateTime(), nullable=True), - sa.Column("cvssVector", sa.String(length=100), nullable=True), - sa.Column("configurations", sa.JSON(), nullable=True), - sa.Column("cwe", sa.String(length=50), nullable=True), - sa.Column("epss", sa.Float(), nullable=True), - sa.Column("epssMetric", sa.JSON(), nullable=True), - sa.Column("exploitabilityScore", sa.Float(), nullable=True), - sa.Column("impact", sa.JSON(), nullable=True), - sa.Column("impactScore", sa.Float(), nullable=True), - sa.Column("lastModified", sa.DateTime(), nullable=True), - sa.Column("modified", sa.DateTime(), nullable=True), - sa.Column("products", sa.JSON(), nullable=True), - sa.Column("published", sa.DateTime(), nullable=True), - sa.Column("references", sa.JSON(), nullable=True), - sa.Column("status", sa.String(length=25), nullable=True), - sa.Column("summary", sa.Text(), nullable=True), - sa.Column("vendors", sa.JSON(), nullable=True), - sa.Column("vulnerable_configuration", sa.JSON(), nullable=True), - sa.Column("vulnerable_configuration_cpe_2_2", sa.JSON(), nullable=True), - sa.Column("vulnerable_configuration_stems", sa.JSON(), nullable=True), - sa.Column("vulnerable_product", sa.JSON(), nullable=True), - sa.Column("vulnerable_product_stems", sa.JSON(), nullable=True), - sa.PrimaryKeyConstraint("_id"), - ) - op.create_index(op.f("ix_cves__id"), "cves", ["_id"], unique=True) - op.create_index(op.f("ix_cves_assigner"), "cves", ["assigner"], unique=False) - op.create_index(op.f("ix_cves_cvss"), "cves", ["cvss"], unique=False) - op.create_index(op.f("ix_cves_cvss3"), "cves", ["cvss3"], unique=False) - op.create_index(op.f("ix_cves_cwe"), "cves", ["cwe"], unique=False) - op.create_index(op.f("ix_cves_epss"), "cves", ["epss"], unique=False) - op.create_index(op.f("ix_cves_id"), "cves", ["id"], unique=True) - op.create_index( - op.f("ix_cves_lastModified"), "cves", ["lastModified"], unique=False - ) - op.create_index(op.f("ix_cves_modified"), "cves", ["modified"], unique=False) - op.create_index(op.f("ix_cves_published"), "cves", ["published"], unique=False) - op.create_index(op.f("ix_cves_status"), "cves", ["status"], unique=False) - op.create_table( - "cwe", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("description", sa.Text(), nullable=True), - sa.Column("name", sa.String(length=250), nullable=True), - sa.Column("status", sa.String(length=25), nullable=True), - sa.Column("weaknessabs", sa.String(length=25), nullable=True), - sa.Column("related_weaknesses", sa.JSON(), nullable=True), - sa.PrimaryKeyConstraint("id"), - ) - op.create_index(op.f("ix_cwe_id"), "cwe", ["id"], unique=True) - op.create_index(op.f("ix_cwe_name"), "cwe", ["name"], unique=False) - op.create_index(op.f("ix_cwe_status"), "cwe", ["status"], unique=False) - op.create_table( - "info", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("db", sa.String(length=25), nullable=True), - sa.Column("lastModified", sa.DateTime(), nullable=True), - sa.PrimaryKeyConstraint("id"), - ) - op.create_table( - "mgmt_blacklist", - sa.Column("id", sa.Integer(), nullable=False), - sa.PrimaryKeyConstraint("id"), - ) - op.create_index(op.f("ix_mgmt_blacklist_id"), "mgmt_blacklist", ["id"], unique=True) - op.create_table( - "mgmt_whitelist", - sa.Column("id", sa.Integer(), nullable=False), - sa.PrimaryKeyConstraint("id"), - ) - op.create_index(op.f("ix_mgmt_whitelist_id"), "mgmt_whitelist", ["id"], unique=True) - op.create_table( - "schema", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("rebuild_needed", sa.Boolean(), nullable=True), - sa.Column("version", sa.Float(), nullable=True), - sa.PrimaryKeyConstraint("id"), - ) - op.create_table( - "via4", - sa.Column("_id", sa.Integer(), nullable=False), - sa.Column("id", sa.String(length=50), nullable=True), - sa.Column("db", sa.String(length=25), nullable=True), - sa.Column("searchables", sa.JSON(), nullable=True), - sa.Column("sources", sa.JSON(), nullable=True), - sa.Column("msbulletin", sa.JSON(), nullable=True), - sa.Column("redhat", sa.JSON(), nullable=True), - sa.Column("refmap", sa.JSON(), nullable=True), - sa.PrimaryKeyConstraint("_id"), - ) - op.create_index(op.f("ix_via4__id"), "via4", ["_id"], unique=True) - op.create_index(op.f("ix_via4_id"), "via4", ["id"], unique=False) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index(op.f("ix_via4_id"), table_name="via4") - op.drop_index(op.f("ix_via4__id"), table_name="via4") - op.drop_table("via4") - op.drop_table("schema") - op.drop_index(op.f("ix_mgmt_whitelist_id"), table_name="mgmt_whitelist") - op.drop_table("mgmt_whitelist") - op.drop_index(op.f("ix_mgmt_blacklist_id"), table_name="mgmt_blacklist") - op.drop_table("mgmt_blacklist") - op.drop_table("info") - op.drop_index(op.f("ix_cwe_status"), table_name="cwe") - op.drop_index(op.f("ix_cwe_name"), table_name="cwe") - op.drop_index(op.f("ix_cwe_id"), table_name="cwe") - op.drop_table("cwe") - op.drop_index(op.f("ix_cves_status"), table_name="cves") - op.drop_index(op.f("ix_cves_published"), table_name="cves") - op.drop_index(op.f("ix_cves_modified"), table_name="cves") - op.drop_index(op.f("ix_cves_lastModified"), table_name="cves") - op.drop_index(op.f("ix_cves_id"), table_name="cves") - op.drop_index(op.f("ix_cves_epss"), table_name="cves") - op.drop_index(op.f("ix_cves_cwe"), table_name="cves") - op.drop_index(op.f("ix_cves_cvss3"), table_name="cves") - op.drop_index(op.f("ix_cves_cvss"), table_name="cves") - op.drop_index(op.f("ix_cves_assigner"), table_name="cves") - op.drop_index(op.f("ix_cves__id"), table_name="cves") - op.drop_table("cves") - op.drop_index(op.f("ix_cpeother_id"), table_name="cpeother") - op.drop_table("cpeother") - op.drop_index(op.f("ix_cpe_vendor"), table_name="cpe") - op.drop_index(op.f("ix_cpe_title"), table_name="cpe") - op.drop_index(op.f("ix_cpe_stem"), table_name="cpe") - op.drop_index(op.f("ix_cpe_product"), table_name="cpe") - op.drop_index(op.f("ix_cpe_padded_version"), table_name="cpe") - op.drop_index(op.f("ix_cpe_lastModified"), table_name="cpe") - op.drop_index(op.f("ix_cpe_id"), table_name="cpe") - op.drop_index(op.f("ix_cpe_deprecated"), table_name="cpe") - op.drop_index(op.f("ix_cpe_cpeName"), table_name="cpe") - op.drop_index(op.f("ix_cpe__id"), table_name="cpe") - op.drop_table("cpe") - op.drop_index(op.f("ix_capec_typical_severity"), table_name="capec") - op.drop_index(op.f("ix_capec_name"), table_name="capec") - op.drop_index(op.f("ix_capec_loa"), table_name="capec") - op.drop_index(op.f("ix_capec_id"), table_name="capec") - op.drop_table("capec") - # ### end Alembic commands ### diff --git a/CveXplore/alembic/versions/ace5a887137a_first_setup.py b/CveXplore/alembic/versions/ace5a887137a_first_setup.py new file mode 100644 index 00000000..7666932c --- /dev/null +++ b/CveXplore/alembic/versions/ace5a887137a_first_setup.py @@ -0,0 +1,211 @@ +"""First Setup + +Revision ID: ace5a887137a +Revises: +Create Date: 2024-04-24 12:46:33.196785 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'ace5a887137a' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('capec', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('loa', sa.String(length=25), nullable=True), + sa.Column('name', sa.String(length=250), nullable=True), + sa.Column('prerequisites', sa.Text(), nullable=True), + sa.Column('solutions', sa.Text(), nullable=True), + sa.Column('summary', sa.Text(), nullable=True), + sa.Column('typical_severity', sa.String(length=25), nullable=True), + sa.Column('execution_flow', sa.JSON(), nullable=True), + sa.Column('related_capecs', sa.JSON(), nullable=True), + sa.Column('related_weakness', sa.JSON(), nullable=True), + sa.Column('taxonomy', sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_capec_id'), 'capec', ['id'], unique=True) + op.create_index(op.f('ix_capec_loa'), 'capec', ['loa'], unique=False) + op.create_index(op.f('ix_capec_name'), 'capec', ['name'], unique=False) + op.create_index(op.f('ix_capec_typical_severity'), 'capec', ['typical_severity'], unique=False) + op.create_table('cpe', + sa.Column('_id', sa.BigInteger(), nullable=False), + sa.Column('id', sa.String(length=50), nullable=True), + sa.Column('cpeName', sa.String(length=256), nullable=True), + sa.Column('cpeNameId', sa.String(length=50), nullable=True), + sa.Column('created', sa.DateTime(), nullable=True), + sa.Column('deprecated', sa.Boolean(), nullable=True), + sa.Column('deprecatedBy', sa.String(length=50), nullable=True), + sa.Column('lastModified', sa.DateTime(), nullable=True), + sa.Column('padded_version', sa.String(length=128), nullable=True), + sa.Column('product', sa.String(length=128), nullable=True), + sa.Column('stem', sa.String(length=256), nullable=True), + sa.Column('title', sa.String(length=256), nullable=True), + sa.Column('vendor', sa.String(length=256), nullable=True), + sa.Column('version', sa.String(length=128), nullable=True), + sa.PrimaryKeyConstraint('_id') + ) + op.create_index(op.f('ix_cpe__id'), 'cpe', ['_id'], unique=True) + op.create_index(op.f('ix_cpe_cpeName'), 'cpe', ['cpeName'], unique=False) + op.create_index(op.f('ix_cpe_deprecated'), 'cpe', ['deprecated'], unique=False) + op.create_index(op.f('ix_cpe_id'), 'cpe', ['id'], unique=True) + op.create_index(op.f('ix_cpe_lastModified'), 'cpe', ['lastModified'], unique=False) + op.create_index(op.f('ix_cpe_padded_version'), 'cpe', ['padded_version'], unique=False) + op.create_index(op.f('ix_cpe_product'), 'cpe', ['product'], unique=False) + op.create_index(op.f('ix_cpe_stem'), 'cpe', ['stem'], unique=False) + op.create_index(op.f('ix_cpe_title'), 'cpe', ['title'], unique=False) + op.create_index(op.f('ix_cpe_vendor'), 'cpe', ['vendor'], unique=False) + op.create_table('cpeother', + sa.Column('id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_cpeother_id'), 'cpeother', ['id'], unique=True) + op.create_table('cves', + sa.Column('_id', sa.BigInteger(), nullable=False), + sa.Column('id', sa.String(length=50), nullable=True), + sa.Column('access', sa.JSON(), nullable=True), + sa.Column('assigner', sa.String(length=50), nullable=True), + sa.Column('cvss', sa.Float(), nullable=True), + sa.Column('cvss3', sa.Float(), nullable=True), + sa.Column('cvssSource', sa.String(length=50), nullable=True), + sa.Column('cvssTime', sa.DateTime(), nullable=True), + sa.Column('cvssVector', sa.String(length=100), nullable=True), + sa.Column('configurations', sa.JSON(), nullable=True), + sa.Column('cwe', sa.String(length=50), nullable=True), + sa.Column('epss', sa.Float(), nullable=True), + sa.Column('epssMetric', sa.JSON(), nullable=True), + sa.Column('exploitabilityScore', sa.Float(), nullable=True), + sa.Column('impact', sa.JSON(), nullable=True), + sa.Column('impactScore', sa.Float(), nullable=True), + sa.Column('lastModified', sa.DateTime(), nullable=True), + sa.Column('modified', sa.DateTime(), nullable=True), + sa.Column('products', sa.JSON(), nullable=True), + sa.Column('published', sa.DateTime(), nullable=True), + sa.Column('references', sa.JSON(), nullable=True), + sa.Column('status', sa.String(length=25), nullable=True), + sa.Column('summary', sa.Text(), nullable=True), + sa.Column('vendors', sa.JSON(), nullable=True), + sa.Column('vulnerable_configuration', sa.JSON(), nullable=True), + sa.Column('vulnerable_configuration_cpe_2_2', sa.JSON(), nullable=True), + sa.Column('vulnerable_configuration_stems', sa.JSON(), nullable=True), + sa.Column('vulnerable_product', sa.JSON(), nullable=True), + sa.Column('vulnerable_product_stems', sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint('_id') + ) + op.create_index(op.f('ix_cves__id'), 'cves', ['_id'], unique=True) + op.create_index(op.f('ix_cves_assigner'), 'cves', ['assigner'], unique=False) + op.create_index(op.f('ix_cves_cvss'), 'cves', ['cvss'], unique=False) + op.create_index(op.f('ix_cves_cvss3'), 'cves', ['cvss3'], unique=False) + op.create_index(op.f('ix_cves_cwe'), 'cves', ['cwe'], unique=False) + op.create_index(op.f('ix_cves_epss'), 'cves', ['epss'], unique=False) + op.create_index(op.f('ix_cves_id'), 'cves', ['id'], unique=True) + op.create_index(op.f('ix_cves_lastModified'), 'cves', ['lastModified'], unique=False) + op.create_index(op.f('ix_cves_modified'), 'cves', ['modified'], unique=False) + op.create_index(op.f('ix_cves_published'), 'cves', ['published'], unique=False) + op.create_index(op.f('ix_cves_status'), 'cves', ['status'], unique=False) + op.create_table('cwe', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('name', sa.String(length=250), nullable=True), + sa.Column('status', sa.String(length=25), nullable=True), + sa.Column('weaknessabs', sa.String(length=25), nullable=True), + sa.Column('related_weaknesses', sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_cwe_id'), 'cwe', ['id'], unique=True) + op.create_index(op.f('ix_cwe_name'), 'cwe', ['name'], unique=False) + op.create_index(op.f('ix_cwe_status'), 'cwe', ['status'], unique=False) + op.create_table('info', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('db', sa.String(length=25), nullable=True), + sa.Column('lastModified', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('mgmt_blacklist', + sa.Column('id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_mgmt_blacklist_id'), 'mgmt_blacklist', ['id'], unique=True) + op.create_table('mgmt_whitelist', + sa.Column('id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_mgmt_whitelist_id'), 'mgmt_whitelist', ['id'], unique=True) + op.create_table('schema', + sa.Column('_id', sa.Integer(), nullable=False), + sa.Column('rebuild_needed', sa.Boolean(), nullable=True), + sa.Column('version', sa.Float(), nullable=True), + sa.PrimaryKeyConstraint('_id') + ) + op.create_table('via4', + sa.Column('_id', sa.Integer(), nullable=False), + sa.Column('id', sa.String(length=50), nullable=True), + sa.Column('db', sa.String(length=25), nullable=True), + sa.Column('searchables', sa.JSON(), nullable=True), + sa.Column('sources', sa.JSON(), nullable=True), + sa.Column('msbulletin', sa.JSON(), nullable=True), + sa.Column('redhat', sa.JSON(), nullable=True), + sa.Column('refmap', sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint('_id') + ) + op.create_index(op.f('ix_via4__id'), 'via4', ['_id'], unique=True) + op.create_index(op.f('ix_via4_id'), 'via4', ['id'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_via4_id'), table_name='via4') + op.drop_index(op.f('ix_via4__id'), table_name='via4') + op.drop_table('via4') + op.drop_table('schema') + op.drop_index(op.f('ix_mgmt_whitelist_id'), table_name='mgmt_whitelist') + op.drop_table('mgmt_whitelist') + op.drop_index(op.f('ix_mgmt_blacklist_id'), table_name='mgmt_blacklist') + op.drop_table('mgmt_blacklist') + op.drop_table('info') + op.drop_index(op.f('ix_cwe_status'), table_name='cwe') + op.drop_index(op.f('ix_cwe_name'), table_name='cwe') + op.drop_index(op.f('ix_cwe_id'), table_name='cwe') + op.drop_table('cwe') + op.drop_index(op.f('ix_cves_status'), table_name='cves') + op.drop_index(op.f('ix_cves_published'), table_name='cves') + op.drop_index(op.f('ix_cves_modified'), table_name='cves') + op.drop_index(op.f('ix_cves_lastModified'), table_name='cves') + op.drop_index(op.f('ix_cves_id'), table_name='cves') + op.drop_index(op.f('ix_cves_epss'), table_name='cves') + op.drop_index(op.f('ix_cves_cwe'), table_name='cves') + op.drop_index(op.f('ix_cves_cvss3'), table_name='cves') + op.drop_index(op.f('ix_cves_cvss'), table_name='cves') + op.drop_index(op.f('ix_cves_assigner'), table_name='cves') + op.drop_index(op.f('ix_cves__id'), table_name='cves') + op.drop_table('cves') + op.drop_index(op.f('ix_cpeother_id'), table_name='cpeother') + op.drop_table('cpeother') + op.drop_index(op.f('ix_cpe_vendor'), table_name='cpe') + op.drop_index(op.f('ix_cpe_title'), table_name='cpe') + op.drop_index(op.f('ix_cpe_stem'), table_name='cpe') + op.drop_index(op.f('ix_cpe_product'), table_name='cpe') + op.drop_index(op.f('ix_cpe_padded_version'), table_name='cpe') + op.drop_index(op.f('ix_cpe_lastModified'), table_name='cpe') + op.drop_index(op.f('ix_cpe_id'), table_name='cpe') + op.drop_index(op.f('ix_cpe_deprecated'), table_name='cpe') + op.drop_index(op.f('ix_cpe_cpeName'), table_name='cpe') + op.drop_index(op.f('ix_cpe__id'), table_name='cpe') + op.drop_table('cpe') + op.drop_index(op.f('ix_capec_typical_severity'), table_name='capec') + op.drop_index(op.f('ix_capec_name'), table_name='capec') + op.drop_index(op.f('ix_capec_loa'), table_name='capec') + op.drop_index(op.f('ix_capec_id'), table_name='capec') + op.drop_table('capec') + # ### end Alembic commands ### diff --git a/CveXplore/cli_cmds/tasks_cmds/commands.py b/CveXplore/cli_cmds/tasks_cmds/commands.py index 941d42e1..f6de9605 100644 --- a/CveXplore/cli_cmds/tasks_cmds/commands.py +++ b/CveXplore/cli_cmds/tasks_cmds/commands.py @@ -4,12 +4,12 @@ from CveXplore.cli_cmds.mutex_options.mutex import Mutex from CveXplore.core.general.constants import task_status_rev_types from CveXplore.core.general.utils import ( - datetimeToTimestring, + datetime_to_timestring, set_ansi_color_red, set_ansi_color_green, set_ansi_color_magenta, set_ansi_color_yellow, - timestampTOdatetimestring, + timestamp_to_datetimestring, ) from CveXplore.errors.tasks import TaskNotFoundError @@ -121,11 +121,11 @@ def scheduled_cmd(ctx, list, delete, toggle, purge, results): if task["enabled"] else set_ansi_color_red(task["enabled"]) ), - f"{set_ansi_color_magenta(datetimeToTimestring(task['last_run_at']))} " + f"{set_ansi_color_magenta(datetime_to_timestring(task['last_run_at']))} " f"{last_run_result} " f"[{set_ansi_color_yellow(task['total_run_count'])}]", set_ansi_color_magenta( - datetimeToTimestring(task["next_run_at"]) + datetime_to_timestring(task["next_run_at"]) ), ] ) @@ -179,7 +179,7 @@ def scheduled_cmd(ctx, list, delete, toggle, purge, results): [ x, set_ansi_color_magenta( - timestampTOdatetimestring(each["inserted"]) + timestamp_to_datetimestring(each["inserted"]) ), each["task_id"], each["state"], diff --git a/CveXplore/common/data_source_connection.py b/CveXplore/common/data_source_connection.py index 56a26c35..3ec5b922 100644 --- a/CveXplore/common/data_source_connection.py +++ b/CveXplore/common/data_source_connection.py @@ -45,7 +45,7 @@ def datasource_connection(self) -> DatabaseConnectionBase: ).database_connection else: return DatabaseConnection( - database_type=self.config.DATASOURCE_TYPE, + database_type=self.config.DATASOURCE_TYPE.lower(), database_init_parameters=self.config.DATASOURCE_CONNECTION_DETAILS, ).database_connection diff --git a/CveXplore/core/celery_task_handler/task_handler.py b/CveXplore/core/celery_task_handler/task_handler.py index 1a7b7954..df7f2c82 100644 --- a/CveXplore/core/celery_task_handler/task_handler.py +++ b/CveXplore/core/celery_task_handler/task_handler.py @@ -18,7 +18,7 @@ from CveXplore.celery_app.cvexplore_daemon import app from CveXplore.common.config import Configuration -from CveXplore.core.general.utils import timestampTOdatetime +from CveXplore.core.general.utils import timestamp_to_datetime from CveXplore.core.logging.logger_class import AppLogger from CveXplore.errors.tasks import TaskNotFoundError @@ -790,7 +790,7 @@ def show_scheduled_tasks(self) -> list[Task]: ret_list.append( Task( **task_data.to_dict(), - next_run_at=timestampTOdatetime(int(next_run)), + next_run_at=timestamp_to_datetime(int(next_run)), ) ) diff --git a/CveXplore/core/database_maintenance/download_handler.py b/CveXplore/core/database_maintenance/download_handler.py index 7c472925..3f8c2251 100644 --- a/CveXplore/core/database_maintenance/download_handler.py +++ b/CveXplore/core/database_maintenance/download_handler.py @@ -22,8 +22,8 @@ from urllib3 import Retry from CveXplore.common.config import Configuration -from CveXplore.core.general.utils import sanitize from CveXplore.core.database_maintenance.worker_q import WorkerQueue +from CveXplore.core.general.utils import sanitize from ..database_indexer.db_indexer import DatabaseIndexer from ..logging.handlers.cve_explore_rfh import CveExploreUpdateRfhHandler from ..logging.handlers.cve_explore_stream import CveExploreUpdateStreamHandler @@ -68,7 +68,7 @@ def __init__( self.do_process = True database = DatabaseConnection( - database_type=self.config.DATASOURCE_TYPE, + database_type=self.config.DATASOURCE_TYPE.lower(), database_init_parameters=self.config.DATASOURCE_CONNECTION_DETAILS, ).database_connection @@ -438,7 +438,11 @@ def drop_collection(self, col: str): return self.database[col].drop() def get_table_names(self): - return self.database.list_collection_names() + # if self.config.DATASOURCE_TYPE.lower() == datasources.MONGODB: + # return self.database.list_collection_names() + # if self.config.DATASOURCE_TYPE.lower() == datasources.MYSQL: + # return self.database[self.feed_type.lower()].list_collection_names() + return self.database[self.feed_type.lower()].list_collection_names() def set_col_field(self, collection: str, field: str, data: dict): self.database[collection].update_one( diff --git a/CveXplore/core/database_maintenance/main_updater.py b/CveXplore/core/database_maintenance/main_updater.py index 51187fde..1bc5a13a 100644 --- a/CveXplore/core/database_maintenance/main_updater.py +++ b/CveXplore/core/database_maintenance/main_updater.py @@ -141,6 +141,17 @@ def populate( self.logger.info(f"Upgrading database schema to latest head, if needed...") self.database_migrator.db_upgrade() + if limit is not None or get_id is not None: + if populate_source is None: + raise ValueError( + "populate_source is not defined, you need to provide a source when providing a limit" + " or get_id parameter!" + ) + if populate_source not in ["cve", "cpe"]: + raise ValueError( + f"The source '{populate_source}' does not support the limit or get_id parameter!" + ) + if populate_source is not None: if not isinstance(populate_source, str | list): raise ValueError("Wrong 'populate_source' parameter type received!") diff --git a/CveXplore/core/database_maintenance/sources_process.py b/CveXplore/core/database_maintenance/sources_process.py index 24b8712b..394763e7 100644 --- a/CveXplore/core/database_maintenance/sources_process.py +++ b/CveXplore/core/database_maintenance/sources_process.py @@ -10,17 +10,18 @@ from dateutil.parser import parse as parse_datetime from tqdm import tqdm -from CveXplore.core.database_maintenance.db_action import DatabaseAction from CveXplore.core.database_maintenance.api_handlers import NVDApiHandler from CveXplore.core.database_maintenance.content_handlers import ( CapecHandler, CWEHandler, ) +from CveXplore.core.database_maintenance.db_action import DatabaseAction from CveXplore.core.database_maintenance.file_handlers import ( XMLFileHandler, JSONFileHandler, CSVFileHandler, ) +from CveXplore.core.general.datasources import datasources from CveXplore.errors.apis import ApiDataRetrievalFailed, ApiMaxRetryError date = datetime.datetime.now() @@ -273,7 +274,7 @@ def populate(self, **kwargs): self.process_downloads(**kwargs) - if self.config.DATASOURCE_TYPE == "mongodb": + if self.config.DATASOURCE_TYPE.lower() == datasources.MONGODB: self.database_indexer.create_indexes(collection=self.feed_type.lower()) self.logger.info("Finished CPE database population") @@ -832,7 +833,7 @@ def populate(self, **kwargs): self.process_downloads(**kwargs) - if self.config.DATASOURCE_TYPE == "mongodb": + if self.config.DATASOURCE_TYPE.lower() == datasources.MONGODB: self.database_indexer.create_indexes(collection=self.feed_type.lower()) self.logger.info("Finished CVE database population") diff --git a/CveXplore/core/database_maintenance/worker_q.py b/CveXplore/core/database_maintenance/worker_q.py index ebb659b6..9dbeeba9 100644 --- a/CveXplore/core/database_maintenance/worker_q.py +++ b/CveXplore/core/database_maintenance/worker_q.py @@ -1,11 +1,14 @@ from queue import Empty, Queue +from CveXplore.common.config import Configuration from CveXplore.core.database_maintenance.db_action import DatabaseAction +from CveXplore.core.general.datasources import datasources class WorkerQueue(Queue): def __init__(self, name: str, maxsize: int = 0): super().__init__(maxsize) + self.config = Configuration self.name = name self.maxsize = maxsize @@ -25,7 +28,10 @@ def __next__(self): item = self.get(timeout=1) if item is not None: if isinstance(item, DatabaseAction): - item = item.entry + if self.config.DATASOURCE_TYPE.lower() == datasources.MONGODB: + item = item.entry + else: + item = item.doc return item else: raise StopIteration diff --git a/CveXplore/core/database_version/db_version_checker.py b/CveXplore/core/database_version/db_version_checker.py index bbaaa500..32f88c3b 100644 --- a/CveXplore/core/database_version/db_version_checker.py +++ b/CveXplore/core/database_version/db_version_checker.py @@ -2,8 +2,8 @@ import os from CveXplore.core.database_maintenance.update_base_class import UpdateBaseClass -from CveXplore.database_models.models import CveXploreModel from CveXplore.database.connection.base.db_connection_base import DatabaseConnectionBase +from CveXplore.database_models.models import CveXploreModel from CveXplore.errors import DatabaseSchemaVersionError runPath = os.path.dirname(os.path.realpath(__file__)) diff --git a/CveXplore/core/general/datasources.py b/CveXplore/core/general/datasources.py index 1e57ff9a..5e3590ea 100644 --- a/CveXplore/core/general/datasources.py +++ b/CveXplore/core/general/datasources.py @@ -1 +1,7 @@ -supported_datasources = {"mongodb", "api", "mysql"} +import collections + +supported_datasources = {"MONGODB", "API", "MYSQL"} + +datasources = collections.namedtuple("datasources", "MONGODB, API, MYSQL")( + "mongodb", "api", "mysql" +) diff --git a/CveXplore/core/general/utils.py b/CveXplore/core/general/utils.py index b04aee6c..feb57b89 100644 --- a/CveXplore/core/general/utils.py +++ b/CveXplore/core/general/utils.py @@ -4,6 +4,8 @@ import colors import pymongo +from CveXplore.database_models.models import CveXploreModel + def sanitize(x: list | pymongo.cursor.Cursor): if isinstance(x, pymongo.cursor.Cursor): @@ -11,12 +13,14 @@ def sanitize(x: list | pymongo.cursor.Cursor): if type(x) == list: for y in x: sanitize(y) + if isinstance(x, CveXploreModel): + x = x.to_dict() if x and "_id" in x: x.pop("_id") return x -def timestampTOdatetime(timestamp): +def timestamp_to_datetime(timestamp): """ Method that will take the provided timestamp and converts it into a date time object @@ -30,15 +34,15 @@ def timestampTOdatetime(timestamp): return value -def datetimeTOtimestamp(date_time_object): +def datetime_to_timestamp(date_time_object): return calendar.timegm(date_time_object.utctimetuple()) -def datetimeToTimestring(datetime_object): - return timestampTOdatetimestring(datetimeTOtimestamp(datetime_object)) +def datetime_to_timestring(datetime_object): + return timestamp_to_datetimestring(datetime_to_timestamp(datetime_object)) -def timestampTOdatetimestring(timestamp, vis=False): +def timestamp_to_datetimestring(timestamp, vis=False): """ Method that will take the provided timestamp and converts it into a RFC3339 date time string diff --git a/CveXplore/database/connection/base/db_connection_base.py b/CveXplore/database/connection/base/db_connection_base.py index bd40eabe..05c76663 100644 --- a/CveXplore/database/connection/base/db_connection_base.py +++ b/CveXplore/database/connection/base/db_connection_base.py @@ -23,3 +23,7 @@ def dbclient(self): @abstractmethod def set_handlers_for_collections(self): raise NotImplementedError + + @abstractmethod + def list_collection_names(self): + raise NotImplementedError diff --git a/CveXplore/database/connection/dummy/dummy.py b/CveXplore/database/connection/dummy/dummy.py index 6cbe632f..4eaba237 100644 --- a/CveXplore/database/connection/dummy/dummy.py +++ b/CveXplore/database/connection/dummy/dummy.py @@ -11,5 +11,8 @@ def __init__(self, **kwargs): def dbclient(self): return self._dbclient + def list_collection_names(self): + pass + def set_handlers_for_collections(self): pass diff --git a/CveXplore/database/connection/mongodb/mongo_db.py b/CveXplore/database/connection/mongodb/mongo_db.py index 715eb044..bb55ee8f 100644 --- a/CveXplore/database/connection/mongodb/mongo_db.py +++ b/CveXplore/database/connection/mongodb/mongo_db.py @@ -58,6 +58,9 @@ def __init__( def dbclient(self): return self._dbclient + def list_collection_names(self): + return self._dbclient.list_collection_names() + def set_handlers_for_collections(self): for each in self.dbclient.list_collection_names(): if not hasattr(self, each): diff --git a/CveXplore/database/connection/sqlbase/sql_base.py b/CveXplore/database/connection/sqlbase/sql_base.py index 02b2e580..cc4feb38 100644 --- a/CveXplore/database/connection/sqlbase/sql_base.py +++ b/CveXplore/database/connection/sqlbase/sql_base.py @@ -1,12 +1,14 @@ +from CveXplore.core.general.datasources import datasources from CveXplore.database.connection.base.db_connection_base import DatabaseConnectionBase from CveXplore.database_models.models import CveXploreBase from CveXplore.errors import DatabaseConnectionException class SQLBaseConnection(DatabaseConnectionBase): + def __init__(self, **kwargs): super().__init__(logger_name=__name__) - if self.config.DATASOURCE_TYPE != "mongodb": + if self.config.DATASOURCE_TYPE.lower() != datasources.MONGODB: from CveXplore.database.connection.sqlbase.sql_client import SQLClient self._dbclient = { @@ -34,8 +36,11 @@ def __init__(self, **kwargs): def dbclient(self): return self._dbclient + def list_collection_names(self): + return list(self._dbclient.keys()) + def set_handlers_for_collections(self): - if self.config.DATASOURCE_TYPE != "mongodb": + if self.config.DATASOURCE_TYPE.lower() != datasources.MONGODB: from CveXplore.database.connection.sqlbase.sql_client import SQLClient for each in list(CveXploreBase.metadata.tables.keys()): diff --git a/CveXplore/database/connection/sqlbase/sql_client.py b/CveXplore/database/connection/sqlbase/sql_client.py index 0d444c43..7e65fb4f 100644 --- a/CveXplore/database/connection/sqlbase/sql_client.py +++ b/CveXplore/database/connection/sqlbase/sql_client.py @@ -1,6 +1,8 @@ from sqlalchemy import insert, text from sqlalchemy.exc import IntegrityError +from CveXplore.database.connection.sqlbase.connection import Session +from CveXplore.database.connection.sqlbase.sql_client_base import SQLClientBase from CveXplore.database_models.models import ( Cpe, Info, @@ -13,8 +15,6 @@ MgmtBlacklist, MgmtWhitelist, ) -from CveXplore.database.connection.sqlbase.connection import Session -from CveXplore.database.connection.sqlbase.sql_client_base import SQLClientBase class SQLClient(SQLClientBase): @@ -41,6 +41,9 @@ def __init__(self, collection_name: str): def __repr__(self): return f"<< SQLClient:{self.collection_name} >>" + def list_collection_names(self): + return list(self.model_mapping.keys()) + def bulk_write(self, write_entries: list, ordered: bool = False): with self.session() as session: session.execute( @@ -101,13 +104,30 @@ def find(self, query_dict: dict, *args, **kwargs): if len(query_dict) == 0: data = session.query(self.obj_ref).all() return data - elif len(query_dict) == 1: - query_key, query_value = list(query_dict.items())[0] - data = ( - session.query(self.obj_ref) - .filter(getattr(self.obj_ref, query_key) == query_value) - .all() - ) + else: + q = session.query(self.obj_ref) + for attr, value in query_dict.items(): + if isinstance(value, dict): + for k, v in value.items(): + if k == "$lte": + q = q.filter( + getattr(self.obj_ref, attr) <= value["$lte"] + ) + elif k == "$gte": + q = q.filter( + getattr(self.obj_ref, attr) >= value["$gte"] + ) + elif k == "$lt": + q = q.filter(getattr(self.obj_ref, attr) < value["$lt"]) + elif k == "$gt": + q = q.filter(getattr(self.obj_ref, attr) > value["$gt"]) + else: + raise ValueError(f"Unsupported key {k} -> {value}") + else: + q = q.filter(getattr(self.obj_ref, attr) == value) + + data = q.all() + return data def find_one(self, query_dict: dict, *args, **kwargs): @@ -115,11 +135,28 @@ def find_one(self, query_dict: dict, *args, **kwargs): if len(query_dict) == 0: data = session.query(self.obj_ref).first() return data - elif len(query_dict) == 1: - query_key, query_value = list(query_dict.items())[0] - data = ( - session.query(self.obj_ref) - .filter(getattr(self.obj_ref, query_key) == query_value) - .first() - ) + else: + q = session.query(self.obj_ref) + for attr, value in query_dict.items(): + if isinstance(value, dict): + for k, v in value.items(): + if k == "$lte": + q = q.filter( + getattr(self.obj_ref, attr) <= value["$lte"] + ) + elif k == "$gte": + q = q.filter( + getattr(self.obj_ref, attr) >= value["$gte"] + ) + elif k == "$lt": + q = q.filter(getattr(self.obj_ref, attr) < value["$lt"]) + elif k == "$gt": + q = q.filter(getattr(self.obj_ref, attr) > value["$gt"]) + else: + raise ValueError(f"Unsupported key {k} -> {value}") + else: + q = q.filter(getattr(self.obj_ref, attr) == value) + + data = q.first() + return data diff --git a/CveXplore/database/connection/sqlbase/sql_client_base.py b/CveXplore/database/connection/sqlbase/sql_client_base.py index 685536c5..c57fde6d 100644 --- a/CveXplore/database/connection/sqlbase/sql_client_base.py +++ b/CveXplore/database/connection/sqlbase/sql_client_base.py @@ -40,3 +40,7 @@ def find(self, *args, **kwargs): @abstractmethod def find_one(self, *args, **kwargs): raise NotImplementedError + + @abstractmethod + def list_collection_names(self): + raise NotImplementedError diff --git a/CveXplore/database_models/models.py b/CveXplore/database_models/models.py index 32cdc5ea..d709e5e7 100644 --- a/CveXplore/database_models/models.py +++ b/CveXplore/database_models/models.py @@ -58,14 +58,14 @@ class Schema(CveXploreModel): """ __tablename__ = "schema" - id = Column(Integer, primary_key=True, doc="ID of the record") + _id = Column(Integer, primary_key=True, doc="ID of the record") rebuild_needed = Column( Boolean, default=False, doc="Flag to indicate if a rebuild is needed" ) version = Column(Float, doc="Version of the database schema") def __repr__(self): - return f"<< Schema: {self.id} >>" + return f"<< Schema: {self._id} >>" class Capec(CveXploreModel): @@ -104,7 +104,7 @@ class Cpe(CveXploreModel): BigInteger, primary_key=True, unique=True, index=True, doc="ID of the record" ) id = Column(String(50), unique=True, index=True, doc="ID of the CPE") - cpeName = Column(String(50), index=True, doc="Name of the CPE") + cpeName = Column(String(256), index=True, doc="Name of the CPE") cpeNameId = Column(String(50), doc="Unique ID number of the CPE") created = Column(DateTime, doc="Creation time of the CPE") deprecated = Column( @@ -116,13 +116,13 @@ class Cpe(CveXploreModel): deprecatedBy = Column(String(50), doc="Organization that deprecated the CPE") lastModified = Column(DateTime, index=True, doc="Last modified time of the CPE") padded_version = Column( - String(50), index=True, doc="Left zero padded version of the CPE" + String(128), index=True, doc="Left zero padded version of the CPE" ) - product = Column(String(50), index=True, doc="Product of the CPE") - stem = Column(String(50), index=True, doc="Stem of the CPE") - title = Column(String(150), index=True, doc="Title of the CPE") - vendor = Column(String(50), index=True, doc="Vendor of the CPE") - version = Column(String(50), doc="Version of the CPE") + product = Column(String(128), index=True, doc="Product of the CPE") + stem = Column(String(256), index=True, doc="Stem of the CPE") + title = Column(String(256), index=True, doc="Title of the CPE") + vendor = Column(String(256), index=True, doc="Vendor of the CPE") + version = Column(String(128), doc="Version of the CPE") def __repr__(self): return f"<< Cpe: {self.id} >>" diff --git a/CveXplore/errors/database.py b/CveXplore/errors/database.py index 081e8623..e6dc0c68 100644 --- a/CveXplore/errors/database.py +++ b/CveXplore/errors/database.py @@ -18,5 +18,9 @@ class UpdateSourceNotFound(DatabaseException): pass +class PopulateSourceNotFound(DatabaseException): + pass + + class DatabaseSchemaVersionError(DatabaseException): pass diff --git a/CveXplore/main.py b/CveXplore/main.py index 1f907bef..4e52c857 100644 --- a/CveXplore/main.py +++ b/CveXplore/main.py @@ -99,7 +99,7 @@ def __init__(self, **kwargs): for each in kwargs: setattr(self.config, each.upper(), kwargs[each]) - self._datasource_type = self.config.DATASOURCE_TYPE + self._datasource_type = self.config.DATASOURCE_TYPE.lower() self._datasource_connection_details = self.config.DATASOURCE_CONNECTION_DETAILS @@ -112,9 +112,10 @@ def __init__(self, **kwargs): f"Using {self.datasource_type} as datasource, connection details: {self.datasource_connection_details}" ) - if self.datasource_type not in supported_datasources: + if self.datasource_type.upper() not in supported_datasources: raise UnsupportedDatasourceException( - f"Unsupported datasource selected: '{self.datasource_type}'; currently supported: {supported_datasources}" + f"Unsupported datasource selected: '{self.datasource_type}'; " + f"currently supported: {[x.lower() for x in supported_datasources]}" ) if self.datasource_type == "api" and self.datasource_connection_details is None: