Further refactoring of mongodb constants

- undoing some too-eager changes from earlier

dev_utils/mongo_hooks.py

@@ -22,10 +22,10 @@
 def normalize_file(file_dict, task_id):
     """Pull out the detonation-independent attributes of the given file and
     return an UpdateOne object usable by bulk_write to upsert a
-    document into the MONGODB_COLLECTION_FILES collection with its _id set to the FILE_KEY of
+    document into the FILES_COLL collection with its _id set to the FILE_KEY of
     the file. The given file_dict is updated in place to remove those
     attributes and add a FILE_REF_KEY key containing the FILE_KEY that can be
-    used as a lookup in the MONGODB_COLLECTION_FILES collection.
+    used as a lookup in the FILES_COLL collection.
     If the file has already been "normalized," then it is not modified and
     None is returned.
     """
@@ -86,7 +86,7 @@ def normalize_files(report):
 
 @mongo_hook(mongo_find, ANALYSIS_COLL)
 def denormalize_files_from_reports(reports):
-    """Pull the file info from the MONGODB_COLLECTION_FILES collection in to associated parts of
+    """Pull the file info from the FILES_COLL collection in to associated parts of
     the reports.
     """
     # Make sure we have a list whose objects we can modify in place instead of a mongo
@@ -127,7 +127,7 @@ def denormalize_files_from_reports(reports):
 
 @mongo_hook(mongo_find_one, ANALYSIS_COLL)
 def denormalize_files(report):
-    """Pull the file info from the MONGODB_COLLECTION_FILES collection in to associated parts of
+    """Pull the file info from the FILES_COLL collection in to associated parts of
     the report.
     """
     denormalize_files_from_reports([report])
@@ -147,7 +147,7 @@ def remove_task_references_from_files(task_ids):
 
 
 def delete_unused_file_docs():
-    """Delete entries in the MONGODB_COLLECTION_FILES collection that are no longer
+    """Delete entries in the FILES_COLL collection that are no longer
     referenced by any analysis tasks. This should typically be invoked
     via utils/cleaners.py in a cron job.
     """

docs/book/src/development/code_style.rst

@@ -160,7 +160,7 @@ When catching an exception and accessing its handle, use ``as e``::
     except Exception, something:
         bar()
 
-It's a good practice to use "e" instead of "e.message" or "e.msg".
+It's a good practice to use "e" instead of "e.message".
 
 Documentation
 =============

lib/cuckoo/common/cape_utils.py

@@ -10,7 +10,6 @@
 from lib.cuckoo.common.constants import CUCKOO_ROOT
 from lib.cuckoo.common.objects import File
 from lib.cuckoo.common.path_utils import path_read_file
-from modules.reporting.mongodb_constants import ANALYSIS_COLL
 
 try:
     import pydeep
@@ -44,6 +43,7 @@
 
 if repconf.mongodb.enabled:
     from dev_utils.mongodb import mongo_find_one
+    from modules.reporting.mongodb_constants import ANALYSIS_COLL
 
 if repconf.elasticsearchdb.enabled:
     from dev_utils.elasticsearchdb import elastic_handler, get_analysis_index

lib/cuckoo/common/compare.py

@@ -6,12 +6,12 @@
 from typing import Dict
 
 from lib.cuckoo.common.config import Config
-from modules.reporting.mongodb_constants import CALLS_COLL, ID_KEY
 
 repconf = Config("reporting")
 
 if repconf.mongodb.enabled:
     from dev_utils.mongodb import mongo_find_one
+    from modules.reporting.mongodb_constants import CALLS_COLL, ID_KEY
 
 if repconf.elasticsearchdb.enabled:
     from dev_utils.elasticsearchdb import get_analysis_index, get_calls_index, get_query_by_info_id

lib/cuckoo/common/web_utils.py

@@ -50,7 +50,6 @@
     Task,
 )
 from lib.cuckoo.core.rooter import _load_socks5_operational, vpns
-from modules.reporting.mongodb_constants import ANALYSIS_COLL, FILE_REF_KEY, FILES_COLL, NORMALIZED_FILE_FIELDS
 
 _current_dir = os.path.abspath(os.path.dirname(__file__))
 CUCKOO_ROOT = os.path.normpath(os.path.join(_current_dir, "..", "..", ".."))
@@ -97,6 +96,7 @@
 
 if repconf.mongodb.enabled:
     from dev_utils.mongodb import mongo_aggregate, mongo_find, mongo_find_one
+    from modules.reporting.mongodb_constants import ANALYSIS_COLL, FILE_REF_KEY, FILES_COLL, NORMALIZED_FILE_FIELDS
 
 es_as_db = False
 essearch = False

lib/cuckoo/core/database.py

@@ -34,7 +34,6 @@
 from lib.cuckoo.common.objects import PCAP, URL, File, Static
 from lib.cuckoo.common.path_utils import path_delete, path_exists
 from lib.cuckoo.common.utils import bytes2str, create_folder, get_options
-from modules.reporting.mongodb_constants import ANALYSIS_COLL
 
 try:
     from sqlalchemy import (
@@ -125,6 +124,8 @@
 
 if repconf.mongodb.enabled:
     from dev_utils.mongodb import mongo_find
+    from modules.reporting.mongodb_constants import ANALYSIS_COLL
+
 if repconf.elasticsearchdb.enabled:
     from dev_utils.elasticsearchdb import elastic_handler, get_analysis_index
 

modules/reporting/mongodb.py

@@ -37,7 +37,6 @@
 MONGOSIZELIMIT = 0x1000000
 MEGABYTE = 0x100000
 
-
 log = logging.getLogger(__name__)
 
 

modules/reporting/report_doc.py

@@ -11,7 +11,6 @@
 from lib.cuckoo.common.config import Config
 from lib.cuckoo.common.objects import File
 from lib.cuckoo.common.path_utils import path_exists
-from modules.reporting.mongodb_constants import CALLS_COLL
 
 log = logging.getLogger(__name__)
 repconf = Config("reporting")
@@ -21,6 +20,7 @@
 
 if repconf.mongodb.enabled:
     from dev_utils.mongodb import mongo_insert_one
+    from modules.reporting.mongodb_constants import CALLS_COLL
 elif repconf.elasticsearchdb.enabled:
     from elasticsearch.helpers import parallel_bulk
 

utils/process.py

@@ -49,7 +49,6 @@
 )
 from lib.cuckoo.core.plugins import RunProcessing, RunReporting, RunSignatures
 from lib.cuckoo.core.startup import ConsoleHandler, check_linux_dist, init_modules
-from modules.reporting.mongodb_constants import ANALYSIS_COLL, CALLS_COLL, ID_KEY
 
 cfg = Config()
 logconf = Config("logging")
@@ -60,6 +59,7 @@
     from bson.objectid import ObjectId
 
     from dev_utils.mongodb import mongo_find, mongo_find_one
+    from modules.reporting.mongodb_constants import ANALYSIS_COLL, CALLS_COLL, ID_KEY
 
 if repconf.elasticsearchdb.enabled and not repconf.elasticsearchdb.searchonly:
     from elasticsearch.exceptions import RequestError as ESRequestError

web/analysis/views.py

@@ -35,7 +35,6 @@
 from lib.cuckoo.common.utils import delete_folder, yara_detected
 from lib.cuckoo.common.web_utils import category_all_files, my_rate_minutes, my_rate_seconds, perform_search, rateblock, statistics
 from lib.cuckoo.core.database import TASK_PENDING, Database, Task
-from modules.reporting.mongodb_constants import ANALYSIS_COLL, CALLS_COLL
 from modules.reporting.report_doc import CHUNK_CALL_SIZE
 
 try:
@@ -142,6 +141,7 @@
     from bson.objectid import ObjectId
 
     from dev_utils.mongodb import mongo_aggregate, mongo_delete_data, mongo_find, mongo_find_one, mongo_update_one
+    from modules.reporting.mongodb_constants import ANALYSIS_COLL, CALLS_COLL
 
 es_as_db = False
 essearch = False

web/apiv2/views.py

@@ -55,7 +55,6 @@
 )
 from lib.cuckoo.core.database import TASK_RECOVERED, TASK_RUNNING, Database, Task, _Database
 from lib.cuckoo.core.rooter import _load_socks5_operational, vpns
-from modules.reporting.mongodb_constants import ANALYSIS_COLL
 
 try:
     import psutil
@@ -99,6 +98,7 @@
 
 if repconf.mongodb.enabled:
     from dev_utils.mongodb import mongo_delete_data, mongo_find, mongo_find_one, mongo_find_one_and_update
+    from modules.reporting.mongodb_constants import ANALYSIS_COLL
 
 es_as_db = False
 if repconf.elasticsearchdb.enabled and not repconf.elasticsearchdb.searchonly: