Merge branch 'shadowserver-dynamic-config' into develop

CHANGELOG.md

@@ -124,11 +124,13 @@
   - added support for `Subject NOT LIKE` queries,
   - added support for multiple values in ticket subject queries.
 - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante).
+- `intelmq.bots.collectors.shadowserver.collector_reports_api`:
+  - The 'json' option is no longer supported as the 'csv' option provides better performance.
 
 #### Parsers
 - `intelmq.bots.parsers.shadowserver._config`:
   - Reset detected `feedname` at shutdown to re-detect the feedname on reloads (PR#2361 by @elsif2, fixes #2360).
-- `intelmq.bots.parsers.shadowserver._config`:
+  - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases. 
   - Added 'IPv6-Vulnerable-Exchange' alias and 'Accessible-WS-Discovery-Service' report. (PR#2338)
   - Removed unused `p0f_genre` and `p0f_detail` from the 'DNS-Open-Resolvers' report. (PR#2338)
   - Added 'Accessible-SIP' report. (PR#2348)

intelmq/bots/collectors/shadowserver/collector_reports_api.py

@@ -34,15 +34,13 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin):
             A list of strings or a comma-separated list of the mailing lists you want to process.
         types (list):
             A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined').
-        file_format (str): File format to download ('csv' or 'json').  The default is 'json' for compatibility. Using 'csv' is recommended for best performance.
     """
 
     country = None
     api_key = None
     secret = None
     types = None
     reports = None
-    file_format = None
     rate_limit: int = 86400
     redis_cache_db: int = 12
     redis_cache_host: str = "127.0.0.1"  # TODO: type could be ipadress
@@ -66,15 +64,15 @@ def init(self):
             self.logger.warn("Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0.")
             self._report_list.append(self.country)
 
-        if self.file_format is not None:
-            if not (self.file_format == 'csv' or self.file_format == 'json'):
-                raise ValueError('Invalid file_format')
-        else:
-            self.file_format = 'json'
-            self.logger.info("For best performance, set 'file_format' to 'csv' and use intelmq.bots.parsers.shadowserver.parser.")
-
         self.preamble = f'{{ "apikey": "{self.api_key}" '
 
+    def check(parameters: dict):
+        for key in parameters:
+            if key == 'file_format':
+                return [["error", "The file_format parameter is no longer supported. All reports are CSV."]]
+            elif key == 'country':
+                return [["warning", "Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0."]]
+
     def _headers(self, data):
         return {'HMAC2': hmac.new(self.secret.encode(), data.encode('utf-8'), digestmod=hashlib.sha256).hexdigest()}
 
@@ -123,11 +121,7 @@ def _report_download(self, reportid: str):
         data = self.preamble
         data += f',"id": "{reportid}"}}'
         self.logger.debug('Downloading report with data: %s.', data)
-
-        if (self.file_format == 'json'):
-            response = self.http_session().post(APIROOT + 'reports/download', data=data, headers=self._headers(data))
-        else:
-            response = self.http_session().get(DLROOT + reportid)
+        response = self.http_session().get(DLROOT + reportid)
         response.raise_for_status()
 
         return response.text
@@ -144,7 +138,7 @@ def process(self):
 
         for item in reportslist:
             filename = item['file']
-            filename_fixed = FILENAME_PATTERN.sub('.' + self.file_format, filename, count=1)
+            filename_fixed = FILENAME_PATTERN.sub('.csv', filename, count=1)
             if self.cache_get(filename):
                 self.logger.debug('Processed file %r (fixed: %r) already.', filename, filename_fixed)
                 continue