diff --git a/CHANGELOG.md b/CHANGELOG.md index daceddd..93bfb7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 3.1.0 + +- Added field to make batch size configurable for IMAP inputs + ## 3.0.2 - Lowercase keys and values in policy_published and records diff --git a/README.md b/README.md index 82dde58..e706a52 100644 --- a/README.md +++ b/README.md @@ -332,6 +332,7 @@ This add-on is maintained by Jorrit Folmer. These people and organisations have - Christopher G Andrews (ChristopherGAndrews) - John (john-9c54a80b) - Mike Kolk +- Samuel Haper (sharperer) - Steve Myers (stmyers) - Steven Hilton (malvidin) - [SMTware](https://www.smtware.com/en/services) diff --git a/README/inputs.conf.spec b/README/inputs.conf.spec index d391682..a6cddec 100644 --- a/README/inputs.conf.spec +++ b/README/inputs.conf.spec @@ -1,12 +1,3 @@ -[dmarc_imap://] -global_account = Use the account configured in the setup tab -imap_server = Connect to the specified IMAP server with TLS (port 993) -resolve_ip = Resolve the source_ip field in the DMARC aggregate reports. -validate_xml = Validate the aggregate reports against the DMARC XSD. Results are included in the field vendor_rua_xsd_validation. -validate_dkim = (Beta) Validate the DKIM signatures in the mail headers. Results are currently only available in DEBUG log. -imap_mailbox = Select the IMAP mailbox to poll. Default: INBOX -output_format = - [dmarc_pop3://] global_account = pop3_server = Connect to the specified POP3 server with TLS (port 995) @@ -20,4 +11,14 @@ dmarc_directory = Directory containing DMARC aggregate reports quiet_time = Ignore files that have a modification time of less than n seconds ago. resolve_ip = Resolve the source_ip field in the DMARC XML aggregate report validate_xml = Validate the aggregate report XML against the DMARC XSD. Results are included in the field vendor_rua_xsd_validation. -output_format = \ No newline at end of file +output_format = + +[dmarc_imap://] +global_account = Use the account configured in the setup tab +imap_server = Connect to the specified IMAP server with TLS (port 993) +resolve_ip = Resolve the source_ip field in the DMARC aggregate reports. +validate_xml = Validate the aggregate reports against the DMARC XSD. Results are included in the field vendor_rua_xsd_validation. +validate_dkim = (Beta) Validate the DKIM signatures in the mail headers. Results are currently only available in DEBUG log. +imap_mailbox = Select the IMAP mailbox to poll. Default: INBOX +output_format = +batch_size = Max number of messages to fetch per batch to prevent connection timeouts and resets \ No newline at end of file diff --git a/app.manifest b/app.manifest index 3588a6f..aa2eae8 100644 --- a/app.manifest +++ b/app.manifest @@ -5,7 +5,7 @@ "id": { "group": null, "name": "TA-dmarc", - "version": "3.0.2" + "version": "3.0.3" }, "author": [ { @@ -50,4 +50,4 @@ "Enterprise": "*" } } -} +} \ No newline at end of file diff --git a/appserver/static/js/build/globalConfig.json b/appserver/static/js/build/globalConfig.json index 53a15fd..2eedd3e 100644 --- a/appserver/static/js/build/globalConfig.json +++ b/appserver/static/js/build/globalConfig.json @@ -1,128 +1,8 @@ { "pages": { - "configuration": { - "title": "Configuration", - "description": "Set up your add-on", - "tabs": [ - { - "title": "Account", - "entity": [ - { - "label": "Account name", - "type": "text", - "required": true, - "validators": [ - { - "maxLength": 50, - "type": "string", - "errorMsg": "Length of Account name should be between 1 and 50", - "minLength": 1 - }, - { - "type": "regex", - "errorMsg": "Account name must start with a letter and followed by alphabetic letters, digits or underscores.", - "pattern": "^[a-zA-Z]\\w*$" - } - ], - "help": "Enter a unique name for this account.", - "field": "name" - }, - { - "label": "Username", - "type": "text", - "required": true, - "validators": [ - { - "maxLength": 200, - "type": "string", - "errorMsg": "Length of username should be between 1 and 200", - "minLength": 1 - } - ], - "help": "Enter the username for this account.", - "options": { - "placeholder": "Enter the username here" - }, - "field": "username" - }, - { - "label": "Password", - "type": "text", - "required": true, - "encrypted": true, - "validators": [ - { - "maxLength": 8192, - "type": "string", - "errorMsg": "Length of password should be between 1 and 8192", - "minLength": 1 - } - ], - "help": "Enter the password for this account.", - "field": "password" - } - ], - "name": "account", - "table": { - "actions": [ - "edit", - "delete", - "clone" - ], - "header": [ - { - "label": "Account name", - "field": "name" - }, - { - "label": "Username", - "field": "username" - } - ] - } - }, - { - "title": "Logging", - "entity": [ - { - "label": "Log level", - "options": { - "disableSearch": true, - "autoCompleteFields": [ - { - "label": "DEBUG", - "value": "DEBUG" - }, - { - "label": "INFO", - "value": "INFO" - }, - { - "label": "WARNING", - "value": "WARNING" - }, - { - "label": "ERROR", - "value": "ERROR" - }, - { - "label": "CRITICAL", - "value": "CRITICAL" - } - ] - }, - "type": "singleSelect", - "defaultValue": "INFO", - "field": "loglevel" - } - ], - "name": "logging" - } - ] - }, "inputs": { - "title": "Inputs", "description": "Manage your data inputs", + "title": "Inputs", "table": { "actions": [ "edit", @@ -130,6 +10,24 @@ "delete", "clone" ], + "header": [ + { + "label": "Name", + "field": "name" + }, + { + "label": "Interval", + "field": "interval" + }, + { + "label": "Index", + "field": "index" + }, + { + "label": "Status", + "field": "disabled" + } + ], "moreInfo": [ { "label": "Name", @@ -152,8 +50,8 @@ "field": "global_account" }, { - "label": "IMAP server", - "field": "imap_server" + "label": "POP3 server", + "field": "pop3_server" }, { "label": "Resolve IP", @@ -167,18 +65,10 @@ "label": "Validate DKIM", "field": "validate_dkim" }, - { - "label": "IMAP mailbox", - "field": "imap_mailbox" - }, { "label": "Output format", "field": "output_format" }, - { - "label": "POP3 server", - "field": "pop3_server" - }, { "label": "Directory", "field": "dmarc_directory" @@ -186,39 +76,35 @@ { "label": "Quiet time", "field": "quiet_time" - } - ], - "header": [ - { - "label": "Name", - "field": "name" }, { - "label": "Interval", - "field": "interval" + "label": "IMAP server", + "field": "imap_server" }, { - "label": "Index", - "field": "index" + "label": "IMAP mailbox", + "field": "imap_mailbox" }, { - "label": "Status", - "field": "disabled" + "label": "Batch size", + "field": "batch_size" } ] }, "services": [ { - "title": "DMARC imap", + "title": "DMARC pop3", "entity": [ { "label": "Name", "type": "text", + "field": "name", + "help": "Enter a unique name for the data input", "required": true, "validators": [ { - "type": "regex", "errorMsg": "Input Name must start with a letter and followed by alphabetic letters, digits or underscores.", + "type": "regex", "pattern": "^[a-zA-Z]\\w*$" }, { @@ -227,28 +113,27 @@ "errorMsg": "Length of input name should be between 1 and 100", "minLength": 1 } - ], - "help": "Enter a unique name for the data input", - "field": "name" + ] }, { "label": "Interval", "type": "text", + "field": "interval", + "help": "Time interval of input in seconds.", "required": true, "validators": [ { - "type": "regex", "errorMsg": "Interval must be an integer.", + "type": "regex", "pattern": "^\\-[1-9]\\d*$|^\\d*$" } - ], - "help": "Time interval of input in seconds.", - "field": "interval" + ] }, { "label": "Index", "type": "singleSelect", - "required": true, + "field": "index", + "defaultValue": "default", "validators": [ { "maxLength": 80, @@ -258,27 +143,26 @@ } ], "options": { - "endpointUrl": "data/indexes", "createSearchChoice": true, + "endpointUrl": "data/indexes", "blackList": "^_.*$" }, - "defaultValue": "default", - "field": "index" + "required": true }, { "label": "Global Account", + "help": "", + "field": "global_account", "type": "singleSelect", - "required": true, - "help": "Use the account configured in the setup tab", "options": { "referenceName": "account" }, - "field": "global_account" + "required": true }, { - "label": "IMAP server", - "type": "text", - "required": true, + "label": "POP3 server", + "help": "Connect to the specified POP3 server with TLS (port 995)", + "field": "pop3_server", "validators": [ { "maxLength": 8192, @@ -287,54 +171,39 @@ "minLength": 0 } ], - "help": "Connect to the specified IMAP server with TLS (port 993)", - "field": "imap_server" + "type": "text", + "required": false }, { "label": "Resolve IP", - "type": "checkbox", - "required": false, "help": "Resolve the source_ip field in the DMARC aggregate reports.", + "field": "resolve_ip", "defaultValue": true, - "field": "resolve_ip" + "type": "checkbox", + "required": false }, { "label": "Validate XML", - "type": "checkbox", - "required": false, "help": "Validate the aggregate reports against the DMARC XSD. Results are included in the field vendor_rua_xsd_validation.", + "field": "validate_xml", "defaultValue": true, - "field": "validate_xml" + "type": "checkbox", + "required": false }, { "label": "Validate DKIM", "help": "(Beta) Validate the DKIM signatures in the mail headers. Results are currently only available in DEBUG log.", - "type": "checkbox", + "field": "validate_dkim", "required": false, - "field": "validate_dkim" - }, - { - "label": "IMAP mailbox", - "type": "text", - "required": true, - "validators": [ - { - "maxLength": 8192, - "type": "string", - "errorMsg": "Max length of text input is 8192", - "minLength": 0 - } - ], - "help": "Select the IMAP mailbox to poll. Default: INBOX", - "defaultValue": "INBOX", - "field": "imap_mailbox" + "type": "checkbox" }, { "label": "Output format", + "help": "", + "field": "output_format", + "defaultValue": "json", "type": "singleSelect", - "required": true, "options": { - "disableSearch": true, "autoCompleteFields": [ { "label": "JSON", @@ -344,26 +213,27 @@ "label": "KV (legacy)", "value": "kv" } - ] + ], + "disableSearch": true }, - "help": "", - "defaultValue": "json", - "field": "output_format" + "required": false } ], - "name": "dmarc_imap" + "name": "dmarc_pop3" }, { - "title": "DMARC pop3", + "title": "DMARC directory", "entity": [ { "label": "Name", "type": "text", + "field": "name", + "help": "Enter a unique name for the data input", "required": true, "validators": [ { - "type": "regex", "errorMsg": "Input Name must start with a letter and followed by alphabetic letters, digits or underscores.", + "type": "regex", "pattern": "^[a-zA-Z]\\w*$" }, { @@ -372,28 +242,27 @@ "errorMsg": "Length of input name should be between 1 and 100", "minLength": 1 } - ], - "help": "Enter a unique name for the data input", - "field": "name" + ] }, { "label": "Interval", "type": "text", + "field": "interval", + "help": "Time interval of input in seconds.", "required": true, "validators": [ { - "type": "regex", "errorMsg": "Interval must be an integer.", + "type": "regex", "pattern": "^\\-[1-9]\\d*$|^\\d*$" } - ], - "help": "Time interval of input in seconds.", - "field": "interval" + ] }, { "label": "Index", "type": "singleSelect", - "required": true, + "field": "index", + "defaultValue": "default", "validators": [ { "maxLength": 80, @@ -403,27 +272,32 @@ } ], "options": { - "endpointUrl": "data/indexes", "createSearchChoice": true, + "endpointUrl": "data/indexes", "blackList": "^_.*$" }, - "defaultValue": "default", - "field": "index" + "required": true }, { - "label": "Global Account", - "type": "singleSelect", - "required": true, - "help": "", - "options": { - "referenceName": "account" - }, - "field": "global_account" + "label": "Directory", + "help": "Directory containing DMARC aggregate reports", + "field": "dmarc_directory", + "validators": [ + { + "maxLength": 8192, + "type": "string", + "errorMsg": "Max length of text input is 8192", + "minLength": 0 + } + ], + "type": "text", + "required": true }, { - "label": "POP3 server", - "type": "text", - "required": false, + "label": "Quiet time", + "help": "Ignore files that have a modification time of less than n seconds ago.", + "field": "quiet_time", + "defaultValue": "10", "validators": [ { "maxLength": 8192, @@ -432,38 +306,32 @@ "minLength": 0 } ], - "help": "Connect to the specified POP3 server with TLS (port 995)", - "field": "pop3_server" + "type": "text", + "required": true }, { "label": "Resolve IP", - "type": "checkbox", - "required": false, - "help": "Resolve the source_ip field in the DMARC aggregate reports.", + "help": "Resolve the source_ip field in the DMARC XML aggregate report", + "field": "resolve_ip", "defaultValue": true, - "field": "resolve_ip" + "type": "checkbox", + "required": false }, { "label": "Validate XML", - "type": "checkbox", - "required": false, - "help": "Validate the aggregate reports against the DMARC XSD. Results are included in the field vendor_rua_xsd_validation.", + "help": "Validate the aggregate report XML against the DMARC XSD. Results are included in the field vendor_rua_xsd_validation.", + "field": "validate_xml", "defaultValue": true, - "field": "validate_xml" - }, - { - "label": "Validate DKIM", - "help": "(Beta) Validate the DKIM signatures in the mail headers. Results are currently only available in DEBUG log.", "type": "checkbox", - "required": false, - "field": "validate_dkim" + "required": false }, { "label": "Output format", + "help": "", + "field": "output_format", + "defaultValue": "json", "type": "singleSelect", - "required": false, "options": { - "disableSearch": true, "autoCompleteFields": [ { "label": "JSON", @@ -473,26 +341,27 @@ "label": "KV (legacy)", "value": "kv" } - ] + ], + "disableSearch": true }, - "help": "", - "defaultValue": "json", - "field": "output_format" + "required": true } ], - "name": "dmarc_pop3" + "name": "dmarc_directory" }, { - "title": "DMARC directory", + "title": "DMARC imap", "entity": [ { "label": "Name", "type": "text", + "field": "name", + "help": "Enter a unique name for the data input", "required": true, "validators": [ { - "type": "regex", "errorMsg": "Input Name must start with a letter and followed by alphabetic letters, digits or underscores.", + "type": "regex", "pattern": "^[a-zA-Z]\\w*$" }, { @@ -501,28 +370,27 @@ "errorMsg": "Length of input name should be between 1 and 100", "minLength": 1 } - ], - "help": "Enter a unique name for the data input", - "field": "name" + ] }, { "label": "Interval", "type": "text", + "field": "interval", + "help": "Time interval of input in seconds.", "required": true, "validators": [ { - "type": "regex", "errorMsg": "Interval must be an integer.", + "type": "regex", "pattern": "^\\-[1-9]\\d*$|^\\d*$" } - ], - "help": "Time interval of input in seconds.", - "field": "interval" + ] }, { "label": "Index", "type": "singleSelect", - "required": true, + "field": "index", + "defaultValue": "default", "validators": [ { "maxLength": 80, @@ -532,32 +400,26 @@ } ], "options": { - "endpointUrl": "data/indexes", "createSearchChoice": true, + "endpointUrl": "data/indexes", "blackList": "^_.*$" }, - "defaultValue": "default", - "field": "index" + "required": true }, { - "label": "Directory", - "type": "text", - "required": true, - "validators": [ - { - "maxLength": 8192, - "type": "string", - "errorMsg": "Max length of text input is 8192", - "minLength": 0 - } - ], - "help": "Directory containing DMARC aggregate reports", - "field": "dmarc_directory" + "label": "Global Account", + "help": "Use the account configured in the setup tab", + "field": "global_account", + "type": "singleSelect", + "options": { + "referenceName": "account" + }, + "required": true }, { - "label": "Quiet time", - "type": "text", - "required": true, + "label": "IMAP server", + "help": "Connect to the specified IMAP server with TLS (port 993)", + "field": "imap_server", "validators": [ { "maxLength": 8192, @@ -566,32 +428,55 @@ "minLength": 0 } ], - "help": "Ignore files that have a modification time of less than n seconds ago.", - "defaultValue": "10", - "field": "quiet_time" + "type": "text", + "required": true }, { "label": "Resolve IP", - "type": "checkbox", - "required": false, - "help": "Resolve the source_ip field in the DMARC XML aggregate report", + "help": "Resolve the source_ip field in the DMARC aggregate reports.", + "field": "resolve_ip", "defaultValue": true, - "field": "resolve_ip" + "type": "checkbox", + "required": false }, { "label": "Validate XML", + "help": "Validate the aggregate reports against the DMARC XSD. Results are included in the field vendor_rua_xsd_validation.", + "field": "validate_xml", + "defaultValue": true, "type": "checkbox", + "required": false + }, + { + "label": "Validate DKIM", + "help": "(Beta) Validate the DKIM signatures in the mail headers. Results are currently only available in DEBUG log.", + "field": "validate_dkim", "required": false, - "help": "Validate the aggregate report XML against the DMARC XSD. Results are included in the field vendor_rua_xsd_validation.", - "defaultValue": true, - "field": "validate_xml" + "type": "checkbox" + }, + { + "label": "IMAP mailbox", + "help": "Select the IMAP mailbox to poll. Default: INBOX", + "field": "imap_mailbox", + "defaultValue": "INBOX", + "validators": [ + { + "maxLength": 8192, + "type": "string", + "errorMsg": "Max length of text input is 8192", + "minLength": 0 + } + ], + "type": "text", + "required": true }, { "label": "Output format", + "help": "", + "field": "output_format", + "defaultValue": "json", "type": "singleSelect", - "required": true, "options": { - "disableSearch": true, "autoCompleteFields": [ { "label": "JSON", @@ -601,23 +486,158 @@ "label": "KV (legacy)", "value": "kv" } - ] + ], + "disableSearch": true }, - "help": "", - "defaultValue": "json", - "field": "output_format" + "required": true + }, + { + "label": "Batch size", + "help": "Max number of messages to fetch per batch to prevent connection timeouts and resets", + "field": "batch_size", + "defaultValue": "100", + "validators": [ + { + "maxLength": 8192, + "type": "string", + "errorMsg": "Max length of text input is 8192", + "minLength": 0 + } + ], + "type": "text", + "required": false } ], - "name": "dmarc_directory" + "name": "dmarc_imap" } ] + }, + "configuration": { + "tabs": [ + { + "title": "Account", + "table": { + "actions": [ + "edit", + "delete", + "clone" + ], + "header": [ + { + "label": "Account name", + "field": "name" + }, + { + "label": "Username", + "field": "username" + } + ] + }, + "name": "account", + "entity": [ + { + "label": "Account name", + "type": "text", + "field": "name", + "help": "Enter a unique name for this account.", + "required": true, + "validators": [ + { + "maxLength": 50, + "type": "string", + "errorMsg": "Length of Account name should be between 1 and 50", + "minLength": 1 + }, + { + "errorMsg": "Account name must start with a letter and followed by alphabetic letters, digits or underscores.", + "type": "regex", + "pattern": "^[a-zA-Z]\\w*$" + } + ] + }, + { + "label": "Username", + "type": "text", + "field": "username", + "help": "Enter the username for this account.", + "options": { + "placeholder": "Enter the username here" + }, + "validators": [ + { + "maxLength": 200, + "type": "string", + "errorMsg": "Length of username should be between 1 and 200", + "minLength": 1 + } + ], + "required": true + }, + { + "label": "Password", + "type": "text", + "field": "password", + "encrypted": true, + "help": "Enter the password for this account.", + "validators": [ + { + "maxLength": 8192, + "type": "string", + "errorMsg": "Length of password should be between 1 and 8192", + "minLength": 1 + } + ], + "required": true + } + ] + }, + { + "title": "Logging", + "entity": [ + { + "label": "Log level", + "type": "singleSelect", + "field": "loglevel", + "defaultValue": "INFO", + "options": { + "autoCompleteFields": [ + { + "label": "DEBUG", + "value": "DEBUG" + }, + { + "label": "INFO", + "value": "INFO" + }, + { + "label": "WARNING", + "value": "WARNING" + }, + { + "label": "ERROR", + "value": "ERROR" + }, + { + "label": "CRITICAL", + "value": "CRITICAL" + } + ], + "disableSearch": true + } + } + ], + "name": "logging" + } + ], + "description": "Set up your add-on", + "title": "Configuration" } }, "meta": { + "version": "3.0.0", "apiVersion": "3.0.0", - "name": "TA-dmarc", "displayName": "TA-dmarc add-on for Splunk", - "version": "3.0.0", + "name": "TA-dmarc", "restRoot": "TA_dmarc" } } \ No newline at end of file diff --git a/bin/TA_dmarc_rh_dmarc_imap.py b/bin/TA_dmarc_rh_dmarc_imap.py index 6c42bac..8aedecc 100644 --- a/bin/TA_dmarc_rh_dmarc_imap.py +++ b/bin/TA_dmarc_rh_dmarc_imap.py @@ -88,6 +88,16 @@ default='json', validator=None ), + field.RestField( + 'batch_size', + required=False, + encrypted=False, + default='100', + validator=validator.String( + min_len=0, + max_len=8192, + ) + ), field.RestField( 'disabled', diff --git a/bin/dmarc/imap2dir.py b/bin/dmarc/imap2dir.py index 7419dcd..fc92e87 100644 --- a/bin/dmarc/imap2dir.py +++ b/bin/dmarc/imap2dir.py @@ -34,7 +34,7 @@ class Imap2Dir: """ - def __init__(self, helper, opt_imap_server, tmp_dir, opt_use_ssl, opt_global_account, opt_imap_mailbox, opt_validate_dkim): + def __init__(self, helper, opt_imap_server, tmp_dir, opt_use_ssl, opt_global_account, opt_imap_mailbox, opt_validate_dkim, opt_batch_size): # Instance variables: self.helper = helper self.opt_imap_server = opt_imap_server @@ -42,6 +42,7 @@ def __init__(self, helper, opt_imap_server, tmp_dir, opt_use_ssl, opt_global_acc self.opt_validate_dkim = opt_validate_dkim self.opt_use_ssl = opt_use_ssl self.opt_global_account = opt_global_account + self.opt_batch_size = 100 if opt_batch_size == None else opt_batch_size self.tmp_dir = tmp_dir self.server = None @@ -85,11 +86,11 @@ def get_dmarc_messages(self): def get_dmarc_message_bodies(self, messages): """ Return the full message bodies from the list of message uids """ - fetch_size=100 + fetch_size=self.opt_batch_size response = dict() messageslist = list(messages) for x in range(0,len(messageslist),fetch_size): - self.helper.log_debug('get_dmarc_message_bodies: getting messages %s to %s' % (str(x),str(min(x+fetch_size,len(messageslist))))) + self.helper.log_info('get_dmarc_message_bodies: getting messages %s to %s' % (str(x),str(min(x+fetch_size,len(messageslist))))) response.update(self.server.fetch(set(messageslist[x:x+fetch_size]), ['RFC822'])) return response diff --git a/bin/dmarc_imap.py b/bin/dmarc_imap.py index ae4f501..7614932 100644 --- a/bin/dmarc_imap.py +++ b/bin/dmarc_imap.py @@ -74,6 +74,10 @@ def get_scheme(self): description="", required_on_create=True, required_on_edit=False)) + scheme.add_argument(smi.Argument("batch_size", title="Batch size", + description="Max number of messages to fetch per batch to prevent connection timeouts and resets", + required_on_create=False, + required_on_edit=False)) return scheme def get_app_name(self): diff --git a/bin/input_module_dmarc_imap.py b/bin/input_module_dmarc_imap.py index 8aaa787..a147f3b 100644 --- a/bin/input_module_dmarc_imap.py +++ b/bin/input_module_dmarc_imap.py @@ -19,10 +19,11 @@ def validate_input(helper, definition): opt_use_ssl = True opt_global_account = definition.parameters.get('global_account', None) opt_validate_dkim = definition.parameters.get('validate_dkim', None) + opt_batch_size = int(definition.parameters.get('batch_size', None)) try: tmp_dir = create_tmp_dir(helper) - i2d = Imap2Dir(helper, opt_imap_server, tmp_dir, opt_use_ssl, opt_global_account, opt_imap_mailbox, opt_validate_dkim) + i2d = Imap2Dir(helper, opt_imap_server, tmp_dir, opt_use_ssl, opt_global_account, opt_imap_mailbox, opt_validate_dkim, opt_batch_size) i2d.get_imap_connectivity() finally: remove_tmp_dir(helper, tmp_dir) @@ -38,12 +39,13 @@ def collect_events(helper, ew): opt_validate_xml = helper.get_arg('validate_xml') opt_validate_dkim = helper.get_arg('validate_dkim') opt_output_format = helper.get_arg('output_format') + opt_batch_size = int(helper.get_arg('batch_size')) loglevel = helper.get_log_level() helper.set_log_level(loglevel) tmp_dir = create_tmp_dir(helper) - i2d = Imap2Dir(helper, opt_imap_server, tmp_dir, opt_use_ssl, opt_global_account, opt_imap_mailbox, opt_validate_dkim) + i2d = Imap2Dir(helper, opt_imap_server, tmp_dir, opt_use_ssl, opt_global_account, opt_imap_mailbox, opt_validate_dkim, opt_batch_size) try: filelist = i2d.process_incoming() if len(filelist)>0: diff --git a/default/inputs.conf b/default/inputs.conf index f4c6696..c5be98e 100644 --- a/default/inputs.conf +++ b/default/inputs.conf @@ -18,6 +18,7 @@ validate_xml = True validate_dkim = False imap_mailbox = INBOX output_format = json +batch_size = 100 [dmarc_pop3] index = default