diff --git a/includes/extractions/ai/config.yaml b/includes/extractions/ai/config.yaml index 40ce36b..4ad488b 100644 --- a/includes/extractions/ai/config.yaml +++ b/includes/extractions/ai/config.yaml @@ -12,6 +12,7 @@ ai_cryptocurrency_btc_wallet: created_by: DOGESEC version: 1.0.0 prompt_base: 'Extract all Bitcoin Wallet hashes from the text.' + prompt_helper: prompt_conversion: '' test_cases: generic_cryptocurrency_btc_wallet ignore_extractions: @@ -28,6 +29,7 @@ ai_cryptocurrency_btc_transaction: created_by: DOGESEC version: 1.0.0 prompt_base: 'Extract all Cryptocurrency Bitcoin Transaction hashes from the text.' + prompt_helper: prompt_conversion: '' test_cases: generic_cryptocurrency_btc_transaction ignore_extractions: @@ -46,10 +48,9 @@ ai_phone_number: created_by: DOGESEC version: 1.0.0 prompt_base: 'Extract all phone numbers from the text.' + prompt_helper: prompt_conversion: 'If possible, please convert the number to the E.164 standard with the correct country code. Remove any whitespace from the final value.' test_cases: generic_phone_number - ignore_extractions: - - stix_mapping: phone-number ####### County extractions ####### @@ -63,11 +64,10 @@ ai_country_alpha2: modified: 2020-01-01 created_by: DOGESEC version: 1.0.0 - prompt_base: 'Extract all countries from the text, including countries printed as IS0-3166 Alpha2 and Alpha3 codes.' - prompt_conversion: 'Convert all extractions to IS0-3166 Alpha2 codes.' + prompt_base: 'Extract all countries described in the text, including countries printed as IS0-3166 Alpha2 and Alpha3 codes.' + prompt_helper: 'If you are unsure, you can read more about the standard here: https://www.iso.org/iso-3166-country-codes.html' + prompt_conversion: 'Convert all country extractions to their corresponding IS0-3166 Alpha2 codes.' test_cases: ai_country_alpha2 - ignore_extractions: - - stix_mapping: location ####### MITRE ATT&CK ####### @@ -81,11 +81,10 @@ ai_mitre_attack_enterprise: modified: 2020-01-01 created_by: DOGESEC version: 1.0.0 - prompt_base: 'Extract all MITRE ATT&CK Enterprise tactics, techniques, groups, data sources, mitigations, software, and campaigns described in the text. Do not include MITRE ATT&CK ICS or MITRE ATT&CK Mobile in the results.' + prompt_base: 'Extract all references to MITRE ATT&CK Enterprise tactics, techniques, groups, data sources, mitigations, software, and campaigns described in the text. These references may not be explicit in the text so you should be careful to account for the natural language of the text your analysis. Do not include MITRE ATT&CK ICS or MITRE ATT&CK Mobile in the results.' + prompt_helper: 'If you are unsure, you can learn more about MITRE ATT&CK Enterprise here: https://attack.mitre.org/matrices/enterprise/' prompt_conversion: 'Convert all extractions into the corresponding ATT&CK ID.' test_cases: generic_mitre_attack_enterprise - ignore_extractions: - - stix_mapping: ctibutler-mitre-attack-enterprise-id ai_mitre_attack_mobile: @@ -97,11 +96,10 @@ ai_mitre_attack_mobile: modified: 2020-01-01 created_by: DOGESEC version: 1.0.0 - prompt_base: 'Extract all MITRE ATT&CK Mobile tactics, techniques, groups, data sources, mitigations, software, and campaigns described in the text. Do not include MITRE ATT&CK ICS or MITRE ATT&CK Enterprise in the results.' + prompt_base: 'Extract all references to MITRE ATT&CK Mobile tactics, techniques, groups, data sources, mitigations, software, and campaigns described in the text. These references may not be explicit in the text so you should be careful to account for the natural language of the text your analysis. Do not include MITRE ATT&CK ICS or MITRE ATT&CK Enterprise in the results.' + prompt_helper: 'If you are unsure, you can learn more about MITRE ATT&CK Enterprise here: https://attack.mitre.org/matrices/mobile/' prompt_conversion: 'Convert all extractions into the corresponding ATT&CK ID.' test_cases: generic_mitre_attack_mobile - ignore_extractions: - - stix_mapping: ctibutler-mitre-attack-mobile-id ai_mitre_attack_ics: @@ -113,11 +111,10 @@ ai_mitre_attack_ics: modified: 2020-01-01 created_by: DOGESEC version: 1.0.0 - prompt_base: 'Extract all MITRE ATT&CK ICS tactics, techniques, groups, data sources, mitigations, software, and campaigns described in the text. Do not include MITRE ATT&CK Mobile or MITRE ATT&CK Enterprise in the results.' + prompt_base: 'Extract all references to MITRE ATT&CK ICS tactics, techniques, groups, data sources, mitigations, software, and campaigns described in the text. These references may not be explicit in the text so you should be careful to account for the natural language of the text your analysis. Do not include MITRE ATT&CK Mobile or MITRE ATT&CK Enterprise in the results.' + prompt_helper: 'If you are unsure, you can learn more about MITRE ATT&CK Enterprise here: https://attack.mitre.org/matrices/ics/' prompt_conversion: 'Convert all extractions into the corresponding ATT&CK ID.' test_cases: generic_mitre_attack_ics - ignore_extractions: - - stix_mapping: ctibutler-mitre-attack-ics-id ####### MITRE CAPEC ####### @@ -132,10 +129,9 @@ ai_mitre_capec: created_by: DOGESEC version: 1.0.0 prompt_base: 'Extract all references to a MITRE CAPEC object.' + prompt_helper: 'If you are unsure, you can learn more about MITRE CAPEC here: https://capec.mitre.org/' prompt_conversion: 'Convert all extractions into the corresponding CAPEC ID in the format `CAPEC-ID`' test_cases: generic_mitre_capec - ignore_extractions: - - stix_mapping: ctibutler-mitre-capec-id ####### MITRE CWE ####### @@ -150,8 +146,7 @@ ai_mitre_cwe: created_by: DOGESEC version: 1.0.0 prompt_base: 'Extract all references to a MITRE CWE object.' + prompt_helper: 'If you are unsure, you can learn more about MITRE CAPEC here: https://cwe.mitre.org/' prompt_conversion: 'Convert all extractions into the corresponding CWE ID in the format `CWE-ID`' test_cases: generic_mitre_cwe - ignore_extractions: - - stix_mapping: ctibutler-mitre-cwe-id \ No newline at end of file diff --git a/includes/extractions/ai/schema.yaml b/includes/extractions/ai/schema.yaml deleted file mode 100644 index dbbea75..0000000 --- a/includes/extractions/ai/schema.yaml +++ /dev/null @@ -1,15 +0,0 @@ -SLUG: # REQUIRED: machine friendly name of extraction, must only contain characters `a-z`,`0-9`,`_`,`-` - type: # [dictionary] REQUIRED: should always be `ai` - name: # [string] REQUIRED: human friendly name. max 72 characters. - description: # [string] OPTIONAL: human friendly description. max 270 characters. - notes: # [string] OPTIONAL: designed for more developer friendly notes that do not suit the description field. - created: # [date] REQUIRED: YYYY-MM-DD of creation time. - modified: # [date] REQUIRED: YYYY-MM-DD of creation time. - created_by: # [string] REQUIRED: name of author , must only contain characters `a-z`,`0-9`,`_`,`-` - version: # [number] REQUIRED: version number in format N.N.N - prompt_base: # [string] REQUIRED: Value to be used for the prompt. max 500 characters. - prompt_conversion: # [string] OPTIONAL: Additional prompt to convert what the LLM extracts into another value (e.g. turn name into an ID) - test_cases: # [test extraction slug] REQUIRED: test cases for this extraction. Slugs found in tests/test_cases.yaml - ignore_extractions: - - # [list of extraction slugs] OPTIONAL: defines slugs of other extraction types that should be ignored for any part of the matched string that they match too. e.g. ignore AI domain extractions (ai_domain_name_only) for google.com if url extraction matches https://google.com/something. This is dynamic, and you add/remove entries to enable/disable ignores - stix_mapping: # [dictionary] REQUIRED: STIX object extraction should map to. Must be supported STIX type \ No newline at end of file diff --git a/includes/extractions/lookup/config.yaml b/includes/extractions/lookup/config.yaml index 808032f..888b689 100644 --- a/includes/extractions/lookup/config.yaml +++ b/includes/extractions/lookup/config.yaml @@ -13,8 +13,6 @@ lookup_mitre_attack_enterprise_id: created_by: DOGESEC version: 1.0.0 test_cases: generic_mitre_attack_enterprise - ignore_extractions: - - stix_mapping: ctibutler-mitre-attack-enterprise-id lookup_mitre_attack_enterprise_name: @@ -28,8 +26,6 @@ lookup_mitre_attack_enterprise_name: created_by: DOGESEC version: 1.0.0 test_cases: generic_mitre_attack_enterprise_name - ignore_extractions: - - stix_mapping: ctibutler-mitre-attack-enterprise-name lookup_mitre_attack_mobile_id: @@ -43,8 +39,6 @@ lookup_mitre_attack_mobile_id: created_by: DOGESEC version: 1.0.0 test_cases: generic_mitre_attack_mobile - ignore_extractions: - - stix_mapping: ctibutler-mitre-attack-mobile-id lookup_mitre_attack_mobile_name: @@ -58,8 +52,6 @@ lookup_mitre_attack_mobile_name: created_by: DOGESEC version: 1.0.0 test_cases: generic_mitre_attack_mobile_name - ignore_extractions: - - stix_mapping: ctibutler-mitre-attack-mobile-name lookup_mitre_attack_ics_id: @@ -73,8 +65,6 @@ lookup_mitre_attack_ics_id: created_by: DOGESEC version: 1.0.0 test_cases: generic_mitre_attack_ics - ignore_extractions: - - stix_mapping: ctibutler-mitre-attack-ics-id lookup_mitre_attack_ics_name: @@ -88,8 +78,6 @@ lookup_mitre_attack_ics_name: created_by: DOGESEC version: 1.0.0 test_cases: generic_mitre_attack_ics_name - ignore_extractions: - - stix_mapping: ctibutler-mitre-attack-ics-name ####### MITRE CAPEC ####### @@ -105,8 +93,6 @@ lookup_mitre_capec_id: created_by: DOGESEC version: 1.0.0 test_cases: generic_mitre_capec - ignore_extractions: - - stix_mapping: ctibutler-mitre-capec-id lookup_mitre_capec_name: @@ -120,8 +106,6 @@ lookup_mitre_capec_name: created_by: DOGESEC version: 1.0.0 test_cases: generic_mitre_capec_name - ignore_extractions: - - stix_mapping: ctibutler-mitre-capec-name ####### MITRE CWE ####### @@ -137,8 +121,6 @@ lookup_mitre_cwe_id: created_by: DOGESEC version: 1.0.0 test_cases: generic_mitre_cwe - ignore_extractions: - - stix_mapping: ctibutler-mitre-cwe-id lookup_mitre_cwe_name: @@ -152,8 +134,6 @@ lookup_mitre_cwe_name: created_by: DOGESEC version: 1.0.0 test_cases: generic_mitre_cwe_name - ignore_extractions: - - stix_mapping: ctibutler-mitre-cwe-name ####### MITRE ATLAS ####### @@ -169,8 +149,6 @@ lookup_mitre_atlas_id: created_by: DOGESEC version: 1.0.0 test_cases: generic_mitre_atlas - ignore_extractions: - - stix_mapping: ctibutler-mitre-atlas-id lookup_mitre_atlas_name: @@ -184,8 +162,6 @@ lookup_mitre_atlas_name: created_by: DOGESEC version: 1.0.0 test_cases: generic_mitre_atlas_name - ignore_extractions: - - stix_mapping: ctibutler-mitre-atlas-name ####### DISARM ####### @@ -201,8 +177,6 @@ lookup_disarm_id: created_by: DOGESEC version: 1.0.0 test_cases: generic_disarm - ignore_extractions: - - stix_mapping: ctibutler-disarm-id lookup_disarm_name: @@ -216,8 +190,6 @@ lookup_disarm_name: created_by: DOGESEC version: 1.0.0 test_cases: generic_disarm_name - ignore_extractions: - - stix_mapping: ctibutler-disarm-name ####### County extractions ####### @@ -233,8 +205,6 @@ lookup_country_alpha2: created_by: DOGESEC version: 1.0.0 test_cases: generic_country_alpha2 - ignore_extractions: - - stix_mapping: ctibutler-location ####### Misc STIX Objects ####### @@ -250,8 +220,6 @@ lookup_attack_pattern: created_by: DOGESEC version: 1.0.0 test_cases: lookup_attack_pattern - ignore_extractions: - - stix_mapping: attack-pattern lookup_campaign: @@ -265,8 +233,6 @@ lookup_campaign: created_by: DOGESEC version: 1.0.0 test_cases: lookup_campaign - ignore_extractions: - - stix_mapping: campaign lookup_course_of_action: @@ -280,8 +246,6 @@ lookup_course_of_action: created_by: DOGESEC version: 1.0.0 test_cases: lookup_course_of_action - ignore_extractions: - - stix_mapping: course-of-action lookup_identity: @@ -295,8 +259,6 @@ lookup_identity: created_by: DOGESEC version: 1.0.0 test_cases: lookup_identity - ignore_extractions: - - stix_mapping: identity identity: lookups/identity.txt @@ -311,8 +273,6 @@ lookup_infrastructure: created_by: DOGESEC version: 1.0.0 test_cases: lookup_infrastructure - ignore_extractions: - - stix_mapping: infrastructure lookup_intrusion_set: @@ -326,8 +286,6 @@ lookup_intrusion_set: created_by: DOGESEC version: 1.0.0 test_cases: lookup_intrusion_set - ignore_extractions: - - stix_mapping: intrusion-set lookup_malware: @@ -341,8 +299,6 @@ lookup_malware: created_by: DOGESEC version: 1.0.0 test_cases: lookup_malware - ignore_extractions: - - stix_mapping: malware lookup_threat_actor: @@ -356,8 +312,6 @@ lookup_threat_actor: created_by: DOGESEC version: 1.0.0 test_cases: lookup_threat_actor - ignore_extractions: - - stix_mapping: threat-actor lookup_tool: @@ -371,6 +325,4 @@ lookup_tool: created_by: DOGESEC version: 1.0.0 test_cases: lookup_tool - ignore_extractions: - - stix_mapping: tool \ No newline at end of file diff --git a/includes/extractions/lookup/schema.yaml b/includes/extractions/lookup/schema.yaml deleted file mode 100644 index 03e9dd2..0000000 --- a/includes/extractions/lookup/schema.yaml +++ /dev/null @@ -1,14 +0,0 @@ -SLUG: # REQUIRED: machine friendly name of extraction, must only contain characters `a-z`,`0-9`,`_`,`-` - type: # [dictionary] REQUIRED: should always be `lookup` - name: # [string] REQUIRED: human friendly name. max 72 characters. - description: # [string] OPTIONAL: human friendly description. max 270 characters. - notes: # [string] OPTIONAL: designed for more developer friendly notes that do not suit the description field. - file: # [path to file from repository root] REQUIRED: the path to the lookup file that should be used. Must be of type txt - created: # [date] REQUIRED: YYYY-MM-DD of creation time. - modified: # [date] REQUIRED: YYYY-MM-DD of creation time. - created_by: # [string] REQUIRED: name of author , must only contain characters `a-z`,`0-9`,`_`,`-` - version: # [number] REQUIRED: version number in format N.N.N - test_cases: # [test extraction slug] REQUIRED: test cases for this extraction. Slugs found in tests/test_cases.yaml - ignore_extractions: - - # [list of extraction slugs] OPTIONAL: defines slugs of other extraction types that should be ignored for any part of the matched string that they match too. e.g. ignore AI domain extractions (ai_domain_name_only) for google.com if url extraction matches https://google.com/something. This is dynamic, and you add/remove entries to enable/disable ignores - stix_mapping: # [dictionary] REQUIRED: STIX object extraction should map to. Must be supported STIX type \ No newline at end of file diff --git a/includes/extractions/pattern/config.yaml b/includes/extractions/pattern/config.yaml index 5ff84bd..eeb7542 100644 --- a/includes/extractions/pattern/config.yaml +++ b/includes/extractions/pattern/config.yaml @@ -12,8 +12,6 @@ pattern_ipv4_address_only: created_by: DOGESEC version: 1.0.0 test_cases: generic_ipv4_address_only - ignore_extractions: - - stix_mapping: ipv4-addr pattern_ipv4_address_cidr: @@ -26,9 +24,6 @@ pattern_ipv4_address_cidr: created_by: DOGESEC version: 1.0.0 test_cases: ipv4_address_cidr - ignore_extractions: - - pattern_ipv4_address_only - - ai_ipv4_address_only stix_mapping: ipv4-addr pattern_ipv4_address_port: @@ -41,9 +36,6 @@ pattern_ipv4_address_port: created_by: DOGESEC version: 1.0.0 test_cases: ipv4_address_port - ignore_extractions: - - pattern_ipv4_address_only - - ai_ipv4_address_only stix_mapping: ipv4-addr-port ####### IPv6 extractions ####### @@ -58,9 +50,6 @@ pattern_ipv6_address_only: created_by: DOGESEC version: 1.0.0 test_cases: generic_ipv6_address_only - ignore_extractions: - - pattern_ipv6_address_cidr - - pattern_ipv6_address_port stix_mapping: ipv6-addr pattern_ipv6_address_cidr: @@ -73,9 +62,6 @@ pattern_ipv6_address_cidr: created_by: DOGESEC version: 1.0.0 test_cases: generic_ipv6_address_cidr - ignore_extractions: - - pattern_ipv6_address_only - - ai_ipv6_address_only stix_mapping: ipv6-addr pattern_ipv6_address_port: @@ -88,9 +74,6 @@ pattern_ipv6_address_port: created_by: DOGESEC version: 1.0.0 test_cases: generic_ipv6_address_port - ignore_extractions: - - pattern_ipv6_address_only - - ai_ipv6_address_only stix_mapping: ipv6-addr-port ####### Domain name extractions ####### @@ -105,8 +88,6 @@ pattern_domain_name_only: created_by: DOGESEC version: 1.0.0 test_cases: generic_domain_name_only - ignore_extractions: - - stix_mapping: domain-name pattern_domain_name_subdomain: @@ -119,9 +100,6 @@ pattern_domain_name_subdomain: created_by: DOGESEC version: 1.0.0 test_cases: generic_domain_name_subdomain - ignore_extractions: - - pattern_domain_name_only - - ai_domain_name_only stix_mapping: domain-name ####### URL extractions ####### @@ -136,8 +114,6 @@ pattern_url: created_by: DOGESEC version: 1.0.0 test_cases: generic_url - ignore_extractions: - - stix_mapping: url pattern_url_file: @@ -150,11 +126,6 @@ pattern_url_file: created_by: DOGESEC version: 1.0.0 test_cases: generic_url_file - ignore_extractions: - - pattern_url - - ai_url - - pattern_url_path - - ai_url_path stix_mapping: url pattern_url_path: @@ -167,8 +138,6 @@ pattern_url_path: created_by: DOGESEC version: 1.0.0 test_cases: generic_url_path - ignore_extractions: - - stix_mapping: url ####### Hostname extractions ####### @@ -183,8 +152,6 @@ pattern_host_name: created_by: DOGESEC version: 1.0.0 test_cases: generic_host_name - ignore_extractions: - - stix_mapping: domain-name pattern_host_name_subdomain: @@ -197,9 +164,6 @@ pattern_host_name_subdomain: created_by: DOGESEC version: 1.0.0 test_cases: generic_host_name_subdomain - ignore_extractions: - - pattern_host_name - - ai_host_name stix_mapping: domain-name pattern_host_name_url: @@ -212,11 +176,6 @@ pattern_host_name_url: created_by: DOGESEC version: 1.0.0 test_cases: generic_host_name_url - ignore_extractions: - - pattern_host_name_subdomain - - ai_host_name_subdomain - - pattern_host_name - - ai_host_name stix_mapping: url pattern_host_name_file: @@ -229,15 +188,6 @@ pattern_host_name_file: created_by: DOGESEC version: 1.0.0 test_cases: generic_host_name_file - ignore_extractions: host_name_path - - pattern_host_name_url - - ai_host_name_url - - pattern_host_name - - ai_host_name - - pattern_host_name_subdomain - - ai_host_name_subdomain - - pattern_host_name_path - - ai_host_name_path stix_mapping: url pattern_host_name_path: @@ -250,13 +200,6 @@ pattern_host_name_path: created_by: DOGESEC version: 1.0.0 test_cases: generic_host_name_path - ignore_extractions: - - pattern_host_name_url - - ai_host_name_url - - pattern_host_name - - ai_host_name - - pattern_host_name_subdomain - - ai_host_name_subdomain stix_mapping: url ####### File name extractions ####### @@ -271,8 +214,6 @@ pattern_file_name: created_by: DOGESEC version: 1.0.0 test_cases: generic_file_name - ignore_extractions: - - stix_mapping: file ####### Directory path extractions ####### @@ -287,8 +228,6 @@ pattern_directory_windows: created_by: DOGESEC version: 1.0.0 test_cases: generic_directory_windows - ignore_extractions: - - stix_mapping: directory pattern_directory_windows_with_file: @@ -301,8 +240,6 @@ pattern_directory_windows_with_file: created_by: DOGESEC version: 1.0.0 test_cases: generic_directory_windows_with_file - ignore_extractions: - - pattern_directory_windows stix_mapping: directory-file pattern_directory_unix: @@ -315,8 +252,6 @@ pattern_directory_unix: created_by: DOGESEC version: 1.0.0 test_cases: generic_directory_unix - ignore_extractions: - - stix_mapping: directory pattern_directory_unix_file: @@ -329,8 +264,6 @@ pattern_directory_unix_file: created_by: DOGESEC version: 1.0.0 test_cases: generic_directory_unix_file - ignore_extractions: - - pattern_directory_unix stix_mapping: directory-file ####### File hash extractions ####### @@ -345,8 +278,6 @@ pattern_file_hash_md5: created_by: DOGESEC version: 1.0.0 test_cases: generic_file_hash_md5 - ignore_extractions: - - stix_mapping: file-hash pattern_file_hash_sha_1: @@ -359,8 +290,6 @@ pattern_file_hash_sha_1: created_by: DOGESEC version: 1.0.0 test_cases: generic_file_hash_sha_1 - ignore_extractions: - - stix_mapping: file-hash pattern_file_hash_sha_256: @@ -373,8 +302,6 @@ pattern_file_hash_sha_256: created_by: DOGESEC version: 1.0.0 test_cases: generic_file_hash_sha_256 - ignore_extractions: - - stix_mapping: file-hash pattern_file_hash_sha_512: @@ -387,8 +314,6 @@ pattern_file_hash_sha_512: created_by: DOGESEC version: 1.0.0 test_cases: generic_file_hash_sha_512 - ignore_extractions: - - stix_mapping: file-hash ####### Email address extractions ####### @@ -403,9 +328,6 @@ pattern_email_address: created_by: DOGESEC version: 1.0.0 test_cases: generic_email_address - ignore_extractions: - - pattern_domain_name_only - - pattern_domain_name_subdomain stix_mapping: email-addr ####### MAC address extractions ####### @@ -420,8 +342,6 @@ pattern_mac_address: created_by: DOGESEC version: 1.0.0 test_cases: generic_mac_address - ignore_extractions: - - stix_mapping: mac-addr ####### Windows registry key extractions ####### @@ -436,8 +356,6 @@ pattern_windows_registry_key: created_by: DOGESEC version: 1.0.0 test_cases: generic_windows_registry_key - ignore_extractions: - - pattern_directory_windows stix_mapping: windows-registry-key ####### User agent extractions ####### @@ -452,8 +370,6 @@ pattern_user_agent: created_by: DOGESEC version: 1.0.0 test_cases: generic_user_agent - ignore_extractions: - - pattern_directory_unix stix_mapping: user-agent ####### ASN extractions ####### @@ -468,8 +384,6 @@ pattern_autonomous_system_number: created_by: DOGESEC version: 1.0.0 test_cases: generic_autonomous_system_number - ignore_extractions: - - stix_mapping: autonomous-system ####### Cryptocurrency wallet extractions ####### @@ -484,8 +398,6 @@ pattern_cryptocurrency_btc_wallet: created_by: DOGESEC version: 1.0.0 test_cases: generic_cryptocurrency_btc_wallet - ignore_extractions: - - stix_mapping: cryptocurrency-wallet pattern_cryptocurrency_btc_wallet_transaction: @@ -498,8 +410,6 @@ pattern_cryptocurrency_btc_wallet_transaction: created_by: DOGESEC version: 1.0.0 test_cases: generic_cryptocurrency_btc_wallet - ignore_extractions: - - stix_mapping: cryptocurrency-wallet-with-transaction ####### Cryptocurrency transaction extractions ####### @@ -514,8 +424,6 @@ pattern_cryptocurrency_btc_transaction: created_by: DOGESEC version: 1.0.0 test_cases: generic_cryptocurrency_btc_transaction - ignore_extractions: - - stix_mapping: cryptocurrency-transaction ####### CVE extractions ####### @@ -530,8 +438,6 @@ pattern_cve_id: created_by: DOGESEC version: 1.0.0 test_cases: generic_cve_id - ignore_extractions: - - stix_mapping: vulmatch-cve-id ####### CPE extractions ####### @@ -546,8 +452,6 @@ pattern_cpe_uri: created_by: DOGESEC version: 1.0.0 test_cases: generic_cpe_uri - ignore_extractions: - - stix_mapping: vulmatch-cpe-id ####### Bank card extractions ####### @@ -562,8 +466,6 @@ pattern_bank_card_mastercard: created_by: DOGESEC version: 1.0.0 test_cases: generic_bank_card_mastercard - ignore_extractions: - - stix_mapping: bank-card pattern_bank_card_visa: @@ -576,8 +478,6 @@ pattern_bank_card_visa: created_by: DOGESEC version: 1.0.0 test_cases: generic_bank_card_visa - ignore_extractions: - - stix_mapping: bank-card pattern_bank_card_amex: @@ -602,8 +502,6 @@ pattern_bank_card_union_pay: created_by: DOGESEC version: 1.0.0 test_cases: generic_bank_card_union_pay - ignore_extractions: - - stix_mapping: bank-card pattern_bank_card_diners: @@ -616,8 +514,6 @@ pattern_bank_card_diners: created_by: DOGESEC version: 1.0.0 test_cases: generic_bank_card_diners - ignore_extractions: - - stix_mapping: bank-card pattern_bank_card_jcb: @@ -630,8 +526,6 @@ pattern_bank_card_jcb: created_by: DOGESEC version: 1.0.0 test_cases: generic_bank_card_jcb - ignore_extractions: - - stix_mapping: bank-card pattern_bank_card_discover: @@ -644,8 +538,6 @@ pattern_bank_card_discover: created_by: DOGESEC version: 1.0.0 test_cases: generic_bank_card_discover - ignore_extractions: - - stix_mapping: bank-card ####### IBAN Extractions ####### @@ -660,8 +552,6 @@ pattern_iban_number: created_by: DOGESEC version: 1.0.0 test_cases: generic_iban_number - ignore_extractions: - - stix_mapping: bank-account ####### Phone number Extractions ####### @@ -676,6 +566,4 @@ pattern_phone_number: created_by: DOGESEC version: 1.0.0 test_cases: generic_phone_number - ignore_extractions: - - stix_mapping: phone-number \ No newline at end of file diff --git a/includes/extractions/pattern/schema.yaml b/includes/extractions/pattern/schema.yaml deleted file mode 100644 index 1f177cb..0000000 --- a/includes/extractions/pattern/schema.yaml +++ /dev/null @@ -1,13 +0,0 @@ -SLUG: # REQUIRED: machine friendly name of extraction, must only contain characters `a-z`,`0-9`,`_`,`-` - type: # [dictionary] REQUIRED: should always be `pattern` - name: # [string] REQUIRED: human friendly name. max 72 characters. - description: # [string] OPTIONAL: human friendly description. max 270 characters. - notes: # [string] OPTIONAL: designed for more developer friendly notes that do not suit the description field. - created: # [date] REQUIRED: YYYY-MM-DD of creation time. - modified: # [date] REQUIRED: YYYY-MM-DD of creation time. - created_by: # [string] REQUIRED: name of author , must only contain characters `a-z`,`0-9`,`_`,`-` - version: # [number] REQUIRED: version number in format N.N.N - test_cases: # [test extraction slug] REQUIRED: test cases for this extraction. Slugs found in tests/test_cases.yaml - ignore_extractions: - - # [list of extraction slugs] OPTIONAL: defines slugs of other extraction types that should be ignored for any part of the matched string that they match too. e.g. ignore AI domain extractions (ai_domain_name_only) for google.com if url extraction matches https://google.com/something. This is dynamic, and you add/remove entries to enable/disable ignores - stix_mapping: # [dictionary] REQUIRED: STIX object extraction should map to. Must be supported STIX type \ No newline at end of file diff --git a/tests/data/manually_generated_reports/mitre_attack_enterprise_ai_demo.txt b/tests/data/manually_generated_reports/mitre_attack_enterprise_ai_demo.txt new file mode 100644 index 0000000..88e2258 --- /dev/null +++ b/tests/data/manually_generated_reports/mitre_attack_enterprise_ai_demo.txt @@ -0,0 +1 @@ +REvil uses WMI to execute malicious commands to reference a retrieved PE file through a path modification. \ No newline at end of file diff --git a/tests/data/manually_generated_reports/mitre_attack_enterprise_demo.txt b/tests/data/manually_generated_reports/mitre_attack_enterprise_lookup_demo.txt similarity index 100% rename from tests/data/manually_generated_reports/mitre_attack_enterprise_demo.txt rename to tests/data/manually_generated_reports/mitre_attack_enterprise_lookup_demo.txt diff --git a/txt2stix/txt2stix.py b/txt2stix/txt2stix.py index cc60e3d..238d7ca 100644 --- a/txt2stix/txt2stix.py +++ b/txt2stix/txt2stix.py @@ -123,7 +123,7 @@ def parse_args(): parser = argparse.ArgumentParser(description="File Conversion Tool") inf_arg = parser.add_argument("--input_file", "--input-file", required=True, help="The file to be converted. Must be .txt", type=Path) - name_arg = parser.add_argument("--name", required=True, help="Name of the file, max 72 chars", default="stix-out") + name_arg = parser.add_argument("--name", required=True, help="Name of the file, max 124 chars", default="stix-out") parser.add_argument("--created", required=False, default=datetime.now(), help="Allow user to optionally pass --created time in input, which will hardcode the time used in created times") parser.add_argument("--ai_settings_extractions", required=False, type=parse_model, help="(required if AI extraction enabled): passed in format provider:model e.g. openai:gpt4o. Can pass more than one value to get extractions from multiple providers.", metavar="provider[:model]", nargs='+', default=[parse_model('openai')]) parser.add_argument("--ai_settings_relationships", required=False, type=parse_model, help="(required if AI relationship enabled): passed in format `provider:model`. Can only pass one model at this time.", metavar="provider[:model]") @@ -139,8 +139,8 @@ def parse_args(): args = parser.parse_args() if not args.input_file.exists(): raise argparse.ArgumentError(inf_arg, "cannot open file") - if len(args.name) > 72: - raise argparse.ArgumentError(name_arg, "max 72 characters") + if len(args.name) > 124: + raise argparse.ArgumentError(name_arg, "max 124 characters") if args.relationship_mode == 'ai' and not args.ai_settings_relationships: parser.error("relationship_mode is set to AI, --ai_settings_relationships is required")