diff --git a/.env.sample b/.env.sample index 1b63698..4d16d60 100644 --- a/.env.sample +++ b/.env.sample @@ -1,8 +1,14 @@ -INPUT_TOKEN_LIMIT=50 # [REQUIRED] keep in mind the token limit for selected model (which includes both input AND output tokens). For example, if your input limit is 50,000 characters, this could incur up to 25,000 tokens. Assuming your selected model allows for 64,000 tokens, you will therefore be able to obtain an output of over 39,000 tokens. +INPUT_TOKEN_LIMIT=50 # [REQUIRED] for AI modes. keep in mind the token limit for selected model (which includes both input AND output tokens). For example, if your input limit is 50,000 characters, this could incur up to 25,000 tokens. Assuming your selected model allows for 64,000 tokens, you will therefore be able to obtain an output of over 39,000 tokens. OPENAI_API_KEY= # [REQUIRED IF USING AI MODES] get it from https://platform.openai.com/api-keys OPENAI_MODEL=gpt-4 # [REQUIRED IF USING AI MODES] choose an OpenAI model of your choice. Ensure the input/output token count meets requirements (and adjust INPUT_TOKEN_LIMIT accordingly). List of models here: https://platform.openai.com/docs/models ARANGODB_HOST_URL=https://database.ctibutler.com:8529 # [REQUIRED] user can also self host -ARANGODB_USERNAME= # [REQUIRED] user must have write privileges +ARANGODB_USERNAME= # [REQUIRED] user must have write privileges to the ARANGODB_DATABASE specified ARANGODB_PASSWORD= # [REQUIRED] password for specified username ARANGODB_DATABASE=cti_database # [REQUIRED] database where collections are held, if using CTI Butler is cti_database -BIN_LIST_API_KEY= #[OPTIONAL] needed for extracting credit card information \ No newline at end of file +BIN_LIST_API_KEY= #[OPTIONAL] needed for extracting credit card information +## CTIBUTLER FOR ATT&CK, CAPEC, AND CWE LOOKUPS +CTIBUTLER_HOST= # [REQUIRED] e.g. http://localhost:8006/ +CTIBUTLER_APIKEY= #[OPTIONAL] if using https://app.ctibutler.com +## VULMATCH FOR CVE AD CPE LOOKUPS +VULMATCH_HOST= # [REQUIRED] e.g. http://localhost:8005/ +VULMATCH_APIKEY= #[OPTIONAL] if using https://app.vulmatch.com \ No newline at end of file diff --git a/README.md b/README.md index d4943f2..c6ef916 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ ## Overview +![txt2stix](docs/txt2stix.png) + txt2stix is a Python script that is designed to identify and extract IoCs and TTPs from text files, identify the relationships between them, convert them to STIX 2.1 objects, and output as a STIX 2.1 bundle. The general design goal of txt2stix was to keep it flexible, but simple, so that new extractions could be added or modified over time. @@ -9,12 +11,13 @@ The general design goal of txt2stix was to keep it flexible, but simple, so that In short txt2stix; 1. takes a txt file input -2. rewrites file with enabled aliases -3. extracts observables for enabled extractions (and ignores any whitelisted values) -4. converts extracted observables to STIX 2.1 objects -5. generates the relationships between extracted observables -6. converts extracted relationships to STIX 2.1 SRO objects -7. outputs a STIX 2.1 bundle +2. (optional) rewrites file with enabled aliases +3. extracts observables for enabled extractions (ai, pattern, or lookup) +4. (optional) removes any extractions that match whitelists +5. converts extracted observables to STIX 2.1 objects +6. generates the relationships between extracted observables (ai, standard) +7. converts extracted relationships to STIX 2.1 SRO objects +8. outputs a STIX 2.1 bundle ## tl;dr @@ -22,87 +25,6 @@ In short txt2stix; [Watch the demo](https://www.youtube.com/watch?v=TWVGCou9oGk). -## The problem - -More-and-more organisations are standardising the way the represent threat intelligence using the STIX 2.1 data model. - -As a result, an increasing number of SIEMs, SOARs, TIPs, etc. have native STIX 2.1 support. - -However, authoring STIX 2.1 content can be laborious. I have seen analysts manually copy and paste data from reports, blogs, emails, and other sources into STIX 2.1 Objects. - -In many cases these Observables (IOCs) can be automatically detected in plain text using defined patterns. - -For example, an IPv4 observable has a specific pattern that can be identified using regular expressions. This regular expression will match an IPv4 observable; - -```regex -^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$ -``` - -Similarly, the following regular expression will capture URLs; - -```regex -^(https?|ftp|file)://.+$ -``` - -Both of these examples ([here](https://www.oreilly.com/library/view/regular-expressions-cookbook/9780596802837/ch07s16.html) and [here](https://www.oreilly.com/library/view/regular-expressions-cookbook/9781449327453/ch08s01.html), respectively) are taken from the brilliant Regular Expressions Cookbook (2nd edition) by Jan Goyvaerts and Steven Levithan. - -Now this isn't rocket science, and indeed there are already quite a few open source tools that contain regular expressions for extracting Observables in this way; - -* [IoC extractor](https://github.com/ninoseki/ioc-extractor): An npm package for extracting common IoC (Indicator of Compromise) -* [IOC Finder](https://github.com/fhightower/ioc-finder): Simple, effective, and modular package for parsing Observables (indicators of compromise (IOCs), network data, and other, security related information) from text. -* [cacador](https://github.com/sroberts/cacador): Indicator Extractor -* [iocextract](https://github.com/InQuest/python-iocextract): Defanged Indicator of Compromise (IOC) Extractor. -* [Cyobstract](https://github.com/cmu-sei/cyobstract): A tool to extract structured cyber information from incident reports. - -However, we wanted a more modularised extraction logic, especially to take advantage of the new accessibility of AI. - -## Concepts - -Here is an overview of how the txt2stix processes txt files into STIX 2.1 bundles: - -https://miro.com/app/board/uXjVKEyFzB8=/ - -### Extractions - -This is the logic that actually extracts the text from the input document. - -txt2stix has 3 types of extracions; - -1. AI: uses an LLM to extracts the data based on a prompt - * when to use: contextual types data that can't be easily detected using patterns - * when not to use: when costs are an issue, when user will not review output for errors -2. Pattern: all extractions will be performed by regular expressions (or via existing Python libraries). - * when to use: for pattern based - * when not to use: when costs are an issue, where user will not -3. Lookups: file2txt will compare strings in input document against a list of strings in lookups - * when to use: for specialist data not easily detected in patterns - * when not to use: for large amounts of data (in the lookup) - -### Relationships - -This is how extractions are joined together using STIX SROs. - -There are 2 relationship modes in txt2stix; - -* `ai`: Rich relationships created by LLM between extractions. -* `standard`: Basic relationships created from extractions back to master Report object generated. - -### Aliases - -In many cases two extractions might be related to the same thing. For example, the extraction `USA` and `United States` and `United States of America` are all referring to the same thing. - -Aliases normalise the input text before extractions happen so that the same extraction is used. e.g. changing `United States` -> `USA`. - -### Whitelists - -In many cases files will have IoC extractions that are not malicious. e.g. `google.com` (and thus they don't want them to appear in final bundle). - -Whitelists provide a list of values to be compared to extractions. If a whitelist value matches an extraction, that extraction is removed and any relationships where is the `source_ref` or `target_ref` are also removed so that a user does not see them. - -Design decision: This is done after extractions to save tokens with AI providers (otherwise might be easily passing 10000+ more tokens to the AI). - -Note, whitelists are designed to be simplistic in txt2stix. If you want more advanced removal of potential benign extractions you should use another tool, like a Threat Intelligence Platform. - ## Usage ### Setup @@ -128,20 +50,6 @@ cp .env.sample .env You can new set the correct values in `.env`. -A quick note on OPEN_AI and ARANGODB variables.... - -`OPENAI_*` properties are required should you want to use AI based extractions or AI relationship mode. If left blank, you can use pattern extractions and standard relationship modes only. - -`ARANGODB_*` properties are required if you want to use MITRE ATT&CK, MITRE CWE, MITRE CAPEC, NVD CPE, or NVD CVE extractions. You must define an ArangoDB instance with the required data in the expected format in order for these extraction types to work. - -You can populate your own instance of ArangoDB with the required data by using the scripts referenced in the [stix2arango](https://github.com/muchdogesec/stix2arango) quickstart. - -**Make life simpler for yourself...** - -If you do not want to backfill, maintain, or support your own ArangoDB STIX objects check out CTI Butler which provides a fully manage database of these objects you can use with txt2stix. - -https://www.ctibutler.com/ - ### Usage ```shell @@ -187,7 +95,7 @@ Currently it is not possible to easily add any other types of extractions (witho ## Detailed documentation -If you would like to understand how txt2stix works in more detail, please refer to the documentation in `/doc`. +If you would like to understand how txt2stix works in more detail, please refer to the documentation in `/docs/README.md`. This documentation is paticularly helpful to read for those of you wanting to add your own custom extractions. diff --git a/docs/README.md b/docs/README.md index c73a286..4478a53 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,12 +1,71 @@ -# Extractions Types +# Docs + +## Overview + +More-and-more organisations are standardising the way the represent threat intelligence using the STIX 2.1 data model. + +As a result, an increasing number of SIEMs, SOARs, TIPs, etc. have native STIX 2.1 support. + +However, authoring STIX 2.1 content can be laborious. I have seen analysts manually copy and paste data from reports, blogs, emails, and other sources into STIX 2.1 Objects. + +In many cases these Observables (IOCs) can be automatically detected in plain text using defined patterns. + +For example, an IPv4 observable has a specific pattern that can be identified using regular expressions. This regular expression will match an IPv4 observable; + +```regex +^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$ +``` + +Similarly, the following regular expression will capture URLs; + +```regex +^(https?|ftp|file)://.+$ +``` + +Now this isn't rocket science, and indeed there are already quite a few open source tools that contain regular expressions for extracting Observables in this way. + +However, we wanted a more modularised extraction logic, especially to take advantage of the new accessibility of AI. + +## Concepts + +Here is an overview of how the txt2stix processes txt files into STIX 2.1 bundles: -## Aliases +https://miro.com/app/board/uXjVKEyFzB8=/ -In many cases two extractions in the same text document might be related to the same thing. +### Extractions -For example, the extraction `USA` and `United States` and `United States of America` are all referring to the same thing. +This is the logic that actually extracts the text from the input document. -To account for this, aliases can be used. Aliases are applied before extractions. Essentially the first step of processing is to replace the alias values, with the desired value. +txt2stix has 3 types of extracions; + +1. AI: uses an LLM to extracts the data based on a prompt + * when to use: contextual types data that can't be easily detected using patterns + * when not to use: when costs are an issue, when user will not review output for errors +2. Pattern: all extractions will be performed by regular expressions (or via existing Python libraries). + * when to use: for pattern based + * when not to use: when costs are an issue, where user will not +3. Lookups: file2txt will compare strings in input document against a list of strings in lookups + * when to use: for specialist data not easily detected in patterns + * when not to use: for large amounts of data (in the lookup) + +### Relationships + +This is how extractions are joined together using STIX SROs. + +There are 2 relationship modes in txt2stix; + +* `ai`: NLP based relationships created by LLM between extractions. +* `standard`: Basic relationships created from extractions back to master Report object generated. + +### Aliases + +In many cases two extractions might be related to the same thing. For example, the extraction `USA` and `United States` and `United States of America` are all referring to the same thing. + +Aliases normalise the input text before extractions happen so that the same extraction is used. e.g. changing `United States` -> `USA`. + +Aliases are applied before extractions. Essentially the first step of processing is to replace the alias values, with the desired value. + +The aliaases are set in the `includes/extractions/config.yaml` To demonstrate, lets say the alias config file (in `aliases/`) looks like so; @@ -33,7 +92,21 @@ ALB,AL DZA,DZ ``` -This will turn all references of AFG in the input document to AF in the version of the text document sent for extractions/relationship generation. +This will turn all references of AFG in the inp + +### Whitelists + +In many cases files will have IoC extractions that are not malicious. e.g. `google.com` (and thus they don't want them to appear in final bundle). + +Whitelists provide a list of values to be compared to extractions. If a whitelist value matches an extraction, that extraction is removed and any relationships where is the `source_ref` or `target_ref` are also removed so that a user does not see them. + +Design decision: This is done after extractions to save tokens with AI providers (otherwise might be easily passing 10000+ more tokens to the AI). + +Note, whitelists are designed to be simplistic in txt2stix. If you want more advanced removal of potential benign extractions you should use another tool, like a Threat Intelligence Platform. + + +# Extractions Types + ## Extractions @@ -104,46 +177,7 @@ Design decision: this does not apply to AI mode extractions (but still applies f AI extractions work by analysing the users text file input and extracting date (keywords / phrases from it). -#### A note on modularisation of AI mode - -Whilst the txt2stix MVP is designed for OpenAI, it is not coupled to it. - -In future it is likely other modules (other LLMs) will be available to use with txt2stix. - -### Data input - -When considering a users text inputs, it is very important to be aware of GPT tokens. - -Tokens are shared between prompt and completion. For example, if your prompt is 28,000 tokens using GPT-4, your completion can be 4,768 tokens at most (gpt-4-32k) support 32,768 tokens max). - -[Currently a user can only use the gpt-4-32k typel](https://platform.openai.com/docs/typels/gpt-4). - -We enforce input limits (that can be changed at any time) in the `.env` file under `INPUT_CHARACHTER_LIMIT=`. Default is 32,0000. - -### Prompt engineering - -In the `extractions/ai/` directory is a file called `config.yaml` - -For AI extractions there are a few special properties used to engineer the prompt for an extractions - -In the context of the extraction the key properties are: - -* `prompt_base` (required): the base prompt that will be used to perform the extraction. -* `prompt_conversion` (optional): can be used to fine tune the extraction and modify the output of the extraction - -Both values will be used to create the prompt (if they exist). -#### Prompt 1 - -This simply provides an instruction to the AI to remember the text. It includes a full copy of the txt file uploaded by the user AFTER any aliases have been applied. - -It asks the AI to provide a one word response, either "successful" or "unsuccessful". If unsuccessful returned the script will fail with an error. If successful, the script will continue. - -#### Prompt 2 - -Assuming the word "successful" is returned from prompt 1, a second prompt will be constructed using the `prompt_base` and `prompt_conversion` settings for the `ai` type extractions a user has enabled. - -The script asks the AI to produce ONLY a structured extractions JSON output containing information about each extraction detected. ## Mixing of extraction types @@ -159,2113 +193,23 @@ Ultimately extraction mode is run in sequence, e.g. for the above At the end of this process, 3 extractions json documents will exist. These are then concatenated into a single extractions JSON before being passed onto the next step, generate relationship JSON. -# Commons - -Much of the core logic for txt2stix is the same regardless of extractions used. This page describes the commonalities for all inputs and extractions. - -## TLPs SMOs - -At script run time user can set `tlp_level`; either white, green, amber or red. - -Each mode maps to a STIX marking definition object; - -* Clear: `marking-definition--94868c89-83c2-464b-929b-a1a8aa3c8487` -* Green: `marking-definition--bab4a63c-aed9-4cf5-a766-dfca5abac2bb` -* Amber: `marking-definition--55d920b0-5e8b-4f79-9ee9-91f868d9b421` -* Amber+Strict: `marking-definition--939a9414-2ddd-4d32-a0cd-375ea402b003` -* Red: `marking-definition--e828b379-4e03-4974-9ac4-e53a884c97c1` - -Depending on the value set by user, the generated STIX objects will contain a `marking-definition--` reference in the `object_marking_refs` field to the corresponding TLP level. - -## Marking definitions - -All objects created by txt2stix also have a standard marking definition in the `object_marking_refs` property. - -The default STIX 2.1 marking definition for txt2stix is always used and imported from https://raw.githubusercontent.com/muchdogesec/stix4doge/main/objects/marking-definition/txt2stix.json - -This object will be printed in the final bundle, and in all objects generated by txt2stix in the `object_marking_refs` property. - -## Identities (`identity`) SDOs - -txt2stix assigns a `created_by_ref` property to all SDOs and SROs it creates to their `created_by_ref` field. - -The default STIX 2.1 identity for txt2stix is imported from https://raw.githubusercontent.com/muchdogesec/stix4doge/main/objects/identity/txt2stix.json - -This object will be printed in the final bundle, and the ID will be used for all `created_by_ref` generated by txt2stix. - -Often a user will want to use a custom ID of their own to generate objects. They can do this using the `use_identity` flag in the CLI. - -Two things will happen if a custom identity is used in this way. The identity JSON object will be printed in the final bundle, and the ID will be used for all `created_by_ref` generated by txt2stix. - -## Report SDO (`report`) - -All files uploaded are represented as a unique [STIX Report SDO](https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html#_n8bjzg1ysgdq) that take the following structure; - -```json -{ - "type": "report", - "spec_version": "2.1", - "id": "report--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "name": "", - "description": "", - "confidence": "", - "published": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "labels": [ - "" - ], - "external_references": [ - { - "source_name": "txt2stix Report MD5", - "external_id": "" - } - ], - "object_refs": [ - "" - ] -} -``` - -Note, the `object_refs` contains all references that are referenced by objects in the report (SDOs, SROs, SCOs, marking definitions, etc.). This includes extracted objects (i.e. Indicator SDOs, Vulnerability SDOs, Software SCOs, Relationship SROs etc.). The only object it does not include is the Note object created automatically. - -`` should match the report object UUID, e.g. if this was - -## Note SDO (`note`) - -All files uploaded have a [Note SDOs](https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html#_gudodcg1sbb9) created to capture - -* the input settings -* the extraction JSON -* the relationships JSON (only generated in AI relationship mode because standard mode relationships are hardcoded) - -Note objects are always marked with TLP red `marking-definition--e828b379-4e03-4974-9ac4-e53a884c97c1` as the content in them does not usually want to be shared beyond the user running the script. - -Each object takes the following structure; - -### the input settings - -```json -{ - "type": "note", - "spec_version": "2.1", - "id": "note--", - "created": "", - "modified": "", - "abstract": "txt2stix Config: ", - "content": "", - "object_refs": ["report--"], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - } - ], - "object_marking_refs": [ - "marking-definition--e828b379-4e03-4974-9ac4-e53a884c97c1", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ] -} -``` - -### the extraction JSON - -```json -{ - "type": "note", - "spec_version": "2.1", - "id": "note--", - "created": "", - "modified": "", - "abstract": "txt2stix Extractions: ", - "content": "", - "object_refs": ["report--"], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - } - ], - "object_marking_refs": [ - "marking-definition--e828b379-4e03-4974-9ac4-e53a884c97c1", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ] -} -``` - -### the relationships JSON (only generated in AI relationship mode) - -```json -{ - "type": "note", - "spec_version": "2.1", - "id": "note--", - "created": "", - "modified": "", - "abstract": "txt2stix Relationships: ", - "content": ">", - "object_refs": ["report--"], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - } - ], - "object_marking_refs": [ - "marking-definition--e828b379-4e03-4974-9ac4-e53a884c97c1", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ] -} -``` - -## STIX Mapping - -All extractions detected by txt2stix, regardless of mode are converted to STIX. - -txt2stix uses the [STIX2 Python library](https://pypi.org/project/stix2/) for all STIX 2.1 object generation (except for imported objects). - -In the extractions document, a `stix-mapping` property exists for all extractions. - -Below shows how the data extracted is mapped to a set of STIX objects for each `stix-mapping` type. - -e.g. if an extraction with a `stix-mapping: ipv4-address` is triggered, the `ipv4-addr` STIX mapping shown below is used. - -### Standard Relationships for objects created by extractions - -Generally speaking, but not in every case, an extraction will create; - -1. An Indicator SDO with a pattern to detect the identified IOC (using a STIX patter) -2. An SCO (referenced in the Indicator SDO pattern) -3. An SRO (between SCO [2.] and SDO [1.]) -4. An SRO (between to SDOs for an extraction) - -These are described below. - -Note, this is different to relationship generation logic for relationship mode set (see relationship-modes.md). The relationships described in this document are created for all modes. - -### stix-mapping: ipv4-addr - -2 objects created: - -* `indicator` -* `ipv4-addr` (with `relationship` to `indicator`) - -Relationship modes: - -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: ipv4-addr is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": "ipv4: ", - "pattern_type": "stix", - "pattern": "[ ipv4-addr:value = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -```json -{ - "type": "ipv4-addr", - "spec_version": "2.1", - "id": "ipv4-addr--", - "value": "" -} -``` - -### stix-mapping: ipv4-addr-port - -3 objects created: - -* `indicator` -* `ipv4-addr` (with `relationship` to `indicator`) -* `network-traffic` - -Relationship modes: - -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: ipv4-addr is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": "ipv4: ", - "pattern_type": "stix", - "pattern": "[ ipv4-addr:value = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -```json -{ - "type": "ipv4-addr", - "spec_version": "2.1", - "id": "ipv4-addr--", - "value": "" -} -``` - -When a port is reported, it could be either a source or destination port. - -Generally, threat intel research reports cover destination ports when reported with an IP (they report on what was seen). Therefore, a conscious decision was made that txt2stix always classifies IPs with port numbers as showing destination ports. -```json -{ - "type": "network-traffic", - "spec_version": "2.1", - "id": "network-traffic--", - "dst_ref": "ipv4-addr--", - "dst_port": "", - "protocols": [ - "ipv4" - ] -} -``` - -### stix-mapping: ipv6-addr - -2 objects created: - -* `indicator` -* `ipv6-addr` (with `relationship` to `indicator`) - -Relationship modes: - -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: ipv6-addr is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": "ipv6: ", - "pattern_type": "stix", - "pattern": "[ ipv6-addr:value = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -```json -{ - "type": "ipv6-addr", - "spec_version": "2.1", - "id": "ipv4-addr--", - "value": "" -} -``` - -### stix-mapping: ipv6-addr-port - -3 objects created: - -* `indicator` -* `ipv6-addr` (with `relationship` to `indicator`) -* `network-traffic` - -Relationship modes: - -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: ipv6-addr is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": "ipv6: ", - "pattern_type": "stix", - "pattern": "[ ipv6-addr:value = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -```json -{ - "type": "ipv6-addr", - "spec_version": "2.1", - "id": "ipv4-addr--", - "value": "" -} -``` - -```json -{ - "type": "network-traffic", - "spec_version": "2.1", - "id": "network-traffic--", - "dst_ref": "ipv6-addr--", - "dst_port": "", - "protocols": [ - "ipv4" - ] -} -``` - -### stix-mapping: domain-name - -2 objects created: - -* `indicator` -* `domain-name` (with `relationship` to `indicator`) - -Relationship modes: - -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: domain-name is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": "Domain: ", - "pattern_type": "stix", - "pattern": "[ domain-name:value = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -```json -{ - "type": "domain-name", - "spec_version": "2.1", - "id": "domain-name--", - "value": "" -} -``` - -### stix-mapping: url - -2 objects created: - -* `indicator` -* `url` (with `relationship` to `indicator`) - -Relationship modes: - -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: url is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": "URL: ", - "pattern_type": "stix", - "pattern": "[ url:value = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -```json -{ - "type": "url", - "spec_version": "2.1", - "id": "url--", - "value": "" -} -``` - -### stix-mapping: file - -2 objects created: - -* `indicator` -* `file` (with `relationship` to `indicator`) - -Relationship modes: - -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: file is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": "File name: ", - "pattern_type": "stix", - "pattern": "[ file:name = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -```json -{ - "type": "file", - "spec_version": "2.1", - "id": "file--", - "name": "" -} -``` - -### stix-mapping: directory - -2 objects created: - -* `indicator` -* `directory` (with `relationship` to `indicator`) - -Relationship modes: -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: directory is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": "Directory: ", - "pattern_type": "stix", - "pattern": "[ directory:path = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -```json -{ - "type": "directory", - "spec_version": "2.1", - "id": "directory--", - "path": "" -} -``` - -### stix-mapping: directory-file - -3 objects created: - -* `indicator` -* `directory` (with `relationship` to `indicator`) -* `file` - -Relationship modes: - -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: directory is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": "Directory: ", - "pattern_type": "stix", - "pattern": "[ directory:path = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -```json -{ - "type": "directory", - "spec_version": "2.1", - "id": "directory--", - "path": "", - -} -``` - -```json -{ - "type": "file", - "spec_version": "2.1", - "id": "file--", - "name": "" -} -``` - -Note, we decided not to add the `parent_directory_ref` to the file object, as we may see the same file in different directories on different txt2stix runs. As the ID contributing properties of the file object do not consider `parent_directory_ref` it means all filenames with different paths will always have the same ID, which is not what we wanted. Thus, we standardise the filenames to always include the same properties, and use an SRO to join it to the directory. - -```json -{ - "type": "relationship", - "spec_version": "2.1", - "id": "relationship--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "relationship_type": "directory", - "source_ref": "file--", - "target_ref": "directory--", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ] -} -``` - -### stix-mapping: file-hash - -2 objects created: - -* `indicator` -* `file` (with `relationship` to `indicator`) - -Relationship modes: - -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: file is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": ": ", - "pattern_type": "stix", - "pattern": "[ file:hashes. = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -```json -{ - "type": "file", - "spec_version": "2.1", - "id": "file--", - "hashes": { - "": "" - } -} -``` - -### stix-mapping: email-addr - -2 objects created: - -* `indicator` -* `email-addr` (with `relationship` to `indicator`) - -Relationship modes: - -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: email-addr is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": "Email Address: ", - "pattern_type": "stix", - "pattern": "[ email-addr:value = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -```json -{ - "type": "email-addr", - "spec_version": "2.1", - "id": "email-addr--", - "value": "" -} -``` - -### stix-mapping: mac-addr - -2 objects created: - -* `indicator` -* `mac-addr` (with `relationship` to `indicator`) - -Relationship modes: +# Relationship modes -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: mac-addr is connected as source or target object (depending on if extraction is source or target) +A user can set the relationship mode at the command line level, depending on the mode set, relationships objects will be created in a certain way as described in this document... -```json -{ - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": "MAC Address: ", - "pattern_type": "stix", - "pattern": "[ mac-addr:value = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` +Regardless of mode, all modes create a relationships json in the format; ```json -{ - "type": "mac-addr", - "spec_version": "2.1", - "id": "mac-addr--", - "value": "" -} +{"relationships":{"relationship_0":{"source_ref":"","target_ref":"","relationship_type":""},"relationship_n":{"source_ref":"","target_ref":"","relationship_type":""}}} ``` -### stix-mapping: windows-registry-key - -2 objects created: - -* `indicator` -* `windows-registry-key` (with `relationship` to `indicator`) - -Relationship modes: +Where: -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: windows-registry-key is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": "Windows Registry Key: ", - "pattern_type": "stix", - "pattern": "[ windows-registry-key:key = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -```json -{ - "type": "windows-registry-key", - "spec_version": "2.1", - "id": "windows-registry-key--", - "key": "" -} -``` - -### stix-mapping: autonomous-system - -2 objects created: - -* `indicator` -* [`autonomous-system`](https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html#_27gux0aol9e3) (with `relationship` to `indicator`) - -Relationship modes: - -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: autonomous-system is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": "AS", - "pattern_type": "stix", - "pattern": "[ autonomous-system:number = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -```json -{ - "type": "autonomous-system", - "spec_version": "2.1", - "id": "autonomous-system--", - "number": "" -} -``` - -### stix-mapping: user-agent - -3 objects created: - -* `indicator` -* `user-agent` (with `relationship` to `indicator`) -* User agent extension definition: https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions/scos/user-agent.json - -Relationship modes: - -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: user-agent is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": "User Agent: ", - "pattern_type": "stix", - "pattern": "[ user-agent:string = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -```json -{ - "type": "user-agent", - "spec_version": "2.1", - "id": "user-agent--", - "string": "", - "extensions": { - "extension-definition--7ca5afee-0e4e-5813-b643-de51538658cc" : { - "extension_type" : "new-sco" - } - } -} -``` - -### stix-mapping: cryptocurrency-wallet - -3 objects created: - -* `indicator` -* `cryptocurrency-wallet` (with `relationship` to `indicator`) -* Crypto wallet extension definition: https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions/scos/cryptocurrency-wallet.json - -Relationship modes: - -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: cryptocurrency-wallet is connected as source or target object (depending on if extraction is source or target) - -```json - { - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": " Wallet: ", - "pattern_type": "stix", - "pattern": "[ cryptocurrency-wallet:address = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -The `cryptocurrency-wallet` object is generated by [crypto2stix](https://github.com/muchdogesec/crypto2stix). - -The crypto2stix equivilant command is; - -```shell -python3 crypto2stix.py --wallet HASH --wallet_only -``` - -Which only generates the wallet object. - -### stix-mapping: cryptocurrency-wallet-with-transaction - -At least 4 objects created: - -* `indicator` -* `cryptocurrency-wallet` (with `relationship` to `indicator`) -* `cryptocurrency-transaction` for all crypto transactions that exist related to the wallet (is not always generated if lookup unsuccessful) -* Crypto wallet extension definition: https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions/scos/cryptocurrency-wallet.json -* Crypto transaction extension definition: https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions/scos/cryptocurrency-transaction.json - -Relationship modes: - -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: cryptocurrency-wallet is connected as source or target object (depending on if extraction is source or target) - -```json - { - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": " Wallet: ", - "pattern_type": "stix", - "pattern": "[ cryptocurrency-wallet:address = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -The `cryptocurrency-transaction` object is generated by [crypto2stix](https://github.com/muchdogesec/crypto2stix). - -The crypto2stix equivilant command is; - -```shell -python3 crypto2stix.py --wallet HASH --transactions_only -``` - -Which generates the wallet object and any transactions associated with it. - -### stix-mapping: cryptocurrency-transaction - -5 objects created: - -* `indicator` -* `cryptocurrency-transaction` (with `relationship` to `indicator`) -* `cryptocurrency-wallet` for wallets seen in transaction identified by crypto2stix (is not always generated if lookup unsuccessful) -* Crypto transaction extension definition: https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions/scos/cryptocurrency-transaction.json -* Crypto wallet extension definition: https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions/scos/cryptocurrency-wallet.json - -Relationship modes: - -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: cryptocurrency-transaction is connected as source or target object (depending on if extraction is source or target) - -```json - { - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": " Transaction: ", - "pattern_type": "stix", - "pattern": "[ cryptocurrency-transaction:hash = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -```json -{ - "type": "cryptocurrency-transaction", - "spec_version": "2.1", - "id": "cryptocurrency-transaction--", - "currency_symbol": "", - "hash": "", - "timestamp": "2022-10-02T15:22:21Z", - "extensions": { - "extension-definition--151d042d-4dcf-5e44-843f-1024440318e5" : { - "extension_type" : "new-sco" - } - } -} -``` - -The `cryptocurrency-transaction` object is generated by [crypto2stix](https://github.com/muchdogesec/crypto2stix). - -The crypto2stix equivilant command is; - -```shell -python3 crypto2stix.py --transaction HASH -``` - -This will also generate all `cryptocurrency-wallets` seen in the transaction. - -### stix-mapping: bank-card - -3 objects created: - -* `indicator` -* `bank-card` (with `relationship` to `indicator`) -* `identity` (with `relationship` to `bank-card`) generated by creditcard2stix (is not always generated if lookup unsuccessful) -* Bank card extension definition: https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions/scos/bank-card.json - -Relationship modes: - -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: bank-card is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": ": ", - "pattern_type": "stix", - "pattern": "[ bank-card:number = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -The `bank-card` object is generated by [creditcard2stix](https://github.com/muchdogesec/creditcard2stix). This will require users to enter an `BIN_LIST_API_KEY` in the `.env` file. - -### stix-mapping: bank-account - -3 objects created: - -* `indicator` -* `bank-account` (with `relationship` to `indicator`) -* Bank account extension definition: https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions/scos/bank-account.json - -Relationship modes: - -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: bank-card is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": "Bank account: ", - "pattern_type": "stix", - "pattern": "[ bank-account:iban_number = '' ]", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -```json -{ - "type": "bank-account", - "spec_version": "2.1", - "id": "bank-account--", - "iban_number": "", - "extensions": { - "extension-definition--f19f3291-6a84-5674-b311-d75a925d5bd9": { - "extension_type" : "new-sco" - } - } -} -``` - -To ensure duplicate `bank-account` objects are not created for the same values, a UUIDv5 address is generated for the ID as follows; - -* Namespace = `00abedb4-aa42-466c-9c01-fed23315a9b7` (this is the default MITRE namespace used in the stix2 python lib https://github.com/oasis-open/cti-python-stix2/blob/50fd81fd6ba4f26824a864319305bc298e89bb45/stix2/base.py#L29) -* Value = `` - -### stix-mapping: phone-number - -3 objects created: - -* `indicator` -* `phone-number` (with `relationship` to `indicator`) -* Phone number extension definition: https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions/scos/phone-number.json - -Relationship modes: - -* Standard relationship SRO: Indicator is connected to Report -* AI mode relationship SROs: phone-number is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "indicator", - "spec_version": "2.1", - "id": "indicator--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "indicator_types": [ - "unknown" - ], - "name": "Phone Number: ", - "pattern_type": "stix", - "pattern": "[ phone-number:number = ''", - "valid_from": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -```json -{ - "type": "phone-number", - "spec_version": "2.1", - "id": "phone-number--", - "number": "", - "extensions": { - "extension-definition--14a97ee2-e666-5ada-a6bd-b7177f79e211" : { - "extension_type" : "new-sco" - } - } -} -``` - -To ensure duplicate `phone-number` objects are not created for the same values, a UUIDv5 address is generated for the ID as follows; - -* Namespace = `00abedb4-aa42-466c-9c01-fed23315a9b7` (this is the default MITRE namespace used in the stix2 python lib https://github.com/oasis-open/cti-python-stix2/blob/50fd81fd6ba4f26824a864319305bc298e89bb45/stix2/base.py#L29) -* Value = `` - -### stix-mapping: attack-pattern - -1 object created: - -* `attack-pattern` - -Relationship modes: - -* Standard relationship SRO: Attack Pattern is connected to Report -* AI mode relationship SROs: attack-pattern is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "attack-pattern", - "spec_version": "2.1", - "id": "campaign--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "name": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -### stix-mapping: campaign - -1 object created: - -* `campaign` - -Relationship modes: - -* Standard relationship SRO: Campaign is connected to Report -* AI mode relationship SROs: campaign is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "campaign", - "spec_version": "2.1", - "id": "campaign--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "name": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -### stix-mapping: course-of-action - -1 object created: - -* `course-of-action` - -Relationship modes: - -* Standard relationship SRO: Course of Action is connected to Report -* AI mode relationship SROs: course-of-action is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "course-of-action", - "spec_version": "2.1", - "id": "course-of-action--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "name": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -### stix-mapping: infrastructure - -1 object created: - -* `infrastructure` - -Relationship modes: - -* Standard relationship SRO: Infrastructure is connected to Report -* AI mode relationship SROs: infrastructure is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type":"infrastructure", - "spec_version": "2.1", - "id":"infrastructure--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "name": "", - "infrastructure_types": ["unknown"], - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -### stix-mapping: intrusion-set - -1 object created: - -* `intrusion-set` - -Relationship modes: - -* Standard relationship SRO: Intrusion Set is connected to Report -* AI mode relationship SROs: intrusion-set is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "intrusion-set", - "spec_version": "2.1", - "id": "intrusion-set--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "name": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -### stix-mapping: malware - -1 object created: - -* `malware` - -Relationship modes: - -* Standard relationship SRO: Malware is connected to Report -* AI mode relationship SROs: malware is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "malware", - "spec_version": "2.1", - "id": "malware--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "name": "", - "malware_types": ["unknown"], - "is_family": true, - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -### stix-mapping: threat-actor - -1 object created: - -* `threat-actor` - -Relationship modes: - -* Standard relationship SRO: Threat Actor is connected to Report -* AI mode relationship SROs: threat-actor is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "threat-actor", - "spec_version": "2.1", - "id": "threat-actor--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "name": "", - "threat_actor_types": "unknown", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -### stix-mapping: tool - -1 object created: - -* `tool` - -Relationship modes: - -* Standard relationship SRO: Tool is connected to Report -* AI mode relationship SROs: tool is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "tool", - "spec_version": "2.1", - "id": "tool--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "name": "", - "tool_types": "unknown", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -### stix-mapping: identity - -1 object created: - -* `identity` - -Relationship modes: - -* Standard relationship SRO: Identity is connected to Report -* AI mode relationship SROs: identity is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "identity", - "spec_version": "2.1", - "id": "identity--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "name": "", - "identity_class": "unspecified", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -### stix-mapping: location - -1 object created: - -* `location` - -Relationship modes: - -* Standard relationship SRO: Location is connected to Report -* AI mode relationship SROs: location is connected as source or target object (depending on if extraction is source or target) - -```json -{ - "type": "location", - "spec_version": "2.1", - "id": "location--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "name": "Country: ", - "country": "", - "object_marking_refs": [ - "marking-definition--", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - }, - { - "source_name": "txt2stix extraction ID", - "external_id": "_" - } - ] -} -``` - -### Externally generated STIX objects - -Some objects created for extractions do not need to be generated by txt2stix, they can be looked up from an external database. - -txt2stix is designed to work with a store of intellignece in ArangoDB uploaded using stix2arango: https://github.com/signalscorps/stix2arango - -The data is expected to be imported using: https://github.com/signalscorps/stix2arango/blob/main/design/mvp/backfill.md - -If no ArangoDB values are set in the `.env` file, the following extractions will not work. - -All request to the ArangoDB API require a JWT. This can be obtained - -```shell -curl -X 'POST' \ - 'http://:/_open/auth' \ - -H 'accept: application/json' \ - -H 'Content-Type: application/json' \ - -d '{ - "username": "", - "password": "" -}' -``` - -Respone will be in format; - -```shell -{"jwt":""} -``` - -The token will need to be renewed with every session. - -The actual query (shown below for each extraction) can then be executed as follows; - -```shell -curl -X 'POST' \ - 'http://:/_db//_api/cursor' \ - -H 'accept: application/json' \ - -H 'Content-Type: application/json' \ - -H 'Authorization: bearer ' \ - -d '{ "query" : "" }' -``` - -This will return the actual STIX object in the `result` property. - -The queries set are always designed to return one result, so the printed STIX object can be imported directly (we also set `"batchSize": 1` for the same reason, but this is not really needed) - -### stix-mapping: mitre-attack-enterprise-id - -N object created (some IDs match multiple objects): - -txt2stix extracts Mitre ATT&CK IDs. These can be passed as `` in the query below; - -```sql -FOR doc IN mitre_attack_enterprise_vertex_collection - FILTER IS_ARRAY(doc.external_references) - FOR external_references IN doc.external_references - FILTER external_references.external_id == '' - RETURN UNSET(doc, '_key', '_bundle_id', '_file_name', '_id', '_is_latest', '_record_created', '_record_md5_hash', '_record_modified', '_rev', '_stix2arango_note') -``` - -Relationship modes: - -* Standard relationship SRO: Imported object(s) is connected to Report -* AI mode relationship SROs: all imported objects are connected as source or target object (depending on if extraction is source or target) - -### stix-mapping: mitre-attack-mobile-id - -N object created (some IDs match multiple objects): - -txt2stix extracts Mitre ATT&CK IDs. These can be passed as `` in the query below; - -```sql -FOR doc IN mitre_attack_mobile_vertex_collection - FILTER IS_ARRAY(doc.external_references) - FOR external_references IN doc.external_references - FILTER external_references.external_id == '' - RETURN UNSET(doc, '_key', '_bundle_id', '_file_name', '_id', '_is_latest', '_record_created', '_record_md5_hash', '_record_modified', '_rev', '_stix2arango_note') -``` - -Relationship modes: - -* Standard relationship SRO: Imported object(s) is connected to Report -* AI mode relationship SROs: all imported objects are connected as source or target object (depending on if extraction is source or target) - -### stix-mapping: mitre-attack-ics-id - -N object created (some IDs match multiple objects): - -txt2stix extracts Mitre ATT&CK IDs. These can be passed as `` in the query below; - -```sql -FOR doc IN mitre_attack_ics_vertex_collection - FILTER IS_ARRAY(doc.external_references) - FOR external_references IN doc.external_references - FILTER external_references.external_id == '' - RETURN UNSET(doc, '_key', '_bundle_id', '_file_name', '_id', '_is_latest', '_record_created', '_record_md5_hash', '_record_modified', '_rev', '_stix2arango_note') -``` - -Relationship modes: - -* Standard relationship SRO: Imported object(s) is connected to Report -* AI mode relationship SROs: all imported objects are c - -### stix-mapping: mitre-capec-id - -1 object created: - -These can be passed as `` in the query below; - -```sql -FOR doc IN mitre_capec_vertex_collection - FILTER doc.type == 'attack-pattern' - AND IS_ARRAY(doc.external_references) - FOR external_references IN doc.external_references - FILTER external_references.external_id == '' - RETURN UNSET( doc, '_key', '_bundle_id', '_file_name', '_id', '_is_latest', '_record_created', '_record_md5_hash', '_record_modified', '_rev', '_stix2arango_note') -``` - -Relationship modes: - -* Standard relationship SRO: Imported object is connected to Report -* AI mode relationship SROs: all imported objects are connected as source or target object (depending on if extraction is source or target) - -### stix-mapping: mitre-cwe-id - -2 object created: - -* Weakness extension definition: https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions/sdos/weakness.json -* The weakness object described below - -These can be passed as `` in the query below; - -```sql -FOR doc IN mitre_cwe_vertex_collection - FILTER doc.type == 'weakness' - AND IS_ARRAY(doc.external_references) - FOR external_references IN doc.external_references - FILTER external_references.external_id == '' - RETURN UNSET( doc, '_key', '_bundle_id', '_file_name', '_id', '_is_latest', '_record_created', '_record_md5_hash', '_record_modified', '_rev', '_stix2arango_note') -``` - -Relationship modes: - -* Standard relationship SRO: Imported object is connected to Report -* AI mode relationship SROs: all imported objects are connected as source or target object (depending on if extraction is source or target) - -### stix-mapping: cve-id - -1 object created: - -These can be passed as `` in the query below; - -```sql -FOR doc IN nvd_cve_vertex_collection - FILTER doc.type == 'vulnerability' - AND doc.name == '' - RETURN UNSET( doc, '_key', '_bundle_id', '_file_name', '_id', '_is_latest', '_record_created', '_record_md5_hash', '_record_modified', '_rev', '_stix2arango_note') -``` - -Relationship modes: - -* Standard relationship SRO: Imported object is connected to Report -* AI mode relationship SROs: all imported objects are connected as source or target object (depending on if extraction is source or target) - -### stix-mapping: cpe-id - -1 object created: - -These can be passed as `` in the query below; - -```sql -FOR doc IN nvd_cpe_vertex_collection - FILTER doc.type == 'software' - AND doc.cpe == '' - RETURN UNSET( doc, '_key', '_bundle_id', '_file_name', '_id', '_is_latest', '_record_created', '_record_md5_hash', '_record_modified', '_rev', '_stix2arango_note') -``` - -Relationship modes: - -* Standard relationship SRO: Imported object is connected to Report -* AI mode relationship SROs: all imported objects are connected as source or target object (depending on if extraction is source or target) - -## Bundle (output) - -The output of txt2stix is a STIX bundle file. - -This bundle takes the format; - -```json -{ - "type": "bundle", - "id": "bundle--", - "objects": [ - "" - ] -} -``` - -The objects include all SROs generated for the input. - -The filename of the bundle takes the format: `bundle--.json` - -# Relationship modes - -A user can set the relationship mode at the command line level, depending on the mode set, relationships objects will be created in a certain way as described in this document... - -Regardless of mode, all modes create a relationships json in the format; - -```json -{"relationships":{"relationship_0":{"source_ref":"","target_ref":"","relationship_type":""},"relationship_n":{"source_ref":"","target_ref":"","relationship_type":""}}} -``` - -Where: - -* `source_ref`: is the id for the source extraction for the relationship (e.g. extraction_1). -* `target_ref`: is the index for the target extraction for the relationship (e.g. extraction_2). -* relationship_type: is a description of the relationship between target and source. +* `source_ref`: is the id for the source extraction for the relationship (e.g. extraction_1). +* `target_ref`: is the index for the target extraction for the relationship (e.g. extraction_2). +* relationship_type: is a description of the relationship between target and source. Only one type of relationship mode can be used per script run, so only one relationship json is ever produced. -## Standard relationship mode - -In standard mode, only one SRO is created for each extraction back to the source report object created for the job. Even so, the script still creates a relationships JSON for the job (but it will contain no data because no relationships between extracted data). - -Read the extraction definition in extraction-commons.md to see what objects for an extraction (because there are often >1) that will be linked back to the original report at this step. - -### STIX SROs - -```json -{ - "type": "relationship", - "spec_version": "2.1", - "id": "relationship--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "relationship_type": "extracted-from", - "source_ref": "", - "target_ref": "report--", - "object_marking_refs": [ - "marking-definition--" - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - } - ] -} -``` - -## AI relationship mode - -This mode takes the extractions generated and passes them, along with the text of the report input (after aliases applied). - -Which is then used in a series of prompts... - -### Prompts - -#### Prompt 1 (skipped if NO `ai_` extractions applied) - -This simply provides an instruction to the AI to remember the text. It includes a full copy of the txt file uploaded by the user AFTER any aliases have been applied. - -It asks the AI to provide a one word response, either "successful" or "unsuccessful". If unsuccessful returned the script will fail with an error. If successful, the script will continue. - -It is the same as prompt 1 for extractions of `ai_` type, thus this prompt IS NOT used if any AI extraction performed as this prompt will have already been sent (and resending will use up more tokens when not required). - -#### Prompt 2 (skipped if ONLY `ai_` extractions applied) - -Prompt 2 passed the final extractions JSON and asks the prompt to remember the extractions for the next step. - -It asks the AI to provide a one word response, either "successful" or "unsuccessful". If unsuccessful returned the script will fail with an error. If successful, the script will continue. - -If ONLY `ai_` extractions used (that is, NO lookup or pattern extractions done) then this prompt is skipped, as the AI will already have the full extractions JSON file it created. - -#### Prompt 3 - -Assuming the word "successful" is returned from prompt 2, the script asks the AI to produce ONLY a structured relationship JSON output containing information about links between extractions. - -In this prompt, the list of supported relationship types, defined in `helpers/stix_relationship_types.txt`, are also passed to the AI to limit the `relationship_type` descriptions it makes. - -### STIX SROs - -The LLM generates a JSON file describing how objects created from extractions are connected. - -Keep in mind, an extraction generally, but not always creates at least two non-relationship objects, an SDO and an SCO. You should read the extraction definition in extraction-commons.md to see what objects will be linked using the relationship for an extraction. - -The Relationship will be generated as follows; - -```json -{ - "type": "relationship", - "spec_version": "2.1", - "id": "relationship--", - "created_by_ref": "identity--", - "created": "", - "modified": "", - "relationship_type": "", - "source_ref": "", - "target_ref": "", - "object_marking_refs": [ - "marking-definition--" - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" - ], - "external_references": [ - { - "source_name": "txt2stix job ID", - "external_id": "" - } - ] -} -``` - -Note, a source object can be linked to many target objects in the same report. Thus, a source object might create many SROs, representing links to target objects. - -Similarly, in the case of MITRE ATT&CK an extraction might import one or more STIX objects. In this case, an SRO should be created for each imported object to the source/target object identified. \ No newline at end of file diff --git a/docs/stix-mapping.md b/docs/stix-mapping.md new file mode 100644 index 0000000..7a9443c --- /dev/null +++ b/docs/stix-mapping.md @@ -0,0 +1,1959 @@ +# STIX mapping + +## Standard Objects + +### TLPs SMOs + +At script run time user can set `tlp_level`; either white, green, amber or red. + +Each mode maps to a STIX marking definition object; + +* Clear: `marking-definition--94868c89-83c2-464b-929b-a1a8aa3c8487` +* Green: `marking-definition--bab4a63c-aed9-4cf5-a766-dfca5abac2bb` +* Amber: `marking-definition--55d920b0-5e8b-4f79-9ee9-91f868d9b421` +* Amber+Strict: `marking-definition--939a9414-2ddd-4d32-a0cd-375ea402b003` +* Red: `marking-definition--e828b379-4e03-4974-9ac4-e53a884c97c1` + +Depending on the value set by user, the generated STIX objects will contain a `marking-definition--` reference in the `object_marking_refs` field to the corresponding TLP level. + +### Marking definitions + +All objects created by txt2stix also have a standard marking definition in the `object_marking_refs` property. + +The default STIX 2.1 marking definition for txt2stix is always used and imported from https://raw.githubusercontent.com/muchdogesec/stix4doge/main/objects/marking-definition/txt2stix.json + +This object will be printed in the final bundle, and in all objects generated by txt2stix in the `object_marking_refs` property. + +### Identities (`identity`) SDOs + +txt2stix assigns a `created_by_ref` property to all SDOs and SROs it creates to their `created_by_ref` field. + +The default STIX 2.1 identity for txt2stix is imported from https://raw.githubusercontent.com/muchdogesec/stix4doge/main/objects/identity/txt2stix.json + +This object will be printed in the final bundle, and the ID will be used for all `created_by_ref` generated by txt2stix. + +Often a user will want to use a custom ID of their own to generate objects. They can do this using the `use_identity` flag in the CLI. + +Two things will happen if a custom identity is used in this way. The identity JSON object will be printed in the final bundle, and the ID will be used for all `created_by_ref` generated by txt2stix. + +### Report SDO (`report`) + +All files uploaded are represented as a unique [STIX Report SDO](https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html#_n8bjzg1ysgdq) that take the following structure; + +```json +{ + "type": "report", + "spec_version": "2.1", + "id": "report--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "name": "", + "description": "", + "confidence": "", + "published": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "labels": [ + "" + ], + "external_references": [ + { + "source_name": "txt2stix Report MD5", + "external_id": "" + } + ], + "object_refs": [ + "" + ] +} +``` + +Note, the `object_refs` contains all references that are referenced by objects in the report (SDOs, SROs, SCOs, marking definitions, etc.). This includes extracted objects (i.e. Indicator SDOs, Vulnerability SDOs, Software SCOs, Relationship SROs etc.). The only object it does not include is the Note object created automatically. + +`` should match the report object UUID, e.g. if this was + +## STIX Mapping (locally created objects) + +All extractions detected by txt2stix are converted to STIX. + +In the extractions config (`includes/extractions/ai|lookup|pattern/config.yaml)`, a `stix-mapping` property exists for all extractions. + +Below shows how the data extracted is mapped to a set of STIX objects for each `stix-mapping` type. + +### stix-mapping: `ipv4-addr` + +Objects always created: + +* `indicator` +* `ipv4-addr` +* `relationship` (`ipv4-addr` `related-to` `indicator`) + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: ipv4-addr is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": "ipv4: ", + "pattern_type": "stix", + "pattern": "[ ipv4-addr:value = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +```json +{ + "type": "ipv4-addr", + "spec_version": "2.1", + "id": "ipv4-addr--", + "value": "" +} +``` + +### stix-mapping: `ipv4-addr-port` + +Objects always created: + +* `indicator` +* `ipv4-addr` +* `relationship` (`ipv4-addr` `related-to` `indicator`) +* `network-traffic` + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: ipv4-addr is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": "ipv4: ", + "pattern_type": "stix", + "pattern": "[ ipv4-addr:value = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +```json +{ + "type": "ipv4-addr", + "spec_version": "2.1", + "id": "ipv4-addr--", + "value": "" +} +``` + +When a port is reported, it could be either a source or destination port. + +Generally, threat intel research reports cover destination ports when reported with an IP (they report on what was seen). Therefore, a conscious decision was made that txt2stix always classifies IPs with port numbers as showing destination ports. + +```json +{ + "type": "network-traffic", + "spec_version": "2.1", + "id": "network-traffic--", + "dst_ref": "ipv4-addr--", + "dst_port": "", + "protocols": [ + "ipv4" + ] +} +``` + +### stix-mapping: `ipv6-addr` + +Objects always created: + +* `indicator` +* `ipv6-addr` +* `relationship` (`ipv6-addr` `related-to` `indicator`) + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: ipv6-addr is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": "ipv6: ", + "pattern_type": "stix", + "pattern": "[ ipv6-addr:value = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +```json +{ + "type": "ipv6-addr", + "spec_version": "2.1", + "id": "ipv4-addr--", + "value": "" +} +``` + +### stix-mapping: `ipv6-addr-port` + +Objects always created: + +* `indicator` +* `ipv6-addr` +* `relationship` (`ipv6-addr` `related-to` `indicator`) +* `network-traffic` + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: ipv6-addr is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": "ipv6: ", + "pattern_type": "stix", + "pattern": "[ ipv6-addr:value = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +```json +{ + "type": "ipv6-addr", + "spec_version": "2.1", + "id": "ipv4-addr--", + "value": "" +} +``` + +```json +{ + "type": "network-traffic", + "spec_version": "2.1", + "id": "network-traffic--", + "dst_ref": "ipv6-addr--", + "dst_port": "", + "protocols": [ + "ipv4" + ] +} +``` + +### stix-mapping: `domain-name` + +Objects always created: + +* `indicator` +* `domain-name` +* `relationship` (`domain-name` `related-to` `indicator`) + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: domain-name is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": "Domain: ", + "pattern_type": "stix", + "pattern": "[ domain-name:value = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +```json +{ + "type": "domain-name", + "spec_version": "2.1", + "id": "domain-name--", + "value": "" +} +``` + +### stix-mapping: `url` + +Objects always created: + +* `indicator` +* `url` +* `relationship` (`url` `related-to` `indicator`) + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: url is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": "URL: ", + "pattern_type": "stix", + "pattern": "[ url:value = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +```json +{ + "type": "url", + "spec_version": "2.1", + "id": "url--", + "value": "" +} +``` + +### stix-mapping: `file` + +Objects always created: + +* `indicator` +* `file` +* `relationship` (`file` `related-to` `indicator`) + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: file is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": "File name: ", + "pattern_type": "stix", + "pattern": "[ file:name = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +```json +{ + "type": "file", + "spec_version": "2.1", + "id": "file--", + "name": "" +} +``` + +### stix-mapping: `directory` + +Objects always created: + +* `indicator` +* `directory` +* `relationship` (`directory` `related-to` `indicator`) + +Relationship mode object generation behaviour: + +* Standard relationship SRO:`indicator` `extracted-from` `report` +* AI mode relationship SROs: directory is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": "Directory: ", + "pattern_type": "stix", + "pattern": "[ directory:path = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +```json +{ + "type": "directory", + "spec_version": "2.1", + "id": "directory--", + "path": "" +} +``` + +### stix-mapping: `directory-file` + +Objects always created: + +* `indicator` +* `directory` +* `relationship` (`directory` `related-to` `indicator`) +* `file` + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: directory is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": "Directory: ", + "pattern_type": "stix", + "pattern": "[ directory:path = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +```json +{ + "type": "directory", + "spec_version": "2.1", + "id": "directory--", + "path": "", + +} +``` + +```json +{ + "type": "file", + "spec_version": "2.1", + "id": "file--", + "name": "" +} +``` + +Note, we decided not to add the `parent_directory_ref` to the file object, as we may see the same file in different directories on different txt2stix runs. As the ID contributing properties of the file object do not consider `parent_directory_ref` it means all filenames with different paths will always have the same ID, which is not what we wanted. Thus, we standardise the filenames to always include the same properties, and use an SRO to join it to the directory. + +```json +{ + "type": "relationship", + "spec_version": "2.1", + "id": "relationship--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "relationship_type": "directory", + "source_ref": "file--", + "target_ref": "directory--", + "description": " is in the directory ", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ] +} +``` + +### stix-mapping: `file-hash` + +Objects always created: + +* `indicator` +* `file` +* `relationship` (`file` `related-to` `indicator`) + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: file is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": ": ", + "pattern_type": "stix", + "pattern": "[ file:hashes. = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +```json +{ + "type": "file", + "spec_version": "2.1", + "id": "file--", + "hashes": { + "": "" + } +} +``` + +### stix-mapping: `email-addr` + +Objects always created: + +* `indicator` +* `email-addr` +* `relationship` (`email-addr` `related-to` `indicator`) + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: email-addr is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": "Email Address: ", + "pattern_type": "stix", + "pattern": "[ email-addr:value = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +```json +{ + "type": "email-addr", + "spec_version": "2.1", + "id": "email-addr--", + "value": "" +} +``` + +### stix-mapping: `mac-addr` + +Objects always created: + +* `indicator` +* `mac-addr` +* `relationship` (`mac-addr` `related-to` `indicator`) + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: mac-addr is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": "MAC Address: ", + "pattern_type": "stix", + "pattern": "[ mac-addr:value = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +```json +{ + "type": "mac-addr", + "spec_version": "2.1", + "id": "mac-addr--", + "value": "" +} +``` + +### stix-mapping: `windows-registry-key` + +Objects always created: + +* `indicator` +* `windows-registry-key` +* `relationship` (`windows-registry-key` `related-to` `indicator`) + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: windows-registry-key is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": "Windows Registry Key: ", + "pattern_type": "stix", + "pattern": "[ windows-registry-key:key = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +```json +{ + "type": "windows-registry-key", + "spec_version": "2.1", + "id": "windows-registry-key--", + "key": "" +} +``` + +### stix-mapping: `autonomous-system` + +Objects always created: + +* `indicator` +* `autonomous-system` +* `relationship` (`autonomous-system` `related-to` `indicator`) + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: autonomous-system is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": "AS", + "pattern_type": "stix", + "pattern": "[ autonomous-system:number = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +```json +{ + "type": "autonomous-system", + "spec_version": "2.1", + "id": "autonomous-system--", + "number": "" +} +``` + +### stix-mapping: `user-agent` + +Objects always created: + +* `indicator` +* `user-agent` +* `relationship` (`user-agent` `related-to` `indicator`) +* `extensions-definition`: https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions/scos/user-agent.json + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: user-agent is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": "User Agent: ", + "pattern_type": "stix", + "pattern": "[ user-agent:string = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +```json +{ + "type": "user-agent", + "spec_version": "2.1", + "id": "user-agent--", + "string": "", + "extensions": { + "extension-definition--7ca5afee-0e4e-5813-b643-de51538658cc" : { + "extension_type" : "new-sco" + } + } +} +``` + +### stix-mapping: `cryptocurrency-wallet` + +Objects always created: + +* `indicator` +* `cryptocurrency-wallet` +* `relationship` (`cryptocurrency-wallet` `related-to` `indicator`) +* `extensions-definition`: https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions/scos/cryptocurrency-wallet.json + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: cryptocurrency-wallet is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json + { + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": " Wallet: ", + "pattern_type": "stix", + "pattern": "[ cryptocurrency-wallet:address = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +The `cryptocurrency-wallet` object is generated by [crypto2stix](https://github.com/muchdogesec/crypto2stix). + +The crypto2stix equivilant command is; + +```shell +python3 crypto2stix.py --wallet HASH --wallet_only +``` + +Which only generates the wallet object. + +### stix-mapping: `cryptocurrency-wallet-with-transaction` + +Objects always created: + +* `indicator` +* `cryptocurrency-wallet` +* `relationship` (`cryptocurrency-wallet` `related-to` `indicator`) +* `cryptocurrency-transaction` for all crypto transactions that exist related to the wallet (is not always generated if lookup unsuccessful) +* `extensions-definition`: https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions/scos/cryptocurrency-wallet.json +* `extensions-definition`: https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions/scos/cryptocurrency-transaction.json + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: cryptocurrency-wallet is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json + { + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": " Wallet: ", + "pattern_type": "stix", + "pattern": "[ cryptocurrency-wallet:address = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +The `cryptocurrency-transaction` object is generated by [crypto2stix](https://github.com/muchdogesec/crypto2stix). + +The crypto2stix equivilant command is; + +```shell +python3 crypto2stix.py --wallet HASH --transactions_only +``` + +Which generates the wallet object and any transactions associated with it. + +### stix-mapping: `cryptocurrency-transaction` + +Objects always created: + +* `indicator` +* `cryptocurrency-transaction` +* `relationship` (`cryptocurrency-transaction` `related-to` `indicator`) +* `cryptocurrency-wallet` for wallets seen in transaction identified by crypto2stix (is not always generated if lookup unsuccessful) +* `extensions-definition`: https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions/scos/cryptocurrency-transaction.json +* `extensions-definition`: https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions/scos/cryptocurrency-wallet.json + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: cryptocurrency-transaction is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json + { + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": " Transaction: ", + "pattern_type": "stix", + "pattern": "[ cryptocurrency-transaction:hash = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +```json +{ + "type": "cryptocurrency-transaction", + "spec_version": "2.1", + "id": "cryptocurrency-transaction--", + "currency_symbol": "", + "hash": "", + "timestamp": "2022-10-02T15:22:21Z", + "extensions": { + "extension-definition--151d042d-4dcf-5e44-843f-1024440318e5" : { + "extension_type" : "new-sco" + } + } +} +``` + +The `cryptocurrency-transaction` object is generated by [crypto2stix](https://github.com/muchdogesec/crypto2stix). + +The crypto2stix equivilant command is; + +```shell +python3 crypto2stix.py --transaction HASH +``` + +This will also generate all `cryptocurrency-wallets` seen in the transaction. + +### stix-mapping: `bank-card` + +Objects always created: + +* `indicator` +* `bank-card` (with `relationship` to `indicator`) +* `relationship` (`bank-card` `related-to` `indicator`) +* `identity` (with `relationship` to `bank-card`) generated by creditcard2stix (is not always generated if lookup unsuccessful) +* `extension-definition`: https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions/scos/bank-card.json + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: bank-card is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": ": ", + "pattern_type": "stix", + "pattern": "[ bank-card:number = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +The `bank-card` object is generated by [creditcard2stix](https://github.com/muchdogesec/creditcard2stix). This will require users to enter an `BIN_LIST_API_KEY` in the `.env` file. + +### stix-mapping: `bank-account` + +Objects always created: + +* `indicator` +* `bank-account` +* `relationship` (`bank-card` `related-to` `indicator`) +* `extension-definition`: https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions/scos/bank-account.json + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: bank-card is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": "Bank account: ", + "pattern_type": "stix", + "pattern": "[ bank-account:iban_number = '' ]", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +```json +{ + "type": "bank-account", + "spec_version": "2.1", + "id": "bank-account--", + "iban_number": "", + "extensions": { + "extension-definition--f19f3291-6a84-5674-b311-d75a925d5bd9": { + "extension_type" : "new-sco" + } + } +} +``` + +To ensure duplicate `bank-account` objects are not created for the same values, a UUIDv5 address is generated for the ID as follows; + +* Namespace = `00abedb4-aa42-466c-9c01-fed23315a9b7` (this is the default MITRE namespace used in the stix2 python lib https://github.com/oasis-open/cti-python-stix2/blob/50fd81fd6ba4f26824a864319305bc298e89bb45/stix2/base.py#L29) +* Value = `` + +### stix-mapping: `phone-number` + +Objects always created: + +* `indicator` +* `phone-number` +* `relationship` (`phone-number` `related-to` `indicator`) +* `extension-definition`: https://raw.githubusercontent.com/muchdogesec/stix2extensions/main/extension-definitions/scos/phone-number.json + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `indicator` `extracted-from` `report` +* AI mode relationship SROs: phone-number is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "indicator_types": [ + "unknown" + ], + "name": "Phone Number: ", + "pattern_type": "stix", + "pattern": "[ phone-number:number = ''", + "valid_from": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +```json +{ + "type": "phone-number", + "spec_version": "2.1", + "id": "phone-number--", + "number": "", + "extensions": { + "extension-definition--14a97ee2-e666-5ada-a6bd-b7177f79e211" : { + "extension_type" : "new-sco" + } + } +} +``` + +To ensure duplicate `phone-number` objects are not created for the same values, a UUIDv5 address is generated for the ID as follows; + +* Namespace = `00abedb4-aa42-466c-9c01-fed23315a9b7` (this is the default MITRE namespace used in the stix2 python lib https://github.com/oasis-open/cti-python-stix2/blob/50fd81fd6ba4f26824a864319305bc298e89bb45/stix2/base.py#L29) +* Value = `` + +### stix-mapping: `attack-pattern` + +Objects created: + +* `attack-pattern` + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `attack-pattern` `extracted-from` `report` +* AI mode relationship SROs: attack-pattern is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "attack-pattern", + "spec_version": "2.1", + "id": "campaign--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "name": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +### stix-mapping: `campaign` + +Objects created: + +* `campaign` + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `campaign` `extracted-from` `report` +* AI mode relationship SROs: campaign is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "campaign", + "spec_version": "2.1", + "id": "campaign--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "name": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +### stix-mapping: `course-of-action` + +Objects created: + +* `course-of-action` + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `course-of-action` `extracted-from` `report` +* AI mode relationship SROs: course-of-action is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "course-of-action", + "spec_version": "2.1", + "id": "course-of-action--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "name": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +### stix-mapping: `infrastructure` + +Objects created: + +* `infrastructure` + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `infrastructure` `extracted-from` `report` +* AI mode relationship SROs: infrastructure is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type":"infrastructure", + "spec_version": "2.1", + "id":"infrastructure--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "name": "", + "infrastructure_types": ["unknown"], + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +### stix-mapping: `intrusion-set` + +Objects created: + +* `intrusion-set` + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `intrusion-set` `extracted-from` `report` +* AI mode relationship SROs: intrusion-set is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "intrusion-set", + "spec_version": "2.1", + "id": "intrusion-set--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "name": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +### stix-mapping: `malware` + +Objects created: + +* `malware` + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `malware` `extracted-from` `report` +* AI mode relationship SROs: malware is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "malware", + "spec_version": "2.1", + "id": "malware--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "name": "", + "malware_types": ["unknown"], + "is_family": true, + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +### stix-mapping: `threat-actor` + +Objects created: + +* `threat-actor` + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `threat-actor` `extracted-from` `report` +* AI mode relationship SROs: threat-actor is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "threat-actor", + "spec_version": "2.1", + "id": "threat-actor--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "name": "", + "threat_actor_types": "unknown", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +### stix-mapping: `tool` + +Objects created: + +* `tool` + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `tool` `extracted-from` `report` +* AI mode relationship SROs: tool is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "tool", + "spec_version": "2.1", + "id": "tool--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "name": "", + "tool_types": "unknown", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +### stix-mapping: `identity` + +Objects created: + +* `identity` + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `identity` `extracted-from` `report` +* AI mode relationship SROs: identity is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "identity", + "spec_version": "2.1", + "id": "identity--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "name": "", + "identity_class": "unspecified", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +### stix-mapping: `location` + +Objects created: + +* `location` + +Relationship mode object generation behaviour: + +* Standard relationship SRO: `location` `extracted-from` `report` +* AI mode relationship SROs: location is connected as `source_ref` or `target_ref` to 0 or more objects based on AI analysis + +```json +{ + "type": "location", + "spec_version": "2.1", + "id": "location--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "name": "Country: ", + "country": "", + "object_marking_refs": [ + "marking-definition--", + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + }, + { + "source_name": "txt2stix extraction ID", + "external_id": "_" + } + ] +} +``` + +## STIX Mapping (remote created objects) + +Some objects created for extractions do not need to be generated by txt2stix, they can be looked up from an external databases. + +### stix-mapping: `ctibutler-mitre-attack-enterprise-id` + +Takes the extracted ID and passes it to + +```shell +GET CTIBUTLER_HOST/api/v1/attack-enterprise/objects/:attack_id/ +``` + +`CTIBUTLER_APIKEY` in request passed if set. + +All the objects returned are imported. + +Relationship mode object generation behaviour: + +* Standard relationship SRO: Imported object(s) is (`source_ref`) object connected to Report with type `extracted-from` +* AI mode relationship SROs: all imported objects are connected as source or target object (depending on if extraction is source or target) + +### stix-mapping: `ctibutler-mitre-attack-mobile-id` + +Takes the extracted ID and passes it to + +```shell +GET CTIBUTLER_HOST/api/v1/attack-mobile/objects/:attack_id/ +``` + +All the objects returned are imported. + +Relationship mode object generation behaviour: + +* Standard relationship SRO: Imported object(s) is (`source_ref`) object connected to Report with type `extracted-from` +* AI mode relationship SROs: all imported objects are connected as source or target object (depending on if extraction is source or target) + +### stix-mapping: `ctibutler-mitre-attack-ics-id` + +Takes the extracted ID and passes it to + +```shell +GET CTIBUTLER_HOST/api/v1/attack-ics/objects/:attack_id/ +``` + +`CTIBUTLER_APIKEY` in request passed if set. + +All the objects returned are imported. + +Relationship mode object generation behaviour: + +* Standard relationship SRO: Imported object(s) is (`source_ref`) object connected to Report with type `extracted-from` +* AI mode relationship SROs: all imported objects are connected as source or target object (depending on if extraction is source or target) + +### stix-mapping: `ctibutler-mitre-capec-id` + +Takes the extracted ID and passes it to + +```shell +GET CTIBUTLER_HOST/api/v1/capec/objects/:capec_id/ +``` + +`CTIBUTLER_APIKEY` in request passed if set. + +All the objects returned are imported. + +Relationship mode object generation behaviour: + +* Standard relationship SRO: Imported object(s) is (`source_ref`) object connected to Report with type `extracted-from` +* AI mode relationship SROs: all imported `course-of-action` / `attack-pattern` objects are connected as source or target object (depending on if extraction is source or target) + +### stix-mapping: `ctibutler-mitre-cwe-id` + +Takes the extracted ID and passes it to + +```shell +GET CTIBUTLER_HOST/api/v1/cwe/objects/:cwe_id/ +``` + +`CTIBUTLER_APIKEY` in request passed if set. + +Relationship mode object generation behaviour: + +* Standard relationship SRO: Imported `weakness` object is (`source_ref`) object connected to Report +* AI mode relationship SROs: all imported objects are connected as source or target object (depending on if extraction is source or target) + +### stix-mapping: `ctibutler-mitre-atlas-id` + +Takes the extracted ID and passes it to + +```shell +GET CTIBUTLER_HOST/api/v1/atlas/objects/:atlas_id/ +``` + +`CTIBUTLER_APIKEY` in request passed if set. + +All the objects returned are imported. + +Relationship mode object generation behaviour: + +* Standard relationship SRO: Imported object(s) is (`source_ref`) object connected to Report with type `extracted-from` +* AI mode relationship SROs: all imported objects are connected as source or target object (depending on if extraction is source or target) + +### stix-mapping: `vulmatch-cve-id` + +Takes the extracted ID and passes it to + +```shell +GET VULMATCH_HOST/api/v1/cve/objects/:cve_id/ +``` + +`VULMATCH_APIKEY` in request passed if set. + +Relationship mode object generation behaviour: + +* Standard relationship SRO: Imported `vulnerability` object is (`source_ref`) object connected to Report +* AI mode relationship SROs: all imported objects are connected as source or target object (depending on if extraction is source or target) + +### stix-mapping: `vulmatch-cpe-id` + +Takes the extracted ID and passes it to + +```shell +GET VULMATCH_HOST/api/v1/cpe/objects/:cpe_id/ +``` + +`VULMATCH_APIKEY` in request passed if set. + +Relationship mode object generation behaviour: + +* Standard relationship SRO: Imported `software` object is (`source_ref`) object connected to Report +* AI mode relationship SROs: all imported objects are connected as source or target object (depending on if extraction is source or target) + +## Relationship objects + +### Fixed SCO to Indicator relationships for extractions + +In some extractions, SROs are created to link extractions (SCO linked to Indicator) + +These relationships are modelled as follows; + +```json +{ + "type": "relationship", + "spec_version": "2.1", + "id": "relationship--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "relationship_type": "", + "source_ref": "", + "target_ref": "indicator--", + "description": " is found in ", + "object_marking_refs": [ + "marking-definition--" + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + } + ] +} +``` + +### Relationships generated in `standard` relationship mode + +In standard mode, one SRO is created for each extraction back to the source report object created for the job. + +```json +{ + "type": "relationship", + "spec_version": "2.1", + "id": "relationship--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "relationship_type": "extracted-from", + "source_ref": "", + "target_ref": "report--", + "description": " is found in ", + "object_marking_refs": [ + "marking-definition--" + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + } + ] +} +``` + +### Relationships generated in `ai` relationship mode + +The LLM generates a JSON file describing how objects created from extractions are connected. + +The Relationship SRO created for each relationship identified will be generated as follows; + +```json +{ + "type": "relationship", + "spec_version": "2.1", + "id": "relationship--", + "created_by_ref": "identity--", + "created": "", + "modified": "", + "relationship_type": "", + "source_ref": "", + "target_ref": "", + "description": " ", + "object_marking_refs": [ + "marking-definition--" + "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5" + ], + "external_references": [ + { + "source_name": "txt2stix job ID", + "external_id": "" + } + ] +} +``` + +Note, a source object can be linked to many target objects in the same report. Thus, a source object might create many SROs, representing links to target objects. + +Similarly, in the case of MITRE ATT&CK an extraction might import one or more STIX objects. In this case, an SRO should be created for each imported object to the source/target object identified. + +## Bundle (output) + +The output of txt2stix is a STIX bundle file. + +This bundle takes the format; + +```json +{ + "type": "bundle", + "id": "bundle--", + "objects": [ + "" + ] +} +``` + +The objects include all SROs generated for the input. + +The filename of the bundle takes the format: `bundle--.json` \ No newline at end of file diff --git a/docs/txt2stix.png b/docs/txt2stix.png new file mode 100644 index 0000000..5af75de Binary files /dev/null and b/docs/txt2stix.png differ diff --git a/includes/extractions/_README.md b/includes/extractions/_README.md deleted file mode 100644 index 86c996d..0000000 --- a/includes/extractions/_README.md +++ /dev/null @@ -1,42 +0,0 @@ -## Importing external data - -Check what values exist in ArangoDB for generating positive tests (because test server does not always have complete records)... - -### CWEs - -```sql -FOR doc IN mitre_cwe_vertex_collection - FILTER doc._stix2arango_note != "automatically imported on collection creation" - AND doc.type == "weakness" - LIMIT 10 - LET cweExternalId = ( - FOR ref IN doc.external_references - FILTER ref.source_name == "cwe" - RETURN ref.external_id - ) - RETURN { - "id": doc._id, - "cweExternalId": cweExternalId - } -``` - -### CVEs - -```sql -FOR doc IN nvd_cve_vertex_collection - FILTER doc._stix2arango_note != "automatically imported on collection creation" - AND doc.type == "vulnerability" - SORT doc.modified DESC - LIMIT 10 - RETURN doc.name -``` - -### CPEs - -```sql -FOR doc IN nvd_cpe_vertex_collection - FILTER doc._stix2arango_note != "automatically imported on collection creation" - AND doc.type == "software" - LIMIT 10 - RETURN doc.cpe -``` \ No newline at end of file diff --git a/includes/extractions/ai/config.yaml b/includes/extractions/ai/config.yaml index d1b7c23..40ce36b 100644 --- a/includes/extractions/ai/config.yaml +++ b/includes/extractions/ai/config.yaml @@ -83,10 +83,10 @@ ai_mitre_attack_enterprise: version: 1.0.0 prompt_base: 'Extract all MITRE ATT&CK Enterprise tactics, techniques, groups, data sources, mitigations, software, and campaigns described in the text. Do not include MITRE ATT&CK ICS or MITRE ATT&CK Mobile in the results.' prompt_conversion: 'Convert all extractions into the corresponding ATT&CK ID.' - test_cases: ai_mitre_attack_enterprise + test_cases: generic_mitre_attack_enterprise ignore_extractions: - - stix_mapping: mitre-attack-enterprise-id + stix_mapping: ctibutler-mitre-attack-enterprise-id ai_mitre_attack_mobile: type: ai @@ -99,10 +99,10 @@ ai_mitre_attack_mobile: version: 1.0.0 prompt_base: 'Extract all MITRE ATT&CK Mobile tactics, techniques, groups, data sources, mitigations, software, and campaigns described in the text. Do not include MITRE ATT&CK ICS or MITRE ATT&CK Enterprise in the results.' prompt_conversion: 'Convert all extractions into the corresponding ATT&CK ID.' - test_cases: ai_mitre_attack_mobile + test_cases: generic_mitre_attack_mobile ignore_extractions: - - stix_mapping: mitre-attack-mobile-id + stix_mapping: ctibutler-mitre-attack-mobile-id ai_mitre_attack_ics: type: ai @@ -115,10 +115,10 @@ ai_mitre_attack_ics: version: 1.0.0 prompt_base: 'Extract all MITRE ATT&CK ICS tactics, techniques, groups, data sources, mitigations, software, and campaigns described in the text. Do not include MITRE ATT&CK Mobile or MITRE ATT&CK Enterprise in the results.' prompt_conversion: 'Convert all extractions into the corresponding ATT&CK ID.' - test_cases: ai_mitre_attack_ics + test_cases: generic_mitre_attack_ics ignore_extractions: - - stix_mapping: mitre-attack-ics-id + stix_mapping: ctibutler-mitre-attack-ics-id ####### MITRE CAPEC ####### @@ -133,10 +133,10 @@ ai_mitre_capec: version: 1.0.0 prompt_base: 'Extract all references to a MITRE CAPEC object.' prompt_conversion: 'Convert all extractions into the corresponding CAPEC ID in the format `CAPEC-ID`' - test_cases: ai_mitre_capec + test_cases: generic_mitre_capec ignore_extractions: - - stix_mapping: mitre-capec-id + stix_mapping: ctibutler-mitre-capec-id ####### MITRE CWE ####### @@ -151,7 +151,7 @@ ai_mitre_cwe: version: 1.0.0 prompt_base: 'Extract all references to a MITRE CWE object.' prompt_conversion: 'Convert all extractions into the corresponding CWE ID in the format `CWE-ID`' - test_cases: ai_mitre_cwe + test_cases: generic_mitre_cwe ignore_extractions: - - stix_mapping: mitre-cwe-id \ No newline at end of file + stix_mapping: ctibutler-mitre-cwe-id \ No newline at end of file diff --git a/includes/extractions/ai/schema.yaml b/includes/extractions/ai/schema.yaml index 5bd3889..dbbea75 100644 --- a/includes/extractions/ai/schema.yaml +++ b/includes/extractions/ai/schema.yaml @@ -9,7 +9,7 @@ SLUG: # REQUIRED: machine friendly name of extraction, must only contain charact version: # [number] REQUIRED: version number in format N.N.N prompt_base: # [string] REQUIRED: Value to be used for the prompt. max 500 characters. prompt_conversion: # [string] OPTIONAL: Additional prompt to convert what the LLM extracts into another value (e.g. turn name into an ID) - test_cases: # [test extraction slug] REQUIRED: test cases for this extraction. Slugs found in extractions/test_cases/test_data.yaml + test_cases: # [test extraction slug] REQUIRED: test cases for this extraction. Slugs found in tests/test_cases.yaml ignore_extractions: - # [list of extraction slugs] OPTIONAL: defines slugs of other extraction types that should be ignored for any part of the matched string that they match too. e.g. ignore AI domain extractions (ai_domain_name_only) for google.com if url extraction matches https://google.com/something. This is dynamic, and you add/remove entries to enable/disable ignores stix_mapping: # [dictionary] REQUIRED: STIX object extraction should map to. Must be supported STIX type \ No newline at end of file diff --git a/includes/extractions/alias/schema.yaml b/includes/extractions/alias/schema.yaml index a298403..ee3098b 100644 --- a/includes/extractions/alias/schema.yaml +++ b/includes/extractions/alias/schema.yaml @@ -8,4 +8,4 @@ SLUG: # REQUIRED: machine friendly name of alias, must only contain characters ` modified: # [date] REQUIRED: YYYY-MM-DD of creation time. created_by: # [string] REQUIRED: name of author , must only contain characters `a-z`,`0-9`,`_`,`-` version: # [number] REQUIRED: version number in format N.N.N - test_cases: # [test extraction slug] REQUIRED: test cases for this extraction. Slugs found in extractions/test_cases/test_data.yaml \ No newline at end of file + test_cases: # [test extraction slug] REQUIRED: test cases for this extraction. Slugs found in tests/test_cases.yaml \ No newline at end of file diff --git a/includes/extractions/lookup/config.yaml b/includes/extractions/lookup/config.yaml index bcba56d..638b08b 100644 --- a/includes/extractions/lookup/config.yaml +++ b/includes/extractions/lookup/config.yaml @@ -5,8 +5,8 @@ lookup_mitre_attack_enterprise_id: type: lookup name: 'MITRE ATT&CK Enterprise IDs' - description: '' - notes: 'Uses lookup ' + description: 'Extracts MITRE ATT&CK Enterprise IDs from text.' + notes: 'Currently uses v15.1' file: 'lookups/mitre_attack_enterprise_id.txt' created: 2020-01-01 modified: 2020-01-01 @@ -15,13 +15,13 @@ lookup_mitre_attack_enterprise_id: test_cases: generic_mitre_attack_enterprise ignore_extractions: - - stix_mapping: mitre-attack-enterprise-id + stix_mapping: ctibutler-mitre-attack-enterprise-id lookup_mitre_attack_mobile_id: type: lookup name: 'MITRE ATT&CK Mobile IDs' - description: '' - notes: '' + description: 'Extracts MITRE ATT&CK Mobile IDs from text.' + notes: 'Currently uses v15.1' file: 'lookups/mitre_attack_mobile_id.txt' created: 2020-01-01 modified: 2020-01-01 @@ -30,12 +30,13 @@ lookup_mitre_attack_mobile_id: test_cases: generic_mitre_attack_mobile ignore_extractions: - - stix_mapping: mitre-attack-mobile-id + stix_mapping: ctibutler-mitre-attack-mobile-id lookup_mitre_attack_ics_id: type: lookup name: 'MITRE ATT&CK ICS IDs' - description: '' + description: 'Extracts MITRE ATT&CK ICS IDs from text.' + notes: 'Currently uses v15.1' file: 'lookups/mitre_attack_ics_id.txt' created: 2020-01-01 modified: 2020-01-01 @@ -44,15 +45,15 @@ lookup_mitre_attack_ics_id: test_cases: generic_mitre_attack_ics ignore_extractions: - - stix_mapping: mitre-attack-ics-id + stix_mapping: ctibutler-mitre-attack-ics-id ####### MITRE CAPEC ####### lookup_mitre_capec_id: type: lookup name: 'MITRE CAPEC IDs' - description: '' - notes: '' + description: 'Extracts MITRE CAPEC IDs from text.' + notes: 'Currently uses v3.9' file: 'lookups/mitre_capec_id.txt' created: 2020-01-01 modified: 2020-01-01 @@ -61,15 +62,15 @@ lookup_mitre_capec_id: test_cases: generic_mitre_capec ignore_extractions: - - stix_mapping: mitre-capec-id + stix_mapping: ctibutler-mitre-capec-id ####### MITRE CWE ####### lookup_mitre_cwe_id: type: lookup name: MITRE CWE IDs - description: '' - notes: '' + description: 'Extracts MITRE CWE IDs from text.' + notes: 'Currently uses v4.15' file: 'lookups/mitre_cwe_id.txt' created: 2020-01-01 modified: 2020-01-01 @@ -78,7 +79,24 @@ lookup_mitre_cwe_id: test_cases: generic_mitre_cwe ignore_extractions: - - stix_mapping: mitre-cwe-id + stix_mapping: ctibutler-mitre-cwe-id + +####### MITRE ATLAS ####### + +lookup_mitre_atlas_id: + type: lookup + name: MITRE ATLAS IDs + description: 'Extracts MITRE ATLAS IDs from text.' + notes: 'Currently uses v4.5.2' + file: 'lookups/mitre_atlas_id.txt' + created: 2020-01-01 + modified: 2020-01-01 + created_by: DOGESEC + version: 1.0.0 + test_cases: generic_mitre_atlas + ignore_extractions: + - + stix_mapping: ctibutler-mitre-atlas-id ####### County extractions ####### diff --git a/includes/extractions/lookup/schema.yaml b/includes/extractions/lookup/schema.yaml index bc900f8..03e9dd2 100644 --- a/includes/extractions/lookup/schema.yaml +++ b/includes/extractions/lookup/schema.yaml @@ -8,7 +8,7 @@ SLUG: # REQUIRED: machine friendly name of extraction, must only contain charact modified: # [date] REQUIRED: YYYY-MM-DD of creation time. created_by: # [string] REQUIRED: name of author , must only contain characters `a-z`,`0-9`,`_`,`-` version: # [number] REQUIRED: version number in format N.N.N - test_cases: # [test extraction slug] REQUIRED: test cases for this extraction. Slugs found in extractions/test_cases/test_data.yaml + test_cases: # [test extraction slug] REQUIRED: test cases for this extraction. Slugs found in tests/test_cases.yaml ignore_extractions: - # [list of extraction slugs] OPTIONAL: defines slugs of other extraction types that should be ignored for any part of the matched string that they match too. e.g. ignore AI domain extractions (ai_domain_name_only) for google.com if url extraction matches https://google.com/something. This is dynamic, and you add/remove entries to enable/disable ignores stix_mapping: # [dictionary] REQUIRED: STIX object extraction should map to. Must be supported STIX type \ No newline at end of file diff --git a/includes/extractions/pattern/config.yaml b/includes/extractions/pattern/config.yaml index 9f14e29..5ff84bd 100644 --- a/includes/extractions/pattern/config.yaml +++ b/includes/extractions/pattern/config.yaml @@ -532,7 +532,7 @@ pattern_cve_id: test_cases: generic_cve_id ignore_extractions: - - stix_mapping: cve-id + stix_mapping: vulmatch-cve-id ####### CPE extractions ####### @@ -548,7 +548,7 @@ pattern_cpe_uri: test_cases: generic_cpe_uri ignore_extractions: - - stix_mapping: cpe-id + stix_mapping: vulmatch-cpe-id ####### Bank card extractions ####### diff --git a/includes/extractions/pattern/schema.yaml b/includes/extractions/pattern/schema.yaml index 3daa124..1f177cb 100644 --- a/includes/extractions/pattern/schema.yaml +++ b/includes/extractions/pattern/schema.yaml @@ -7,7 +7,7 @@ SLUG: # REQUIRED: machine friendly name of extraction, must only contain charact modified: # [date] REQUIRED: YYYY-MM-DD of creation time. created_by: # [string] REQUIRED: name of author , must only contain characters `a-z`,`0-9`,`_`,`-` version: # [number] REQUIRED: version number in format N.N.N - test_cases: # [test extraction slug] REQUIRED: test cases for this extraction. Slugs found in extractions/test_cases/test_data.yaml + test_cases: # [test extraction slug] REQUIRED: test cases for this extraction. Slugs found in tests/test_cases.yaml ignore_extractions: - # [list of extraction slugs] OPTIONAL: defines slugs of other extraction types that should be ignored for any part of the matched string that they match too. e.g. ignore AI domain extractions (ai_domain_name_only) for google.com if url extraction matches https://google.com/something. This is dynamic, and you add/remove entries to enable/disable ignores stix_mapping: # [dictionary] REQUIRED: STIX object extraction should map to. Must be supported STIX type \ No newline at end of file diff --git a/includes/lookups/_README.md b/includes/lookups/_README.md index 8c2adc5..a9525d5 100644 --- a/includes/lookups/_README.md +++ b/includes/lookups/_README.md @@ -7,20 +7,22 @@ FOR doc IN mitre_cwe_vertex_collection FILTER doc._is_latest == true AND doc._stix2arango_note == "v4.15" AND IS_ARRAY(doc.external_references) + AND doc.x_mitre_deprecated != true + AND doc.revoked != true FOR reference IN doc.external_references FILTER reference.source_name == "cwe" SORT reference.external_id ASC RETURN reference.external_id ``` -(964 results in v4.15) - Generate `mitre_capec_id.txt`: ```sql FOR doc IN mitre_capec_vertex_collection FILTER doc._is_latest == true AND doc._stix2arango_note == "v3.9" + AND doc.x_mitre_deprecated != true + AND doc.revoked != true AND IS_ARRAY(doc.external_references) FOR reference IN doc.external_references FILTER reference.source_name == "capec" @@ -28,8 +30,6 @@ FOR doc IN mitre_capec_vertex_collection RETURN reference.external_id ``` -(615 results in v3.9) - Generate `mitre_attack_enterprise_id.txt` ```sql @@ -37,6 +37,8 @@ FOR doc IN mitre_attack_enterprise_vertex_collection FILTER doc._is_latest == true AND doc._stix2arango_note == "v15.1" AND doc.type != "x-mitre-matrix" + AND doc.x_mitre_deprecated != true + AND doc.revoked != true AND IS_ARRAY(doc.external_references) FOR reference IN doc.external_references FILTER reference.source_name == "mitre-attack" @@ -44,8 +46,6 @@ FOR doc IN mitre_attack_enterprise_vertex_collection RETURN reference.external_id ``` -(1991 results in v15.1) - Generate `mitre_attack_ics_id.txt` ```sql @@ -53,6 +53,8 @@ FOR doc IN mitre_attack_ics_vertex_collection FILTER doc._is_latest == true AND doc._stix2arango_note == "v15.1" AND doc.type != "x-mitre-matrix" + AND doc.x_mitre_deprecated != true + AND doc.revoked != true AND IS_ARRAY(doc.external_references) FOR reference IN doc.external_references FILTER reference.source_name == "mitre-attack" @@ -60,8 +62,6 @@ FOR doc IN mitre_attack_ics_vertex_collection RETURN reference.external_id ``` -(233 results in v15.1) - Generate `mitre_attack_mobile_id.txt` ```sql @@ -69,6 +69,8 @@ FOR doc IN mitre_attack_mobile_vertex_collection FILTER doc._is_latest == true AND doc._stix2arango_note == "v15.1" AND doc.type != "x-mitre-matrix" + AND doc.x_mitre_deprecated != true + AND doc.revoked != true AND IS_ARRAY(doc.external_references) FOR reference IN doc.external_references FILTER reference.source_name == "mitre-attack" @@ -76,4 +78,18 @@ FOR doc IN mitre_attack_mobile_vertex_collection RETURN reference.external_id ``` -(333 results in v15.1) \ No newline at end of file +Generate `mitre_atlas_id.txt` + +```sql +FOR doc IN mitre_atlas_vertex_collection + FILTER doc._is_latest == true + AND doc._stix2arango_note == "v4.5.2" + AND doc.type != "x-mitre-matrix" + AND doc.x_mitre_deprecated != true + AND doc.revoked != true + AND IS_ARRAY(doc.external_references) + FOR reference IN doc.external_references + FILTER reference.source_name == "mitre-atlas" + SORT reference.external_id ASC + RETURN reference.external_id +``` diff --git a/includes/lookups/mitre_atlas_id.txt b/includes/lookups/mitre_atlas_id.txt new file mode 100644 index 0000000..e11ba3f --- /dev/null +++ b/includes/lookups/mitre_atlas_id.txt @@ -0,0 +1,116 @@ +AML.M0000 +AML.M0001 +AML.M0002 +AML.M0003 +AML.M0004 +AML.M0005 +AML.M0006 +AML.M0007 +AML.M0008 +AML.M0009 +AML.M0010 +AML.M0011 +AML.M0012 +AML.M0013 +AML.M0014 +AML.M0015 +AML.M0016 +AML.M0017 +AML.M0018 +AML.M0019 +AML.T0000 +AML.T0000.000 +AML.T0000.001 +AML.T0000.002 +AML.T0001 +AML.T0002 +AML.T0002.000 +AML.T0002.001 +AML.T0003 +AML.T0004 +AML.T0005 +AML.T0005.000 +AML.T0005.001 +AML.T0005.002 +AML.T0006 +AML.T0007 +AML.T0008 +AML.T0008.000 +AML.T0008.001 +AML.T0010 +AML.T0010.000 +AML.T0010.001 +AML.T0010.002 +AML.T0010.003 +AML.T0011 +AML.T0011.000 +AML.T0012 +AML.T0013 +AML.T0014 +AML.T0015 +AML.T0016 +AML.T0016.000 +AML.T0016.001 +AML.T0017 +AML.T0017.000 +AML.T0018 +AML.T0018.000 +AML.T0018.001 +AML.T0019 +AML.T0020 +AML.T0021 +AML.T0024 +AML.T0024.000 +AML.T0024.001 +AML.T0024.002 +AML.T0025 +AML.T0029 +AML.T0031 +AML.T0034 +AML.T0035 +AML.T0036 +AML.T0037 +AML.T0040 +AML.T0041 +AML.T0042 +AML.T0043 +AML.T0043.000 +AML.T0043.001 +AML.T0043.002 +AML.T0043.003 +AML.T0043.004 +AML.T0044 +AML.T0046 +AML.T0047 +AML.T0048 +AML.T0048.000 +AML.T0048.001 +AML.T0048.002 +AML.T0048.003 +AML.T0048.004 +AML.T0049 +AML.T0050 +AML.T0051 +AML.T0051.000 +AML.T0051.001 +AML.T0052 +AML.T0052.000 +AML.T0053 +AML.T0054 +AML.T0055 +AML.T0056 +AML.T0057 +AML.TA0000 +AML.TA0001 +AML.TA0002 +AML.TA0003 +AML.TA0004 +AML.TA0005 +AML.TA0006 +AML.TA0007 +AML.TA0008 +AML.TA0009 +AML.TA0010 +AML.TA0011 +AML.TA0012 +AML.TA0013" \ No newline at end of file diff --git a/includes/lookups/mitre_attack_enterprise_id.txt b/includes/lookups/mitre_attack_enterprise_id.txt index f3b0619..42c94a8 100644 --- a/includes/lookups/mitre_attack_enterprise_id.txt +++ b/includes/lookups/mitre_attack_enterprise_id.txt @@ -1,1991 +1,1586 @@ -C0001, -C0002, -C0004, -C0005, -C0006, -C0007, -C0010, -C0011, -C0012, -C0013, -C0014, -C0015, -C0016, -C0017, -C0018, -C0021, -C0022, -C0023, -C0024, -C0025, -C0026, -C0027, -C0028, -C0029, -C0030, -C0032, -C0033, -C0034, -DS0001, -DS0002, -DS0003, -DS0004, -DS0005, -DS0006, -DS0007, -DS0008, -DS0009, -DS0010, -DS0011, -DS0012, -DS0013, -DS0014, -DS0015, -DS0016, -DS0017, -DS0018, -DS0019, -DS0020, -DS0021, -DS0022, -DS0023, -DS0024, -DS0025, -DS0026, -DS0027, -DS0028, -DS0029, -DS0030, -DS0031, -DS0032, -DS0033, -DS0034, -DS0035, -DS0036, -DS0037, -DS0038, -G0001, -G0002, -G0003, -G0004, -G0005, -G0006, -G0007, -G0008, -G0009, -G0010, -G0011, -G0012, -G0013, -G0014, -G0015, -G0016, -G0017, -G0018, -G0019, -G0020, -G0021, -G0022, -G0023, -G0024, -G0025, -G0026, -G0027, -G0028, -G0029, -G0030, -G0031, -G0032, -G0033, -G0034, -G0035, -G0036, -G0037, -G0038, -G0039, -G0040, -G0041, -G0042, -G0043, -G0044, -G0045, -G0046, -G0047, -G0048, -G0049, -G0050, -G0051, -G0052, -G0053, -G0054, -G0055, -G0056, -G0057, -G0058, -G0058, -G0059, -G0060, -G0061, -G0062, -G0063, -G0064, -G0065, -G0066, -G0067, -G0068, -G0069, -G0070, -G0071, -G0072, -G0073, -G0074, -G0075, -G0076, -G0077, -G0078, -G0079, -G0080, -G0081, -G0082, -G0083, -G0084, -G0085, -G0086, -G0087, -G0088, -G0089, -G0090, -G0091, -G0092, -G0093, -G0094, -G0095, -G0096, -G0097, -G0098, -G0099, -G0100, -G0101, -G0102, -G0103, -G0104, -G0105, -G0106, -G0107, -G0108, -G0112, -G0114, -G0115, -G0116, -G0117, -G0118, -G0119, -G0120, -G0121, -G0122, -G0123, -G0124, -G0125, -G0126, -G0127, -G0128, -G0129, -G0130, -G0131, -G0132, -G0133, -G0134, -G0135, -G0136, -G0137, -G0138, -G0139, -G0140, -G0141, -G0142, -G0143, -G1001, -G1002, -G1003, -G1004, -G1005, -G1006, -G1007, -G1008, -G1009, -G1011, -G1012, -G1013, -G1014, -G1015, -G1016, -G1017, -G1018, -G1019, -G1020, -G1021, -G1022, -G1023, -G1024, -G1026, -G1028, -M1013, -M1015, -M1016, -M1017, -M1018, -M1019, -M1020, -M1021, -M1022, -M1024, -M1025, -M1026, -M1027, -M1028, -M1029, -M1030, -M1031, -M1032, -M1033, -M1034, -M1035, -M1036, -M1037, -M1038, -M1039, -M1040, -M1041, -M1042, -M1043, -M1044, -M1045, -M1046, -M1047, -M1048, -M1049, -M1050, -M1051, -M1052, -M1053, -M1054, -M1055, -M1056, -M1057, -S0001, -S0002, -S0003, -S0004, -S0005, -S0006, -S0007, -S0008, -S0009, -S0010, -S0011, -S0012, -S0013, -S0014, -S0015, -S0016, -S0017, -S0018, -S0019, -S0020, -S0021, -S0022, -S0023, -S0024, -S0025, -S0026, -S0027, -S0028, -S0029, -S0030, -S0031, -S0032, -S0033, -S0034, -S0035, -S0036, -S0037, -S0038, -S0039, -S0040, -S0041, -S0042, -S0043, -S0044, -S0045, -S0046, -S0047, -S0048, -S0049, -S0050, -S0051, -S0052, -S0053, -S0054, -S0055, -S0056, -S0057, -S0058, -S0059, -S0060, -S0061, -S0062, -S0063, -S0064, -S0065, -S0066, -S0067, -S0068, -S0069, -S0070, -S0071, -S0072, -S0073, -S0074, -S0075, -S0076, -S0077, -S0078, -S0079, -S0080, -S0081, -S0082, -S0083, -S0084, -S0085, -S0086, -S0087, -S0088, -S0089, -S0090, -S0091, -S0092, -S0093, -S0094, -S0095, -S0096, -S0097, -S0098, -S0099, -S0100, -S0101, -S0102, -S0103, -S0104, -S0105, -S0106, -S0107, -S0108, -S0109, -S0110, -S0111, -S0112, -S0113, -S0114, -S0115, -S0116, -S0117, -S0118, -S0119, -S0120, -S0121, -S0122, -S0123, -S0124, -S0125, -S0126, -S0127, -S0128, -S0129, -S0130, -S0131, -S0132, -S0133, -S0134, -S0135, -S0136, -S0137, -S0138, -S0139, -S0140, -S0141, -S0142, -S0143, -S0144, -S0145, -S0146, -S0147, -S0148, -S0149, -S0150, -S0151, -S0152, -S0153, -S0154, -S0154, -S0155, -S0156, -S0157, -S0158, -S0159, -S0160, -S0161, -S0162, -S0163, -S0164, -S0165, -S0166, -S0167, -S0168, -S0169, -S0170, -S0171, -S0172, -S0173, -S0174, -S0175, -S0176, -S0177, -S0178, -S0179, -S0180, -S0181, -S0182, -S0183, -S0184, -S0185, -S0186, -S0187, -S0188, -S0189, -S0190, -S0191, -S0192, -S0193, -S0194, -S0195, -S0196, -S0197, -S0198, -S0199, -S0200, -S0201, -S0202, -S0203, -S0204, -S0205, -S0206, -S0207, -S0208, -S0209, -S0210, -S0211, -S0212, -S0213, -S0214, -S0215, -S0216, -S0217, -S0218, -S0219, -S0220, -S0221, -S0222, -S0223, -S0224, -S0225, -S0226, -S0227, -S0228, -S0229, -S0230, -S0231, -S0232, -S0233, -S0234, -S0235, -S0236, -S0237, -S0238, -S0239, -S0240, -S0241, -S0242, -S0243, -S0244, -S0245, -S0246, -S0247, -S0248, -S0249, -S0250, -S0251, -S0252, -S0253, -S0254, -S0255, -S0256, -S0257, -S0258, -S0259, -S0260, -S0261, -S0262, -S0263, -S0264, -S0265, -S0266, -S0267, -S0268, -S0269, -S0270, -S0271, -S0272, -S0273, -S0274, -S0275, -S0276, -S0277, -S0278, -S0279, -S0280, -S0281, -S0282, -S0283, -S0284, -S0302, -S0330, -S0331, -S0332, -S0333, -S0334, -S0335, -S0336, -S0337, -S0338, -S0339, -S0340, -S0341, -S0342, -S0343, -S0344, -S0345, -S0346, -S0347, -S0348, -S0349, -S0350, -S0351, -S0352, -S0353, -S0354, -S0355, -S0356, -S0357, -S0358, -S0359, -S0360, -S0361, -S0362, -S0363, -S0364, -S0365, -S0366, -S0367, -S0368, -S0369, -S0370, -S0371, -S0372, -S0373, -S0374, -S0375, -S0376, -S0377, -S0378, -S0379, -S0380, -S0381, -S0382, -S0383, -S0384, -S0385, -S0386, -S0387, -S0388, -S0389, -S0390, -S0391, -S0393, -S0394, -S0395, -S0396, -S0397, -S0398, -S0400, -S0401, -S0402, -S0404, -S0409, -S0410, -S0412, -S0413, -S0414, -S0415, -S0416, -S0417, -S0428, -S0430, -S0431, -S0433, -S0434, -S0435, -S0436, -S0437, -S0438, -S0439, -S0441, -S0442, -S0443, -S0444, -S0445, -S0446, -S0447, -S0448, -S0449, -S0450, -S0451, -S0452, -S0453, -S0454, -S0455, -S0456, -S0457, -S0458, -S0459, -S0460, -S0461, -S0462, -S0464, -S0465, -S0466, -S0467, -S0468, -S0469, -S0470, -S0471, -S0472, -S0473, -S0475, -S0476, -S0477, -S0481, -S0482, -S0483, -S0484, -S0486, -S0487, -S0488, -S0491, -S0492, -S0493, -S0495, -S0496, -S0497, -S0498, -S0499, -S0500, -S0501, -S0502, -S0503, -S0504, -S0508, -S0511, -S0512, -S0513, -S0514, -S0515, -S0516, -S0517, -S0518, -S0519, -S0520, -S0521, -S0526, -S0527, -S0528, -S0530, -S0531, -S0532, -S0533, -S0534, -S0537, -S0538, -S0543, -S0546, -S0547, -S0552, -S0553, -S0554, -S0556, -S0559, -S0560, -S0561, -S0562, -S0564, -S0565, -S0567, -S0568, -S0569, -S0570, -S0572, -S0574, -S0575, -S0576, -S0578, -S0579, -S0581, -S0582, -S0583, -S0584, -S0585, -S0586, -S0587, -S0588, -S0589, -S0590, -S0591, -S0592, -S0593, -S0594, -S0595, -S0596, -S0597, -S0598, -S0599, -S0600, -S0601, -S0603, -S0604, -S0605, -S0606, -S0607, -S0608, -S0609, -S0610, -S0611, -S0612, -S0613, -S0614, -S0615, -S0616, -S0617, -S0618, -S0622, -S0623, -S0624, -S0625, -S0626, -S0627, -S0628, -S0629, -S0630, -S0631, -S0632, -S0633, -S0634, -S0635, -S0636, -S0637, -S0638, -S0639, -S0640, -S0641, -S0642, -S0643, -S0644, -S0645, -S0646, -S0647, -S0648, -S0649, -S0650, -S0651, -S0652, -S0653, -S0654, -S0657, -S0658, -S0659, -S0660, -S0661, -S0662, -S0663, -S0664, -S0665, -S0666, -S0667, -S0668, -S0669, -S0670, -S0671, -S0672, -S0673, -S0674, -S0677, -S0678, -S0679, -S0680, -S0681, -S0682, -S0683, -S0684, -S0685, -S0686, -S0687, -S0688, -S0689, -S0690, -S0691, -S0692, -S0693, -S0694, -S0695, -S0696, -S0697, -S0698, -S0699, -S1011, -S1012, -S1013, -S1014, -S1015, -S1016, -S1017, -S1018, -S1019, -S1020, -S1021, -S1022, -S1023, -S1024, -S1025, -S1026, -S1027, -S1028, -S1029, -S1030, -S1031, -S1032, -S1033, -S1034, -S1035, -S1037, -S1039, -S1040, -S1041, -S1042, -S1043, -S1044, -S1046, -S1047, -S1048, -S1049, -S1050, -S1051, -S1052, -S1053, -S1058, -S1059, -S1060, -S1063, -S1064, -S1065, -S1066, -S1068, -S1070, -S1071, -S1072, -S1073, -S1074, -S1075, -S1076, -S1078, -S1081, -S1084, -S1085, -S1086, -S1087, -S1088, -S1089, -S1090, -S1091, -S1096, -S1097, -S1099, -S1100, -S1101, -S1102, -S1104, -S1105, -S1106, -S1107, -S1108, -S1109, -S1110, -S1111, -S1112, -S1113, -S1114, -S1115, -S1116, -S1117, -S1118, -S1119, -S1120, -S1121, -S1122, -S1123, -S1124, -S1125, -S1129, -S9000, -T1001, -T1001, -T1001.001, -T1001.002, -T1001.003, -T1002, -T1002, -T1003, -T1003, -T1003.001, -T1003.002, -T1003.003, -T1003.004, -T1003.005, -T1003.006, -T1003.007, -T1003.008, -T1004, -T1004, -T1005, -T1005, -T1006, -T1006, -T1007, -T1007, -T1008, -T1008, -T1009, -T1009, -T1010, -T1010, -T1011, -T1011, -T1011.001, -T1012, -T1012, -T1013, -T1013, -T1014, -T1014, -T1015, -T1015, -T1016, -T1016, -T1016.001, -T1016.002, -T1017, -T1017, -T1018, -T1018, -T1019, -T1019, -T1020, -T1020, -T1020.001, -T1021, -T1021, -T1021.001, -T1021.002, -T1021.003, -T1021.004, -T1021.005, -T1021.006, -T1021.007, -T1021.008, -T1022, -T1022, -T1023, -T1023, -T1024, -T1024, -T1025, -T1025, -T1026, -T1026, -T1027, -T1027, -T1027.001, -T1027.002, -T1027.003, -T1027.004, -T1027.005, -T1027.006, -T1027.007, -T1027.008, -T1027.009, -T1027.010, -T1027.011, -T1027.012, -T1027.013, -T1028, -T1028, -T1029, -T1029, -T1030, -T1030, -T1031, -T1031, -T1032, -T1032, -T1033, -T1033, -T1034, -T1034, -T1035, -T1035, -T1036, -T1036, -T1036.001, -T1036.002, -T1036.003, -T1036.004, -T1036.005, -T1036.006, -T1036.007, -T1036.008, -T1036.009, -T1037, -T1037, -T1037.001, -T1037.002, -T1037.003, -T1037.004, -T1037.005, -T1038, -T1038, -T1039, -T1039, -T1040, -T1040, -T1041, -T1041, -T1042, -T1042, -T1043, -T1043, -T1044, -T1044, -T1045, -T1045, -T1046, -T1046, -T1047, -T1047, -T1048, -T1048, -T1048.001, -T1048.002, -T1048.003, -T1049, -T1049, -T1050, -T1050, -T1051, -T1051, -T1052, -T1052, -T1052.001, -T1053, -T1053, -T1053.001, -T1053.002, -T1053.003, -T1053.004, -T1053.005, -T1053.006, -T1053.007, -T1054, -T1054, -T1055, -T1055, -T1055.001, -T1055.002, -T1055.003, -T1055.004, -T1055.005, -T1055.008, -T1055.009, -T1055.011, -T1055.012, -T1055.013, -T1055.014, -T1055.015, -T1056, -T1056, -T1056.001, -T1056.002, -T1056.003, -T1056.004, -T1057, -T1057, -T1058, -T1058, -T1059, -T1059, -T1059.001, -T1059.002, -T1059.003, -T1059.004, -T1059.005, -T1059.006, -T1059.007, -T1059.008, -T1059.009, -T1059.010, -T1060, -T1060, -T1061, -T1061, -T1062, -T1062, -T1063, -T1063, -T1064, -T1064, -T1065, -T1065, -T1066, -T1066, -T1067, -T1067, -T1068, -T1068, -T1069, -T1069, -T1069.001, -T1069.002, -T1069.003, -T1070, -T1070, -T1070.001, -T1070.002, -T1070.003, -T1070.004, -T1070.005, -T1070.006, -T1070.007, -T1070.008, -T1070.009, -T1071, -T1071, -T1071.001, -T1071.002, -T1071.003, -T1071.004, -T1072, -T1072, -T1073, -T1073, -T1074, -T1074, -T1074.001, -T1074.002, -T1075, -T1075, -T1076, -T1076, -T1077, -T1077, -T1078, -T1078, -T1078.001, -T1078.002, -T1078.003, -T1078.004, -T1079, -T1079, -T1080, -T1080, -T1081, -T1081, -T1082, -T1082, -T1083, -T1083, -T1084, -T1084, -T1085, -T1085, -T1086, -T1086, -T1087, -T1087, -T1087.001, -T1087.002, -T1087.003, -T1087.004, -T1088, -T1088, -T1089, -T1089, -T1090, -T1090, -T1090.001, -T1090.002, -T1090.003, -T1090.004, -T1091, -T1091, -T1092, -T1092, -T1093, -T1093, -T1094, -T1094, -T1095, -T1095, -T1096, -T1096, -T1097, -T1097, -T1098, -T1098, -T1098.001, -T1098.002, -T1098.003, -T1098.004, -T1098.005, -T1098.006, -T1099, -T1099, -T1100, -T1100, -T1101, -T1101, -T1102, -T1102, -T1102.001, -T1102.002, -T1102.003, -T1103, -T1103, -T1104, -T1104, -T1105, -T1105, -T1106, -T1106, -T1107, -T1107, -T1108, -T1108, -T1109, -T1109, -T1110, -T1110, -T1110.001, -T1110.002, -T1110.003, -T1110.004, -T1111, -T1111, -T1112, -T1112, -T1113, -T1113, -T1114, -T1114, -T1114.001, -T1114.002, -T1114.003, -T1115, -T1115, -T1116, -T1116, -T1117, -T1117, -T1118, -T1118, -T1119, -T1119, -T1120, -T1120, -T1121, -T1121, -T1122, -T1122, -T1123, -T1123, -T1124, -T1124, -T1125, -T1125, -T1126, -T1126, -T1127, -T1127, -T1127.001, -T1128, -T1128, -T1129, -T1129, -T1130, -T1130, -T1131, -T1131, -T1132, -T1132, -T1132.001, -T1132.002, -T1133, -T1133, -T1134, -T1134, -T1134.001, -T1134.002, -T1134.003, -T1134.004, -T1134.005, -T1135, -T1135, -T1136, -T1136, -T1136.001, -T1136.002, -T1136.003, -T1137, -T1137, -T1137.001, -T1137.002, -T1137.003, -T1137.004, -T1137.005, -T1137.006, -T1138, -T1138, -T1139, -T1139, -T1140, -T1140, -T1141, -T1141, -T1142, -T1142, -T1143, -T1143, -T1144, -T1144, -T1145, -T1145, -T1146, -T1146, -T1147, -T1147, -T1148, -T1148, -T1149, -T1149, -T1150, -T1150, -T1151, -T1151, -T1152, -T1152, -T1153, -T1153, -T1154, -T1154, -T1155, -T1155, -T1156, -T1156, -T1157, -T1157, -T1158, -T1158, -T1159, -T1159, -T1160, -T1160, -T1161, -T1161, -T1162, -T1162, -T1163, -T1163, -T1164, -T1164, -T1165, -T1165, -T1166, -T1166, -T1167, -T1168, -T1168, -T1169, -T1169, -T1170, -T1170, -T1171, -T1171, -T1172, -T1172, -T1173, -T1173, -T1174, -T1174, -T1175, -T1175, -T1176, -T1176, -T1177, -T1177, -T1178, -T1178, -T1179, -T1179, -T1180, -T1180, -T1181, -T1181, -T1182, -T1182, -T1183, -T1183, -T1184, -T1184, -T1185, -T1185, -T1186, -T1186, -T1187, -T1187, -T1188, -T1188, -T1189, -T1189, -T1190, -T1190, -T1191, -T1191, -T1192, -T1192, -T1193, -T1193, -T1194, -T1194, -T1195, -T1195, -T1195.001, -T1195.002, -T1195.003, -T1196, -T1196, -T1197, -T1197, -T1198, -T1198, -T1199, -T1199, -T1200, -T1200, -T1201, -T1201, -T1202, -T1202, -T1203, -T1203, -T1204, -T1204, -T1204.001, -T1204.002, -T1204.003, -T1205, -T1205, -T1205.001, -T1205.002, -T1206, -T1206, -T1207, -T1207, -T1208, -T1208, -T1209, -T1209, -T1210, -T1210, -T1211, -T1211, -T1212, -T1212, -T1213, -T1213, -T1213.001, -T1213.002, -T1213.003, -T1214, -T1214, -T1215, -T1215, -T1216, -T1216, -T1216.001, -T1216.002, -T1217, -T1217, -T1218, -T1218, -T1218.001, -T1218.002, -T1218.003, -T1218.004, -T1218.005, -T1218.007, -T1218.008, -T1218.009, -T1218.010, -T1218.011, -T1218.012, -T1218.013, -T1218.014, -T1218.015, -T1219, -T1219, -T1220, -T1220, -T1221, -T1221, -T1222, -T1222, -T1222.001, -T1222.002, -T1223, -T1223, -T1480, -T1480, -T1480.001, -T1482, -T1482, -T1483, -T1483, -T1484, -T1484, -T1484.001, -T1484.002, -T1485, -T1486, -T1486, -T1487, -T1488, -T1488, -T1489, -T1489, -T1490, -T1490, -T1491, -T1491, -T1491.001, -T1491.002, -T1492, -T1492, -T1493, -T1493, -T1494, -T1494, -T1495, -T1495, -T1496, -T1496, -T1497, -T1497, -T1497.001, -T1497.002, -T1497.003, -T1498, -T1498, -T1498.001, -T1498.002, -T1499, -T1499, -T1499.001, -T1499.002, -T1499.003, -T1499.004, -T1500, -T1500, -T1501, -T1501, -T1502, -T1503, -T1504, -T1505, -T1505.001, -T1505.002, -T1505.003, -T1505.004, -T1505.005, -T1506, -T1514, -T1518, -T1518.001, -T1519, -T1522, -T1525, -T1526, -T1527, -T1528, -T1529, -T1530, -T1531, -T1534, -T1535, -T1536, -T1537, -T1538, -T1539, -T1542, -T1542.001, -T1542.002, -T1542.003, -T1542.004, -T1542.005, -T1543, -T1543.001, -T1543.002, -T1543.003, -T1543.004, -T1543.005, -T1546, -T1546.001, -T1546.002, -T1546.003, -T1546.004, -T1546.005, -T1546.006, -T1546.007, -T1546.008, -T1546.009, -T1546.010, -T1546.011, -T1546.012, -T1546.013, -T1546.014, -T1546.015, -T1546.016, -T1547, -T1547.001, -T1547.002, -T1547.003, -T1547.004, -T1547.005, -T1547.006, -T1547.007, -T1547.008, -T1547.009, -T1547.010, -T1547.011, -T1547.012, -T1547.013, -T1547.014, -T1547.015, -T1548, -T1548.001, -T1548.002, -T1548.003, -T1548.004, -T1548.005, -T1548.006, -T1550, -T1550.001, -T1550.002, -T1550.003, -T1550.004, -T1552, -T1552.001, -T1552.002, -T1552.003, -T1552.004, -T1552.005, -T1552.006, -T1552.007, -T1552.008, -T1553, -T1553.001, -T1553.002, -T1553.003, -T1553.004, -T1553.005, -T1553.006, -T1554, -T1555, -T1555.001, -T1555.002, -T1555.003, -T1555.004, -T1555.005, -T1555.006, -T1556, -T1556.001, -T1556.002, -T1556.003, -T1556.004, -T1556.005, -T1556.006, -T1556.007, -T1556.008, -T1556.009, -T1557, -T1557.001, -T1557.002, -T1557.003, -T1558, -T1558.001, -T1558.002, -T1558.003, -T1558.004, -T1559, -T1559.001, -T1559.002, -T1559.003, -T1560, -T1560.001, -T1560.002, -T1560.003, -T1561, -T1561.001, -T1561.002, -T1562, -T1562.001, -T1562.002, -T1562.003, -T1562.004, -T1562.006, -T1562.007, -T1562.008, -T1562.009, -T1562.010, -T1562.011, -T1562.012, -T1563, -T1563.001, -T1563.002, -T1564, -T1564.001, -T1564.002, -T1564.003, -T1564.004, -T1564.005, -T1564.006, -T1564.007, -T1564.008, -T1564.009, -T1564.010, -T1564.011, -T1564.012, -T1565, -T1565.001, -T1565.002, -T1565.003, -T1566, -T1566.001, -T1566.002, -T1566.003, -T1566.004, -T1567, -T1567.001, -T1567.002, -T1567.003, -T1567.004, -T1568, -T1568.001, -T1568.002, -T1568.003, -T1569, -T1569.001, -T1569.002, -T1570, -T1571, -T1572, -T1573, -T1573.001, -T1573.002, -T1574, -T1574.001, -T1574.002, -T1574.004, -T1574.005, -T1574.006, -T1574.007, -T1574.008, -T1574.009, -T1574.010, -T1574.011, -T1574.012, -T1574.013, -T1574.014, -T1578, -T1578.001, -T1578.002, -T1578.003, -T1578.004, -T1578.005, -T1580, -T1583, -T1583.001, -T1583.002, -T1583.003, -T1583.004, -T1583.005, -T1583.006, -T1583.007, -T1583.008, -T1584, -T1584.001, -T1584.002, -T1584.003, -T1584.004, -T1584.005, -T1584.006, -T1584.007, -T1584.008, -T1585, -T1585.001, -T1585.002, -T1585.003, -T1586, -T1586.001, -T1586.002, -T1586.003, -T1587, -T1587.001, -T1587.002, -T1587.003, -T1587.004, -T1588, -T1588.001, -T1588.002, -T1588.003, -T1588.004, -T1588.005, -T1588.006, -T1588.007, -T1589, -T1589.001, -T1589.002, -T1589.003, -T1590, -T1590.001, -T1590.002, -T1590.003, -T1590.004, -T1590.005, -T1590.006, -T1591, -T1591.001, -T1591.002, -T1591.003, -T1591.004, -T1592, -T1592.001, -T1592.002, -T1592.003, -T1592.004, -T1593, -T1593.001, -T1593.002, -T1593.003, -T1594, -T1595, -T1595.001, -T1595.002, -T1595.003, -T1596, -T1596.001, -T1596.002, -T1596.003, -T1596.004, -T1596.005, -T1597, -T1597.001, -T1597.002, -T1598, -T1598.001, -T1598.002, -T1598.003, -T1598.004, -T1599, -T1599.001, -T1600, -T1600.001, -T1600.002, -T1601, -T1601.001, -T1601.002, -T1602, -T1602.001, -T1602.002, -T1606, -T1606.001, -T1606.002, -T1608, -T1608.001, -T1608.002, -T1608.003, -T1608.004, -T1608.005, -T1608.006, -T1609, -T1610, -T1611, -T1612, -T1613, -T1614, -T1614.001, -T1615, -T1619, -T1620, -T1621, -T1622, -T1647, -T1648, -T1649, -T1650, -T1651, -T1652, -T1653, -T1654, -T1656, -T1657, -T1659, -T1665, -TA0001, -TA0002, -TA0003, -TA0004, -TA0005, -TA0006, -TA0007, -TA0008, -TA0009, -TA0010, -TA0011, -TA0040, -TA0042, +C0001 +C0002 +C0004 +C0005 +C0006 +C0007 +C0010 +C0011 +C0012 +C0013 +C0014 +C0015 +C0016 +C0017 +C0018 +C0021 +C0022 +C0023 +C0024 +C0025 +C0026 +C0027 +C0028 +C0029 +C0030 +C0032 +C0033 +C0034 +DS0001 +DS0002 +DS0003 +DS0004 +DS0005 +DS0006 +DS0007 +DS0008 +DS0009 +DS0010 +DS0011 +DS0012 +DS0013 +DS0014 +DS0015 +DS0016 +DS0017 +DS0018 +DS0019 +DS0020 +DS0021 +DS0022 +DS0023 +DS0024 +DS0025 +DS0026 +DS0027 +DS0028 +DS0029 +DS0030 +DS0032 +DS0033 +DS0034 +DS0035 +DS0036 +DS0037 +DS0038 +G0001 +G0002 +G0003 +G0004 +G0005 +G0006 +G0007 +G0008 +G0009 +G0010 +G0011 +G0012 +G0013 +G0016 +G0017 +G0018 +G0019 +G0020 +G0021 +G0022 +G0023 +G0024 +G0025 +G0026 +G0027 +G0028 +G0029 +G0030 +G0032 +G0033 +G0034 +G0035 +G0036 +G0037 +G0038 +G0039 +G0040 +G0041 +G0043 +G0044 +G0045 +G0046 +G0047 +G0048 +G0049 +G0050 +G0051 +G0052 +G0053 +G0054 +G0055 +G0056 +G0059 +G0060 +G0061 +G0062 +G0063 +G0064 +G0065 +G0066 +G0067 +G0068 +G0069 +G0070 +G0071 +G0073 +G0075 +G0076 +G0077 +G0078 +G0079 +G0080 +G0081 +G0082 +G0083 +G0084 +G0085 +G0087 +G0088 +G0089 +G0090 +G0091 +G0092 +G0093 +G0094 +G0095 +G0096 +G0097 +G0098 +G0099 +G0100 +G0102 +G0103 +G0105 +G0106 +G0107 +G0108 +G0112 +G0114 +G0115 +G0117 +G0119 +G0120 +G0121 +G0122 +G0123 +G0124 +G0125 +G0126 +G0127 +G0128 +G0129 +G0130 +G0131 +G0133 +G0134 +G0135 +G0136 +G0137 +G0138 +G0139 +G0140 +G0142 +G0143 +G1001 +G1002 +G1003 +G1004 +G1005 +G1006 +G1007 +G1008 +G1009 +G1011 +G1012 +G1013 +G1014 +G1015 +G1016 +G1017 +G1018 +G1019 +G1020 +G1021 +G1022 +G1023 +G1024 +G1026 +G1028 +M1013 +M1015 +M1016 +M1017 +M1018 +M1019 +M1020 +M1021 +M1022 +M1024 +M1025 +M1026 +M1027 +M1028 +M1029 +M1030 +M1031 +M1032 +M1033 +M1034 +M1035 +M1036 +M1037 +M1038 +M1039 +M1040 +M1041 +M1042 +M1043 +M1044 +M1045 +M1046 +M1047 +M1048 +M1049 +M1050 +M1051 +M1052 +M1053 +M1054 +M1055 +M1056 +M1057 +S0001 +S0002 +S0003 +S0004 +S0005 +S0006 +S0007 +S0008 +S0009 +S0010 +S0011 +S0012 +S0013 +S0014 +S0015 +S0016 +S0017 +S0018 +S0019 +S0020 +S0021 +S0022 +S0023 +S0024 +S0025 +S0026 +S0027 +S0028 +S0029 +S0030 +S0031 +S0032 +S0033 +S0034 +S0035 +S0036 +S0037 +S0038 +S0039 +S0040 +S0041 +S0042 +S0043 +S0044 +S0045 +S0046 +S0047 +S0048 +S0049 +S0050 +S0051 +S0052 +S0053 +S0054 +S0055 +S0056 +S0057 +S0058 +S0059 +S0060 +S0061 +S0062 +S0063 +S0064 +S0065 +S0066 +S0067 +S0068 +S0069 +S0070 +S0071 +S0072 +S0073 +S0074 +S0075 +S0076 +S0077 +S0078 +S0079 +S0080 +S0081 +S0082 +S0083 +S0084 +S0085 +S0086 +S0087 +S0088 +S0089 +S0090 +S0091 +S0092 +S0093 +S0094 +S0095 +S0096 +S0097 +S0098 +S0099 +S0100 +S0101 +S0102 +S0103 +S0104 +S0105 +S0106 +S0107 +S0108 +S0109 +S0110 +S0111 +S0112 +S0113 +S0114 +S0115 +S0116 +S0117 +S0118 +S0119 +S0120 +S0121 +S0122 +S0123 +S0124 +S0125 +S0126 +S0127 +S0128 +S0129 +S0130 +S0131 +S0132 +S0133 +S0134 +S0135 +S0136 +S0137 +S0138 +S0139 +S0140 +S0141 +S0142 +S0143 +S0144 +S0145 +S0146 +S0147 +S0148 +S0149 +S0150 +S0151 +S0152 +S0153 +S0154 +S0155 +S0156 +S0157 +S0158 +S0159 +S0160 +S0161 +S0162 +S0163 +S0164 +S0165 +S0166 +S0167 +S0168 +S0169 +S0170 +S0171 +S0172 +S0173 +S0174 +S0175 +S0176 +S0177 +S0178 +S0179 +S0180 +S0181 +S0182 +S0183 +S0184 +S0185 +S0186 +S0187 +S0188 +S0189 +S0190 +S0191 +S0192 +S0193 +S0194 +S0195 +S0196 +S0197 +S0198 +S0199 +S0200 +S0201 +S0202 +S0203 +S0204 +S0205 +S0206 +S0207 +S0208 +S0210 +S0211 +S0212 +S0213 +S0214 +S0215 +S0216 +S0217 +S0218 +S0219 +S0220 +S0221 +S0222 +S0223 +S0224 +S0225 +S0226 +S0227 +S0228 +S0229 +S0230 +S0231 +S0232 +S0233 +S0234 +S0235 +S0236 +S0237 +S0238 +S0239 +S0240 +S0241 +S0242 +S0243 +S0244 +S0245 +S0246 +S0247 +S0248 +S0249 +S0250 +S0251 +S0252 +S0253 +S0254 +S0255 +S0256 +S0257 +S0258 +S0259 +S0260 +S0261 +S0262 +S0263 +S0264 +S0265 +S0266 +S0267 +S0268 +S0269 +S0270 +S0271 +S0272 +S0273 +S0274 +S0275 +S0276 +S0277 +S0278 +S0279 +S0280 +S0281 +S0282 +S0283 +S0284 +S0302 +S0330 +S0331 +S0332 +S0333 +S0334 +S0335 +S0336 +S0337 +S0338 +S0339 +S0340 +S0341 +S0342 +S0343 +S0344 +S0345 +S0346 +S0347 +S0348 +S0349 +S0350 +S0351 +S0352 +S0353 +S0354 +S0355 +S0356 +S0357 +S0358 +S0359 +S0360 +S0361 +S0362 +S0363 +S0364 +S0365 +S0366 +S0367 +S0368 +S0369 +S0370 +S0371 +S0372 +S0373 +S0374 +S0375 +S0376 +S0377 +S0378 +S0379 +S0380 +S0381 +S0382 +S0383 +S0384 +S0385 +S0386 +S0387 +S0388 +S0389 +S0390 +S0391 +S0393 +S0394 +S0395 +S0396 +S0397 +S0398 +S0400 +S0401 +S0402 +S0404 +S0409 +S0410 +S0412 +S0413 +S0414 +S0415 +S0416 +S0417 +S0428 +S0430 +S0431 +S0433 +S0434 +S0435 +S0436 +S0437 +S0438 +S0439 +S0441 +S0442 +S0443 +S0444 +S0445 +S0446 +S0447 +S0448 +S0449 +S0450 +S0451 +S0452 +S0453 +S0454 +S0455 +S0456 +S0457 +S0458 +S0459 +S0460 +S0461 +S0462 +S0464 +S0465 +S0466 +S0467 +S0468 +S0469 +S0470 +S0471 +S0472 +S0473 +S0475 +S0476 +S0477 +S0481 +S0482 +S0483 +S0484 +S0486 +S0487 +S0488 +S0491 +S0492 +S0493 +S0495 +S0496 +S0497 +S0498 +S0499 +S0500 +S0501 +S0502 +S0503 +S0504 +S0508 +S0511 +S0512 +S0513 +S0514 +S0515 +S0516 +S0517 +S0518 +S0519 +S0520 +S0521 +S0526 +S0527 +S0528 +S0530 +S0531 +S0532 +S0533 +S0534 +S0537 +S0538 +S0543 +S0546 +S0547 +S0552 +S0553 +S0554 +S0556 +S0559 +S0560 +S0561 +S0562 +S0564 +S0565 +S0567 +S0568 +S0569 +S0570 +S0572 +S0574 +S0575 +S0576 +S0578 +S0579 +S0581 +S0582 +S0583 +S0584 +S0585 +S0586 +S0587 +S0588 +S0589 +S0590 +S0591 +S0592 +S0593 +S0594 +S0595 +S0596 +S0597 +S0598 +S0599 +S0600 +S0601 +S0603 +S0604 +S0605 +S0606 +S0607 +S0608 +S0610 +S0611 +S0612 +S0613 +S0614 +S0615 +S0616 +S0617 +S0618 +S0622 +S0623 +S0624 +S0625 +S0626 +S0627 +S0628 +S0629 +S0630 +S0631 +S0632 +S0633 +S0634 +S0635 +S0636 +S0637 +S0638 +S0639 +S0640 +S0641 +S0642 +S0643 +S0644 +S0645 +S0646 +S0647 +S0648 +S0649 +S0650 +S0651 +S0652 +S0653 +S0654 +S0657 +S0658 +S0659 +S0660 +S0661 +S0662 +S0663 +S0664 +S0665 +S0666 +S0667 +S0668 +S0669 +S0670 +S0671 +S0672 +S0673 +S0674 +S0677 +S0678 +S0679 +S0680 +S0681 +S0682 +S0683 +S0684 +S0685 +S0686 +S0687 +S0688 +S0689 +S0690 +S0691 +S0692 +S0693 +S0694 +S0695 +S0696 +S0697 +S0698 +S0699 +S1011 +S1012 +S1013 +S1014 +S1015 +S1016 +S1017 +S1018 +S1019 +S1020 +S1021 +S1022 +S1023 +S1024 +S1025 +S1026 +S1027 +S1028 +S1029 +S1030 +S1031 +S1032 +S1033 +S1034 +S1035 +S1037 +S1039 +S1040 +S1041 +S1042 +S1043 +S1044 +S1046 +S1047 +S1048 +S1049 +S1050 +S1051 +S1052 +S1053 +S1058 +S1059 +S1060 +S1063 +S1064 +S1065 +S1066 +S1068 +S1070 +S1071 +S1072 +S1073 +S1074 +S1075 +S1076 +S1078 +S1081 +S1084 +S1085 +S1086 +S1087 +S1088 +S1089 +S1090 +S1091 +S1096 +S1097 +S1099 +S1100 +S1101 +S1102 +S1104 +S1105 +S1106 +S1107 +S1108 +S1109 +S1110 +S1111 +S1112 +S1113 +S1114 +S1115 +S1116 +S1117 +S1118 +S1119 +S1120 +S1121 +S1122 +S1123 +S1124 +S1125 +S1129 +T1001 +T1001.001 +T1001.002 +T1001.003 +T1003 +T1003.001 +T1003.002 +T1003.003 +T1003.004 +T1003.005 +T1003.006 +T1003.007 +T1003.008 +T1005 +T1006 +T1007 +T1008 +T1010 +T1011 +T1011.001 +T1012 +T1014 +T1016 +T1016.001 +T1016.002 +T1018 +T1020 +T1020.001 +T1021 +T1021.001 +T1021.002 +T1021.003 +T1021.004 +T1021.005 +T1021.006 +T1021.007 +T1021.008 +T1025 +T1027 +T1027.001 +T1027.002 +T1027.003 +T1027.004 +T1027.005 +T1027.006 +T1027.007 +T1027.008 +T1027.009 +T1027.010 +T1027.011 +T1027.012 +T1027.013 +T1029 +T1030 +T1033 +T1036 +T1036.001 +T1036.002 +T1036.003 +T1036.004 +T1036.005 +T1036.006 +T1036.007 +T1036.008 +T1036.009 +T1037 +T1037.001 +T1037.002 +T1037.003 +T1037.004 +T1037.005 +T1039 +T1040 +T1041 +T1046 +T1047 +T1048 +T1048.001 +T1048.002 +T1048.003 +T1049 +T1052 +T1052.001 +T1053 +T1053.002 +T1053.003 +T1053.005 +T1053.006 +T1053.007 +T1055 +T1055.001 +T1055.002 +T1055.003 +T1055.004 +T1055.005 +T1055.008 +T1055.009 +T1055.011 +T1055.012 +T1055.013 +T1055.014 +T1055.015 +T1056 +T1056.001 +T1056.002 +T1056.003 +T1056.004 +T1057 +T1059 +T1059.001 +T1059.002 +T1059.003 +T1059.004 +T1059.005 +T1059.006 +T1059.007 +T1059.008 +T1059.009 +T1059.010 +T1068 +T1069 +T1069.001 +T1069.002 +T1069.003 +T1070 +T1070.001 +T1070.002 +T1070.003 +T1070.004 +T1070.005 +T1070.006 +T1070.007 +T1070.008 +T1070.009 +T1071 +T1071.001 +T1071.002 +T1071.003 +T1071.004 +T1072 +T1074 +T1074.001 +T1074.002 +T1078 +T1078.001 +T1078.002 +T1078.003 +T1078.004 +T1080 +T1082 +T1083 +T1087 +T1087.001 +T1087.002 +T1087.003 +T1087.004 +T1090 +T1090.001 +T1090.002 +T1090.003 +T1090.004 +T1091 +T1092 +T1095 +T1098 +T1098.001 +T1098.002 +T1098.003 +T1098.004 +T1098.005 +T1098.006 +T1102 +T1102.001 +T1102.002 +T1102.003 +T1104 +T1105 +T1106 +T1110 +T1110.001 +T1110.002 +T1110.003 +T1110.004 +T1111 +T1112 +T1113 +T1114 +T1114.001 +T1114.002 +T1114.003 +T1115 +T1119 +T1120 +T1123 +T1124 +T1125 +T1127 +T1127.001 +T1129 +T1132 +T1132.001 +T1132.002 +T1133 +T1134 +T1134.001 +T1134.002 +T1134.003 +T1134.004 +T1134.005 +T1135 +T1136 +T1136.001 +T1136.002 +T1136.003 +T1137 +T1137.001 +T1137.002 +T1137.003 +T1137.004 +T1137.005 +T1137.006 +T1140 +T1176 +T1185 +T1187 +T1189 +T1190 +T1195 +T1195.001 +T1195.002 +T1195.003 +T1197 +T1199 +T1200 +T1201 +T1202 +T1203 +T1204 +T1204.001 +T1204.002 +T1204.003 +T1205 +T1205.001 +T1205.002 +T1207 +T1210 +T1211 +T1212 +T1213 +T1213.001 +T1213.002 +T1213.003 +T1216 +T1216.001 +T1216.002 +T1217 +T1218 +T1218.001 +T1218.002 +T1218.003 +T1218.004 +T1218.005 +T1218.007 +T1218.008 +T1218.009 +T1218.010 +T1218.011 +T1218.012 +T1218.013 +T1218.014 +T1218.015 +T1219 +T1220 +T1221 +T1222 +T1222.001 +T1222.002 +T1480 +T1480.001 +T1482 +T1484 +T1484.001 +T1484.002 +T1485 +T1486 +T1489 +T1490 +T1491 +T1491.001 +T1491.002 +T1495 +T1496 +T1497 +T1497.001 +T1497.002 +T1497.003 +T1498 +T1498.001 +T1498.002 +T1499 +T1499.001 +T1499.002 +T1499.003 +T1499.004 +T1505 +T1505.001 +T1505.002 +T1505.003 +T1505.004 +T1505.005 +T1518 +T1518.001 +T1525 +T1526 +T1528 +T1529 +T1530 +T1531 +T1534 +T1535 +T1537 +T1538 +T1539 +T1542 +T1542.001 +T1542.002 +T1542.003 +T1542.004 +T1542.005 +T1543 +T1543.001 +T1543.002 +T1543.003 +T1543.004 +T1543.005 +T1546 +T1546.001 +T1546.002 +T1546.003 +T1546.004 +T1546.005 +T1546.006 +T1546.007 +T1546.008 +T1546.009 +T1546.010 +T1546.011 +T1546.012 +T1546.013 +T1546.014 +T1546.015 +T1546.016 +T1547 +T1547.001 +T1547.002 +T1547.003 +T1547.004 +T1547.005 +T1547.006 +T1547.007 +T1547.008 +T1547.009 +T1547.010 +T1547.012 +T1547.013 +T1547.014 +T1547.015 +T1548 +T1548.001 +T1548.002 +T1548.003 +T1548.004 +T1548.005 +T1548.006 +T1550 +T1550.001 +T1550.002 +T1550.003 +T1550.004 +T1552 +T1552.001 +T1552.002 +T1552.003 +T1552.004 +T1552.005 +T1552.006 +T1552.007 +T1552.008 +T1553 +T1553.001 +T1553.002 +T1553.003 +T1553.004 +T1553.005 +T1553.006 +T1554 +T1555 +T1555.001 +T1555.002 +T1555.003 +T1555.004 +T1555.005 +T1555.006 +T1556 +T1556.001 +T1556.002 +T1556.003 +T1556.004 +T1556.005 +T1556.006 +T1556.007 +T1556.008 +T1556.009 +T1557 +T1557.001 +T1557.002 +T1557.003 +T1558 +T1558.001 +T1558.002 +T1558.003 +T1558.004 +T1559 +T1559.001 +T1559.002 +T1559.003 +T1560 +T1560.001 +T1560.002 +T1560.003 +T1561 +T1561.001 +T1561.002 +T1562 +T1562.001 +T1562.002 +T1562.003 +T1562.004 +T1562.006 +T1562.007 +T1562.008 +T1562.009 +T1562.010 +T1562.011 +T1562.012 +T1563 +T1563.001 +T1563.002 +T1564 +T1564.001 +T1564.002 +T1564.003 +T1564.004 +T1564.005 +T1564.006 +T1564.007 +T1564.008 +T1564.009 +T1564.010 +T1564.011 +T1564.012 +T1565 +T1565.001 +T1565.002 +T1565.003 +T1566 +T1566.001 +T1566.002 +T1566.003 +T1566.004 +T1567 +T1567.001 +T1567.002 +T1567.003 +T1567.004 +T1568 +T1568.001 +T1568.002 +T1568.003 +T1569 +T1569.001 +T1569.002 +T1570 +T1571 +T1572 +T1573 +T1573.001 +T1573.002 +T1574 +T1574.001 +T1574.002 +T1574.004 +T1574.005 +T1574.006 +T1574.007 +T1574.008 +T1574.009 +T1574.010 +T1574.011 +T1574.012 +T1574.013 +T1574.014 +T1578 +T1578.001 +T1578.002 +T1578.003 +T1578.004 +T1578.005 +T1580 +T1583 +T1583.001 +T1583.002 +T1583.003 +T1583.004 +T1583.005 +T1583.006 +T1583.007 +T1583.008 +T1584 +T1584.001 +T1584.002 +T1584.003 +T1584.004 +T1584.005 +T1584.006 +T1584.007 +T1584.008 +T1585 +T1585.001 +T1585.002 +T1585.003 +T1586 +T1586.001 +T1586.002 +T1586.003 +T1587 +T1587.001 +T1587.002 +T1587.003 +T1587.004 +T1588 +T1588.001 +T1588.002 +T1588.003 +T1588.004 +T1588.005 +T1588.006 +T1588.007 +T1589 +T1589.001 +T1589.002 +T1589.003 +T1590 +T1590.001 +T1590.002 +T1590.003 +T1590.004 +T1590.005 +T1590.006 +T1591 +T1591.001 +T1591.002 +T1591.003 +T1591.004 +T1592 +T1592.001 +T1592.002 +T1592.003 +T1592.004 +T1593 +T1593.001 +T1593.002 +T1593.003 +T1594 +T1595 +T1595.001 +T1595.002 +T1595.003 +T1596 +T1596.001 +T1596.002 +T1596.003 +T1596.004 +T1596.005 +T1597 +T1597.001 +T1597.002 +T1598 +T1598.001 +T1598.002 +T1598.003 +T1598.004 +T1599 +T1599.001 +T1600 +T1600.001 +T1600.002 +T1601 +T1601.001 +T1601.002 +T1602 +T1602.001 +T1602.002 +T1606 +T1606.001 +T1606.002 +T1608 +T1608.001 +T1608.002 +T1608.003 +T1608.004 +T1608.005 +T1608.006 +T1609 +T1610 +T1611 +T1612 +T1613 +T1614 +T1614.001 +T1615 +T1619 +T1620 +T1621 +T1622 +T1647 +T1648 +T1649 +T1650 +T1651 +T1652 +T1653 +T1654 +T1656 +T1657 +T1659 +T1665 +TA0001 +TA0002 +TA0003 +TA0004 +TA0005 +TA0006 +TA0007 +TA0008 +TA0009 +TA0010 +TA0011 +TA0040 +TA0042 TA0043 \ No newline at end of file diff --git a/includes/lookups/mitre_attack_ics_id.txt b/includes/lookups/mitre_attack_ics_id.txt index 4b02bed..482ac83 100644 --- a/includes/lookups/mitre_attack_ics_id.txt +++ b/includes/lookups/mitre_attack_ics_id.txt @@ -1,233 +1,253 @@ -A0001, -A0002, -A0003, -A0004, -A0005, -A0006, -A0007, -A0008, -A0009, -A0010, -A0011, -A0012, -A0013, -A0014, -C0009, -C0020, -C0025, -C0028, -C0030, -C0031, -C0034, -DS0001, -DS0002, -DS0003, -DS0009, -DS0011, -DS0012, -DS0015, -DS0016, -DS0017, -DS0019, -DS0022, -DS0024, -DS0028, -DS0029, -DS0033, -DS0039, -DS0040, -G0032, -G0034, -G0035, -G0037, -G0046, -G0049, -G0057, -G0064, -G0074, -G0077, -G0082, -G0088, -G0102, -G0115, -G1000, -G1001, -G1027, -M0800, -M0801, -M0802, -M0803, -M0804, -M0805, -M0806, -M0807, -M0808, -M0809, -M0810, -M0811, -M0812, -M0813, -M0814, -M0815, -M0816, -M0817, -M0818, -M0913, -M0915, -M0916, -M0917, -M0918, -M0919, -M0920, -M0921, -M0922, -M0924, -M0926, -M0927, -M0928, -M0930, -M0931, -M0932, -M0934, -M0935, -M0936, -M0937, -M0938, -M0941, -M0942, -M0944, -M0945, -M0946, -M0947, -M0948, -M0949, -M0950, -M0951, -M0953, -M0954, -M1013, -M1017, -M1024, -M1027, -M1028, -M1032, -M1036, -M1038, -M1047, -M1054, -S0038, -S0089, -S0093, -S0143, -S0366, -S0368, -S0372, -S0446, -S0496, -S0603, -S0604, -S0605, -S0606, -S0607, -S0608, -S1000, -S1006, -S1009, -S1010, -S1045, -S1072, -T0800, -T0801, -T0802, -T0803, -T0804, -T0805, -T0806, -T0807, -T0809, -T0811, -T0812, -T0813, -T0814, -T0815, -T0816, -T0817, -T0819, -T0820, -T0821, -T0822, -T0823, -T0826, -T0827, -T0828, -T0829, -T0830, -T0831, -T0832, -T0834, -T0835, -T0836, -T0837, -T0838, -T0839, -T0840, -T0842, -T0843, -T0845, -T0846, -T0847, -T0848, -T0849, -T0851, -T0852, -T0853, -T0855, -T0856, -T0857, -T0858, -T0859, -T0860, -T0861, -T0862, -T0863, -T0864, -T0865, -T0866, -T0867, -T0868, -T0869, -T0871, -T0872, -T0873, -T0874, -T0877, -T0878, -T0879, -T0880, -T0881, -T0882, -T0883, -T0884, -T0885, -T0886, -T0887, -T0888, -T0889, -T0890, -T0891, -T0892, -T0893, -T0894, -T0895, -TA0100, -TA0101, -TA0102, -TA0103, -TA0104, -TA0105, -TA0106, -TA0107, -TA0108, -TA0109, -TA0110, +A0001 +A0002 +A0003 +A0004 +A0005 +A0006 +A0007 +A0008 +A0009 +A0010 +A0011 +A0012 +A0013 +A0014 +C0020 +C0025 +C0028 +C0030 +C0031 +C0034 +DS0001 +DS0002 +DS0003 +DS0009 +DS0011 +DS0012 +DS0015 +DS0016 +DS0017 +DS0019 +DS0022 +DS0024 +DS0028 +DS0029 +DS0033 +DS0039 +DS0040 +G0032 +G0034 +G0035 +G0037 +G0046 +G0049 +G0064 +G0077 +G0082 +G0088 +G0102 +G0115 +G1000 +G1001 +G1027 +M0800 +M0801 +M0802 +M0803 +M0804 +M0805 +M0806 +M0807 +M0808 +M0809 +M0810 +M0811 +M0812 +M0813 +M0814 +M0815 +M0816 +M0817 +M0818 +M0913 +M0915 +M0916 +M0917 +M0918 +M0919 +M0920 +M0921 +M0922 +M0924 +M0926 +M0927 +M0928 +M0930 +M0931 +M0932 +M0934 +M0935 +M0936 +M0937 +M0938 +M0941 +M0942 +M0944 +M0945 +M0946 +M0947 +M0948 +M0949 +M0950 +M0951 +M0953 +M0954 +M1013 +M1015 +M1016 +M1017 +M1018 +M1019 +M1020 +M1021 +M1022 +M1024 +M1026 +M1027 +M1028 +M1030 +M1031 +M1032 +M1034 +M1035 +M1036 +M1037 +M1038 +M1041 +M1042 +M1044 +M1045 +M1046 +M1047 +M1048 +M1049 +M1050 +M1051 +M1053 +M1054 +S0038 +S0089 +S0093 +S0143 +S0366 +S0368 +S0372 +S0446 +S0496 +S0603 +S0604 +S0605 +S0606 +S0607 +S0608 +S1000 +S1006 +S1009 +S1010 +S1045 +S1072 +T0800 +T0801 +T0802 +T0803 +T0804 +T0805 +T0806 +T0807 +T0809 +T0811 +T0812 +T0813 +T0814 +T0815 +T0816 +T0817 +T0819 +T0820 +T0821 +T0822 +T0823 +T0826 +T0827 +T0828 +T0829 +T0830 +T0831 +T0832 +T0834 +T0835 +T0836 +T0837 +T0838 +T0839 +T0840 +T0842 +T0843 +T0845 +T0846 +T0847 +T0848 +T0849 +T0851 +T0852 +T0853 +T0855 +T0856 +T0857 +T0858 +T0859 +T0860 +T0861 +T0862 +T0863 +T0864 +T0865 +T0866 +T0867 +T0868 +T0869 +T0871 +T0872 +T0873 +T0874 +T0877 +T0878 +T0879 +T0880 +T0881 +T0882 +T0883 +T0884 +T0885 +T0886 +T0887 +T0888 +T0889 +T0890 +T0891 +T0892 +T0893 +T0894 +T0895 +TA0100 +TA0101 +TA0102 +TA0103 +TA0104 +TA0105 +TA0106 +TA0107 +TA0108 +TA0109 +TA0110 TA0111 \ No newline at end of file diff --git a/includes/lookups/mitre_attack_mobile_id.txt b/includes/lookups/mitre_attack_mobile_id.txt index 246ff82..a1d8deb 100644 --- a/includes/lookups/mitre_attack_mobile_id.txt +++ b/includes/lookups/mitre_attack_mobile_id.txt @@ -1,333 +1,214 @@ -C0016, -C0033, -DS0009, -DS0013, -DS0017, -DS0029, -DS0041, -DS0042, -G0007, -G0034, -G0056, -G0070, -G0097, -G0112, -G0142, -G1002, -G1006, -G1015, -G1019, -G1028, -G1029, -M1001, -M1002, -M1003, -M1004, -M1005, -M1006, -M1007, -M1008, -M1009, -M1010, -M1011, -M1012, -M1013, -M1014, -M1058, -M1059, -S0182, -S0285, -S0286, -S0287, -S0288, -S0289, -S0290, -S0291, -S0292, -S0293, -S0294, -S0295, -S0297, -S0298, -S0299, -S0300, -S0301, -S0302, -S0303, -S0304, -S0305, -S0306, -S0307, -S0308, -S0309, -S0310, -S0311, -S0312, -S0313, -S0314, -S0315, -S0316, -S0317, -S0318, -S0319, -S0320, -S0321, -S0322, -S0323, -S0324, -S0325, -S0326, -S0327, -S0328, -S0329, -S0399, -S0403, -S0405, -S0406, -S0407, -S0408, -S0411, -S0418, -S0419, -S0420, -S0421, -S0422, -S0423, -S0424, -S0425, -S0426, -S0427, -S0432, -S0440, -S0463, -S0478, -S0479, -S0480, -S0485, -S0489, -S0490, -S0494, -S0505, -S0506, -S0507, -S0509, -S0522, -S0524, -S0525, -S0529, -S0535, -S0536, -S0539, -S0540, -S0544, -S0545, -S0549, -S0550, -S0551, -S0555, -S0558, -S0577, -S0602, -S0655, -S1054, -S1055, -S1056, -S1061, -S1062, -S1067, -S1069, -S1077, -S1079, -S1080, -S1082, -S1083, -S1092, -S1093, -S1094, -S1095, -S1103, -S1126, -S1128, -T1398, -T1399, -T1400, -T1401, -T1402, -T1403, -T1404, -T1405, -T1406, -T1406.001, -T1406.002, -T1407, -T1408, -T1409, -T1410, -T1411, -T1412, -T1413, -T1414, -T1416, -T1417, -T1417.001, -T1417.002, -T1418, -T1418.001, -T1420, -T1421, -T1422, -T1422.001, -T1422.002, -T1423, -T1424, -T1426, -T1427, -T1428, -T1429, -T1430, -T1430.001, -T1430.002, -T1432, -T1433, -T1435, -T1436, -T1437, -T1437.001, -T1438, -T1439, -T1444, -T1446, -T1447, -T1448, -T1449, -T1450, -T1451, -T1452, -T1456, -T1458, -T1461, -T1463, -T1464, -T1465, -T1466, -T1467, -T1468, -T1469, -T1470, -T1471, -T1472, -T1474, -T1474.001, -T1474.002, -T1474.003, -T1475, -T1476, -T1477, -T1478, -T1481, -T1481.001, -T1481.002, -T1481.003, -T1507, -T1508, -T1509, -T1510, -T1512, -T1513, -T1516, -T1517, -T1520, -T1521, -T1521.001, -T1521.002, -T1521.003, -T1523, -T1532, -T1533, -T1540, -T1541, -T1544, -T1575, -T1576, -T1577, -T1579, -T1581, -T1582, -T1603, -T1604, -T1605, -T1616, -T1617, -T1618, -T1623, -T1623.001, -T1624, -T1624.001, -T1625, -T1625.001, -T1626, -T1626.001, -T1627, -T1627.001, -T1628, -T1628.001, -T1628.002, -T1628.003, -T1629, -T1629.001, -T1629.002, -T1629.003, -T1630, -T1630.001, -T1630.002, -T1630.003, -T1631, -T1631.001, -T1632, -T1632.001, -T1633, -T1633.001, -T1634, -T1634.001, -T1635, -T1635.001, -T1636, -T1636.001, -T1636.002, -T1636.003, -T1636.004, -T1637, -T1637.001, -T1638, -T1639, -T1639.001, -T1640, -T1641, -T1641.001, -T1642, -T1643, -T1644, -T1645, -T1646, -T1655, -T1655.001, -T1658, -T1660, -T1661, -T1662, -T1663, -T1664, -TA0027, -TA0028, -TA0029, -TA0030, -TA0031, -TA0032, -TA0033, -TA0034, -TA0035, -TA0036, -TA0037, -TA0038, -TA0039, -TA0041 \ No newline at end of file +C0016 +C0033 +DS0009 +DS0013 +DS0017 +DS0029 +DS0041 +DS0042 +G0007 +G0034 +G0056 +G0070 +G0112 +G0142 +G1002 +G1006 +G1015 +G1019 +G1028 +G1029 +M1001 +M1002 +M1003 +M1004 +M1006 +M1009 +M1010 +M1011 +M1012 +M1013 +M1014 +M1058 +M1059 +S0285 +S0286 +S0287 +S0288 +S0289 +S0290 +S0291 +S0292 +S0294 +S0295 +S0297 +S0298 +S0299 +S0300 +S0301 +S0302 +S0303 +S0304 +S0305 +S0306 +S0307 +S0308 +S0309 +S0310 +S0311 +S0312 +S0313 +S0314 +S0315 +S0316 +S0317 +S0318 +S0319 +S0320 +S0321 +S0322 +S0323 +S0324 +S0325 +S0326 +S0327 +S0328 +S0329 +S0407 +S0408 +S0422 +S0423 +S0427 +S0432 +S0440 +S0479 +S0480 +S0507 +S0509 +S0522 +S0655 +S1054 +S1055 +S1056 +S1061 +S1062 +S1067 +S1069 +S1077 +S1079 +S1080 +S1082 +S1083 +S1092 +S1093 +S1094 +S1095 +S1103 +S1126 +S1128 +T1398 +T1404 +T1406 +T1406.002 +T1407 +T1409 +T1414 +T1417 +T1417.001 +T1417.002 +T1418 +T1418.001 +T1420 +T1422 +T1422.001 +T1422.002 +T1424 +T1428 +T1429 +T1430 +T1430.001 +T1430.002 +T1456 +T1458 +T1461 +T1464 +T1471 +T1474 +T1474.001 +T1474.002 +T1474.003 +T1481 +T1481.001 +T1481.002 +T1481.003 +T1509 +T1512 +T1513 +T1516 +T1517 +T1521.003 +T1541 +T1544 +T1577 +T1582 +T1603 +T1604 +T1616 +T1617 +T1623 +T1623.001 +T1624 +T1624.001 +T1625 +T1625.001 +T1626 +T1626.001 +T1627 +T1627.001 +T1628 +T1628.001 +T1628.003 +T1629 +T1629.001 +T1629.002 +T1629.003 +T1630 +T1630.001 +T1630.002 +T1630.003 +T1631 +T1631.001 +T1632 +T1632.001 +T1633 +T1633.001 +T1634 +T1634.001 +T1635 +T1635.001 +T1636 +T1636.001 +T1636.002 +T1636.003 +T1636.004 +T1637 +T1637.001 +T1638 +T1639 +T1639.001 +T1640 +T1641 +T1641.001 +T1642 +T1643 +T1644 +T1645 +T1646 +T1655 +T1655.001 +T1658 +T1660 +T1661 +T1662 +T1663 +T1664 \ No newline at end of file diff --git a/includes/lookups/mitre_capec_id.txt b/includes/lookups/mitre_capec_id.txt index 7a8422d..3532dbc 100644 --- a/includes/lookups/mitre_capec_id.txt +++ b/includes/lookups/mitre_capec_id.txt @@ -1,615 +1,615 @@ -CAPEC-1, -CAPEC-10, -CAPEC-100, -CAPEC-101, -CAPEC-102, -CAPEC-103, -CAPEC-104, -CAPEC-105, -CAPEC-106, -CAPEC-107, -CAPEC-108, -CAPEC-109, -CAPEC-11, -CAPEC-110, -CAPEC-111, -CAPEC-112, -CAPEC-113, -CAPEC-114, -CAPEC-115, -CAPEC-116, -CAPEC-117, -CAPEC-12, -CAPEC-120, -CAPEC-121, -CAPEC-122, -CAPEC-123, -CAPEC-124, -CAPEC-125, -CAPEC-126, -CAPEC-127, -CAPEC-128, -CAPEC-129, -CAPEC-13, -CAPEC-130, -CAPEC-131, -CAPEC-132, -CAPEC-133, -CAPEC-134, -CAPEC-135, -CAPEC-136, -CAPEC-137, -CAPEC-138, -CAPEC-139, -CAPEC-14, -CAPEC-140, -CAPEC-141, -CAPEC-142, -CAPEC-143, -CAPEC-144, -CAPEC-145, -CAPEC-146, -CAPEC-147, -CAPEC-148, -CAPEC-149, -CAPEC-15, -CAPEC-150, -CAPEC-151, -CAPEC-153, -CAPEC-154, -CAPEC-155, -CAPEC-157, -CAPEC-158, -CAPEC-159, -CAPEC-16, -CAPEC-160, -CAPEC-161, -CAPEC-162, -CAPEC-163, -CAPEC-164, -CAPEC-165, -CAPEC-166, -CAPEC-167, -CAPEC-168, -CAPEC-169, -CAPEC-17, -CAPEC-170, -CAPEC-171, -CAPEC-173, -CAPEC-174, -CAPEC-175, -CAPEC-176, -CAPEC-177, -CAPEC-178, -CAPEC-179, -CAPEC-18, -CAPEC-180, -CAPEC-181, -CAPEC-182, -CAPEC-183, -CAPEC-184, -CAPEC-185, -CAPEC-186, -CAPEC-187, -CAPEC-188, -CAPEC-189, -CAPEC-19, -CAPEC-190, -CAPEC-191, -CAPEC-192, -CAPEC-193, -CAPEC-194, -CAPEC-195, -CAPEC-196, -CAPEC-197, -CAPEC-198, -CAPEC-199, -CAPEC-2, -CAPEC-20, -CAPEC-200, -CAPEC-201, -CAPEC-202, -CAPEC-203, -CAPEC-204, -CAPEC-205, -CAPEC-206, -CAPEC-207, -CAPEC-208, -CAPEC-209, -CAPEC-21, -CAPEC-211, -CAPEC-212, -CAPEC-213, -CAPEC-214, -CAPEC-215, -CAPEC-216, -CAPEC-217, -CAPEC-218, -CAPEC-219, -CAPEC-22, -CAPEC-220, -CAPEC-221, -CAPEC-222, -CAPEC-224, -CAPEC-226, -CAPEC-227, -CAPEC-228, -CAPEC-229, -CAPEC-23, -CAPEC-230, -CAPEC-231, -CAPEC-233, -CAPEC-234, -CAPEC-235, -CAPEC-236, -CAPEC-237, -CAPEC-238, -CAPEC-239, -CAPEC-24, -CAPEC-240, -CAPEC-241, -CAPEC-242, -CAPEC-243, -CAPEC-244, -CAPEC-245, -CAPEC-246, -CAPEC-247, -CAPEC-248, -CAPEC-249, -CAPEC-25, -CAPEC-250, -CAPEC-251, -CAPEC-252, -CAPEC-253, -CAPEC-254, -CAPEC-256, -CAPEC-257, -CAPEC-258, -CAPEC-259, -CAPEC-26, -CAPEC-260, -CAPEC-261, -CAPEC-263, -CAPEC-264, -CAPEC-265, -CAPEC-266, -CAPEC-267, -CAPEC-268, -CAPEC-269, -CAPEC-27, -CAPEC-270, -CAPEC-271, -CAPEC-272, -CAPEC-273, -CAPEC-274, -CAPEC-275, -CAPEC-276, -CAPEC-277, -CAPEC-278, -CAPEC-279, -CAPEC-28, -CAPEC-280, -CAPEC-285, -CAPEC-287, -CAPEC-288, -CAPEC-289, -CAPEC-29, -CAPEC-290, -CAPEC-291, -CAPEC-292, -CAPEC-293, -CAPEC-294, -CAPEC-295, -CAPEC-296, -CAPEC-297, -CAPEC-298, -CAPEC-299, -CAPEC-3, -CAPEC-30, -CAPEC-300, -CAPEC-301, -CAPEC-302, -CAPEC-303, -CAPEC-304, -CAPEC-305, -CAPEC-306, -CAPEC-307, -CAPEC-308, -CAPEC-309, -CAPEC-31, -CAPEC-310, -CAPEC-311, -CAPEC-312, -CAPEC-313, -CAPEC-314, -CAPEC-315, -CAPEC-316, -CAPEC-317, -CAPEC-318, -CAPEC-319, -CAPEC-32, -CAPEC-320, -CAPEC-321, -CAPEC-322, -CAPEC-323, -CAPEC-324, -CAPEC-325, -CAPEC-326, -CAPEC-327, -CAPEC-328, -CAPEC-329, -CAPEC-33, -CAPEC-330, -CAPEC-331, -CAPEC-332, -CAPEC-34, -CAPEC-35, -CAPEC-36, -CAPEC-37, -CAPEC-38, -CAPEC-383, -CAPEC-384, -CAPEC-385, -CAPEC-386, -CAPEC-387, -CAPEC-388, -CAPEC-389, -CAPEC-39, -CAPEC-390, -CAPEC-391, -CAPEC-392, -CAPEC-393, -CAPEC-394, -CAPEC-395, -CAPEC-396, -CAPEC-397, -CAPEC-398, -CAPEC-399, -CAPEC-4, -CAPEC-40, -CAPEC-400, -CAPEC-401, -CAPEC-402, -CAPEC-404, -CAPEC-405, -CAPEC-406, -CAPEC-407, -CAPEC-408, -CAPEC-409, -CAPEC-41, -CAPEC-410, -CAPEC-411, -CAPEC-412, -CAPEC-413, -CAPEC-414, -CAPEC-415, -CAPEC-416, -CAPEC-417, -CAPEC-418, -CAPEC-419, -CAPEC-42, -CAPEC-420, -CAPEC-421, -CAPEC-422, -CAPEC-423, -CAPEC-424, -CAPEC-425, -CAPEC-426, -CAPEC-427, -CAPEC-428, -CAPEC-429, -CAPEC-43, -CAPEC-430, -CAPEC-431, -CAPEC-432, -CAPEC-433, -CAPEC-434, -CAPEC-435, -CAPEC-438, -CAPEC-439, -CAPEC-44, -CAPEC-440, -CAPEC-441, -CAPEC-442, -CAPEC-443, -CAPEC-444, -CAPEC-445, -CAPEC-446, -CAPEC-447, -CAPEC-448, -CAPEC-449, -CAPEC-45, -CAPEC-450, -CAPEC-451, -CAPEC-452, -CAPEC-453, -CAPEC-454, -CAPEC-455, -CAPEC-456, -CAPEC-457, -CAPEC-458, -CAPEC-459, -CAPEC-46, -CAPEC-460, -CAPEC-461, -CAPEC-462, -CAPEC-463, -CAPEC-464, -CAPEC-465, -CAPEC-466, -CAPEC-467, -CAPEC-468, -CAPEC-469, -CAPEC-47, -CAPEC-470, -CAPEC-471, -CAPEC-472, -CAPEC-473, -CAPEC-474, -CAPEC-475, -CAPEC-476, -CAPEC-477, -CAPEC-478, -CAPEC-479, -CAPEC-48, -CAPEC-480, -CAPEC-481, -CAPEC-482, -CAPEC-484, -CAPEC-485, -CAPEC-486, -CAPEC-487, -CAPEC-488, -CAPEC-489, -CAPEC-49, -CAPEC-490, -CAPEC-491, -CAPEC-492, -CAPEC-493, -CAPEC-494, -CAPEC-495, -CAPEC-496, -CAPEC-497, -CAPEC-498, -CAPEC-499, -CAPEC-5, -CAPEC-50, -CAPEC-500, -CAPEC-501, -CAPEC-502, -CAPEC-503, -CAPEC-504, -CAPEC-505, -CAPEC-506, -CAPEC-507, -CAPEC-508, -CAPEC-509, -CAPEC-51, -CAPEC-510, -CAPEC-511, -CAPEC-516, -CAPEC-517, -CAPEC-518, -CAPEC-519, -CAPEC-52, -CAPEC-520, -CAPEC-521, -CAPEC-522, -CAPEC-523, -CAPEC-524, -CAPEC-528, -CAPEC-529, -CAPEC-53, -CAPEC-530, -CAPEC-531, -CAPEC-532, -CAPEC-533, -CAPEC-534, -CAPEC-535, -CAPEC-536, -CAPEC-537, -CAPEC-538, -CAPEC-539, -CAPEC-54, -CAPEC-540, -CAPEC-541, -CAPEC-542, -CAPEC-543, -CAPEC-544, -CAPEC-545, -CAPEC-546, -CAPEC-547, -CAPEC-548, -CAPEC-549, -CAPEC-55, -CAPEC-550, -CAPEC-551, -CAPEC-552, -CAPEC-554, -CAPEC-555, -CAPEC-556, -CAPEC-557, -CAPEC-558, -CAPEC-559, -CAPEC-56, -CAPEC-560, -CAPEC-561, -CAPEC-562, -CAPEC-563, -CAPEC-564, -CAPEC-565, -CAPEC-566, -CAPEC-567, -CAPEC-568, -CAPEC-569, -CAPEC-57, -CAPEC-570, -CAPEC-571, -CAPEC-572, -CAPEC-573, -CAPEC-574, -CAPEC-575, -CAPEC-576, -CAPEC-577, -CAPEC-578, -CAPEC-579, -CAPEC-58, -CAPEC-580, -CAPEC-581, -CAPEC-582, -CAPEC-583, -CAPEC-584, -CAPEC-585, -CAPEC-586, -CAPEC-587, -CAPEC-588, -CAPEC-589, -CAPEC-59, -CAPEC-590, -CAPEC-591, -CAPEC-592, -CAPEC-593, -CAPEC-594, -CAPEC-595, -CAPEC-596, -CAPEC-597, -CAPEC-598, -CAPEC-599, -CAPEC-6, -CAPEC-60, -CAPEC-600, -CAPEC-601, -CAPEC-602, -CAPEC-603, -CAPEC-604, -CAPEC-605, -CAPEC-606, -CAPEC-607, -CAPEC-608, -CAPEC-609, -CAPEC-61, -CAPEC-610, -CAPEC-611, -CAPEC-612, -CAPEC-613, -CAPEC-614, -CAPEC-615, -CAPEC-616, -CAPEC-617, -CAPEC-618, -CAPEC-619, -CAPEC-62, -CAPEC-620, -CAPEC-621, -CAPEC-622, -CAPEC-623, -CAPEC-624, -CAPEC-625, -CAPEC-626, -CAPEC-627, -CAPEC-628, -CAPEC-629, -CAPEC-63, -CAPEC-630, -CAPEC-631, -CAPEC-632, -CAPEC-633, -CAPEC-634, -CAPEC-635, -CAPEC-636, -CAPEC-637, -CAPEC-638, -CAPEC-639, -CAPEC-64, -CAPEC-640, -CAPEC-641, -CAPEC-642, -CAPEC-643, -CAPEC-644, -CAPEC-645, -CAPEC-646, -CAPEC-647, -CAPEC-648, -CAPEC-649, -CAPEC-65, -CAPEC-650, -CAPEC-651, -CAPEC-652, -CAPEC-653, -CAPEC-654, -CAPEC-655, -CAPEC-656, -CAPEC-657, -CAPEC-66, -CAPEC-660, -CAPEC-661, -CAPEC-662, -CAPEC-663, -CAPEC-664, -CAPEC-665, -CAPEC-666, -CAPEC-667, -CAPEC-668, -CAPEC-669, -CAPEC-67, -CAPEC-670, -CAPEC-671, -CAPEC-672, -CAPEC-673, -CAPEC-674, -CAPEC-675, -CAPEC-676, -CAPEC-677, -CAPEC-678, -CAPEC-679, -CAPEC-68, -CAPEC-680, -CAPEC-681, -CAPEC-682, -CAPEC-69, -CAPEC-690, -CAPEC-691, -CAPEC-692, -CAPEC-693, -CAPEC-694, -CAPEC-695, -CAPEC-696, -CAPEC-697, -CAPEC-698, -CAPEC-699, -CAPEC-7, -CAPEC-70, -CAPEC-700, -CAPEC-701, -CAPEC-702, -CAPEC-71, -CAPEC-72, -CAPEC-73, -CAPEC-74, -CAPEC-75, -CAPEC-76, -CAPEC-77, -CAPEC-78, -CAPEC-79, -CAPEC-8, -CAPEC-80, -CAPEC-81, -CAPEC-82, -CAPEC-83, -CAPEC-84, -CAPEC-85, -CAPEC-86, -CAPEC-87, -CAPEC-88, -CAPEC-89, -CAPEC-9, -CAPEC-90, -CAPEC-91, -CAPEC-92, -CAPEC-93, -CAPEC-94, -CAPEC-95, -CAPEC-96, -CAPEC-97, -CAPEC-98, +CAPEC-1 +CAPEC-10 +CAPEC-100 +CAPEC-101 +CAPEC-102 +CAPEC-103 +CAPEC-104 +CAPEC-105 +CAPEC-106 +CAPEC-107 +CAPEC-108 +CAPEC-109 +CAPEC-11 +CAPEC-110 +CAPEC-111 +CAPEC-112 +CAPEC-113 +CAPEC-114 +CAPEC-115 +CAPEC-116 +CAPEC-117 +CAPEC-12 +CAPEC-120 +CAPEC-121 +CAPEC-122 +CAPEC-123 +CAPEC-124 +CAPEC-125 +CAPEC-126 +CAPEC-127 +CAPEC-128 +CAPEC-129 +CAPEC-13 +CAPEC-130 +CAPEC-131 +CAPEC-132 +CAPEC-133 +CAPEC-134 +CAPEC-135 +CAPEC-136 +CAPEC-137 +CAPEC-138 +CAPEC-139 +CAPEC-14 +CAPEC-140 +CAPEC-141 +CAPEC-142 +CAPEC-143 +CAPEC-144 +CAPEC-145 +CAPEC-146 +CAPEC-147 +CAPEC-148 +CAPEC-149 +CAPEC-15 +CAPEC-150 +CAPEC-151 +CAPEC-153 +CAPEC-154 +CAPEC-155 +CAPEC-157 +CAPEC-158 +CAPEC-159 +CAPEC-16 +CAPEC-160 +CAPEC-161 +CAPEC-162 +CAPEC-163 +CAPEC-164 +CAPEC-165 +CAPEC-166 +CAPEC-167 +CAPEC-168 +CAPEC-169 +CAPEC-17 +CAPEC-170 +CAPEC-171 +CAPEC-173 +CAPEC-174 +CAPEC-175 +CAPEC-176 +CAPEC-177 +CAPEC-178 +CAPEC-179 +CAPEC-18 +CAPEC-180 +CAPEC-181 +CAPEC-182 +CAPEC-183 +CAPEC-184 +CAPEC-185 +CAPEC-186 +CAPEC-187 +CAPEC-188 +CAPEC-189 +CAPEC-19 +CAPEC-190 +CAPEC-191 +CAPEC-192 +CAPEC-193 +CAPEC-194 +CAPEC-195 +CAPEC-196 +CAPEC-197 +CAPEC-198 +CAPEC-199 +CAPEC-2 +CAPEC-20 +CAPEC-200 +CAPEC-201 +CAPEC-202 +CAPEC-203 +CAPEC-204 +CAPEC-205 +CAPEC-206 +CAPEC-207 +CAPEC-208 +CAPEC-209 +CAPEC-21 +CAPEC-211 +CAPEC-212 +CAPEC-213 +CAPEC-214 +CAPEC-215 +CAPEC-216 +CAPEC-217 +CAPEC-218 +CAPEC-219 +CAPEC-22 +CAPEC-220 +CAPEC-221 +CAPEC-222 +CAPEC-224 +CAPEC-226 +CAPEC-227 +CAPEC-228 +CAPEC-229 +CAPEC-23 +CAPEC-230 +CAPEC-231 +CAPEC-233 +CAPEC-234 +CAPEC-235 +CAPEC-236 +CAPEC-237 +CAPEC-238 +CAPEC-239 +CAPEC-24 +CAPEC-240 +CAPEC-241 +CAPEC-242 +CAPEC-243 +CAPEC-244 +CAPEC-245 +CAPEC-246 +CAPEC-247 +CAPEC-248 +CAPEC-249 +CAPEC-25 +CAPEC-250 +CAPEC-251 +CAPEC-252 +CAPEC-253 +CAPEC-254 +CAPEC-256 +CAPEC-257 +CAPEC-258 +CAPEC-259 +CAPEC-26 +CAPEC-260 +CAPEC-261 +CAPEC-263 +CAPEC-264 +CAPEC-265 +CAPEC-266 +CAPEC-267 +CAPEC-268 +CAPEC-269 +CAPEC-27 +CAPEC-270 +CAPEC-271 +CAPEC-272 +CAPEC-273 +CAPEC-274 +CAPEC-275 +CAPEC-276 +CAPEC-277 +CAPEC-278 +CAPEC-279 +CAPEC-28 +CAPEC-280 +CAPEC-285 +CAPEC-287 +CAPEC-288 +CAPEC-289 +CAPEC-29 +CAPEC-290 +CAPEC-291 +CAPEC-292 +CAPEC-293 +CAPEC-294 +CAPEC-295 +CAPEC-296 +CAPEC-297 +CAPEC-298 +CAPEC-299 +CAPEC-3 +CAPEC-30 +CAPEC-300 +CAPEC-301 +CAPEC-302 +CAPEC-303 +CAPEC-304 +CAPEC-305 +CAPEC-306 +CAPEC-307 +CAPEC-308 +CAPEC-309 +CAPEC-31 +CAPEC-310 +CAPEC-311 +CAPEC-312 +CAPEC-313 +CAPEC-314 +CAPEC-315 +CAPEC-316 +CAPEC-317 +CAPEC-318 +CAPEC-319 +CAPEC-32 +CAPEC-320 +CAPEC-321 +CAPEC-322 +CAPEC-323 +CAPEC-324 +CAPEC-325 +CAPEC-326 +CAPEC-327 +CAPEC-328 +CAPEC-329 +CAPEC-33 +CAPEC-330 +CAPEC-331 +CAPEC-332 +CAPEC-34 +CAPEC-35 +CAPEC-36 +CAPEC-37 +CAPEC-38 +CAPEC-383 +CAPEC-384 +CAPEC-385 +CAPEC-386 +CAPEC-387 +CAPEC-388 +CAPEC-389 +CAPEC-39 +CAPEC-390 +CAPEC-391 +CAPEC-392 +CAPEC-393 +CAPEC-394 +CAPEC-395 +CAPEC-396 +CAPEC-397 +CAPEC-398 +CAPEC-399 +CAPEC-4 +CAPEC-40 +CAPEC-400 +CAPEC-401 +CAPEC-402 +CAPEC-404 +CAPEC-405 +CAPEC-406 +CAPEC-407 +CAPEC-408 +CAPEC-409 +CAPEC-41 +CAPEC-410 +CAPEC-411 +CAPEC-412 +CAPEC-413 +CAPEC-414 +CAPEC-415 +CAPEC-416 +CAPEC-417 +CAPEC-418 +CAPEC-419 +CAPEC-42 +CAPEC-420 +CAPEC-421 +CAPEC-422 +CAPEC-423 +CAPEC-424 +CAPEC-425 +CAPEC-426 +CAPEC-427 +CAPEC-428 +CAPEC-429 +CAPEC-43 +CAPEC-430 +CAPEC-431 +CAPEC-432 +CAPEC-433 +CAPEC-434 +CAPEC-435 +CAPEC-438 +CAPEC-439 +CAPEC-44 +CAPEC-440 +CAPEC-441 +CAPEC-442 +CAPEC-443 +CAPEC-444 +CAPEC-445 +CAPEC-446 +CAPEC-447 +CAPEC-448 +CAPEC-449 +CAPEC-45 +CAPEC-450 +CAPEC-451 +CAPEC-452 +CAPEC-453 +CAPEC-454 +CAPEC-455 +CAPEC-456 +CAPEC-457 +CAPEC-458 +CAPEC-459 +CAPEC-46 +CAPEC-460 +CAPEC-461 +CAPEC-462 +CAPEC-463 +CAPEC-464 +CAPEC-465 +CAPEC-466 +CAPEC-467 +CAPEC-468 +CAPEC-469 +CAPEC-47 +CAPEC-470 +CAPEC-471 +CAPEC-472 +CAPEC-473 +CAPEC-474 +CAPEC-475 +CAPEC-476 +CAPEC-477 +CAPEC-478 +CAPEC-479 +CAPEC-48 +CAPEC-480 +CAPEC-481 +CAPEC-482 +CAPEC-484 +CAPEC-485 +CAPEC-486 +CAPEC-487 +CAPEC-488 +CAPEC-489 +CAPEC-49 +CAPEC-490 +CAPEC-491 +CAPEC-492 +CAPEC-493 +CAPEC-494 +CAPEC-495 +CAPEC-496 +CAPEC-497 +CAPEC-498 +CAPEC-499 +CAPEC-5 +CAPEC-50 +CAPEC-500 +CAPEC-501 +CAPEC-502 +CAPEC-503 +CAPEC-504 +CAPEC-505 +CAPEC-506 +CAPEC-507 +CAPEC-508 +CAPEC-509 +CAPEC-51 +CAPEC-510 +CAPEC-511 +CAPEC-516 +CAPEC-517 +CAPEC-518 +CAPEC-519 +CAPEC-52 +CAPEC-520 +CAPEC-521 +CAPEC-522 +CAPEC-523 +CAPEC-524 +CAPEC-528 +CAPEC-529 +CAPEC-53 +CAPEC-530 +CAPEC-531 +CAPEC-532 +CAPEC-533 +CAPEC-534 +CAPEC-535 +CAPEC-536 +CAPEC-537 +CAPEC-538 +CAPEC-539 +CAPEC-54 +CAPEC-540 +CAPEC-541 +CAPEC-542 +CAPEC-543 +CAPEC-544 +CAPEC-545 +CAPEC-546 +CAPEC-547 +CAPEC-548 +CAPEC-549 +CAPEC-55 +CAPEC-550 +CAPEC-551 +CAPEC-552 +CAPEC-554 +CAPEC-555 +CAPEC-556 +CAPEC-557 +CAPEC-558 +CAPEC-559 +CAPEC-56 +CAPEC-560 +CAPEC-561 +CAPEC-562 +CAPEC-563 +CAPEC-564 +CAPEC-565 +CAPEC-566 +CAPEC-567 +CAPEC-568 +CAPEC-569 +CAPEC-57 +CAPEC-570 +CAPEC-571 +CAPEC-572 +CAPEC-573 +CAPEC-574 +CAPEC-575 +CAPEC-576 +CAPEC-577 +CAPEC-578 +CAPEC-579 +CAPEC-58 +CAPEC-580 +CAPEC-581 +CAPEC-582 +CAPEC-583 +CAPEC-584 +CAPEC-585 +CAPEC-586 +CAPEC-587 +CAPEC-588 +CAPEC-589 +CAPEC-59 +CAPEC-590 +CAPEC-591 +CAPEC-592 +CAPEC-593 +CAPEC-594 +CAPEC-595 +CAPEC-596 +CAPEC-597 +CAPEC-598 +CAPEC-599 +CAPEC-6 +CAPEC-60 +CAPEC-600 +CAPEC-601 +CAPEC-602 +CAPEC-603 +CAPEC-604 +CAPEC-605 +CAPEC-606 +CAPEC-607 +CAPEC-608 +CAPEC-609 +CAPEC-61 +CAPEC-610 +CAPEC-611 +CAPEC-612 +CAPEC-613 +CAPEC-614 +CAPEC-615 +CAPEC-616 +CAPEC-617 +CAPEC-618 +CAPEC-619 +CAPEC-62 +CAPEC-620 +CAPEC-621 +CAPEC-622 +CAPEC-623 +CAPEC-624 +CAPEC-625 +CAPEC-626 +CAPEC-627 +CAPEC-628 +CAPEC-629 +CAPEC-63 +CAPEC-630 +CAPEC-631 +CAPEC-632 +CAPEC-633 +CAPEC-634 +CAPEC-635 +CAPEC-636 +CAPEC-637 +CAPEC-638 +CAPEC-639 +CAPEC-64 +CAPEC-640 +CAPEC-641 +CAPEC-642 +CAPEC-643 +CAPEC-644 +CAPEC-645 +CAPEC-646 +CAPEC-647 +CAPEC-648 +CAPEC-649 +CAPEC-65 +CAPEC-650 +CAPEC-651 +CAPEC-652 +CAPEC-653 +CAPEC-654 +CAPEC-655 +CAPEC-656 +CAPEC-657 +CAPEC-66 +CAPEC-660 +CAPEC-661 +CAPEC-662 +CAPEC-663 +CAPEC-664 +CAPEC-665 +CAPEC-666 +CAPEC-667 +CAPEC-668 +CAPEC-669 +CAPEC-67 +CAPEC-670 +CAPEC-671 +CAPEC-672 +CAPEC-673 +CAPEC-674 +CAPEC-675 +CAPEC-676 +CAPEC-677 +CAPEC-678 +CAPEC-679 +CAPEC-68 +CAPEC-680 +CAPEC-681 +CAPEC-682 +CAPEC-69 +CAPEC-690 +CAPEC-691 +CAPEC-692 +CAPEC-693 +CAPEC-694 +CAPEC-695 +CAPEC-696 +CAPEC-697 +CAPEC-698 +CAPEC-699 +CAPEC-7 +CAPEC-70 +CAPEC-700 +CAPEC-701 +CAPEC-702 +CAPEC-71 +CAPEC-72 +CAPEC-73 +CAPEC-74 +CAPEC-75 +CAPEC-76 +CAPEC-77 +CAPEC-78 +CAPEC-79 +CAPEC-8 +CAPEC-80 +CAPEC-81 +CAPEC-82 +CAPEC-83 +CAPEC-84 +CAPEC-85 +CAPEC-86 +CAPEC-87 +CAPEC-88 +CAPEC-89 +CAPEC-9 +CAPEC-90 +CAPEC-91 +CAPEC-92 +CAPEC-93 +CAPEC-94 +CAPEC-95 +CAPEC-96 +CAPEC-97 +CAPEC-98 CAPEC-99 \ No newline at end of file diff --git a/includes/lookups/mitre_cwe_id.txt b/includes/lookups/mitre_cwe_id.txt index 144373e..7a66967 100644 --- a/includes/lookups/mitre_cwe_id.txt +++ b/includes/lookups/mitre_cwe_id.txt @@ -1,964 +1,964 @@ -CWE-1004, -CWE-1007, -CWE-102, -CWE-1021, -CWE-1022, -CWE-1023, -CWE-1024, -CWE-1025, -CWE-103, -CWE-1037, -CWE-1038, -CWE-1039, -CWE-104, -CWE-1041, -CWE-1042, -CWE-1043, -CWE-1044, -CWE-1045, -CWE-1046, -CWE-1047, -CWE-1048, -CWE-1049, -CWE-105, -CWE-1050, -CWE-1051, -CWE-1052, -CWE-1053, -CWE-1054, -CWE-1055, -CWE-1056, -CWE-1057, -CWE-1058, -CWE-1059, -CWE-106, -CWE-1060, -CWE-1061, -CWE-1062, -CWE-1063, -CWE-1064, -CWE-1065, -CWE-1066, -CWE-1067, -CWE-1068, -CWE-1069, -CWE-107, -CWE-1070, -CWE-1071, -CWE-1072, -CWE-1073, -CWE-1074, -CWE-1075, -CWE-1076, -CWE-1077, -CWE-1078, -CWE-1079, -CWE-108, -CWE-1080, -CWE-1082, -CWE-1083, -CWE-1084, -CWE-1085, -CWE-1086, -CWE-1087, -CWE-1088, -CWE-1089, -CWE-109, -CWE-1090, -CWE-1091, -CWE-1092, -CWE-1093, -CWE-1094, -CWE-1095, -CWE-1096, -CWE-1097, -CWE-1098, -CWE-1099, -CWE-11, -CWE-110, -CWE-1100, -CWE-1101, -CWE-1102, -CWE-1103, -CWE-1104, -CWE-1105, -CWE-1106, -CWE-1107, -CWE-1108, -CWE-1109, -CWE-111, -CWE-1110, -CWE-1111, -CWE-1112, -CWE-1113, -CWE-1114, -CWE-1115, -CWE-1116, -CWE-1117, -CWE-1118, -CWE-1119, -CWE-112, -CWE-1120, -CWE-1121, -CWE-1122, -CWE-1123, -CWE-1124, -CWE-1125, -CWE-1126, -CWE-1127, -CWE-113, -CWE-114, -CWE-115, -CWE-116, -CWE-1164, -CWE-117, -CWE-1173, -CWE-1174, -CWE-1176, -CWE-1177, -CWE-118, -CWE-1187, -CWE-1188, -CWE-1189, -CWE-119, -CWE-1190, -CWE-1191, -CWE-1192, -CWE-1193, -CWE-12, -CWE-120, -CWE-1204, -CWE-1209, -CWE-121, -CWE-122, -CWE-1220, -CWE-1221, -CWE-1222, -CWE-1223, -CWE-1224, -CWE-1229, -CWE-123, -CWE-1230, -CWE-1231, -CWE-1232, -CWE-1233, -CWE-1234, -CWE-1235, -CWE-1236, -CWE-1239, -CWE-124, -CWE-1240, -CWE-1241, -CWE-1242, -CWE-1243, -CWE-1244, -CWE-1245, -CWE-1246, -CWE-1247, -CWE-1248, -CWE-1249, -CWE-125, -CWE-1250, -CWE-1251, -CWE-1252, -CWE-1253, -CWE-1254, -CWE-1255, -CWE-1256, -CWE-1257, -CWE-1258, -CWE-1259, -CWE-126, -CWE-1260, -CWE-1261, -CWE-1262, -CWE-1263, -CWE-1264, -CWE-1265, -CWE-1266, -CWE-1267, -CWE-1268, -CWE-1269, -CWE-127, -CWE-1270, -CWE-1271, -CWE-1272, -CWE-1273, -CWE-1274, -CWE-1275, -CWE-1276, -CWE-1277, -CWE-1278, -CWE-1279, -CWE-128, -CWE-1280, -CWE-1281, -CWE-1282, -CWE-1283, -CWE-1284, -CWE-1285, -CWE-1286, -CWE-1287, -CWE-1288, -CWE-1289, -CWE-129, -CWE-1290, -CWE-1291, -CWE-1292, -CWE-1293, -CWE-1294, -CWE-1295, -CWE-1296, -CWE-1297, -CWE-1298, -CWE-1299, -CWE-13, -CWE-130, -CWE-1300, -CWE-1301, -CWE-1302, -CWE-1303, -CWE-1304, -CWE-131, -CWE-1310, -CWE-1311, -CWE-1312, -CWE-1313, -CWE-1314, -CWE-1315, -CWE-1316, -CWE-1317, -CWE-1318, -CWE-1319, -CWE-132, -CWE-1320, -CWE-1321, -CWE-1322, -CWE-1323, -CWE-1324, -CWE-1325, -CWE-1326, -CWE-1327, -CWE-1328, -CWE-1329, -CWE-1330, -CWE-1331, -CWE-1332, -CWE-1333, -CWE-1334, -CWE-1335, -CWE-1336, -CWE-1338, -CWE-1339, -CWE-134, -CWE-1341, -CWE-1342, -CWE-135, -CWE-1351, -CWE-1357, -CWE-138, -CWE-1384, -CWE-1385, -CWE-1386, -CWE-1389, -CWE-1390, -CWE-1391, -CWE-1392, -CWE-1393, -CWE-1394, -CWE-1395, -CWE-14, -CWE-140, -CWE-141, -CWE-1419, -CWE-142, -CWE-1420, -CWE-1421, -CWE-1422, -CWE-1423, -CWE-1426, -CWE-143, -CWE-144, -CWE-145, -CWE-146, -CWE-147, -CWE-148, -CWE-149, -CWE-15, -CWE-150, -CWE-151, -CWE-152, -CWE-153, -CWE-154, -CWE-155, -CWE-156, -CWE-157, -CWE-158, -CWE-159, -CWE-160, -CWE-161, -CWE-162, -CWE-163, -CWE-164, -CWE-165, -CWE-166, -CWE-167, -CWE-168, -CWE-170, -CWE-172, -CWE-173, -CWE-174, -CWE-175, -CWE-176, -CWE-177, -CWE-178, -CWE-179, -CWE-180, -CWE-181, -CWE-182, -CWE-183, -CWE-184, -CWE-185, -CWE-186, -CWE-187, -CWE-188, -CWE-190, -CWE-191, -CWE-192, -CWE-193, -CWE-194, -CWE-195, -CWE-196, -CWE-197, -CWE-198, -CWE-20, -CWE-200, -CWE-201, -CWE-202, -CWE-203, -CWE-204, -CWE-205, -CWE-206, -CWE-207, -CWE-208, -CWE-209, -CWE-210, -CWE-211, -CWE-212, -CWE-213, -CWE-214, -CWE-215, -CWE-216, -CWE-217, -CWE-218, -CWE-219, -CWE-22, -CWE-220, -CWE-221, -CWE-222, -CWE-223, -CWE-224, -CWE-225, -CWE-226, -CWE-228, -CWE-229, -CWE-23, -CWE-230, -CWE-231, -CWE-232, -CWE-233, -CWE-234, -CWE-235, -CWE-236, -CWE-237, -CWE-238, -CWE-239, -CWE-24, -CWE-240, -CWE-241, -CWE-242, -CWE-243, -CWE-244, -CWE-245, -CWE-246, -CWE-247, -CWE-248, -CWE-249, -CWE-25, -CWE-250, -CWE-252, -CWE-253, -CWE-256, -CWE-257, -CWE-258, -CWE-259, -CWE-26, -CWE-260, -CWE-261, -CWE-262, -CWE-263, -CWE-266, -CWE-267, -CWE-268, -CWE-269, -CWE-27, -CWE-270, -CWE-271, -CWE-272, -CWE-273, -CWE-274, -CWE-276, -CWE-277, -CWE-278, -CWE-279, -CWE-28, -CWE-280, -CWE-281, -CWE-282, -CWE-283, -CWE-284, -CWE-285, -CWE-286, -CWE-287, -CWE-288, -CWE-289, -CWE-29, -CWE-290, -CWE-291, -CWE-292, -CWE-293, -CWE-294, -CWE-295, -CWE-296, -CWE-297, -CWE-298, -CWE-299, -CWE-30, -CWE-300, -CWE-301, -CWE-302, -CWE-303, -CWE-304, -CWE-305, -CWE-306, -CWE-307, -CWE-308, -CWE-309, -CWE-31, -CWE-311, -CWE-312, -CWE-313, -CWE-314, -CWE-315, -CWE-316, -CWE-317, -CWE-318, -CWE-319, -CWE-32, -CWE-321, -CWE-322, -CWE-323, -CWE-324, -CWE-325, -CWE-326, -CWE-327, -CWE-328, -CWE-329, -CWE-33, -CWE-330, -CWE-331, -CWE-332, -CWE-333, -CWE-334, -CWE-335, -CWE-336, -CWE-337, -CWE-338, -CWE-339, -CWE-34, -CWE-340, -CWE-341, -CWE-342, -CWE-343, -CWE-344, -CWE-345, -CWE-346, -CWE-347, -CWE-348, -CWE-349, -CWE-35, -CWE-350, -CWE-351, -CWE-352, -CWE-353, -CWE-354, -CWE-356, -CWE-357, -CWE-358, -CWE-359, -CWE-36, -CWE-360, -CWE-362, -CWE-363, -CWE-364, -CWE-365, -CWE-366, -CWE-367, -CWE-368, -CWE-369, -CWE-37, -CWE-370, -CWE-372, -CWE-373, -CWE-374, -CWE-375, -CWE-377, -CWE-378, -CWE-379, -CWE-38, -CWE-382, -CWE-383, -CWE-384, -CWE-385, -CWE-386, -CWE-39, -CWE-390, -CWE-391, -CWE-392, -CWE-393, -CWE-394, -CWE-395, -CWE-396, -CWE-397, -CWE-40, -CWE-400, -CWE-401, -CWE-402, -CWE-403, -CWE-404, -CWE-405, -CWE-406, -CWE-407, -CWE-408, -CWE-409, -CWE-41, -CWE-410, -CWE-412, -CWE-413, -CWE-414, -CWE-415, -CWE-416, -CWE-419, -CWE-42, -CWE-420, -CWE-421, -CWE-422, -CWE-423, -CWE-424, -CWE-425, -CWE-426, -CWE-427, -CWE-428, -CWE-43, -CWE-430, -CWE-431, -CWE-432, -CWE-433, -CWE-434, -CWE-435, -CWE-436, -CWE-437, -CWE-439, -CWE-44, -CWE-440, -CWE-441, -CWE-443, -CWE-444, -CWE-446, -CWE-447, -CWE-448, -CWE-449, -CWE-45, -CWE-450, -CWE-451, -CWE-453, -CWE-454, -CWE-455, -CWE-456, -CWE-457, -CWE-458, -CWE-459, -CWE-46, -CWE-460, -CWE-462, -CWE-463, -CWE-464, -CWE-466, -CWE-467, -CWE-468, -CWE-469, -CWE-47, -CWE-470, -CWE-471, -CWE-472, -CWE-473, -CWE-474, -CWE-475, -CWE-476, -CWE-477, -CWE-478, -CWE-479, -CWE-48, -CWE-480, -CWE-481, -CWE-482, -CWE-483, -CWE-484, -CWE-486, -CWE-487, -CWE-488, -CWE-489, -CWE-49, -CWE-491, -CWE-492, -CWE-493, -CWE-494, -CWE-495, -CWE-496, -CWE-497, -CWE-498, -CWE-499, -CWE-5, -CWE-50, -CWE-500, -CWE-501, -CWE-502, -CWE-506, -CWE-507, -CWE-508, -CWE-509, -CWE-51, -CWE-510, -CWE-511, -CWE-512, -CWE-514, -CWE-515, -CWE-516, -CWE-52, -CWE-520, -CWE-521, -CWE-522, -CWE-523, -CWE-524, -CWE-525, -CWE-526, -CWE-527, -CWE-528, -CWE-529, -CWE-53, -CWE-530, -CWE-531, -CWE-532, -CWE-533, -CWE-534, -CWE-535, -CWE-536, -CWE-537, -CWE-538, -CWE-539, -CWE-54, -CWE-540, -CWE-541, -CWE-542, -CWE-543, -CWE-544, -CWE-545, -CWE-546, -CWE-547, -CWE-548, -CWE-549, -CWE-55, -CWE-550, -CWE-551, -CWE-552, -CWE-553, -CWE-554, -CWE-555, -CWE-556, -CWE-558, -CWE-56, -CWE-560, -CWE-561, -CWE-562, -CWE-563, -CWE-564, -CWE-565, -CWE-566, -CWE-567, -CWE-568, -CWE-57, -CWE-570, -CWE-571, -CWE-572, -CWE-573, -CWE-574, -CWE-575, -CWE-576, -CWE-577, -CWE-578, -CWE-579, -CWE-58, -CWE-580, -CWE-581, -CWE-582, -CWE-583, -CWE-584, -CWE-585, -CWE-586, -CWE-587, -CWE-588, -CWE-589, -CWE-59, -CWE-590, -CWE-591, -CWE-592, -CWE-593, -CWE-594, -CWE-595, -CWE-596, -CWE-597, -CWE-598, -CWE-599, -CWE-6, -CWE-600, -CWE-601, -CWE-602, -CWE-603, -CWE-605, -CWE-606, -CWE-607, -CWE-608, -CWE-609, -CWE-61, -CWE-610, -CWE-611, -CWE-612, -CWE-613, -CWE-614, -CWE-615, -CWE-616, -CWE-617, -CWE-618, -CWE-619, -CWE-62, -CWE-620, -CWE-621, -CWE-622, -CWE-623, -CWE-624, -CWE-625, -CWE-626, -CWE-627, -CWE-628, -CWE-636, -CWE-637, -CWE-638, -CWE-639, -CWE-64, -CWE-640, -CWE-641, -CWE-642, -CWE-643, -CWE-644, -CWE-645, -CWE-646, -CWE-647, -CWE-648, -CWE-649, -CWE-65, -CWE-650, -CWE-651, -CWE-652, -CWE-653, -CWE-654, -CWE-655, -CWE-656, -CWE-657, -CWE-66, -CWE-662, -CWE-663, -CWE-664, -CWE-665, -CWE-666, -CWE-667, -CWE-668, -CWE-669, -CWE-67, -CWE-670, -CWE-671, -CWE-672, -CWE-673, -CWE-674, -CWE-675, -CWE-676, -CWE-680, -CWE-681, -CWE-682, -CWE-683, -CWE-684, -CWE-685, -CWE-686, -CWE-687, -CWE-688, -CWE-689, -CWE-69, -CWE-690, -CWE-691, -CWE-692, -CWE-693, -CWE-694, -CWE-695, -CWE-696, -CWE-697, -CWE-698, -CWE-7, -CWE-703, -CWE-704, -CWE-705, -CWE-706, -CWE-707, -CWE-708, -CWE-71, -CWE-710, -CWE-72, -CWE-73, -CWE-732, -CWE-733, -CWE-74, -CWE-749, -CWE-75, -CWE-754, -CWE-755, -CWE-756, -CWE-757, -CWE-758, -CWE-759, -CWE-76, -CWE-760, -CWE-761, -CWE-762, -CWE-763, -CWE-764, -CWE-765, -CWE-766, -CWE-767, -CWE-768, -CWE-769, -CWE-77, -CWE-770, -CWE-771, -CWE-772, -CWE-773, -CWE-774, -CWE-775, -CWE-776, -CWE-777, -CWE-778, -CWE-779, -CWE-78, -CWE-780, -CWE-781, -CWE-782, -CWE-783, -CWE-784, -CWE-785, -CWE-786, -CWE-787, -CWE-788, -CWE-789, -CWE-79, -CWE-790, -CWE-791, -CWE-792, -CWE-793, -CWE-794, -CWE-795, -CWE-796, -CWE-797, -CWE-798, -CWE-799, -CWE-8, -CWE-80, -CWE-804, -CWE-805, -CWE-806, -CWE-807, -CWE-81, -CWE-82, -CWE-820, -CWE-821, -CWE-822, -CWE-823, -CWE-824, -CWE-825, -CWE-826, -CWE-827, -CWE-828, -CWE-829, -CWE-83, -CWE-830, -CWE-831, -CWE-832, -CWE-833, -CWE-834, -CWE-835, -CWE-836, -CWE-837, -CWE-838, -CWE-839, -CWE-84, -CWE-841, -CWE-842, -CWE-843, -CWE-85, -CWE-86, -CWE-862, -CWE-863, -CWE-87, -CWE-88, -CWE-89, -CWE-9, -CWE-90, -CWE-908, -CWE-909, -CWE-91, -CWE-910, -CWE-911, -CWE-912, -CWE-913, -CWE-914, -CWE-915, -CWE-916, -CWE-917, -CWE-918, -CWE-92, -CWE-920, -CWE-921, -CWE-922, -CWE-923, -CWE-924, -CWE-925, -CWE-926, -CWE-927, -CWE-93, -CWE-939, -CWE-94, -CWE-940, -CWE-941, -CWE-942, -CWE-943, -CWE-95, -CWE-96, -CWE-97, -CWE-98, +CWE-1004 +CWE-1007 +CWE-102 +CWE-1021 +CWE-1022 +CWE-1023 +CWE-1024 +CWE-1025 +CWE-103 +CWE-1037 +CWE-1038 +CWE-1039 +CWE-104 +CWE-1041 +CWE-1042 +CWE-1043 +CWE-1044 +CWE-1045 +CWE-1046 +CWE-1047 +CWE-1048 +CWE-1049 +CWE-105 +CWE-1050 +CWE-1051 +CWE-1052 +CWE-1053 +CWE-1054 +CWE-1055 +CWE-1056 +CWE-1057 +CWE-1058 +CWE-1059 +CWE-106 +CWE-1060 +CWE-1061 +CWE-1062 +CWE-1063 +CWE-1064 +CWE-1065 +CWE-1066 +CWE-1067 +CWE-1068 +CWE-1069 +CWE-107 +CWE-1070 +CWE-1071 +CWE-1072 +CWE-1073 +CWE-1074 +CWE-1075 +CWE-1076 +CWE-1077 +CWE-1078 +CWE-1079 +CWE-108 +CWE-1080 +CWE-1082 +CWE-1083 +CWE-1084 +CWE-1085 +CWE-1086 +CWE-1087 +CWE-1088 +CWE-1089 +CWE-109 +CWE-1090 +CWE-1091 +CWE-1092 +CWE-1093 +CWE-1094 +CWE-1095 +CWE-1096 +CWE-1097 +CWE-1098 +CWE-1099 +CWE-11 +CWE-110 +CWE-1100 +CWE-1101 +CWE-1102 +CWE-1103 +CWE-1104 +CWE-1105 +CWE-1106 +CWE-1107 +CWE-1108 +CWE-1109 +CWE-111 +CWE-1110 +CWE-1111 +CWE-1112 +CWE-1113 +CWE-1114 +CWE-1115 +CWE-1116 +CWE-1117 +CWE-1118 +CWE-1119 +CWE-112 +CWE-1120 +CWE-1121 +CWE-1122 +CWE-1123 +CWE-1124 +CWE-1125 +CWE-1126 +CWE-1127 +CWE-113 +CWE-114 +CWE-115 +CWE-116 +CWE-1164 +CWE-117 +CWE-1173 +CWE-1174 +CWE-1176 +CWE-1177 +CWE-118 +CWE-1187 +CWE-1188 +CWE-1189 +CWE-119 +CWE-1190 +CWE-1191 +CWE-1192 +CWE-1193 +CWE-12 +CWE-120 +CWE-1204 +CWE-1209 +CWE-121 +CWE-122 +CWE-1220 +CWE-1221 +CWE-1222 +CWE-1223 +CWE-1224 +CWE-1229 +CWE-123 +CWE-1230 +CWE-1231 +CWE-1232 +CWE-1233 +CWE-1234 +CWE-1235 +CWE-1236 +CWE-1239 +CWE-124 +CWE-1240 +CWE-1241 +CWE-1242 +CWE-1243 +CWE-1244 +CWE-1245 +CWE-1246 +CWE-1247 +CWE-1248 +CWE-1249 +CWE-125 +CWE-1250 +CWE-1251 +CWE-1252 +CWE-1253 +CWE-1254 +CWE-1255 +CWE-1256 +CWE-1257 +CWE-1258 +CWE-1259 +CWE-126 +CWE-1260 +CWE-1261 +CWE-1262 +CWE-1263 +CWE-1264 +CWE-1265 +CWE-1266 +CWE-1267 +CWE-1268 +CWE-1269 +CWE-127 +CWE-1270 +CWE-1271 +CWE-1272 +CWE-1273 +CWE-1274 +CWE-1275 +CWE-1276 +CWE-1277 +CWE-1278 +CWE-1279 +CWE-128 +CWE-1280 +CWE-1281 +CWE-1282 +CWE-1283 +CWE-1284 +CWE-1285 +CWE-1286 +CWE-1287 +CWE-1288 +CWE-1289 +CWE-129 +CWE-1290 +CWE-1291 +CWE-1292 +CWE-1293 +CWE-1294 +CWE-1295 +CWE-1296 +CWE-1297 +CWE-1298 +CWE-1299 +CWE-13 +CWE-130 +CWE-1300 +CWE-1301 +CWE-1302 +CWE-1303 +CWE-1304 +CWE-131 +CWE-1310 +CWE-1311 +CWE-1312 +CWE-1313 +CWE-1314 +CWE-1315 +CWE-1316 +CWE-1317 +CWE-1318 +CWE-1319 +CWE-132 +CWE-1320 +CWE-1321 +CWE-1322 +CWE-1323 +CWE-1324 +CWE-1325 +CWE-1326 +CWE-1327 +CWE-1328 +CWE-1329 +CWE-1330 +CWE-1331 +CWE-1332 +CWE-1333 +CWE-1334 +CWE-1335 +CWE-1336 +CWE-1338 +CWE-1339 +CWE-134 +CWE-1341 +CWE-1342 +CWE-135 +CWE-1351 +CWE-1357 +CWE-138 +CWE-1384 +CWE-1385 +CWE-1386 +CWE-1389 +CWE-1390 +CWE-1391 +CWE-1392 +CWE-1393 +CWE-1394 +CWE-1395 +CWE-14 +CWE-140 +CWE-141 +CWE-1419 +CWE-142 +CWE-1420 +CWE-1421 +CWE-1422 +CWE-1423 +CWE-1426 +CWE-143 +CWE-144 +CWE-145 +CWE-146 +CWE-147 +CWE-148 +CWE-149 +CWE-15 +CWE-150 +CWE-151 +CWE-152 +CWE-153 +CWE-154 +CWE-155 +CWE-156 +CWE-157 +CWE-158 +CWE-159 +CWE-160 +CWE-161 +CWE-162 +CWE-163 +CWE-164 +CWE-165 +CWE-166 +CWE-167 +CWE-168 +CWE-170 +CWE-172 +CWE-173 +CWE-174 +CWE-175 +CWE-176 +CWE-177 +CWE-178 +CWE-179 +CWE-180 +CWE-181 +CWE-182 +CWE-183 +CWE-184 +CWE-185 +CWE-186 +CWE-187 +CWE-188 +CWE-190 +CWE-191 +CWE-192 +CWE-193 +CWE-194 +CWE-195 +CWE-196 +CWE-197 +CWE-198 +CWE-20 +CWE-200 +CWE-201 +CWE-202 +CWE-203 +CWE-204 +CWE-205 +CWE-206 +CWE-207 +CWE-208 +CWE-209 +CWE-210 +CWE-211 +CWE-212 +CWE-213 +CWE-214 +CWE-215 +CWE-216 +CWE-217 +CWE-218 +CWE-219 +CWE-22 +CWE-220 +CWE-221 +CWE-222 +CWE-223 +CWE-224 +CWE-225 +CWE-226 +CWE-228 +CWE-229 +CWE-23 +CWE-230 +CWE-231 +CWE-232 +CWE-233 +CWE-234 +CWE-235 +CWE-236 +CWE-237 +CWE-238 +CWE-239 +CWE-24 +CWE-240 +CWE-241 +CWE-242 +CWE-243 +CWE-244 +CWE-245 +CWE-246 +CWE-247 +CWE-248 +CWE-249 +CWE-25 +CWE-250 +CWE-252 +CWE-253 +CWE-256 +CWE-257 +CWE-258 +CWE-259 +CWE-26 +CWE-260 +CWE-261 +CWE-262 +CWE-263 +CWE-266 +CWE-267 +CWE-268 +CWE-269 +CWE-27 +CWE-270 +CWE-271 +CWE-272 +CWE-273 +CWE-274 +CWE-276 +CWE-277 +CWE-278 +CWE-279 +CWE-28 +CWE-280 +CWE-281 +CWE-282 +CWE-283 +CWE-284 +CWE-285 +CWE-286 +CWE-287 +CWE-288 +CWE-289 +CWE-29 +CWE-290 +CWE-291 +CWE-292 +CWE-293 +CWE-294 +CWE-295 +CWE-296 +CWE-297 +CWE-298 +CWE-299 +CWE-30 +CWE-300 +CWE-301 +CWE-302 +CWE-303 +CWE-304 +CWE-305 +CWE-306 +CWE-307 +CWE-308 +CWE-309 +CWE-31 +CWE-311 +CWE-312 +CWE-313 +CWE-314 +CWE-315 +CWE-316 +CWE-317 +CWE-318 +CWE-319 +CWE-32 +CWE-321 +CWE-322 +CWE-323 +CWE-324 +CWE-325 +CWE-326 +CWE-327 +CWE-328 +CWE-329 +CWE-33 +CWE-330 +CWE-331 +CWE-332 +CWE-333 +CWE-334 +CWE-335 +CWE-336 +CWE-337 +CWE-338 +CWE-339 +CWE-34 +CWE-340 +CWE-341 +CWE-342 +CWE-343 +CWE-344 +CWE-345 +CWE-346 +CWE-347 +CWE-348 +CWE-349 +CWE-35 +CWE-350 +CWE-351 +CWE-352 +CWE-353 +CWE-354 +CWE-356 +CWE-357 +CWE-358 +CWE-359 +CWE-36 +CWE-360 +CWE-362 +CWE-363 +CWE-364 +CWE-365 +CWE-366 +CWE-367 +CWE-368 +CWE-369 +CWE-37 +CWE-370 +CWE-372 +CWE-373 +CWE-374 +CWE-375 +CWE-377 +CWE-378 +CWE-379 +CWE-38 +CWE-382 +CWE-383 +CWE-384 +CWE-385 +CWE-386 +CWE-39 +CWE-390 +CWE-391 +CWE-392 +CWE-393 +CWE-394 +CWE-395 +CWE-396 +CWE-397 +CWE-40 +CWE-400 +CWE-401 +CWE-402 +CWE-403 +CWE-404 +CWE-405 +CWE-406 +CWE-407 +CWE-408 +CWE-409 +CWE-41 +CWE-410 +CWE-412 +CWE-413 +CWE-414 +CWE-415 +CWE-416 +CWE-419 +CWE-42 +CWE-420 +CWE-421 +CWE-422 +CWE-423 +CWE-424 +CWE-425 +CWE-426 +CWE-427 +CWE-428 +CWE-43 +CWE-430 +CWE-431 +CWE-432 +CWE-433 +CWE-434 +CWE-435 +CWE-436 +CWE-437 +CWE-439 +CWE-44 +CWE-440 +CWE-441 +CWE-443 +CWE-444 +CWE-446 +CWE-447 +CWE-448 +CWE-449 +CWE-45 +CWE-450 +CWE-451 +CWE-453 +CWE-454 +CWE-455 +CWE-456 +CWE-457 +CWE-458 +CWE-459 +CWE-46 +CWE-460 +CWE-462 +CWE-463 +CWE-464 +CWE-466 +CWE-467 +CWE-468 +CWE-469 +CWE-47 +CWE-470 +CWE-471 +CWE-472 +CWE-473 +CWE-474 +CWE-475 +CWE-476 +CWE-477 +CWE-478 +CWE-479 +CWE-48 +CWE-480 +CWE-481 +CWE-482 +CWE-483 +CWE-484 +CWE-486 +CWE-487 +CWE-488 +CWE-489 +CWE-49 +CWE-491 +CWE-492 +CWE-493 +CWE-494 +CWE-495 +CWE-496 +CWE-497 +CWE-498 +CWE-499 +CWE-5 +CWE-50 +CWE-500 +CWE-501 +CWE-502 +CWE-506 +CWE-507 +CWE-508 +CWE-509 +CWE-51 +CWE-510 +CWE-511 +CWE-512 +CWE-514 +CWE-515 +CWE-516 +CWE-52 +CWE-520 +CWE-521 +CWE-522 +CWE-523 +CWE-524 +CWE-525 +CWE-526 +CWE-527 +CWE-528 +CWE-529 +CWE-53 +CWE-530 +CWE-531 +CWE-532 +CWE-533 +CWE-534 +CWE-535 +CWE-536 +CWE-537 +CWE-538 +CWE-539 +CWE-54 +CWE-540 +CWE-541 +CWE-542 +CWE-543 +CWE-544 +CWE-545 +CWE-546 +CWE-547 +CWE-548 +CWE-549 +CWE-55 +CWE-550 +CWE-551 +CWE-552 +CWE-553 +CWE-554 +CWE-555 +CWE-556 +CWE-558 +CWE-56 +CWE-560 +CWE-561 +CWE-562 +CWE-563 +CWE-564 +CWE-565 +CWE-566 +CWE-567 +CWE-568 +CWE-57 +CWE-570 +CWE-571 +CWE-572 +CWE-573 +CWE-574 +CWE-575 +CWE-576 +CWE-577 +CWE-578 +CWE-579 +CWE-58 +CWE-580 +CWE-581 +CWE-582 +CWE-583 +CWE-584 +CWE-585 +CWE-586 +CWE-587 +CWE-588 +CWE-589 +CWE-59 +CWE-590 +CWE-591 +CWE-592 +CWE-593 +CWE-594 +CWE-595 +CWE-596 +CWE-597 +CWE-598 +CWE-599 +CWE-6 +CWE-600 +CWE-601 +CWE-602 +CWE-603 +CWE-605 +CWE-606 +CWE-607 +CWE-608 +CWE-609 +CWE-61 +CWE-610 +CWE-611 +CWE-612 +CWE-613 +CWE-614 +CWE-615 +CWE-616 +CWE-617 +CWE-618 +CWE-619 +CWE-62 +CWE-620 +CWE-621 +CWE-622 +CWE-623 +CWE-624 +CWE-625 +CWE-626 +CWE-627 +CWE-628 +CWE-636 +CWE-637 +CWE-638 +CWE-639 +CWE-64 +CWE-640 +CWE-641 +CWE-642 +CWE-643 +CWE-644 +CWE-645 +CWE-646 +CWE-647 +CWE-648 +CWE-649 +CWE-65 +CWE-650 +CWE-651 +CWE-652 +CWE-653 +CWE-654 +CWE-655 +CWE-656 +CWE-657 +CWE-66 +CWE-662 +CWE-663 +CWE-664 +CWE-665 +CWE-666 +CWE-667 +CWE-668 +CWE-669 +CWE-67 +CWE-670 +CWE-671 +CWE-672 +CWE-673 +CWE-674 +CWE-675 +CWE-676 +CWE-680 +CWE-681 +CWE-682 +CWE-683 +CWE-684 +CWE-685 +CWE-686 +CWE-687 +CWE-688 +CWE-689 +CWE-69 +CWE-690 +CWE-691 +CWE-692 +CWE-693 +CWE-694 +CWE-695 +CWE-696 +CWE-697 +CWE-698 +CWE-7 +CWE-703 +CWE-704 +CWE-705 +CWE-706 +CWE-707 +CWE-708 +CWE-71 +CWE-710 +CWE-72 +CWE-73 +CWE-732 +CWE-733 +CWE-74 +CWE-749 +CWE-75 +CWE-754 +CWE-755 +CWE-756 +CWE-757 +CWE-758 +CWE-759 +CWE-76 +CWE-760 +CWE-761 +CWE-762 +CWE-763 +CWE-764 +CWE-765 +CWE-766 +CWE-767 +CWE-768 +CWE-769 +CWE-77 +CWE-770 +CWE-771 +CWE-772 +CWE-773 +CWE-774 +CWE-775 +CWE-776 +CWE-777 +CWE-778 +CWE-779 +CWE-78 +CWE-780 +CWE-781 +CWE-782 +CWE-783 +CWE-784 +CWE-785 +CWE-786 +CWE-787 +CWE-788 +CWE-789 +CWE-79 +CWE-790 +CWE-791 +CWE-792 +CWE-793 +CWE-794 +CWE-795 +CWE-796 +CWE-797 +CWE-798 +CWE-799 +CWE-8 +CWE-80 +CWE-804 +CWE-805 +CWE-806 +CWE-807 +CWE-81 +CWE-82 +CWE-820 +CWE-821 +CWE-822 +CWE-823 +CWE-824 +CWE-825 +CWE-826 +CWE-827 +CWE-828 +CWE-829 +CWE-83 +CWE-830 +CWE-831 +CWE-832 +CWE-833 +CWE-834 +CWE-835 +CWE-836 +CWE-837 +CWE-838 +CWE-839 +CWE-84 +CWE-841 +CWE-842 +CWE-843 +CWE-85 +CWE-86 +CWE-862 +CWE-863 +CWE-87 +CWE-88 +CWE-89 +CWE-9 +CWE-90 +CWE-908 +CWE-909 +CWE-91 +CWE-910 +CWE-911 +CWE-912 +CWE-913 +CWE-914 +CWE-915 +CWE-916 +CWE-917 +CWE-918 +CWE-92 +CWE-920 +CWE-921 +CWE-922 +CWE-923 +CWE-924 +CWE-925 +CWE-926 +CWE-927 +CWE-93 +CWE-939 +CWE-94 +CWE-940 +CWE-941 +CWE-942 +CWE-943 +CWE-95 +CWE-96 +CWE-97 +CWE-98 CWE-99 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index bc6d327..08b1c82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "llama-index>=0.10.50", "pathvalidate>=3.2.0", "phonenumbers>=8.13.39", - "python-arango>=8.0.0", + "requests==2.32.3", "python-dotenv>=1.0.1", "schwifty>=2024.6.1", "stix2extensions @ https://github.com/muchdogesec/stix2extensions/archive/main.zip", diff --git a/tests/README.md b/tests/README.md index 5dbfe75..7cda4a5 100644 --- a/tests/README.md +++ b/tests/README.md @@ -2,5 +2,5 @@ Generate test cases ```shell -python3 includes/extractions/test_cases/generate_simple_extraction_test_cases_txt_files.py +python3 tests/scripts/generate_simple_extraction_test_cases_txt_files.py ``` \ No newline at end of file diff --git a/tests/inputs/extraction_types/ai_country_alpha2.txt b/tests/data/extraction_types/ai_country_alpha2.txt similarity index 100% rename from tests/inputs/extraction_types/ai_country_alpha2.txt rename to tests/data/extraction_types/ai_country_alpha2.txt diff --git a/tests/inputs/extraction_types/ai_mitre_attack_enterprise.txt b/tests/data/extraction_types/ai_mitre_attack_enterprise.txt similarity index 100% rename from tests/inputs/extraction_types/ai_mitre_attack_enterprise.txt rename to tests/data/extraction_types/ai_mitre_attack_enterprise.txt diff --git a/tests/inputs/extraction_types/ai_mitre_attack_ics.txt b/tests/data/extraction_types/ai_mitre_attack_ics.txt similarity index 100% rename from tests/inputs/extraction_types/ai_mitre_attack_ics.txt rename to tests/data/extraction_types/ai_mitre_attack_ics.txt diff --git a/tests/inputs/extraction_types/ai_mitre_attack_mobile.txt b/tests/data/extraction_types/ai_mitre_attack_mobile.txt similarity index 100% rename from tests/inputs/extraction_types/ai_mitre_attack_mobile.txt rename to tests/data/extraction_types/ai_mitre_attack_mobile.txt diff --git a/tests/inputs/extraction_types/ai_mitre_capec.txt b/tests/data/extraction_types/ai_mitre_capec.txt similarity index 100% rename from tests/inputs/extraction_types/ai_mitre_capec.txt rename to tests/data/extraction_types/ai_mitre_capec.txt diff --git a/tests/inputs/extraction_types/ai_mitre_cwe.txt b/tests/data/extraction_types/ai_mitre_cwe.txt similarity index 100% rename from tests/inputs/extraction_types/ai_mitre_cwe.txt rename to tests/data/extraction_types/ai_mitre_cwe.txt diff --git a/tests/inputs/extraction_types/alias_country_iso3166_alpha3_to_alpha2.txt b/tests/data/extraction_types/alias_country_iso3166_alpha3_to_alpha2.txt similarity index 100% rename from tests/inputs/extraction_types/alias_country_iso3166_alpha3_to_alpha2.txt rename to tests/data/extraction_types/alias_country_iso3166_alpha3_to_alpha2.txt diff --git a/tests/inputs/extraction_types/alias_country_name_to_iso2.txt b/tests/data/extraction_types/alias_country_name_to_iso2.txt similarity index 100% rename from tests/inputs/extraction_types/alias_country_name_to_iso2.txt rename to tests/data/extraction_types/alias_country_name_to_iso2.txt diff --git a/tests/inputs/extraction_types/alias_mitre_attack_enterprise_name_to_id.txt b/tests/data/extraction_types/alias_mitre_attack_enterprise_name_to_id.txt similarity index 100% rename from tests/inputs/extraction_types/alias_mitre_attack_enterprise_name_to_id.txt rename to tests/data/extraction_types/alias_mitre_attack_enterprise_name_to_id.txt diff --git a/tests/inputs/extraction_types/alias_mitre_attack_ics_name_to_id.txt b/tests/data/extraction_types/alias_mitre_attack_ics_name_to_id.txt similarity index 100% rename from tests/inputs/extraction_types/alias_mitre_attack_ics_name_to_id.txt rename to tests/data/extraction_types/alias_mitre_attack_ics_name_to_id.txt diff --git a/tests/inputs/extraction_types/alias_mitre_attack_mobile_name_to_id.txt b/tests/data/extraction_types/alias_mitre_attack_mobile_name_to_id.txt similarity index 100% rename from tests/inputs/extraction_types/alias_mitre_attack_mobile_name_to_id.txt rename to tests/data/extraction_types/alias_mitre_attack_mobile_name_to_id.txt diff --git a/tests/inputs/extraction_types/alias_mitre_capec_name_to_id.txt b/tests/data/extraction_types/alias_mitre_capec_name_to_id.txt similarity index 100% rename from tests/inputs/extraction_types/alias_mitre_capec_name_to_id.txt rename to tests/data/extraction_types/alias_mitre_capec_name_to_id.txt diff --git a/tests/inputs/extraction_types/alias_mitre_cwe_name_to_id.txt b/tests/data/extraction_types/alias_mitre_cwe_name_to_id.txt similarity index 100% rename from tests/inputs/extraction_types/alias_mitre_cwe_name_to_id.txt rename to tests/data/extraction_types/alias_mitre_cwe_name_to_id.txt diff --git a/tests/inputs/extraction_types/all_cases.txt b/tests/data/extraction_types/all_cases.txt similarity index 100% rename from tests/inputs/extraction_types/all_cases.txt rename to tests/data/extraction_types/all_cases.txt diff --git a/tests/inputs/extraction_types/generic_autonomous_system_number.txt b/tests/data/extraction_types/generic_autonomous_system_number.txt similarity index 100% rename from tests/inputs/extraction_types/generic_autonomous_system_number.txt rename to tests/data/extraction_types/generic_autonomous_system_number.txt diff --git a/tests/inputs/extraction_types/generic_bank_card_amex.txt b/tests/data/extraction_types/generic_bank_card_amex.txt similarity index 100% rename from tests/inputs/extraction_types/generic_bank_card_amex.txt rename to tests/data/extraction_types/generic_bank_card_amex.txt diff --git a/tests/inputs/extraction_types/generic_bank_card_diners.txt b/tests/data/extraction_types/generic_bank_card_diners.txt similarity index 100% rename from tests/inputs/extraction_types/generic_bank_card_diners.txt rename to tests/data/extraction_types/generic_bank_card_diners.txt diff --git a/tests/inputs/extraction_types/generic_bank_card_discover.txt b/tests/data/extraction_types/generic_bank_card_discover.txt similarity index 100% rename from tests/inputs/extraction_types/generic_bank_card_discover.txt rename to tests/data/extraction_types/generic_bank_card_discover.txt diff --git a/tests/inputs/extraction_types/generic_bank_card_jcb.txt b/tests/data/extraction_types/generic_bank_card_jcb.txt similarity index 100% rename from tests/inputs/extraction_types/generic_bank_card_jcb.txt rename to tests/data/extraction_types/generic_bank_card_jcb.txt diff --git a/tests/inputs/extraction_types/generic_bank_card_mastercard.txt b/tests/data/extraction_types/generic_bank_card_mastercard.txt similarity index 100% rename from tests/inputs/extraction_types/generic_bank_card_mastercard.txt rename to tests/data/extraction_types/generic_bank_card_mastercard.txt diff --git a/tests/inputs/extraction_types/generic_bank_card_union_pay.txt b/tests/data/extraction_types/generic_bank_card_union_pay.txt similarity index 100% rename from tests/inputs/extraction_types/generic_bank_card_union_pay.txt rename to tests/data/extraction_types/generic_bank_card_union_pay.txt diff --git a/tests/inputs/extraction_types/generic_bank_card_visa.txt b/tests/data/extraction_types/generic_bank_card_visa.txt similarity index 100% rename from tests/inputs/extraction_types/generic_bank_card_visa.txt rename to tests/data/extraction_types/generic_bank_card_visa.txt diff --git a/tests/inputs/extraction_types/generic_country_alpha2.txt b/tests/data/extraction_types/generic_country_alpha2.txt similarity index 100% rename from tests/inputs/extraction_types/generic_country_alpha2.txt rename to tests/data/extraction_types/generic_country_alpha2.txt diff --git a/tests/inputs/extraction_types/generic_cpe_uri.txt b/tests/data/extraction_types/generic_cpe_uri.txt similarity index 100% rename from tests/inputs/extraction_types/generic_cpe_uri.txt rename to tests/data/extraction_types/generic_cpe_uri.txt diff --git a/tests/inputs/extraction_types/generic_cryptocurrency_btc_transaction.txt b/tests/data/extraction_types/generic_cryptocurrency_btc_transaction.txt similarity index 100% rename from tests/inputs/extraction_types/generic_cryptocurrency_btc_transaction.txt rename to tests/data/extraction_types/generic_cryptocurrency_btc_transaction.txt diff --git a/tests/inputs/extraction_types/generic_cryptocurrency_btc_wallet.txt b/tests/data/extraction_types/generic_cryptocurrency_btc_wallet.txt similarity index 100% rename from tests/inputs/extraction_types/generic_cryptocurrency_btc_wallet.txt rename to tests/data/extraction_types/generic_cryptocurrency_btc_wallet.txt diff --git a/tests/inputs/extraction_types/generic_cryptocurrency_eth_transaction.txt b/tests/data/extraction_types/generic_cryptocurrency_eth_transaction.txt similarity index 100% rename from tests/inputs/extraction_types/generic_cryptocurrency_eth_transaction.txt rename to tests/data/extraction_types/generic_cryptocurrency_eth_transaction.txt diff --git a/tests/inputs/extraction_types/generic_cryptocurrency_eth_wallet.txt b/tests/data/extraction_types/generic_cryptocurrency_eth_wallet.txt similarity index 100% rename from tests/inputs/extraction_types/generic_cryptocurrency_eth_wallet.txt rename to tests/data/extraction_types/generic_cryptocurrency_eth_wallet.txt diff --git a/tests/inputs/extraction_types/generic_cryptocurrency_xmr_transaction.txt b/tests/data/extraction_types/generic_cryptocurrency_xmr_transaction.txt similarity index 100% rename from tests/inputs/extraction_types/generic_cryptocurrency_xmr_transaction.txt rename to tests/data/extraction_types/generic_cryptocurrency_xmr_transaction.txt diff --git a/tests/inputs/extraction_types/generic_cryptocurrency_xmr_wallet.txt b/tests/data/extraction_types/generic_cryptocurrency_xmr_wallet.txt similarity index 100% rename from tests/inputs/extraction_types/generic_cryptocurrency_xmr_wallet.txt rename to tests/data/extraction_types/generic_cryptocurrency_xmr_wallet.txt diff --git a/tests/inputs/extraction_types/generic_cve_id.txt b/tests/data/extraction_types/generic_cve_id.txt similarity index 100% rename from tests/inputs/extraction_types/generic_cve_id.txt rename to tests/data/extraction_types/generic_cve_id.txt diff --git a/tests/inputs/extraction_types/generic_directory_unix.txt b/tests/data/extraction_types/generic_directory_unix.txt similarity index 100% rename from tests/inputs/extraction_types/generic_directory_unix.txt rename to tests/data/extraction_types/generic_directory_unix.txt diff --git a/tests/inputs/extraction_types/generic_directory_unix_file.txt b/tests/data/extraction_types/generic_directory_unix_file.txt similarity index 100% rename from tests/inputs/extraction_types/generic_directory_unix_file.txt rename to tests/data/extraction_types/generic_directory_unix_file.txt diff --git a/tests/inputs/extraction_types/generic_directory_windows.txt b/tests/data/extraction_types/generic_directory_windows.txt similarity index 100% rename from tests/inputs/extraction_types/generic_directory_windows.txt rename to tests/data/extraction_types/generic_directory_windows.txt diff --git a/tests/inputs/extraction_types/generic_directory_windows_with_file.txt b/tests/data/extraction_types/generic_directory_windows_with_file.txt similarity index 100% rename from tests/inputs/extraction_types/generic_directory_windows_with_file.txt rename to tests/data/extraction_types/generic_directory_windows_with_file.txt diff --git a/tests/inputs/extraction_types/generic_domain_name_only.txt b/tests/data/extraction_types/generic_domain_name_only.txt similarity index 100% rename from tests/inputs/extraction_types/generic_domain_name_only.txt rename to tests/data/extraction_types/generic_domain_name_only.txt diff --git a/tests/inputs/extraction_types/generic_domain_name_subdomain.txt b/tests/data/extraction_types/generic_domain_name_subdomain.txt similarity index 100% rename from tests/inputs/extraction_types/generic_domain_name_subdomain.txt rename to tests/data/extraction_types/generic_domain_name_subdomain.txt diff --git a/tests/inputs/extraction_types/generic_email_address.txt b/tests/data/extraction_types/generic_email_address.txt similarity index 100% rename from tests/inputs/extraction_types/generic_email_address.txt rename to tests/data/extraction_types/generic_email_address.txt diff --git a/tests/inputs/extraction_types/generic_file_hash_md5.txt b/tests/data/extraction_types/generic_file_hash_md5.txt similarity index 100% rename from tests/inputs/extraction_types/generic_file_hash_md5.txt rename to tests/data/extraction_types/generic_file_hash_md5.txt diff --git a/tests/inputs/extraction_types/generic_file_hash_sha_1.txt b/tests/data/extraction_types/generic_file_hash_sha_1.txt similarity index 100% rename from tests/inputs/extraction_types/generic_file_hash_sha_1.txt rename to tests/data/extraction_types/generic_file_hash_sha_1.txt diff --git a/tests/inputs/extraction_types/generic_file_hash_sha_224.txt b/tests/data/extraction_types/generic_file_hash_sha_224.txt similarity index 100% rename from tests/inputs/extraction_types/generic_file_hash_sha_224.txt rename to tests/data/extraction_types/generic_file_hash_sha_224.txt diff --git a/tests/inputs/extraction_types/generic_file_hash_sha_256.txt b/tests/data/extraction_types/generic_file_hash_sha_256.txt similarity index 100% rename from tests/inputs/extraction_types/generic_file_hash_sha_256.txt rename to tests/data/extraction_types/generic_file_hash_sha_256.txt diff --git a/tests/inputs/extraction_types/generic_file_hash_sha_384.txt b/tests/data/extraction_types/generic_file_hash_sha_384.txt similarity index 100% rename from tests/inputs/extraction_types/generic_file_hash_sha_384.txt rename to tests/data/extraction_types/generic_file_hash_sha_384.txt diff --git a/tests/inputs/extraction_types/generic_file_hash_sha_512.txt b/tests/data/extraction_types/generic_file_hash_sha_512.txt similarity index 100% rename from tests/inputs/extraction_types/generic_file_hash_sha_512.txt rename to tests/data/extraction_types/generic_file_hash_sha_512.txt diff --git a/tests/inputs/extraction_types/generic_file_name.txt b/tests/data/extraction_types/generic_file_name.txt similarity index 100% rename from tests/inputs/extraction_types/generic_file_name.txt rename to tests/data/extraction_types/generic_file_name.txt diff --git a/tests/inputs/extraction_types/generic_host_name.txt b/tests/data/extraction_types/generic_host_name.txt similarity index 100% rename from tests/inputs/extraction_types/generic_host_name.txt rename to tests/data/extraction_types/generic_host_name.txt diff --git a/tests/inputs/extraction_types/generic_host_name_file.txt b/tests/data/extraction_types/generic_host_name_file.txt similarity index 100% rename from tests/inputs/extraction_types/generic_host_name_file.txt rename to tests/data/extraction_types/generic_host_name_file.txt diff --git a/tests/inputs/extraction_types/generic_host_name_path.txt b/tests/data/extraction_types/generic_host_name_path.txt similarity index 100% rename from tests/inputs/extraction_types/generic_host_name_path.txt rename to tests/data/extraction_types/generic_host_name_path.txt diff --git a/tests/inputs/extraction_types/generic_host_name_subdomain.txt b/tests/data/extraction_types/generic_host_name_subdomain.txt similarity index 100% rename from tests/inputs/extraction_types/generic_host_name_subdomain.txt rename to tests/data/extraction_types/generic_host_name_subdomain.txt diff --git a/tests/inputs/extraction_types/generic_host_name_url.txt b/tests/data/extraction_types/generic_host_name_url.txt similarity index 100% rename from tests/inputs/extraction_types/generic_host_name_url.txt rename to tests/data/extraction_types/generic_host_name_url.txt diff --git a/tests/inputs/extraction_types/generic_iban_number.txt b/tests/data/extraction_types/generic_iban_number.txt similarity index 100% rename from tests/inputs/extraction_types/generic_iban_number.txt rename to tests/data/extraction_types/generic_iban_number.txt diff --git a/tests/inputs/extraction_types/generic_ipv4_address_cidr.txt b/tests/data/extraction_types/generic_ipv4_address_cidr.txt similarity index 100% rename from tests/inputs/extraction_types/generic_ipv4_address_cidr.txt rename to tests/data/extraction_types/generic_ipv4_address_cidr.txt diff --git a/tests/inputs/extraction_types/generic_ipv4_address_only.txt b/tests/data/extraction_types/generic_ipv4_address_only.txt similarity index 100% rename from tests/inputs/extraction_types/generic_ipv4_address_only.txt rename to tests/data/extraction_types/generic_ipv4_address_only.txt diff --git a/tests/inputs/extraction_types/generic_ipv4_address_port.txt b/tests/data/extraction_types/generic_ipv4_address_port.txt similarity index 100% rename from tests/inputs/extraction_types/generic_ipv4_address_port.txt rename to tests/data/extraction_types/generic_ipv4_address_port.txt diff --git a/tests/inputs/extraction_types/generic_ipv6_address_cidr.txt b/tests/data/extraction_types/generic_ipv6_address_cidr.txt similarity index 100% rename from tests/inputs/extraction_types/generic_ipv6_address_cidr.txt rename to tests/data/extraction_types/generic_ipv6_address_cidr.txt diff --git a/tests/inputs/extraction_types/generic_ipv6_address_only.txt b/tests/data/extraction_types/generic_ipv6_address_only.txt similarity index 100% rename from tests/inputs/extraction_types/generic_ipv6_address_only.txt rename to tests/data/extraction_types/generic_ipv6_address_only.txt diff --git a/tests/inputs/extraction_types/generic_ipv6_address_port.txt b/tests/data/extraction_types/generic_ipv6_address_port.txt similarity index 100% rename from tests/inputs/extraction_types/generic_ipv6_address_port.txt rename to tests/data/extraction_types/generic_ipv6_address_port.txt diff --git a/tests/inputs/extraction_types/generic_mac_address.txt b/tests/data/extraction_types/generic_mac_address.txt similarity index 100% rename from tests/inputs/extraction_types/generic_mac_address.txt rename to tests/data/extraction_types/generic_mac_address.txt diff --git a/tests/inputs/extraction_types/generic_mitre_attack_enterprise.txt b/tests/data/extraction_types/generic_mitre_attack_enterprise.txt similarity index 100% rename from tests/inputs/extraction_types/generic_mitre_attack_enterprise.txt rename to tests/data/extraction_types/generic_mitre_attack_enterprise.txt diff --git a/tests/inputs/extraction_types/generic_mitre_attack_ics.txt b/tests/data/extraction_types/generic_mitre_attack_ics.txt similarity index 100% rename from tests/inputs/extraction_types/generic_mitre_attack_ics.txt rename to tests/data/extraction_types/generic_mitre_attack_ics.txt diff --git a/tests/inputs/extraction_types/generic_mitre_attack_mobile.txt b/tests/data/extraction_types/generic_mitre_attack_mobile.txt similarity index 100% rename from tests/inputs/extraction_types/generic_mitre_attack_mobile.txt rename to tests/data/extraction_types/generic_mitre_attack_mobile.txt diff --git a/tests/inputs/extraction_types/generic_mitre_capec.txt b/tests/data/extraction_types/generic_mitre_capec.txt similarity index 100% rename from tests/inputs/extraction_types/generic_mitre_capec.txt rename to tests/data/extraction_types/generic_mitre_capec.txt diff --git a/tests/inputs/extraction_types/generic_mitre_cwe.txt b/tests/data/extraction_types/generic_mitre_cwe.txt similarity index 100% rename from tests/inputs/extraction_types/generic_mitre_cwe.txt rename to tests/data/extraction_types/generic_mitre_cwe.txt diff --git a/tests/inputs/extraction_types/generic_phone_number.txt b/tests/data/extraction_types/generic_phone_number.txt similarity index 100% rename from tests/inputs/extraction_types/generic_phone_number.txt rename to tests/data/extraction_types/generic_phone_number.txt diff --git a/tests/inputs/extraction_types/generic_url.txt b/tests/data/extraction_types/generic_url.txt similarity index 100% rename from tests/inputs/extraction_types/generic_url.txt rename to tests/data/extraction_types/generic_url.txt diff --git a/tests/inputs/extraction_types/generic_url_file.txt b/tests/data/extraction_types/generic_url_file.txt similarity index 100% rename from tests/inputs/extraction_types/generic_url_file.txt rename to tests/data/extraction_types/generic_url_file.txt diff --git a/tests/inputs/extraction_types/generic_url_path.txt b/tests/data/extraction_types/generic_url_path.txt similarity index 100% rename from tests/inputs/extraction_types/generic_url_path.txt rename to tests/data/extraction_types/generic_url_path.txt diff --git a/tests/inputs/extraction_types/generic_user_agent.txt b/tests/data/extraction_types/generic_user_agent.txt similarity index 100% rename from tests/inputs/extraction_types/generic_user_agent.txt rename to tests/data/extraction_types/generic_user_agent.txt diff --git a/tests/inputs/extraction_types/generic_windows_registry_key.txt b/tests/data/extraction_types/generic_windows_registry_key.txt similarity index 100% rename from tests/inputs/extraction_types/generic_windows_registry_key.txt rename to tests/data/extraction_types/generic_windows_registry_key.txt diff --git a/tests/inputs/extraction_types/lookup_attack_pattern.txt b/tests/data/extraction_types/lookup_attack_pattern.txt similarity index 100% rename from tests/inputs/extraction_types/lookup_attack_pattern.txt rename to tests/data/extraction_types/lookup_attack_pattern.txt diff --git a/tests/inputs/extraction_types/lookup_campaign.txt b/tests/data/extraction_types/lookup_campaign.txt similarity index 100% rename from tests/inputs/extraction_types/lookup_campaign.txt rename to tests/data/extraction_types/lookup_campaign.txt diff --git a/tests/inputs/extraction_types/lookup_course_of_action.txt b/tests/data/extraction_types/lookup_course_of_action.txt similarity index 100% rename from tests/inputs/extraction_types/lookup_course_of_action.txt rename to tests/data/extraction_types/lookup_course_of_action.txt diff --git a/tests/inputs/extraction_types/lookup_identity.txt b/tests/data/extraction_types/lookup_identity.txt similarity index 100% rename from tests/inputs/extraction_types/lookup_identity.txt rename to tests/data/extraction_types/lookup_identity.txt diff --git a/tests/inputs/extraction_types/lookup_infrastructure.txt b/tests/data/extraction_types/lookup_infrastructure.txt similarity index 100% rename from tests/inputs/extraction_types/lookup_infrastructure.txt rename to tests/data/extraction_types/lookup_infrastructure.txt diff --git a/tests/inputs/extraction_types/lookup_intrusion_set.txt b/tests/data/extraction_types/lookup_intrusion_set.txt similarity index 100% rename from tests/inputs/extraction_types/lookup_intrusion_set.txt rename to tests/data/extraction_types/lookup_intrusion_set.txt diff --git a/tests/inputs/extraction_types/lookup_malware.txt b/tests/data/extraction_types/lookup_malware.txt similarity index 100% rename from tests/inputs/extraction_types/lookup_malware.txt rename to tests/data/extraction_types/lookup_malware.txt diff --git a/tests/inputs/extraction_types/lookup_threat_actor.txt b/tests/data/extraction_types/lookup_threat_actor.txt similarity index 100% rename from tests/inputs/extraction_types/lookup_threat_actor.txt rename to tests/data/extraction_types/lookup_threat_actor.txt diff --git a/tests/inputs/extraction_types/lookup_tool.txt b/tests/data/extraction_types/lookup_tool.txt similarity index 100% rename from tests/inputs/extraction_types/lookup_tool.txt rename to tests/data/extraction_types/lookup_tool.txt diff --git a/tests/inputs/extraction_types/whitelist_alexa_top_1000_domains.txt b/tests/data/extraction_types/whitelist_alexa_top_1000_domains.txt similarity index 100% rename from tests/inputs/extraction_types/whitelist_alexa_top_1000_domains.txt rename to tests/data/extraction_types/whitelist_alexa_top_1000_domains.txt diff --git a/tests/inputs/extraction_types/whitelist_examples.txt b/tests/data/extraction_types/whitelist_examples.txt similarity index 100% rename from tests/inputs/extraction_types/whitelist_examples.txt rename to tests/data/extraction_types/whitelist_examples.txt diff --git a/tests/inputs/extraction_types/whitelist_security_vendor_domains.txt b/tests/data/extraction_types/whitelist_security_vendor_domains.txt similarity index 100% rename from tests/inputs/extraction_types/whitelist_security_vendor_domains.txt rename to tests/data/extraction_types/whitelist_security_vendor_domains.txt diff --git a/tests/inputs/manually_generated_reports/basic_relationship.txt b/tests/data/manually_generated_reports/basic_relationship.txt similarity index 100% rename from tests/inputs/manually_generated_reports/basic_relationship.txt rename to tests/data/manually_generated_reports/basic_relationship.txt diff --git a/tests/inputs/manually_generated_reports/char_length_too_long.txt b/tests/data/manually_generated_reports/char_length_too_long.txt similarity index 100% rename from tests/inputs/manually_generated_reports/char_length_too_long.txt rename to tests/data/manually_generated_reports/char_length_too_long.txt diff --git a/tests/inputs/manually_generated_reports/descriptive_for_ai_relationships_1.txt b/tests/data/manually_generated_reports/descriptive_for_ai_relationships_1.txt similarity index 100% rename from tests/inputs/manually_generated_reports/descriptive_for_ai_relationships_1.txt rename to tests/data/manually_generated_reports/descriptive_for_ai_relationships_1.txt diff --git a/tests/inputs/manually_generated_reports/known_whitelist_match.txt b/tests/data/manually_generated_reports/known_whitelist_match.txt similarity index 100% rename from tests/inputs/manually_generated_reports/known_whitelist_match.txt rename to tests/data/manually_generated_reports/known_whitelist_match.txt diff --git a/tests/inputs/manually_generated_reports/mixed_extractions.txt b/tests/data/manually_generated_reports/mixed_extractions.txt similarity index 100% rename from tests/inputs/manually_generated_reports/mixed_extractions.txt rename to tests/data/manually_generated_reports/mixed_extractions.txt diff --git a/tests/inputs/manually_generated_reports/test_ai_hash_error_with_stix2_lib.txt b/tests/data/manually_generated_reports/test_ai_hash_error_with_stix2_lib.txt similarity index 100% rename from tests/inputs/manually_generated_reports/test_ai_hash_error_with_stix2_lib.txt rename to tests/data/manually_generated_reports/test_ai_hash_error_with_stix2_lib.txt diff --git a/tests/inputs/manually_generated_reports/test_aliases.txt b/tests/data/manually_generated_reports/test_aliases.txt similarity index 100% rename from tests/inputs/manually_generated_reports/test_aliases.txt rename to tests/data/manually_generated_reports/test_aliases.txt diff --git a/tests/inputs/manually_generated_reports/test_extraction_escapes.txt b/tests/data/manually_generated_reports/test_extraction_escapes.txt similarity index 100% rename from tests/inputs/manually_generated_reports/test_extraction_escapes.txt rename to tests/data/manually_generated_reports/test_extraction_escapes.txt diff --git a/tests/inputs/real_intel_reports/APT28-Center-of-Storm-2017.txt b/tests/data/real_intel_reports/APT28-Center-of-Storm-2017.txt similarity index 100% rename from tests/inputs/real_intel_reports/APT28-Center-of-Storm-2017.txt rename to tests/data/real_intel_reports/APT28-Center-of-Storm-2017.txt diff --git a/tests/inputs/real_intel_reports/Bitdefender-Labs-Report-X-creat6958-en-EN.txt b/tests/data/real_intel_reports/Bitdefender-Labs-Report-X-creat6958-en-EN.txt similarity index 100% rename from tests/inputs/real_intel_reports/Bitdefender-Labs-Report-X-creat6958-en-EN.txt rename to tests/data/real_intel_reports/Bitdefender-Labs-Report-X-creat6958-en-EN.txt diff --git a/tests/inputs/real_intel_reports/FireEyeAPT39.txt b/tests/data/real_intel_reports/FireEyeAPT39.txt similarity index 100% rename from tests/inputs/real_intel_reports/FireEyeAPT39.txt rename to tests/data/real_intel_reports/FireEyeAPT39.txt diff --git a/tests/inputs/real_intel_reports/France_CERT_APT31_Pakdoor_TLPWHITE.txt b/tests/data/real_intel_reports/France_CERT_APT31_Pakdoor_TLPWHITE.txt similarity index 100% rename from tests/inputs/real_intel_reports/France_CERT_APT31_Pakdoor_TLPWHITE.txt rename to tests/data/real_intel_reports/France_CERT_APT31_Pakdoor_TLPWHITE.txt diff --git a/tests/inputs/real_intel_reports/Group-IB_Ransomware_Uncovered_whitepaper_eng.txt b/tests/data/real_intel_reports/Group-IB_Ransomware_Uncovered_whitepaper_eng.txt similarity index 100% rename from tests/inputs/real_intel_reports/Group-IB_Ransomware_Uncovered_whitepaper_eng.txt rename to tests/data/real_intel_reports/Group-IB_Ransomware_Uncovered_whitepaper_eng.txt diff --git a/tests/inputs/real_intel_reports/JOINT_CSA_HUNTING_RU_INTEL_SNAKE_MALWARE_20230509.txt b/tests/data/real_intel_reports/JOINT_CSA_HUNTING_RU_INTEL_SNAKE_MALWARE_20230509.txt similarity index 100% rename from tests/inputs/real_intel_reports/JOINT_CSA_HUNTING_RU_INTEL_SNAKE_MALWARE_20230509.txt rename to tests/data/real_intel_reports/JOINT_CSA_HUNTING_RU_INTEL_SNAKE_MALWARE_20230509.txt diff --git a/tests/inputs/real_intel_reports/TA22-0126-QAKBOT-analysis-TLP-GREEN.txt b/tests/data/real_intel_reports/TA22-0126-QAKBOT-analysis-TLP-GREEN.txt similarity index 100% rename from tests/inputs/real_intel_reports/TA22-0126-QAKBOT-analysis-TLP-GREEN.txt rename to tests/data/real_intel_reports/TA22-0126-QAKBOT-analysis-TLP-GREEN.txt diff --git a/tests/inputs/real_intel_reports/dinners_card.txt b/tests/data/real_intel_reports/dinners_card.txt similarity index 100% rename from tests/inputs/real_intel_reports/dinners_card.txt rename to tests/data/real_intel_reports/dinners_card.txt diff --git a/tests/inputs/real_intel_reports/mandiant-apt1.txt b/tests/data/real_intel_reports/mandiant-apt1.txt similarity index 100% rename from tests/inputs/real_intel_reports/mandiant-apt1.txt rename to tests/data/real_intel_reports/mandiant-apt1.txt diff --git a/tests/inputs/real_intel_reports/mykings_report_final.txt b/tests/data/real_intel_reports/mykings_report_final.txt similarity index 100% rename from tests/inputs/real_intel_reports/mykings_report_final.txt rename to tests/data/real_intel_reports/mykings_report_final.txt diff --git a/tests/cases-ai-extraction-type.md b/tests/manual-tests/cases-ai-extraction-type.md similarity index 75% rename from tests/cases-ai-extraction-type.md rename to tests/manual-tests/cases-ai-extraction-type.md index 3eb7c60..a5ad42a 100644 --- a/tests/cases-ai-extraction-type.md +++ b/tests/manual-tests/cases-ai-extraction-type.md @@ -3,7 +3,7 @@ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_cryptocurrency_btc_wallet.txt \ + --input_file tests/data/extraction_types/generic_cryptocurrency_btc_wallet.txt \ --name 'Test 1.1.1 ai_cryptocurrency_btc_wallet' \ --tlp_level clear \ --confidence 100 \ @@ -15,7 +15,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_cryptocurrency_btc_transaction.txt \ + --input_file tests/data/extraction_types/generic_cryptocurrency_btc_transaction.txt \ --name 'Test 1.1.2 ai_cryptocurrency_btc_transaction' \ --tlp_level clear \ --confidence 100 \ @@ -27,7 +27,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_cryptocurrency_eth_wallet.txt \ + --input_file tests/data/extraction_types/generic_cryptocurrency_eth_wallet.txt \ --name 'Test 1.1.3 ai_cryptocurrency_eth_wallet' \ --tlp_level clear \ --confidence 100 \ @@ -39,7 +39,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_cryptocurrency_eth_transaction.txt \ + --input_file tests/data/extraction_types/generic_cryptocurrency_eth_transaction.txt \ --name 'Test 1.1.4 ai_cryptocurrency_eth_transaction' \ --tlp_level clear \ --confidence 100 \ @@ -51,7 +51,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_cryptocurrency_xmr_wallet.txt \ + --input_file tests/data/extraction_types/generic_cryptocurrency_xmr_wallet.txt \ --name 'Test 1.1.5 ai_cryptocurrency_xmr_wallet' \ --tlp_level clear \ --confidence 100 \ @@ -63,7 +63,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_cryptocurrency_xmr_transaction.txt \ + --input_file tests/data/extraction_types/generic_cryptocurrency_xmr_transaction.txt \ --name 'Test 1.1.6 ai_cryptocurrency_xmr_transaction' \ --tlp_level clear \ --confidence 100 \ @@ -75,7 +75,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_phone_number.txt \ + --input_file tests/data/extraction_types/generic_phone_number.txt \ --name 'Test 1.1.7 ai_phone_number' \ --tlp_level clear \ --confidence 100 \ @@ -87,7 +87,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/ai_country_alpha2.txt \ + --input_file tests/data/extraction_types/ai_country_alpha2.txt \ --name 'Test 1.1.8 ai_country_alpha2' \ --tlp_level clear \ --confidence 100 \ @@ -99,7 +99,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/ai_mitre_attack_enterprise.txt \ + --input_file tests/data/extraction_types/ai_mitre_attack_enterprise.txt \ --name 'Test 21.1.9 ai_mitre_attack_enterprise' \ --tlp_level clear \ --confidence 100 \ @@ -111,7 +111,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/ai_mitre_attack_mobile.txt \ + --input_file tests/data/extraction_types/ai_mitre_attack_mobile.txt \ --name 'Test 1.1.10 ai_mitre_attack_mobile' \ --tlp_level clear \ --confidence 100 \ @@ -123,7 +123,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/ai_mitre_attack_ics.txt \ + --input_file tests/data/extraction_types/ai_mitre_attack_ics.txt \ --name 'Test 1.1.11 ai_mitre_attack_ics' \ --tlp_level clear \ --confidence 100 \ @@ -135,7 +135,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/ai_mitre_capec.txt \ + --input_file tests/data/extraction_types/ai_mitre_capec.txt \ --name 'Test 1.1.12 ai_mitre_capec' \ --tlp_level clear \ --confidence 100 \ @@ -147,7 +147,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/ai_mitre_cwe.txt \ + --input_file tests/data/extraction_types/ai_mitre_cwe.txt \ --name 'Test 1.1.13 ai_mitre_cwe' \ --tlp_level clear \ --confidence 100 \ diff --git a/tests/cases-ai-relationships.md b/tests/manual-tests/cases-ai-relationships.md similarity index 80% rename from tests/cases-ai-relationships.md rename to tests/manual-tests/cases-ai-relationships.md index 9c5ba62..8736e2a 100644 --- a/tests/cases-ai-relationships.md +++ b/tests/manual-tests/cases-ai-relationships.md @@ -5,7 +5,7 @@ ```shell python3 txt2stix.py \ --relationship_mode ai \ - --input_file tests/inputs/manually_generated_reports/basic_relationship.txt \ + --input_file tests/data/manually_generated_reports/basic_relationship.txt \ --name 'Test 4.0.1 Basic AI relationships' \ --tlp_level clear \ --confidence 100 \ diff --git a/tests/cases-lookup-extraction-type.md b/tests/manual-tests/cases-lookup-extraction-type.md similarity index 80% rename from tests/cases-lookup-extraction-type.md rename to tests/manual-tests/cases-lookup-extraction-type.md index b0f8f25..5384446 100644 --- a/tests/cases-lookup-extraction-type.md +++ b/tests/manual-tests/cases-lookup-extraction-type.md @@ -3,7 +3,7 @@ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_mitre_attack_enterprise.txt \ + --input_file tests/data/extraction_types/generic_mitre_attack_enterprise.txt \ --name 'Test 2.1.1 lookup_mitre_attack_enterprise_id' \ --tlp_level clear \ --confidence 100 \ @@ -27,7 +27,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_mitre_attack_mobile.txt \ + --input_file tests/data/extraction_types/generic_mitre_attack_mobile.txt \ --name 'Test 2.1.2 lookup_mitre_attack_mobile_id' \ --tlp_level clear \ --confidence 100 \ @@ -41,7 +41,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_mitre_attack_ics.txt \ + --input_file tests/data/extraction_types/generic_mitre_attack_ics.txt \ --name 'Test 2.1.3 lookup_mitre_attack_ics_id' \ --tlp_level clear \ --confidence 100 \ @@ -53,7 +53,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_mitre_capec.txt \ + --input_file tests/data/extraction_types/generic_mitre_capec.txt \ --name 'Test 2.1.4 lookup_mitre_capec_id' \ --tlp_level clear \ --confidence 100 \ @@ -65,7 +65,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_mitre_cwe.txt \ + --input_file tests/data/extraction_types/generic_mitre_cwe.txt \ --name 'Test 2.1.5 lookup_mitre_cwe_id' \ --tlp_level clear \ --confidence 100 \ @@ -77,7 +77,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/lookup_attack_pattern.txt \ + --input_file tests/data/extraction_types/lookup_attack_pattern.txt \ --name 'Test 2.1.6 lookup_attack_pattern' \ --tlp_level clear \ --confidence 100 \ @@ -89,7 +89,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/lookup_campaign.txt \ + --input_file tests/data/extraction_types/lookup_campaign.txt \ --name 'Test 2.1.7 lookup_campaign' \ --tlp_level clear \ --confidence 100 \ @@ -101,7 +101,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/lookup_course_of_action.txt \ + --input_file tests/data/extraction_types/lookup_course_of_action.txt \ --name 'Test 2.1.8 lookup_course_of_action' \ --tlp_level clear \ --confidence 100 \ @@ -113,7 +113,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/lookup_identity.txt \ + --input_file tests/data/extraction_types/lookup_identity.txt \ --name 'Test 2.1.9 lookup_identity' \ --tlp_level clear \ --confidence 100 \ @@ -125,7 +125,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/lookup_infrastructure.txt \ + --input_file tests/data/extraction_types/lookup_infrastructure.txt \ --name 'Test 2.1.10 lookup_infrastructure' \ --tlp_level clear \ --confidence 100 \ @@ -137,7 +137,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/lookup_intrusion_set.txt \ + --input_file tests/data/extraction_types/lookup_intrusion_set.txt \ --name 'Test 2.1.11 lookup_intrusion_set' \ --tlp_level clear \ --confidence 100 \ @@ -149,7 +149,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/lookup_malware.txt \ + --input_file tests/data/extraction_types/lookup_malware.txt \ --name 'Test 2.1.12 lookup_malware' \ --tlp_level clear \ --confidence 100 \ @@ -161,7 +161,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/lookup_threat_actor.txt \ + --input_file tests/data/extraction_types/lookup_threat_actor.txt \ --name 'Test 2.1.13 lookup_threat_actor' \ --tlp_level clear \ --confidence 100 \ @@ -173,7 +173,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/lookup_tool.txt \ + --input_file tests/data/extraction_types/lookup_tool.txt \ --name 'Test 2.1.14 lookup_tool' \ --tlp_level clear \ --confidence 100 \ @@ -185,7 +185,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_country_alpha2.txt \ + --input_file tests/data/extraction_types/generic_country_alpha2.txt \ --name 'Test 2.1.15 lookup_country_alpha2' \ --tlp_level clear \ --confidence 100 \ diff --git a/tests/cases-pattern-extraction-type.md b/tests/manual-tests/cases-pattern-extraction-type.md similarity index 76% rename from tests/cases-pattern-extraction-type.md rename to tests/manual-tests/cases-pattern-extraction-type.md index 40df81d..daf11c2 100644 --- a/tests/cases-pattern-extraction-type.md +++ b/tests/manual-tests/cases-pattern-extraction-type.md @@ -3,7 +3,7 @@ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv4_address_only.txt \ + --input_file tests/data/extraction_types/generic_ipv4_address_only.txt \ --name 'Test 3.1.1 pattern_ipv4_address_only' \ --tlp_level clear \ --confidence 100 \ @@ -15,7 +15,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv4_address_cidr.txt \ + --input_file tests/data/extraction_types/generic_ipv4_address_cidr.txt \ --name 'Test 3.1.2 pattern_ipv4_address_cidr' \ --tlp_level clear \ --confidence 100 \ @@ -27,7 +27,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv4_address_port.txt \ + --input_file tests/data/extraction_types/generic_ipv4_address_port.txt \ --name 'Test 3.1.3 pattern_ipv4_address_port' \ --tlp_level clear \ --confidence 100 \ @@ -39,7 +39,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv6_address_only.txt \ + --input_file tests/data/extraction_types/generic_ipv6_address_only.txt \ --name 'Test 3.1.4 pattern_ipv6_address_only' \ --tlp_level clear \ --confidence 100 \ @@ -51,7 +51,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv6_address_cidr.txt \ + --input_file tests/data/extraction_types/generic_ipv6_address_cidr.txt \ --name 'Test 3.1.5 pattern_ipv6_address_cidr' \ --tlp_level clear \ --confidence 100 \ @@ -63,7 +63,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv6_address_port.txt \ + --input_file tests/data/extraction_types/generic_ipv6_address_port.txt \ --name 'Test 3.1.6 pattern_ipv6_address_port' \ --tlp_level clear \ --confidence 100 \ @@ -75,7 +75,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_domain_name_only.txt \ + --input_file tests/data/extraction_types/generic_domain_name_only.txt \ --name 'Test 3.1.7 pattern_domain_name_only' \ --tlp_level clear \ --confidence 100 \ @@ -87,7 +87,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_domain_name_subdomain.txt \ + --input_file tests/data/extraction_types/generic_domain_name_subdomain.txt \ --name 'Test 3.1.8 pattern_domain_name_subdomain' \ --tlp_level clear \ --confidence 100 \ @@ -99,7 +99,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_url.txt \ + --input_file tests/data/extraction_types/generic_url.txt \ --name 'Test 3.1.9 pattern_url' \ --tlp_level clear \ --confidence 100 \ @@ -111,7 +111,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_url_file.txt \ + --input_file tests/data/extraction_types/generic_url_file.txt \ --name 'Test 3.1.10 pattern_url_file' \ --tlp_level clear \ --confidence 100 \ @@ -123,7 +123,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_url_path.txt \ + --input_file tests/data/extraction_types/generic_url_path.txt \ --name 'Test 3.1.11 pattern_url_path' \ --tlp_level clear \ --confidence 100 \ @@ -135,7 +135,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_host_name.txt \ + --input_file tests/data/extraction_types/generic_host_name.txt \ --name 'Test 3.1.12 pattern_host_name' \ --tlp_level clear \ --confidence 100 \ @@ -147,7 +147,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_host_name_subdomain.txt \ + --input_file tests/data/extraction_types/generic_host_name_subdomain.txt \ --name 'Test 3.1.13 pattern_host_name_subdomain' \ --tlp_level clear \ --confidence 100 \ @@ -159,7 +159,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_host_name_url.txt \ + --input_file tests/data/extraction_types/generic_host_name_url.txt \ --name 'Test 3.1.14 pattern_host_name_url' \ --tlp_level clear \ --confidence 100 \ @@ -171,7 +171,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_host_name_file.txt \ + --input_file tests/data/extraction_types/generic_host_name_file.txt \ --name 'Test 3.1.15 pattern_host_name_file' \ --tlp_level clear \ --confidence 100 \ @@ -183,7 +183,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_host_name_path.txt \ + --input_file tests/data/extraction_types/generic_host_name_path.txt \ --name 'Test 3.1.16 pattern_host_name_path' \ --tlp_level clear \ --confidence 100 \ @@ -195,7 +195,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_file_name.txt \ + --input_file tests/data/extraction_types/generic_file_name.txt \ --name 'Test 3.1.17 pattern_file_name' \ --tlp_level clear \ --confidence 100 \ @@ -207,7 +207,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_directory_windows.txt \ + --input_file tests/data/extraction_types/generic_directory_windows.txt \ --name 'Test 3.1.18 pattern_directory_windows' \ --tlp_level clear \ --confidence 100 \ @@ -219,7 +219,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_directory_windows_with_file.txt \ + --input_file tests/data/extraction_types/generic_directory_windows_with_file.txt \ --name 'Test 3.1.19 pattern_directory_windows_with_file' \ --tlp_level clear \ --confidence 100 \ @@ -231,7 +231,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_directory_unix.txt \ + --input_file tests/data/extraction_types/generic_directory_unix.txt \ --name 'Test 3.1.20 pattern_directory_unix' \ --tlp_level clear \ --confidence 100 \ @@ -243,7 +243,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_directory_unix_file.txt \ + --input_file tests/data/extraction_types/generic_directory_unix_file.txt \ --name 'Test 3.1.21 pattern_directory_unix_file' \ --tlp_level clear \ --confidence 100 \ @@ -255,7 +255,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_file_hash_md5.txt \ + --input_file tests/data/extraction_types/generic_file_hash_md5.txt \ --name 'Test 3.1.22 pattern_file_hash_md5' \ --tlp_level clear \ --confidence 100 \ @@ -267,7 +267,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_file_hash_sha_1.txt \ + --input_file tests/data/extraction_types/generic_file_hash_sha_1.txt \ --name 'Test 3.1.23 pattern_file_hash_sha_1' \ --tlp_level clear \ --confidence 100 \ @@ -279,7 +279,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_file_hash_sha_256.txt \ + --input_file tests/data/extraction_types/generic_file_hash_sha_256.txt \ --name 'Test 3.1.24 pattern_file_hash_sha_256' \ --tlp_level clear \ --confidence 100 \ @@ -291,7 +291,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_file_hash_sha_512.txt \ + --input_file tests/data/extraction_types/generic_file_hash_sha_512.txt \ --name 'Test 3.1.25 pattern_file_hash_sha_512' \ --tlp_level clear \ --confidence 100 \ @@ -303,7 +303,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_email_address.txt \ + --input_file tests/data/extraction_types/generic_email_address.txt \ --name 'Test 3.1.26 pattern_email_address' \ --tlp_level clear \ --confidence 100 \ @@ -315,7 +315,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_mac_address.txt \ + --input_file tests/data/extraction_types/generic_mac_address.txt \ --name 'Test 3.1.27 pattern_mac_address' \ --tlp_level clear \ --confidence 100 \ @@ -327,7 +327,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_windows_registry_key.txt \ + --input_file tests/data/extraction_types/generic_windows_registry_key.txt \ --name 'Test 3.1.28 pattern_windows_registry_key' \ --tlp_level clear \ --confidence 100 \ @@ -339,7 +339,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_user_agent.txt \ + --input_file tests/data/extraction_types/generic_user_agent.txt \ --name 'Test 3.1.29 pattern_user_agent' \ --tlp_level clear \ --confidence 100 \ @@ -351,7 +351,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_autonomous_system_number.txt \ + --input_file tests/data/extraction_types/generic_autonomous_system_number.txt \ --name 'Test 3.1.30 pattern_autonomous_system_number' \ --tlp_level clear \ --confidence 100 \ @@ -363,7 +363,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_cryptocurrency_btc_wallet.txt \ + --input_file tests/data/extraction_types/generic_cryptocurrency_btc_wallet.txt \ --name 'Test 3.1.31.1 pattern_cryptocurrency_btc_wallet' \ --tlp_level clear \ --confidence 100 \ @@ -375,7 +375,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_cryptocurrency_btc_wallet.txt \ + --input_file tests/data/extraction_types/generic_cryptocurrency_btc_wallet.txt \ --name 'Test 3.1.31.2 pattern_cryptocurrency_btc_wallet_transaction' \ --tlp_level clear \ --confidence 100 \ @@ -387,7 +387,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_cryptocurrency_btc_transaction.txt \ + --input_file tests/data/extraction_types/generic_cryptocurrency_btc_transaction.txt \ --name 'Test 3.1.31.3 pattern_cryptocurrency_btc_transaction' \ --tlp_level clear \ --confidence 100 \ @@ -399,7 +399,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_cve_id.txt \ + --input_file tests/data/extraction_types/generic_cve_id.txt \ --name 'Test 3.1.32 pattern_cve_id' \ --tlp_level clear \ --confidence 100 \ @@ -411,7 +411,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_cpe_uri.txt \ + --input_file tests/data/extraction_types/generic_cpe_uri.txt \ --name 'Test 3.1.33 pattern_cpe_uri' \ --tlp_level clear \ --confidence 100 \ @@ -423,7 +423,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_bank_card_mastercard.txt \ + --input_file tests/data/extraction_types/generic_bank_card_mastercard.txt \ --name 'Test 3.1.34 pattern_bank_card_mastercard' \ --tlp_level clear \ --confidence 100 \ @@ -435,7 +435,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_bank_card_visa.txt \ + --input_file tests/data/extraction_types/generic_bank_card_visa.txt \ --name 'Test 3.1.35 pattern_bank_card_visa' \ --tlp_level clear \ --confidence 100 \ @@ -447,7 +447,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_bank_card_amex.txt \ + --input_file tests/data/extraction_types/generic_bank_card_amex.txt \ --name 'Test 3.1.36 pattern_bank_card_amex' \ --tlp_level clear \ --confidence 100 \ @@ -459,7 +459,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_bank_card_union_pay.txt \ + --input_file tests/data/extraction_types/generic_bank_card_union_pay.txt \ --name 'Test 3.1.37 pattern_bank_card_union_pay' \ --tlp_level clear \ --confidence 100 \ @@ -471,7 +471,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_bank_card_diners.txt \ + --input_file tests/data/extraction_types/generic_bank_card_diners.txt \ --name 'Test 3.1.38 pattern_bank_card_diners' \ --tlp_level clear \ --confidence 100 \ @@ -483,7 +483,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_bank_card_jcb.txt \ + --input_file tests/data/extraction_types/generic_bank_card_jcb.txt \ --name 'Test 3.1.39 pattern_bank_card_jcb' \ --tlp_level clear \ --confidence 100 \ @@ -495,7 +495,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_bank_card_discover.txt \ + --input_file tests/data/extraction_types/generic_bank_card_discover.txt \ --name 'Test 3.1.40 pattern_bank_card_discover' \ --tlp_level clear \ --confidence 100 \ @@ -507,7 +507,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_iban_number.txt \ + --input_file tests/data/extraction_types/generic_iban_number.txt \ --name 'Test 3.1.41 pattern_iban_number' \ --tlp_level clear \ --confidence 100 \ @@ -519,7 +519,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_phone_number.txt \ + --input_file tests/data/extraction_types/generic_phone_number.txt \ --name 'Test 3.1.42 pattern_phone_number' \ --tlp_level clear \ --confidence 100 \ diff --git a/tests/cases-standard-tests.md b/tests/manual-tests/cases-standard-tests.md similarity index 85% rename from tests/cases-standard-tests.md rename to tests/manual-tests/cases-standard-tests.md index e76be5c..6e4518c 100644 --- a/tests/cases-standard-tests.md +++ b/tests/manual-tests/cases-standard-tests.md @@ -7,7 +7,7 @@ Clear ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv4_address_only.txt \ + --input_file tests/data/extraction_types/generic_ipv4_address_only.txt \ --name 'Test 0.1.1 Clear' \ --tlp_level clear \ --confidence 100 \ @@ -18,7 +18,7 @@ Green ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv4_address_only.txt \ + --input_file tests/data/extraction_types/generic_ipv4_address_only.txt \ --name 'Test 0.1.1 Green' \ --tlp_level green \ --confidence 100 \ @@ -30,7 +30,7 @@ Amber ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv4_address_only.txt \ + --input_file tests/data/extraction_types/generic_ipv4_address_only.txt \ --name 'Test 0.1.1 Amber' \ --tlp_level amber \ --confidence 100 \ @@ -42,7 +42,7 @@ Amber+Strict ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv4_address_only.txt \ + --input_file tests/data/extraction_types/generic_ipv4_address_only.txt \ --name 'Test 0.1.1 Amber+Strict' \ --tlp_level amber_strict \ --confidence 100 \ @@ -54,7 +54,7 @@ Red ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv4_address_only.txt \ + --input_file tests/data/extraction_types/generic_ipv4_address_only.txt \ --name 'Test 0.1.1 Red' \ --tlp_level red \ --confidence 100 \ @@ -66,7 +66,7 @@ Bad TLP value -- should return error ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv4_address_only.txt \ + --input_file tests/data/extraction_types/generic_ipv4_address_only.txt \ --name 'Test 0.1.1 Bad TLP value' \ --tlp_level bad \ --confidence 100 \ @@ -80,7 +80,7 @@ Following should use default identity `identity--9c259ff7-f413-5001-9911-70b4352 ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv4_address_only.txt \ + --input_file tests/data/extraction_types/generic_ipv4_address_only.txt \ --name 'Test 0.1.2 Custom Identity' \ --tlp_level amber_strict \ --confidence 90 \ @@ -96,7 +96,7 @@ Following should PASS as Identity is valid: ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv4_address_only.txt \ + --input_file tests/data/extraction_types/generic_ipv4_address_only.txt \ --name 'Test 0.1.2 Custom Identity' \ --tlp_level amber_strict \ --confidence 90 \ @@ -113,7 +113,7 @@ Following should FAIL as Identity is bad: ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv4_address_only.txt \ + --input_file tests/data/extraction_types/generic_ipv4_address_only.txt \ --name 'Test 0.1.2 Custom Identity' \ --tlp_level amber_strict \ --confidence 90 \ @@ -128,7 +128,7 @@ Confidence score not passed, is valid (no `confidence` property in report) ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv4_address_only.txt \ + --input_file tests/data/extraction_types/generic_ipv4_address_only.txt \ --name 'Test 0.1.3 Confidence score not passed' \ --tlp_level clear \ --use_extractions ai_ipv4_address_only @@ -141,7 +141,7 @@ Bad confidence value (out of range 0-100) -- should return error ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv4_address_only.txt \ + --input_file tests/data/extraction_types/generic_ipv4_address_only.txt \ --name 'Test 0.1.1 Bad Confidence value' \ --tlp_level clear \ --confidence 1000 \ @@ -155,7 +155,7 @@ Set `INPUT_CHARACTER_LIMIT= 50000` in `.env` file and this test should fail: ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/manually_generated_reports/char_length_too_long.txt \ + --input_file tests/data/manually_generated_reports/char_length_too_long.txt \ --name 'Test 0.1.4 File too many chars' \ --tlp_level amber \ --confidence 100 \ @@ -169,7 +169,7 @@ Adding good labels, expect to see 2 labels in report: ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv4_address_only.txt \ + --input_file tests/data/extraction_types/generic_ipv4_address_only.txt \ --name 'Test 0.1.5 Good labels' \ --tlp_level amber \ --labels label1,labels2 \ @@ -182,7 +182,7 @@ One bad label as not `a-z`, `0-9`, should error ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv4_address_only.txt \ + --input_file tests/data/extraction_types/generic_ipv4_address_only.txt \ --name 'Test 0.1.5 Bad labels' \ --tlp_level amber \ --labels label_1,labels2 \ @@ -195,7 +195,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_ipv4_address_only.txt \ + --input_file tests/data/extraction_types/generic_ipv4_address_only.txt \ --name 'Test 0.1.5 Created time' \ --tlp_level green \ --created 2020-01-01T00:00:00.000Z \ @@ -209,7 +209,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode ai \ - --input_file tests/inputs/manually_generated_reports/descriptive_for_ai_relationships_1.txt \ + --input_file tests/data/manually_generated_reports/descriptive_for_ai_relationships_1.txt \ --name 'Test 0.2.1 Lots of descriptive relationships' \ --tlp_level clear \ --confidence 100 \ @@ -223,7 +223,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/manually_generated_reports/test_aliases.txt \ + --input_file tests/data/manually_generated_reports/test_aliases.txt \ --name '0.3.1 Test All Default Aliases in Standard relationships mode' \ --tlp_level clear \ --confidence 100 \ @@ -254,7 +254,7 @@ Same input / extractions, but testing AI relationships... ```shell python3 txt2stix.py \ --relationship_mode ai \ - --input_file tests/inputs/manually_generated_reports/test_aliases.txt \ + --input_file tests/data/manually_generated_reports/test_aliases.txt \ --name '0.3.1 Test All Default Aliases in Standard relationships mode' \ --tlp_level clear \ --confidence 100 \ @@ -271,7 +271,7 @@ Contains `google.com` which matches whitelist. Also contains `signalcorps.com` w ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/whitelist_alexa_top_1000_domains.txt \ + --input_file tests/data/extraction_types/whitelist_alexa_top_1000_domains.txt \ --name '0.4.1 Whitelist with 1 known match, 1 unknown' \ --tlp_level clear \ --confidence 100 \ @@ -290,7 +290,7 @@ Here is the same input, this time without whitelist: ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/whitelist_alexa_top_1000_domains.txt \ + --input_file tests/data/extraction_types/whitelist_alexa_top_1000_domains.txt \ --name '0.4.1 Whitelist with 1 known match, 1 unknown' \ --tlp_level clear \ --confidence 100 \ @@ -308,7 +308,7 @@ Expect ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/whitelist_examples.txt \ + --input_file tests/data/extraction_types/whitelist_examples.txt \ --name '0.4.2 Whitelist of Lookup with 1 known match' \ --tlp_level clear \ --confidence 100 \ @@ -324,7 +324,7 @@ Expect ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/whitelist_examples.txt \ + --input_file tests/data/extraction_types/whitelist_examples.txt \ --name '0.4.2 Whitelist of Lookup with 1 known match' \ --tlp_level clear \ --confidence 100 \ @@ -339,7 +339,7 @@ Expect 0 extractions ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/whitelist_examples.txt \ + --input_file tests/data/extraction_types/whitelist_examples.txt \ --name '0.4.3 Whitelist of AI with 1 known match' \ --tlp_level clear \ --confidence 100 \ @@ -352,7 +352,7 @@ Expect 0 extractions. ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/whitelist_examples.txt \ + --input_file tests/data/extraction_types/whitelist_examples.txt \ --name '0.4.3 Whitelist of AI with 1 known match' \ --tlp_level clear \ --confidence 100 \ @@ -370,7 +370,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_url_path.txt \ + --input_file tests/data/extraction_types/generic_url_path.txt \ --name '0.4.4 Where whitelist value appears in string no whitelist' \ --tlp_level clear \ --confidence 100 \ @@ -383,7 +383,7 @@ https://fortinet.com/blog should extract ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/generic_url_path.txt \ + --input_file tests/data/extraction_types/generic_url_path.txt \ --name '0.4.4 Where whitelist value appears in string with whitelist' \ --tlp_level clear \ --confidence 100 \ @@ -402,7 +402,7 @@ https://github.com/signalscorps/txt2stix/blob/beta-1/design/mvp/extraction-types ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/manually_generated_reports/test_extraction_escapes.txt \ + --input_file tests/data/manually_generated_reports/test_extraction_escapes.txt \ --name '0.5.1 Test extraction escapes in pattern mode' \ --tlp_level clear \ --confidence 100 \ @@ -418,7 +418,7 @@ Here 7 IPs will be extracted. But this may vary depending on how AI is feeling! ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/all_cases.txt \ + --input_file tests/data/extraction_types/all_cases.txt \ --name '0.6.1 All test cases pattern extractions' \ --tlp_level clear \ --confidence 100 \ @@ -430,7 +430,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/all_cases.txt \ + --input_file tests/data/extraction_types/all_cases.txt \ --name '0.6.2 All test cases lookup extractions' \ --tlp_level clear \ --confidence 100 \ @@ -442,7 +442,7 @@ python3 txt2stix.py \ ```shell python3 txt2stix.py \ --relationship_mode standard \ - --input_file tests/inputs/extraction_types/all_cases.txt \ + --input_file tests/data/extraction_types/all_cases.txt \ --name '0.6.3 All test cases ai extractions' \ --tlp_level clear \ --confidence 100 \ diff --git a/includes/extractions/test_cases/generate_simple_extraction_test_cases_txt_files.py b/tests/scripts/generate_simple_extraction_test_cases_txt_files.py similarity index 88% rename from includes/extractions/test_cases/generate_simple_extraction_test_cases_txt_files.py rename to tests/scripts/generate_simple_extraction_test_cases_txt_files.py index 23ccc6c..6249713 100644 --- a/includes/extractions/test_cases/generate_simple_extraction_test_cases_txt_files.py +++ b/tests/scripts/generate_simple_extraction_test_cases_txt_files.py @@ -1,4 +1,4 @@ -# python3 extractions/test_cases/generate_simple_extraction_test_cases_txt_files.py +# python3 tests/scripts/generate_simple_extraction_test_cases_txt_files.py import os import yaml @@ -50,8 +50,8 @@ def write_to_file(file_path, positive_data, negative_data): file.write(f"{item}\n") # Usage -file_path = 'includes/extractions/test_cases/test_data.yaml' # Update this to your file's location -output_dir = 'tests/inputs/extraction_types/' # Update this to your desired output directory +file_path = 'tests/test_cases.yaml' # Update this to your file's location +output_dir = 'tests/data/extraction_types/' # Update this to your desired output directory create_test_case_files(file_path, output_dir) print("Test case files and all_cases.txt created successfully.") \ No newline at end of file diff --git a/includes/extractions/test_cases/test_data.yaml b/tests/test_cases.yaml similarity index 89% rename from includes/extractions/test_cases/test_data.yaml rename to tests/test_cases.yaml index 96b3b44..b42656a 100644 --- a/includes/extractions/test_cases/test_data.yaml +++ b/tests/test_cases.yaml @@ -473,19 +473,6 @@ generic_mitre_attack_enterprise: - 'P1174' # not a valid id - 'SolarWinds Compromise' # is a name -ai_mitre_attack_enterprise: - test_positive_examples: - - 'T1174' # course-of-action--00d7d21b-69d6-4797-88a2-c86f3fc97651 , attack-pattern--b8c5c9dd-a662-479d-9428-ae745872537c - - 'TA0006' # x-mitre-tactic--2558fd61-8c75-4730-94c4-11926db2a263 - - 'TA0011' # x-mitre-tactic--f72804c5-f15a-449e-a5da-2eecd181f813 - - 'G1006' # intrusion-set--cc613a49-9bfa-4e22-98d1-15ffbb03f034 - - 'T1053.005' # attack-pattern--005a06c6-14bf-4118-afa0-ebcd8aebb0c9 - - 'T1040' # attack-pattern--3257eb21-f9a7-4430-8de1-d8b6e288f529 , course-of-action--46b7ef91-4e1d-43c5-a2eb-00fa9444f6f4 - - 'TA0003' # x-mitre-tactic--5bc1d813-693e-4823-9961-abf9af4b0e92 - - 'SolarWinds Compromise' # is name, might be tempremental with AI extractions if good = C0024 campaign--808d6b30-df4e-4341-8248-724da4bac650 - test_negative_examples: - - 'P1174' # not a valid id - generic_mitre_attack_mobile: test_positive_examples: - 'M1013' # course-of-action--25dc1ce8-eb55-4333-ae30-a7cb4f5894a1 @@ -496,29 +483,12 @@ generic_mitre_attack_mobile: - 'P1174' # not a valid id - 'Use Recent OS Version' # is a name -ai_mitre_attack_mobile: - test_positive_examples: - - 'M1013' # course-of-action--25dc1ce8-eb55-4333-ae30-a7cb4f5894a1 - - 'S0505' # malware--3271c107-92c4-442e-9506-e76d62230ee8 - - 'T1630.001' # attack-pattern--0cdd66ad-26ac-4338-a764-4972a1e17ee3 - - 'TA0029' # x-mitre-tactic--3e962de5-3280-43b7-bc10-334fbc1d6fa8 - - 'Use Recent OS Version' # is name, might be tempremental with AI extractions if good = M1006 course-of-action--1553b156-6767-47f7-9eb4-2a692505666d - test_negative_examples: - - 'P1174' # not a valid id - generic_mitre_attack_ics: test_positive_examples: - 'TA0111' # x-mitre-tactic--33752ae7-f875-4f43-bdb6-d8d02d341046 test_negative_examples: - 'Privilege Escalation' # is name -ai_mitre_attack_ics: - test_positive_examples: - - 'TA0111' # x-mitre-tactic--33752ae7-f875-4f43-bdb6-d8d02d341046 - - 'Privilege Escalation' # is name, might be tempremental with AI extractions if good = TA0111 x-mitre-tactic--33752ae7-f875-4f43-bdb6-d8d02d341046 - test_negative_examples: - - 'P1174' # not a valid id - ####### MITRE CAPEC ####### # note we have ai and generic extractions because AI logic has possibility of detecting descriptions of objects, not just their explicit IDs (the limitation of pattern/lookup modes) @@ -530,13 +500,6 @@ generic_mitre_capec: - 'CAPEC-999' # invalid ID - 'Brute Force' # is name -ai_mitre_capec: - test_positive_examples: - - 'CAPEC-110' - - 'Brute Force' - test_negative_examples: - - 'CAPEC-999' # invalid ID - ####### MITRE CWE ####### # note we have ai and generic extractions because AI logic has possibility of detecting descriptions of objects, not just their explicit IDs (the limitation of pattern/lookup modes) @@ -549,13 +512,15 @@ generic_mitre_cwe: - 'CWE-999' # invalid id - 'Destructor' # is name -ai_mitre_cwe: +####### MITRE ATLAS ####### + +generic_mitre_atlas: test_positive_examples: - - 'CWE-1023' - - 'CWE-102' - - 'Destructor' # is name, might be tempremental with AI extractions + - 'AML.M0015' + - 'AML.T0050' test_negative_examples: - - 'CWE-999' # invalid id + - 'AML.T0000' # invalid id + - 'Reconnaissance' # is name ####### Misc STIX Objects ####### diff --git a/txt2stix/ai_session.py b/txt2stix/ai_session.py index 7c135da..ebb31bc 100644 --- a/txt2stix/ai_session.py +++ b/txt2stix/ai_session.py @@ -40,7 +40,15 @@ def calculate_token_count(self, text, model='gpt-4o'): class OpenAIAssistantExtractor(BaseAIExtractor): extract_instruction = textwrap.dedent( - """ + """ + + + You are a cyber-security threat intelligence analyst responsible for analysing intelligence. You have a deep understanding of cybersecurity concepts and threat intelligence. You are responsible for extracting observables and TTPs from documents provided, and understanding the relationships being described that link them. + + + + + Using the file above, you are to extract objects from the body of input (either plaintext or markdown), extractions must be unique! ```json @@ -71,48 +79,97 @@ class OpenAIAssistantExtractor(BaseAIExtractor): Only one JSON object should exist for each unique value. - Only include a valid JSON document in your response and no other text. The JSON document should be minified!. - + IMPORTANT: Only include a valid JSON document in your response and no other text. The JSON document should be minified!. + + + + + + Think about your answer first before you respond. + + If you don't know the answer, reply with DO NOT UNDERSTAND, do not every try to make up an answer. + + """) + relationship_instruction = textwrap.dedent( """ - please logically describe the relationships between the extractions in the following JSON format. + - ```json - [ - { - "source_ref": "", - "target_ref": "", - "relationship_type": "" - }, - { - "source_ref": "", - "target_ref": "", - "relationship_type": "" - } - ] - ``` + You are a cyber-security threat intelligence analyst responsible for analysing intelligence. You have a deep understanding of cybersecurity concepts and threat intelligence. You are responsible for extracting observables and TTPs from documents provided, and understanding the relationships being described that link them. + + + + + + Please capture the relationships between the extractions described in the text using NLP techniques. + + Your response should be in the following JSON format; + + ```json + [ + { + "source_ref": "", + "target_ref": "", + "relationship_type": "" + }, + { + "source_ref": "", + "target_ref": "", + "relationship_type": "" + } + ] + ``` - Where; + Where; - * `source_ref`: is the id for the source extraction for the relationship (e.g. extraction_1). - * `target_ref`: is the index for the target extraction for the relationship (e.g. extraction_2). - * `relationship_type`: is a description of the relationship between target and source. + * `source_ref`: is the id for the source extraction for the relationship (e.g. extraction_1). + * `target_ref`: is the index for the target extraction for the relationship (e.g. extraction_2). + * `relationship_type`: is a description of the relationship between target and source. + IMPORTANT: Only include a valid JSON document in your response and no other text. The JSON document should be minified!. - important: JSON output must be minified! - """ + + + + + Think about your answer first before you respond. + + If you don't know the answer, reply with DO NOT UNDERSTAND, do not every try to make up an answer. + + + """ ) + def __init__(self, model="gpt-4-turbo", filename="txt2stix-file.md") -> None: self.client = OpenAI(timeout=120) self.assistant = self.client.beta.assistants.create( model=model, name="CTI Extractor", instructions=textwrap.dedent(""" - You are a CTI extractor, you are to extract objects from the input file and return JSON reponse! + + + You are a cyber-security threat intelligence analyst responsible for analysing intelligence. You have a deep understanding of cybersecurity concepts and threat intelligence. You are responsible for extracting observables and TTPs from documents provided, and understanding the relationships being described that link them. + + + + + + You are to extract objects from the input file and return JSON reponse! + IMPORTANT - Extractions must be unique - All JSON output must be minified! + + + + + + Think about your answer first before you respond. + + If you don't know the answer, reply with DO NOT UNDERSTAND, do not every try to make up an answer. + + """), tools=[{"type": "file_search"}], ) @@ -129,7 +186,7 @@ def set_document(self, input_text): # vector = self.client.beta.vector_stores.create(expires_after=ExpiresAfter(days=1, anchor='last_active_at'), file_ids=[f['file_id'] for f in self.files]) message = dict( role="assistant", - content="I will be working with this file, all subsequent messages after this will bve working on this file.", + content="I will be working with this file, all subsequent messages after this will be working on this file.", attachments=self.files, ) self.thread = self.client.beta.threads.create(messages=[message]) @@ -195,7 +252,7 @@ def extract_relationships(self, extractions, relationship_types=[]): dict( role="assistant", content=textwrap.dedent(""" - relationship_type must be one of the following values, please pick the most suitable value that logically decribe the relationship between the extractions. + relationship_type must be one of the following values, please pick the most suitable value that logically describe the relationship between the extractions. - {} """).format("\n- ".join(relationship_types)) diff --git a/txt2stix/arangodb.py b/txt2stix/arangodb.py deleted file mode 100644 index 6b8f878..0000000 --- a/txt2stix/arangodb.py +++ /dev/null @@ -1,106 +0,0 @@ -from arango import ArangoClient -import dotenv, json, os -import stix2 - -dotenv.load_dotenv() - - -class ArangoSession: - ATTACK_ID_TABLES = { - "mitre-attack-ics-id": "mitre_attack_ics_vertex_collection", - "mitre-attack-mobile-id": "mitre_attack_mobile_vertex_collection", - "mitre-attack-enterprise-id": "mitre_attack_enterprise_vertex_collection", - } - - def __init__(self) -> None: - host_url, db_name, user, passwd = ( - os.environ["ARANGODB_HOST_URL"], - os.environ["ARANGODB_DATABASE"], - os.environ["ARANGODB_USERNAME"], - os.environ["ARANGODB_PASSWORD"], - ) - self.client = ArangoClient(hosts=host_url) - self.db = self.client.db(db_name, username=user, password=passwd) - - def mitre_attack_id(self, id, stix_mapping): - table = self.ATTACK_ID_TABLES[stix_mapping] - cursor = self.db.aql.execute( - f""" - FOR doc IN {table} - FILTER IS_ARRAY(doc.external_references) - FOR external_references IN doc.external_references - FILTER external_references.external_id == @id - RETURN KEEP(doc, KEYS(doc, true)) - """, - bind_vars=dict(id=id), - ) - return self.to_stix_observables(cursor) - - def mitre_capec_id(self, id): - cursor = self.db.aql.execute( - """ - FOR doc IN mitre_capec_vertex_collection - FILTER doc.type == 'attack-pattern' AND IS_ARRAY(doc.external_references) - FOR external_references IN doc.external_references - FILTER external_references.external_id == @id - RETURN KEEP(doc, KEYS(doc, true)) - """, - bind_vars=dict(id=id), - ) - return self.to_stix_observables(cursor) - - def mitre_cwe_id(self, id): - cursor = self.db.aql.execute( - """ - FOR doc IN mitre_cwe_vertex_collection - FILTER doc.type == 'weakness' AND IS_ARRAY(doc.external_references) - FOR external_references IN doc.external_references - FILTER external_references.external_id == @id - RETURN KEEP(doc, KEYS(doc, true)) - """, - bind_vars=dict(id=id), - ) - return self.to_stix_observables(cursor) - - def cve_id(self, id): - cursor = self.db.aql.execute( - """ - FOR doc IN nvd_cve_vertex_collection - FILTER doc.type == 'vulnerability' - AND doc.name == @id - RETURN KEEP(doc, KEYS(doc, true)) - """, - bind_vars=dict(id=id), - ) - return self.to_stix_observables(cursor) - - def cpe_id(self, id): - cursor = self.db.aql.execute( - """ - FOR doc IN nvd_cpe_vertex_collection - FILTER doc.type == 'software' - AND doc.cpe == @id - RETURN KEEP(doc, KEYS(doc, true)) - """, - bind_vars=dict(id=id), - ) - return self.to_stix_observables(cursor) - - def to_stix_observables(self, cursor): - sdos = [] - for item in cursor: - sdos.append(stix2.parse(item, allow_custom=True)) - return sdos - - -if __name__ == "__main__": - session = ArangoSession() - attack_ids = session.mitre_attack_id("M0948", "mitre-attack-ics-id") - capec_ids = session.mitre_capec_id("CAPEC-1") - cwes = session.mitre_cwe_id("CWE-276") - cpes = session.cpe_id( - "cpe:2.3:a:wpeka:wp_cookie_consent:2.2.9:*:*:*:-:wordpress:*:*" - ) - cves = session.cve_id("CVE-2001-0308") - print(attack_ids, capec_ids, cwes, cpes, cves) - print("done") diff --git a/txt2stix/indicator.py b/txt2stix/indicator.py index 0efb08a..2037c03 100644 --- a/txt2stix/indicator.py +++ b/txt2stix/indicator.py @@ -1,25 +1,15 @@ from __future__ import annotations import os -import json, re -from stix2.parsing import dict_to_stix2, parse as parse_stix -from stix2 import IPv4Address, CustomObject, CustomObservable, File, HashConstant +import re +from stix2.parsing import dict_to_stix2 +from stix2 import HashConstant from stix2.v21.vocab import HASHING_ALGORITHM from stix2.patterns import _HASH_REGEX as HASHING_ALGORITHM_2 -from urllib.parse import urlparse from ipaddress import ip_address from pathlib import PurePosixPath, PureWindowsPath import phonenumbers from phonenumbers import geocoder import logging -from stix2extensions import ( - BankAccount, - BankCard, - CryptocurrencyTransaction, - CryptocurrencyWallet, - Phonenumber, - UserAgent, - Weakness, -) from stix2extensions.tools import creditcard2stix, crypto2stix from typing import TYPE_CHECKING @@ -30,7 +20,7 @@ from .common import MinorExcption -from .arangodb import ArangoSession +from .retriever import retrieve_stix_objects logger = logging.getLogger("txt2stix.indicator") @@ -86,12 +76,12 @@ def build_observables( ): value = extracted["value"] - arango_objects = arangodb_check(stix_mapping, value) - if arango_objects: - return arango_objects, [sdo["id"] for sdo in arango_objects] - if arango_objects == []: + retrieved_objects = retrieve_stix_objects(stix_mapping, value) + if retrieved_objects: + return retrieved_objects, [sdo["id"] for sdo in retrieved_objects] + if retrieved_objects == []: logger.error( - f"could not find `{stix_mapping}` with value `{value}` in ArangoDB" + f"could not find `{stix_mapping}` with id=`{value}` in remote" ) return [], [] @@ -106,7 +96,7 @@ def build_observables( ) stix_objects.append( - bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to") + bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to", description=f"{stix_objects[1]['value']} is related to {indicator['name']}") ) if stix_mapping == "ipv4-addr-port": @@ -119,7 +109,7 @@ def build_observables( ) id = stix_objects[-1].id stix_objects.append( - bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to") + bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to", description=f"{stix_objects[1]['value']} is related to {indicator['name']}") ) stix_objects.append( @@ -142,7 +132,7 @@ def build_observables( dict_to_stix2({"type": "ipv6-addr", "spec_version": "2.1", "value": value}) ) stix_objects.append( - bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to") + bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to", description=f"{stix_objects[1]['value']} is related to {indicator['name']}") ) if stix_mapping == "ipv6-addr-port": @@ -155,7 +145,7 @@ def build_observables( ) id = stix_objects[-1].id stix_objects.append( - bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to") + bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to", description=f"{stix_objects[1]['value']} is related to {indicator['name']}") ) stix_objects.append( dict_to_stix2( @@ -179,7 +169,7 @@ def build_observables( ) ) stix_objects.append( - bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to") + bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to", description=f"{value} is related to {indicator['name']}") ) if stix_mapping == "url": @@ -190,7 +180,7 @@ def build_observables( dict_to_stix2({"type": "url", "spec_version": "2.1", "value": value}) ) stix_objects.append( - bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to") + bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to", description=f"{value} is related to {indicator['name']}") ) if stix_mapping == "file": @@ -201,7 +191,7 @@ def build_observables( dict_to_stix2({"type": "file", "spec_version": "2.1", "name": value}) ) stix_objects.append( - bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to") + bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to", description=f"{value} is related to {indicator['name']}") ) if stix_mapping == "directory": @@ -212,7 +202,7 @@ def build_observables( dict_to_stix2({"type": "directory", "spec_version": "2.1", "path": value}) ) stix_objects.append( - bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to") + bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to", description=f"{value} is related to {indicator['name']}") ) if stix_mapping == "directory-file": @@ -226,7 +216,7 @@ def build_observables( ) dir = stix_objects[-1] stix_objects.append( - bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to") + bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to", description=f"{value} is related to {indicator['name']}") ) stix_objects.append( @@ -234,7 +224,7 @@ def build_observables( ) file = stix_objects[-1] - stix_objects.append(bundler.new_relationship(file.id, dir.id, "directory")) + stix_objects.append(bundler.new_relationship(file.id, dir.id, "directory", description=f"{value} directory {indicator['name']}")) if stix_mapping == "file-hash": file_hash_type = find_hash_type(value, extractor.name) or extractor.slug @@ -254,7 +244,7 @@ def build_observables( ) ) stix_objects.append( - bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to") + bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to", description=f"{value} is related to {indicator['name']}") ) if stix_mapping == "email-addr": @@ -265,7 +255,7 @@ def build_observables( dict_to_stix2({"type": "email-addr", "spec_version": "2.1", "value": value}) ) stix_objects.append( - bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to") + bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to", description=f"{value} is related to {indicator['name']}") ) if stix_mapping == "mac-addr": @@ -276,7 +266,7 @@ def build_observables( dict_to_stix2({"type": "mac-addr", "spec_version": "2.1", "value": value}) ) stix_objects.append( - bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to") + bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to", description=f"{value} is related to {indicator['name']}") ) if stix_mapping == "windows-registry-key": @@ -289,7 +279,7 @@ def build_observables( ) ) stix_objects.append( - bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to") + bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to", description=f"{value} is related to {indicator['name']}") ) if stix_mapping == "user-agent": @@ -302,7 +292,7 @@ def build_observables( ) ) stix_objects.append( - bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to") + bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to", description=f"{value} is related to {indicator['name']}") ) if stix_mapping == "autonomous-system": @@ -319,7 +309,7 @@ def build_observables( ) ) stix_objects.append( - bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to") + bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to", description=f"{value} is related to {indicator['name']}") ) if stix_mapping == "cryptocurrency-wallet": @@ -337,7 +327,7 @@ def build_observables( stix_objects.append(wallet_obj) stix_objects.extend(other_objects) stix_objects.append( - bundler.new_relationship(wallet_obj.id, indicator["id"], "related-to") + bundler.new_relationship(wallet_obj.id, indicator["id"], "related-to", description=f"{value} is related to {indicator['name']}") ) return stix_objects, [wallet_obj.id] @@ -353,7 +343,7 @@ def build_observables( stix_objects.append(txn_object) stix_objects.extend(other_objects) stix_objects.append( - bundler.new_relationship(txn_object.id, indicator["id"], "related-to") + bundler.new_relationship(txn_object.id, indicator["id"], "related-to", description=f"{value} is related to {indicator['name']}") ) return stix_objects, [txn_object.id] @@ -373,7 +363,7 @@ def build_observables( stix_objects.append(wallet_obj) stix_objects.extend(other_objects) stix_objects.append( - bundler.new_relationship(wallet_obj.id, indicator["id"], "related-to") + bundler.new_relationship(wallet_obj.id, indicator["id"], "related-to", description=f"{value} is related to {indicator['name']}") ) return stix_objects, [wallet_obj.id] if stix_mapping == "bank-card": @@ -397,7 +387,7 @@ def build_observables( indicator["pattern"] = f"[ bank-card:number = { repr(value) } ]" stix_objects.append( - bundler.new_relationship(card_object["id"], indicator["id"], "related-to") + bundler.new_relationship(card_object["id"], indicator["id"], "related-to", description=f"{value} is related to {indicator['name']}") ) return stix_objects, [card_object["id"]] @@ -419,7 +409,7 @@ def build_observables( ) ) stix_objects.append( - bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to") + bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to", description=f"{value} is related to {indicator['name']}") ) if stix_mapping == "phone-number": @@ -441,7 +431,7 @@ def build_observables( ) ) stix_objects.append( - bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to") + bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to", description=f"{value} is related to {indicator['name']}") ) if stix_mapping == "attack-pattern": @@ -612,7 +602,7 @@ def build_observables( ) ) stix_objects.append( - bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to") + bundler.new_relationship(stix_objects[1].id, indicator["id"], "related-to", description=f"{value} is related to {indicator['name']}") ) RELATABLE = [ @@ -658,25 +648,4 @@ def build_observables( return stix_objects, relationships -def arangodb_check(stix_mapping, id): - try: - s = ArangoSession() - if stix_mapping in s.ATTACK_ID_TABLES: - return s.mitre_attack_id(id, stix_mapping) - if stix_mapping == "mitre-capec-id": - return s.mitre_capec_id(id) - if stix_mapping == "mitre-cwe-id": - return s.mitre_cwe_id(id) - if stix_mapping == "cve-id": - return s.cve_id(id) - if stix_mapping == "cpe-id": - return s.cpe_id(id) - except Exception as e: - pass - return None - - -# print(build_indicator("ipv4-addr", {}, "192.168.0.1")) -# print(build_indicator("ipv4-addr-port", {}, "192.168.0.1:80")) -# print(build_indicator("cryptocurrency-wallet", {}, "192.168.0.1:80")) -# print(get_country_code("2349027338509")) + diff --git a/txt2stix/retriever.py b/txt2stix/retriever.py new file mode 100644 index 0000000..1baee1f --- /dev/null +++ b/txt2stix/retriever.py @@ -0,0 +1,83 @@ +from urllib.parse import urljoin +import dotenv, os +import stix2 +import requests + +dotenv.load_dotenv() + + +class STIXObjectRetriever: + api_root = os.environ['CTIBUTLER_HOST'] + api_key = os.environ['CTIBUTLER_HOST'] + + def __init__(self, host="ctibutler") -> None: + if host == "ctibutler": + self.api_root = os.environ['CTIBUTLER_HOST'] + self.api_key = os.environ['CTIBUTLER_APIKEY'] + elif host == "vulmatch": + self.api_root = os.environ['VULMATCH_HOST'] + self.api_key = os.environ['VULMATCH_APIKEY'] + else: + raise NotImplementedError("The type `%s` is not supported", host) + + def get_attack_objects(self, matrix, attack_id): + endpoint = urljoin(self.api_root, f"/api/v1/attack-{matrix}/objects/{attack_id}/") + return self.get_objects(endpoint) + + def get_capec_object(self, capec_id): + return self.get_objects(urljoin(self.api_root, f"/api/v1/capec/objects/{capec_id}/")) + + def get_cwe_object(self, cwe_id): + return self.get_objects(urljoin(self.api_root, f"/api/v1/cwe/objects/{cwe_id}/")) + + def get_cve_object(self, cve_id): + return self.get_objects(urljoin(self.api_root, f"/api/v1/cve/objects/{cve_id}/"), 'vulnerabilities') + + def get_cpe_object(self, cpe_id): + return self.get_objects(urljoin(self.api_root, f"/api/v1/cpe/objects/{cpe_id}/")) + + def get_objects(self, endpoint, key='objects'): + s = requests.Session() + s.headers.update({ + "Authority": f"Bearer {self.api_key}" + }) + data = [] + page = 1 + while True: + resp = s.get(endpoint, params=dict(page=page, page_size=1000)) + if resp.status_code != 200: + break + d = resp.json() + if len(d[key]) == 0: + break + data.extend(d[key]) + page+=1 + if d['page_results_count'] < d['page_size']: + break + return data + +def retrieve_stix_objects(stix_mapping: str, id, host=None): + try: + if not host: + host, stix_mapping = stix_mapping.split('-', 1) + retreiver = STIXObjectRetriever(host) + match stix_mapping: + case 'mitre-attack-ics-id': + return retreiver.get_attack_objects('ics', id) + case 'mitre-attack-mobile-id': + return retreiver.get_attack_objects('mobile', id) + case 'mitre-attack-enterprise-id': + return retreiver.get_attack_objects('enterprise', id) + case "mitre-capec-id": + return retreiver.get_capec_object(id) + case "mitre-cwe-id": + return retreiver.get_cwe_object(id) + case "cve-id": + return retreiver.get_cve_object(id) + case "cpe-id": + return retreiver.get_cpe_object(id) + case _: + raise NotImplementedError(f"pair {(host, stix_mapping)=} not implemented") + except Exception as e: + pass + return None diff --git a/txt2stix/stix.py b/txt2stix/stix.py index 44e38f1..f01dd85 100644 --- a/txt2stix/stix.py +++ b/txt2stix/stix.py @@ -151,6 +151,7 @@ class txt2stixBundler: object_marking_refs = [] uuid = None id_map = dict() + id_value_map = dict() # this identity is https://raw.githubusercontent.com/muchdogesec/stix4doge/main/objects/identity/txt2stix.json default_identity = Identity( type="identity", @@ -196,17 +197,21 @@ def __init__( extractors, labels, job_id=None, - created=dt.now(), + report_id=None, + created=None, ) -> None: - self.created = created + self.created = created or dt.now() self.whitelisted_values = set() self.whitelisted_refs = set() self.all_extractors = extractors self.identity = identity or self.default_identity self.tlp_level = TLP_LEVEL.get(tlp_level) - self.uuid = str( - uuid.uuid5(UUID_NAMESPACE, f"{self.identity.id}+{self.created}+{name}") - ) + if report_id: + self.uuid = report_id + else: + self.uuid = str( + uuid.uuid5(UUID_NAMESPACE, f"{self.identity.id}+{self.created}+{name}") + ) self.job_id = f"report--{self.uuid}" self.report = Report( @@ -267,27 +272,7 @@ def add_extension(self, object): def load_object_from_json(url): resp = requests.get(url) return dict_to_stix2(resp.json()) - - def add_note(self, content, type): - content, _ = codecs.utf_8_encode(content) - note = Note( - created=self.report.created, - modified=self.report.modified, - abstract=f"txt2stix {type}: {self.job_id}", - content=base64.b64encode(content).decode(), - object_refs=[self.report.id], - created_by_ref=self.report.created_by_ref, - object_marking_refs=[ - "marking-definition--e828b379-4e03-4974-9ac4-e53a884c97c1", - "marking-definition--f92e15d9-6afc-5ae2-bb3e-85a1fd83a3b5", - ], - external_references=[ - {"source_name": "txt2stix job ID", "external_id": self.job_id} - ], - ) - - self.bundle.objects.append(note) - + def add_ref(self, sdo): self.add_extension(sdo) sdo_id = sdo["id"] @@ -295,6 +280,41 @@ def add_ref(self, sdo): self.report.object_refs.append(sdo_id) self.bundle.objects.append(sdo) + # match t := sdo['type']: + # case 'indicator' | 'file': + # sdo_value = sdo['name'] + # case 'ipv4-addr': + # sdo_value = sdo['value'] + # case 'directory': + # sdo_value = sdo['path'] + # case 'windows-registry-key': + # sdo_value = sdo['key'] + # case 'user-agent': + # sdo_value = sdo['string'] + # case 'autonomous-system' | 'bank-card': + # sdo_value = sdo['number'] + # case 'cryptocurrency-wallet': + # sdo_value = sdo['address'] + # case 'cryptocurrency-transaction': + # sdo_value = sdo['hash'] + # case _: + # sdo_value = "{NOTEXTRACTED}" + + sdo_value = "" + for key in ['name', 'value', 'path', 'key', 'string', 'number', 'iban_number', 'address', 'hashes']: + if v := sdo.get(key): + sdo_value = v + break + else: + if refs := sdo.get('external_references', []): + sdo_value = refs[0]['external_id'] + else: + sdo_value = "{NOTEXTRACTED}" + + + self.id_value_map[sdo_id] = sdo_value + + def add_indicator(self, extracted_dict, add_standard_relationship): extractor = self.all_extractors[extracted_dict["type"]] stix_mapping = extractor.stix_mapping @@ -303,7 +323,7 @@ def add_indicator(self, extracted_dict, add_standard_relationship): if extracted_value in self.whitelisted_values: self.whitelisted_refs.add(extracted_id) return - # print(stix_mapping, gpt_out) + indicator = { "type": "indicator", "id": self.indicator_id_from_value(extracted_value, stix_mapping), @@ -369,13 +389,17 @@ def add_ai_relationship(self, gpt_out): for source_ref in self.id_map.get(gpt_out["source_ref"], []): for target_ref in self.id_map.get(gpt_out["target_ref"], []): self.add_standard_relationship( - source_ref, target_ref, gpt_out["relationship_type"] + source_ref, target_ref, gpt_out["relationship_type"], ) def add_standard_relationship(self, source_ref, target_ref, relationship_type): - self.add_ref(self.new_relationship(source_ref, target_ref, relationship_type)) + descriptor = ' '.join(relationship_type.split('-')) + self.add_ref(self.new_relationship( + source_ref, target_ref, relationship_type, + description=f"{self.id_value_map.get(source_ref, source_ref)} {descriptor} {self.id_value_map.get(target_ref, target_ref)}" + )) - def new_relationship(self, source_ref, target_ref, relationship_type): + def new_relationship(self, source_ref, target_ref, relationship_type, description=None): return Relationship( id="relationship--" + str( @@ -388,6 +412,7 @@ def new_relationship(self, source_ref, target_ref, relationship_type): relationship_type=relationship_type, created_by_ref=self.report.created_by_ref, created=self.report.created, + description=description, modified=self.report.modified, object_marking_refs=self.report.object_marking_refs, allow_custom=True, diff --git a/txt2stix/txt2stix.py b/txt2stix/txt2stix.py index f978172..8635f64 100644 --- a/txt2stix/txt2stix.py +++ b/txt2stix/txt2stix.py @@ -112,6 +112,7 @@ def parse_args(): parser.add_argument("--created", required=False, default=datetime.now(), help="Allow user to optionally pass --created time in input, which will hardcode the time used in created times") parser.add_argument("--labels", type=parse_labels) parser.add_argument("--relationship_mode", choices=["ai", "standard"], required=True) + parser.add_argument("--report_id", type=uuid.UUID, required=False, help="id to use instead of automatically generated `{name}+{created}`", metavar="[Valid UUID]") parser.add_argument("--confidence", type=range_type(0,100), default=None, help="value between 0-100. Default if not passed is null.", metavar="[0-100]") parser.add_argument("--tlp_level", "--tlp-level", choices=TLP_LEVEL.levels().keys(), default="clear", help="TLP level, default is clear") parser.add_argument("--use_extractions", "--use-extractions", default={}, type=functools.partial(parse_extractors_globbed, "extractor", all_extractors), help="Specify extraction types from the default/local extractions .yaml file", metavar="EXTRACTION1,EXTRACTION2") @@ -134,10 +135,8 @@ def parse_args(): REQUIRED_ENV_VARIABLES = [ "INPUT_TOKEN_LIMIT", - "ARANGODB_HOST_URL", - "ARANGODB_USERNAME", - "ARANGODB_PASSWORD", - "ARANGODB_DATABASE", + "CTIBUTLER_HOST", + "VULMATCH_HOST", ] def load_env(input_length): dotenv.load_dotenv() @@ -147,6 +146,13 @@ def load_env(input_length): # if input_length > int(os.environ["INPUT_TOKEN_LIMIT"]): # raise FatalException(f"input_file length ({input_length}) exceeds character limit ({os.environ['INPUT_TOKEN_LIMIT']})") + +def log_notes(content, type): + logging.debug(f" ========================= {type} ========================= ") + logging.debug(f" ========================= {"+"*len(type)} ========================= ") + logging.debug(json.dumps(content, sort_keys=True, indent=4)) + logging.debug(f" ========================= {"-"*len(type)} ========================= ") + def extract_all(bundler: txt2stixBundler, extractors_map, aliased_input, ai_extractor: BaseAIExtractor=None): all_extracts = dict() if extractors_map.get("lookup"): @@ -179,7 +185,7 @@ def extract_all(bundler: txt2stixBundler, extractors_map, aliased_input, ai_extr except BaseException as e: logging.exception("AI extraction failed", exc_info=True) - bundler.add_note(json.dumps(all_extracts), "Extractions") + log_notes(all_extracts, "Extractions") return all_extracts def extract_relationships_with_ai(bundler: txt2stixBundler, aliased_input, all_extracts, ai_extractor_session: BaseAIExtractor): @@ -188,7 +194,7 @@ def extract_relationships_with_ai(bundler: txt2stixBundler, aliased_input, all_e ai_extractor_session.set_document(aliased_input) relationship_types = (INCLUDES_PATH/"helpers/stix_relationship_types.txt").read_text().splitlines() relationships = ai_extractor_session.extract_relationships(all_extracts, relationship_types) - bundler.add_note(json.dumps(relationships), "Relationships") + log_notes(relationships, "Relationships") bundler.process_relationships(relationships) except BaseException as e: logging.exception("Relationship processing failed: %s", e) @@ -208,8 +214,8 @@ def main(): load_env(len(aliased_input)) - bundler = txt2stixBundler(args.name, args.use_identity, args.tlp_level, aliased_input, args.confidence, args.all_extractors, args.labels, created=args.created) - bundler.add_note(json.dumps(sys.argv), "Config") + bundler = txt2stixBundler(args.name, args.use_identity, args.tlp_level, aliased_input, args.confidence, args.all_extractors, args.labels, created=args.created, report_id=args.report_id) + log_notes(sys.argv, "Config") convo_str = None bundler.whitelisted_values = args.use_whitelist