From 5163382e3c6ea332afa28cbbd4c8155b5de905c9 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Fri, 24 Nov 2023 15:55:21 +1000 Subject: [PATCH] chore: resolve doctest issues (#555) Signed-off-by: Nathan Nguyen --- pyproject.toml | 1 + src/macaron/config/defaults.py | 8 ++- .../output_reporter/jinja2_extensions.py | 15 ++-- src/macaron/output_reporter/results.py | 19 ++++- .../policy_engine/souffle_code_generator.py | 7 +- .../slsa_analyzer/git_service/api_client.py | 72 +++++++++++-------- src/macaron/slsa_analyzer/git_url.py | 5 +- src/macaron/slsa_analyzer/registry.py | 4 +- src/macaron/util.py | 4 +- 9 files changed, 88 insertions(+), 47 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 76ffa2f1a..403bb93b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -234,6 +234,7 @@ minversion = "7.0" addopts = "--verbose --doctest-modules -ra --cov macaron" # Consider adding --pdb doctest_optionflags = "IGNORE_EXCEPTION_DETAIL" testpaths = [ + "src/macaron", # this is included to run doctest on examples "tests", ] env = [ diff --git a/src/macaron/config/defaults.py b/src/macaron/config/defaults.py index 658920442..3e11b6049 100644 --- a/src/macaron/config/defaults.py +++ b/src/macaron/config/defaults.py @@ -65,7 +65,7 @@ def get_list( -------- Given the following ``defaults.ini`` - .. code-block:: + .. code-block:: ini [git] allowed_hosts = @@ -73,8 +73,10 @@ def get_list( boo.com gitlab.com host com - >>> config_parser.get_list("git", "allowed_hosts") - ['github.com', 'boo.com gitlab.com', 'host com'] + .. code-block:: python3 + + allowed_hosts = config_parser.get_list("git", "allowed_hosts") + # allowed_hosts == ['github.com', 'boo.com gitlab.com', 'host com'] """ try: value = self.get(section, item) diff --git a/src/macaron/output_reporter/jinja2_extensions.py b/src/macaron/output_reporter/jinja2_extensions.py index e4ef836af..76848a086 100644 --- a/src/macaron/output_reporter/jinja2_extensions.py +++ b/src/macaron/output_reporter/jinja2_extensions.py @@ -118,14 +118,13 @@ def j2_filter_get_flatten_dict(data: Any, has_key: bool = False) -> dict | Any: Examples -------- >>> j2_filter_get_flatten_dict( - { - "A": [1,2,3], - "B": { - "C": ["blah", "bar", "foo"] - } - } - ) - {'A': {'0': 1, '1': 2, '2': 3}, 'B': {'C': {'0': 'blah', '1': 'bar', '2': 'foo'}}} + ... { + ... "A": [1, 2, 3], + ... "B": { + ... "C": ["blah", "bar", "foo"], + ... }, + ... }) + {'A': {0: 1, 1: 2, 2: 3}, 'B': {'C': {0: 'blah', 1: 'bar', 2: 'foo'}}} """ if isinstance(data, (str, int, bool, float)): if has_key: diff --git a/src/macaron/output_reporter/results.py b/src/macaron/output_reporter/results.py index f4ac9cf05..b6c99e8e6 100644 --- a/src/macaron/output_reporter/results.py +++ b/src/macaron/output_reporter/results.py @@ -98,8 +98,23 @@ def get_summary(self) -> dict: Examples -------- - >>> record.get_summary() - {'id': 'apache/maven', 'description': 'Analysis completed', 'report': 'apache.html', 'status': SCMStatus.AVAILABLE} + >>> from pprint import pprint + >>> from macaron.config.target_config import Configuration + >>> from macaron.output_reporter.scm import SCMStatus + >>> from macaron.output_reporter.results import Record + >>> record = Record( + ... record_id="apache/maven", + ... description="Analysis completed", + ... pre_config=Configuration({}), + ... status=SCMStatus.MISSING_SCM, + ... context=None, + ... dependencies=[], + ... ) + >>> pprint(record.get_summary()) + {'description': 'Analysis completed', + 'id': 'apache/maven', + 'repo_url_status': , + 'report': ''} """ return { "id": self.record_id, diff --git a/src/macaron/policy_engine/souffle_code_generator.py b/src/macaron/policy_engine/souffle_code_generator.py index 03bb2e733..a59124dcc 100644 --- a/src/macaron/policy_engine/souffle_code_generator.py +++ b/src/macaron/policy_engine/souffle_code_generator.py @@ -95,8 +95,11 @@ def table_to_declaration(table: Table) -> str: Examples -------- - >>> tbl = Table("example", Column("id", Integer), Column("hello", String) - >>> assert table_to_declaration(tbl) == '.decl "example" (id: number, hello: symbol)' + >>> from sqlalchemy import Column, MetaData, Table + >>> from sqlalchemy.sql.sqltypes import Boolean, Integer, String, Text + >>> metadata = MetaData() + >>> tbl = Table("_example", metadata, Column("id", Integer, nullable=False), Column("hello", String)) + >>> assert table_to_declaration(tbl) == '.decl example (id: number, hello: symbol)' Parameters ---------- diff --git a/src/macaron/slsa_analyzer/git_service/api_client.py b/src/macaron/slsa_analyzer/git_service/api_client.py index f431e33e2..575ae290c 100644 --- a/src/macaron/slsa_analyzer/git_service/api_client.py +++ b/src/macaron/slsa_analyzer/git_service/api_client.py @@ -193,10 +193,12 @@ def get_repo_workflow_data(self, full_name: str, workflow_name: str) -> dict: The following call to this method will perform a query to ``https://api.github.com/repos/owner/repo/actions/workflows/build.yml`` - >>> gh_client.get_repo_workflow_data( - full_name="owner/repo", - workflow_name="build.yml" - ) + .. code-block: python3 + + gh_client.get_repo_workflow_data( + full_name="owner/repo", + workflow_name="build.yml", + ) """ logger.debug("Query for %s workflow in repo %s", workflow_name, full_name) url = f"{GhAPIClient._REPO_END_POINT}/{full_name}/actions/workflows/{workflow_name}" @@ -239,12 +241,14 @@ def get_workflow_runs( ``https://api.github/com/repos/owner/repo/actions/runs?1&branch=master&created=>= 2022-03-11T16:44:40Z&per_page=100`` - >>> gh_client.get_workflow_runs( - full_name="owner/repo", - branch_name="master", - created_after="2022-03-11T16:44:40Z", - page=1 - ) + .. code-block: python3 + + gh_client.get_workflow_runs( + full_name="owner/repo", + branch_name="master", + created_after="2022-03-11T16:44:40Z", + page=1, + ) """ logger.debug("Query for runs data in repo %s", full_name) query_params: dict = { @@ -290,10 +294,12 @@ def get_workflow_run_jobs(self, full_name: str, run_id: str) -> dict: ``https://api.github/com/repos/{full_name}/ actions/runs//jobs`` - >>> gh_client.get_workflow_run_jobs( - full_name="owner/repo", - run_id= - ) + .. code-block: python3 + + gh_client.get_workflow_run_jobs( + full_name="owner/repo", + run_id=, + ) """ logger.debug("Query GitHub to get run jobs for %s with run ID %s", full_name, run_id) @@ -325,10 +331,12 @@ def get_workflow_run_for_date_time_range(self, full_name: str, datetime_range: s The following call to this method will perform a query to ``https://api.github/com/repos/owner/repo/actions/runs?created=2022-11-05T20:38:40..2022-11-05T20:38:58`` - >>> e.g., gh_client.get_workflow_run_for_date_time_range( - full_name="owner/repo", - created=2022-11-05T20:38:40..2022-11-05T20:38:58 - ) + .. code-block: python3 + + gh_client.get_workflow_run_for_date_time_range( + full_name="owner/repo", + created=2022-11-05T20:38:40..2022-11-05T20:38:58, + ) """ logger.debug("Query GitHub to get run details for %s at %s", full_name, datetime_range) query_params = {"created": datetime_range} @@ -362,10 +370,12 @@ def get_commit_data_from_hash(self, full_name: str, commit_hash: str) -> dict: The following call to this method will perform a query to: ``https://api.github.com/repos/owner/repo/commits/6dcb09b5b57875f334f61aebed695e2e4193db5e`` - >>> gh_client.get_commit_data_from_hash( - full_name="owner/repo", - commit_hash="6dcb09b5b57875f334f61aebed695e2e4193db5e" - ) + .. code-block:: python3 + + gh_client.get_commit_data_from_hash( + full_name="owner/repo", + commit_hash="6dcb09b5b57875f334f61aebed695e2e4193db5e", + ) """ logger.debug("Query for commit %s 's data in repo %s", commit_hash, full_name) url = f"{GhAPIClient._REPO_END_POINT}/{full_name}/commits/{commit_hash}" @@ -396,10 +406,12 @@ def search(self, target: str, query: str) -> dict: The following call to this method will perform a query to: ``https://api.github.com/search/code?q=addClass+in:file+language:js+repo:jquery/jquery`` - >>> gh_client.search( - target="repositories", - query="q=addClass+in:file+language:js+repo:jquery/jquery" - ) + .. code-block:: python3 + + gh_client.search( + target="repositories", + query="q=addClass+in:file+language:js+repo:jquery/jquery", + ) """ logger.debug("Search %s with query: %s", target, query) url = f"{GhAPIClient._SEARCH_END_POINT}/{target}?{query}" @@ -457,7 +469,9 @@ def get_repo_data(self, full_name: str) -> dict: -------- To get the repo data from repository ``apache/maven``: - >>> gh_client.get_repo_data("apache/maven") + .. code-block:: python3 + + gh_client.get_repo_data("apache/maven") """ logger.debug("Get data of repository %s", full_name) url = f"{GhAPIClient._REPO_END_POINT}/{full_name}" @@ -488,7 +502,8 @@ def get_file_link(self, full_name: str, commit_sha: str, file_path: str) -> str: Examples -------- - >>> get_files_link("owner/repo", "5aaaaa43caabbdbc26c254df8f3aaa7bb3f4ec01", ".travis_ci.yml") + >>> api_client = GhAPIClient(profile={"headers": "", "query": []}) + >>> api_client.get_file_link("owner/repo", "5aaaaa43caabbdbc26c254df8f3aaa7bb3f4ec01", ".travis_ci.yml") 'https://github.com/owner/repo/blob/5aaaaa43caabbdbc26c254df8f3aaa7bb3f4ec01/.travis_ci.yml' """ return f"https://github.com/{full_name}/blob/{commit_sha}/{file_path}" @@ -508,6 +523,7 @@ def get_relative_path_of_workflow(self, workflow_name: str) -> str: Examples -------- + >>> api_client = GhAPIClient(profile={"headers": "", "query": []}) >>> api_client.get_relative_path_of_workflow("build.yaml") '.github/workflows/build.yaml' """ diff --git a/src/macaron/slsa_analyzer/git_url.py b/src/macaron/slsa_analyzer/git_url.py index 8a4c25391..61c322c9a 100644 --- a/src/macaron/slsa_analyzer/git_url.py +++ b/src/macaron/slsa_analyzer/git_url.py @@ -348,7 +348,10 @@ def get_repo_complete_name_from_url(url: str) -> str: Examples -------- - >>> get_repo_complete_name_from_url("github.com/apache/maven") + >>> from macaron.config.defaults import load_defaults + >>> load_defaults("") + True + >>> get_repo_complete_name_from_url("https://github.com/apache/maven") 'github.com/apache/maven' """ remote_url = get_remote_vcs_url(url) diff --git a/src/macaron/slsa_analyzer/registry.py b/src/macaron/slsa_analyzer/registry.py index acf67d8b7..5cb28d423 100644 --- a/src/macaron/slsa_analyzer/registry.py +++ b/src/macaron/slsa_analyzer/registry.py @@ -291,10 +291,10 @@ def _validate_check_id_format(check_id: Any) -> bool: Examples -------- - >>> Register._validate_check_id_format("mcn_the_check_name_123") + >>> Registry._validate_check_id_format("mcn_the_check_name_123") True - >>> Register._validate_check_id_format("Some_Thing', '', '%(*$)") + >>> Registry._validate_check_id_format("Some_Thing', '', '%(*$)") False """ if (not isinstance(check_id, str)) or (not Registry._id_format.match(check_id)): diff --git a/src/macaron/util.py b/src/macaron/util.py index 526c60c0b..f5e584b45 100644 --- a/src/macaron/util.py +++ b/src/macaron/util.py @@ -232,7 +232,9 @@ def copy_file_bulk(file_list: list, src_path: str, target_path: str) -> bool: -------- ``file.txt`` will be copied from ``src/foo/bar/file.txt`` to ``target/foo/bar/file.txt`` - >>> copy_file_bulk(['foo/bar/file.txt'], 'src', 'target') + .. code-block:: python3 + + copy_file_bulk(["foo/bar/file.txt"], "src", "target") """ for file in file_list: file_src_path = os.path.join(src_path, file)