Skip to content

Commit

Permalink
chore: improve cyclonedx-mvn bom file check to include files with cus… (
Browse files Browse the repository at this point in the history
#524)

Signed-off-by: Ben Selwyn-Smith <[email protected]>
  • Loading branch information
benmss authored Nov 24, 2023
1 parent 2d7ff97 commit 2971708
Show file tree
Hide file tree
Showing 21 changed files with 973 additions and 262 deletions.
14 changes: 7 additions & 7 deletions scripts/dev_scripts/integration_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ echo "apache/maven: Analyzing with PURL and repository path without dependency r
echo -e "----------------------------------------------------------------------------------\n"
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/purl/maven/maven.json
JSON_RESULT=$WORKSPACE/output/reports/maven/apache/maven/maven.json
$RUN_MACARON analyze -purl pkg:maven/apache/maven -rp https://github.com/apache/maven -b master -d 6767f2500f1d005924ccff27f04350c253858a84 --skip-deps || log_fail
$RUN_MACARON analyze -purl pkg:maven/apache/maven -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail

check_or_update_expected_output $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail

Expand All @@ -213,7 +213,7 @@ JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/maven/maven.json
JSON_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/maven.json
DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_apache_maven.json
DEP_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/dependencies.json
$RUN_MACARON analyze -rp https://github.com/apache/maven -b master -d 6767f2500f1d005924ccff27f04350c253858a84 || log_fail
$RUN_MACARON analyze -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b || log_fail

check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail

Expand All @@ -226,7 +226,7 @@ SBOM_FILE=$WORKSPACE/tests/dependency_analyzer/cyclonedx/resources/apache_maven_
DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/apache_maven_with_sbom_provided.json
DEP_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/dependencies.json

$RUN_MACARON analyze -rp https://github.com/apache/maven -b master -d 6767f2500f1d005924ccff27f04350c253858a84 -sbom "$SBOM_FILE" || log_fail
$RUN_MACARON analyze -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b -sbom "$SBOM_FILE" || log_fail

check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail

Expand Down Expand Up @@ -350,7 +350,7 @@ echo "Test using the default template file."
echo -e "----------------------------------------------------------------------------------\n"
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/maven/maven.json
JSON_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/maven.json
$RUN_MACARON analyze -rp https://github.com/apache/maven --skip-deps -b master -d 6767f2500f1d005924ccff27f04350c253858a84 -g $WORKSPACE/src/macaron/output_reporter/templates/macaron.html || log_fail
$RUN_MACARON analyze -rp https://github.com/apache/maven --skip-deps -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b -g $WORKSPACE/src/macaron/output_reporter/templates/macaron.html || log_fail

check_or_update_expected_output $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail

Expand Down Expand Up @@ -399,7 +399,7 @@ JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/maven/maven.json
JSON_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/maven.json
DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_apache_maven.json
DEP_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/dependencies.json
$RUN_MACARON -lr $WORKSPACE/output/git_repos/github_com analyze -rp apache/maven -b master -d 6767f2500f1d005924ccff27f04350c253858a84 || log_fail
$RUN_MACARON -lr $WORKSPACE/output/git_repos/github_com analyze -rp apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b || log_fail

check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail
check_or_update_expected_output $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
Expand Down Expand Up @@ -435,7 +435,7 @@ declare -a COMPARE_FILES=(
"mockito.json"
)

$RUN_MACARON -lr $WORKSPACE/output/git_repos/github_com/ analyze -rp apache/maven -b master -d 6767f2500f1d005924ccff27f04350c253858a84 --skip-deps || log_fail
$RUN_MACARON -lr $WORKSPACE/output/git_repos/github_com/ analyze -rp apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail
for i in "${COMPARE_FILES[@]}"
do
check_or_update_expected_output $COMPARE_JSON_OUT $JSON_RESULT_DIR/$i $JSON_EXPECT_DIR/$i || log_fail
Expand All @@ -451,7 +451,7 @@ git clone $WORKSPACE/output/git_repos/github_com/apache/maven $WORKSPACE/output/
JSON_EXPECTED=$WORKSPACE/output/reports/local_repos/maven/maven.json
HTML_EXPECTED=$WORKSPACE/output/reports/local_repos/maven/maven.html

$RUN_MACARON -lr $WORKSPACE/output/git_repos/local_repos/ analyze -rp test_repo -b master -d 6767f2500f1d005924ccff27f04350c253858a84 --skip-deps || log_fail
$RUN_MACARON -lr $WORKSPACE/output/git_repos/local_repos/ analyze -rp test_repo -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail

# We don't compare the report content because the remote_path fields in the reports are undeterministic when running
# this test locally and running it in the GitHub Actions runner. We only check if the reports are generated as
Expand Down
6 changes: 3 additions & 3 deletions scripts/dev_scripts/integration_tests_docker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ echo -e "-----------------------------------------------------------------------
JSON_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/maven.json
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/maven/maven.json

$RUN_MACARON_SCRIPT -lr $WORKSPACE/output/git_repos/github_com analyze -r apache/maven -b master -d 6767f2500f1d005924ccff27f04350c253858a84 --skip-deps || log_fail
$RUN_MACARON_SCRIPT -lr $WORKSPACE/output/git_repos/github_com analyze -r apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail
python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail

echo -e "\n----------------------------------------------------------------------------------"
Expand Down Expand Up @@ -87,7 +87,7 @@ SBOM_FILE=$WORKSPACE/tests/dependency_analyzer/cyclonedx/resources/apache_maven_
DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/apache_maven_with_sbom_provided.json
DEP_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/dependencies.json

$RUN_MACARON_SCRIPT analyze -rp https://github.com/apache/maven -b master -d 6767f2500f1d005924ccff27f04350c253858a84 -sbom $SBOM_FILE || log_fail
$RUN_MACARON_SCRIPT analyze -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b -sbom $SBOM_FILE || log_fail

python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail

Expand All @@ -96,7 +96,7 @@ echo "apache/maven: Analyzing with PURL and repository path without dependency r
echo -e "----------------------------------------------------------------------------------\n"
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/purl/maven/maven.json
JSON_RESULT=$WORKSPACE/output/reports/maven/apache/maven/maven.json
$RUN_MACARON_SCRIPT analyze -purl pkg:maven/apache/maven -rp https://github.com/apache/maven -b master -d 6767f2500f1d005924ccff27f04350c253858a84 --skip-deps || log_fail
$RUN_MACARON_SCRIPT analyze -purl pkg:maven/apache/maven -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail

python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail

Expand Down
2 changes: 1 addition & 1 deletion src/macaron/dependency_analyzer/cyclonedx.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def deserialize_bom_json(file_path: Path) -> dict:
# TODO: use the official CycloneDX library to validate and deserialize BOM files
# once merged: https://github.com/CycloneDX/cyclonedx-python-lib/pull/290
if not os.path.exists(file_path):
raise CycloneDXParserError(f"Unable to locate the BOM file: {str(file_path)}.")
raise CycloneDXParserError(f"Unable to locate any BOM files at: {str(file_path.parent)}.")

with open(file_path, encoding="utf8") as file:
try:
Expand Down
31 changes: 30 additions & 1 deletion src/macaron/dependency_analyzer/cyclonedx_mvn.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,13 @@ def get_cmd(self) -> list:
def collect_dependencies(self, dir_path: str) -> dict[str, DependencyInfo]:
"""Process the dependency JSON files and collect direct dependencies.
We allow the dependency JSON files to be accepted as long as there is only one JSON file in the target
directory. If a file with the expected name is found, it is accepted, otherwise any lone file is accepted
instead. This is because projects can be configured to produce a custom named SBOM, which cannot be
overridden if included at the parent POM level. The presence of multiple JSON files within a target directory
differs too greatly from the expectations of the plugin's output. It is for this reason that an error is
thrown in such cases.
Parameters
----------
dir_path : str
Expand All @@ -64,14 +71,36 @@ def collect_dependencies(self, dir_path: str) -> dict[str, DependencyInfo]:
"""
# Load the top level file separately as it has different content.
top_path = Path(os.path.join(dir_path, "target", self.file_name))
top_path_altered = False
if not os.path.exists(top_path):
# Check for other JSON files.
possible_paths = glob.glob(os.path.join(dir_path, "target", "*.json"))
if not possible_paths:
logger.debug("No JSON files found in target directory.")
return {}
if len(possible_paths) > 1:
logger.debug("Too many JSON SBOM files found. Expected: 1, Found: %s", len(possible_paths))
return {}
top_path = Path(possible_paths[0])
top_path_altered = True

# Collect all the dependency files recursively.
child_paths = [
Path(path)
for path in glob.glob(os.path.join(dir_path, "**", "target", self.file_name), recursive=True)
for path in glob.glob(
os.path.join(dir_path, "**", "target", "*.json" if top_path_altered else self.file_name), recursive=True
)
if Path(path) != top_path
]

# Ensure recursively found SBOMs are at most one per directory.
child_sbom_dir_names = set()
for path in child_paths:
child_sbom_dir_names.add(path.parent)
if len(child_sbom_dir_names) != len(child_paths):
logger.debug("Only one JSON SBOM file is permitted per child directory.")
return {}

# Check if the root BOM has been analyzed before as a child BOM.
self.visited_deps.update(child_paths)
if top_path in self.visited_deps:
Expand Down
4 changes: 2 additions & 2 deletions tests/dependency_analyzer/configurations/maven_config.yaml
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved.
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.

target:
id: apache/maven
branch: master
digest: 6767f2500f1d005924ccff27f04350c253858a84
digest: 3fc399318edef0d5ba593723a24fff64291d6f9b
path: https://github.com/apache/maven.git
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
{
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
{
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
{
"bomFormat" : "CycloneDX",
"specVersion" : "1.4",
"serialNumber" : "urn:uuid:53576e41-735f-4da4-9249-7f63234ebd94",
"version" : 1,
"metadata" : {
"timestamp" : "2023-10-23T00:57:55Z",
"tools" : [
{
"vendor" : "OWASP Foundation",
"name" : "CycloneDX Maven plugin",
"version" : "2.6.2",
"hashes" : [
{
"alg" : "MD5",
"content" : "ff29fc50797fce0b33058a6b2b283f64"
},
{
"alg" : "SHA-1",
"content" : "597e59ebf21c3b8bfb1faeb622569df324eca956"
},
{
"alg" : "SHA-256",
"content" : "3cf9130fcac45a7beb6df2ae9c3fc9c062d1fddd0731d6a302968586f0aa586e"
},
{
"alg" : "SHA-384",
"content" : "8111a6788c959305af23daecbc79defd4478c1e274cba65bfe860e09b30cd9fe29822d5d3d3eea608e4926a9418f92e3"
},
{
"alg" : "SHA-512",
"content" : "2bea87b7bcd70897bf46a28a806b6064a6708d0a45e884e1ceddc25f97ca7bdf4ed190f30d9a28cc9416b6c66176d518c5876fd25bc06bdcb00d39367215e56e"
},
{
"alg" : "SHA3-256",
"content" : "f0f7b771749955e7898665c2fff8f4f2cd734d9cbe4d29883292db772f1be00e"
},
{
"alg" : "SHA3-384",
"content" : "a87d4c18bac4d48a46c0b8611ab92934e457fcd55bd4d39dbc9c4e5044d2736d3bda991c43d67b0987eddcf4c88510ff"
},
{
"alg" : "SHA3-512",
"content" : "90c38f168600787fc90b7e37e743b386b7296bceb10152190de6e30e0f251da3e01698d1b1e11ad84f207532b5a0743aac105f3c5006ff4607d21f30c9ea779f"
}
]
}
],
"component" : {
"group" : "com.example",
"name" : "cyclonedx-test",
"version" : "1.0-SNAPSHOT",
"licenses" : [ ],
"purl" : "pkg:maven/com.example/[email protected]?type=jar",
"type" : "library",
"bom-ref" : "pkg:maven/com.example/[email protected]?type=jar"
}
},
"components" : [
{
"group" : "com.example",
"name" : "cyclonedx-test-dep",
"version" : "1",
"scope" : "optional",
"hashes" : [
{
"alg" : "MD5",
"content" : "c7b63da4c25c163825cca671e7899fbe"
},
{
"alg" : "SHA-1",
"content" : "5aa25ee1bf1ffd60b76f16fe0a8edd76f870958c"
},
{
"alg" : "SHA-256",
"content" : "c38cef49f7676227c1d4cf98e59b96f7a6bf33704d10314d83d682acd2b47d10"
},
{
"alg" : "SHA-384",
"content" : "7afa5feaa7d3a4ca4ecba7d4bd1b093e75be2ee2a25eefbc5fd90eb8b9a4712fa1a720265765a28d858fc64412dbed2b"
},
{
"alg" : "SHA-512",
"content" : "bf69097c4c0d165e5521a918ee79c1e5e211e9e74410d48042994c4c6cf5788cf4d62129e7c0d7a22294835178398c91c31929ce6861068c71ea14059f6f6e56"
},
{
"alg" : "SHA3-256",
"content" : "ba7656644f127c4b10d53c777aee2ed023ac3caf7f420ecb4ca48a909d775a17"
},
{
"alg" : "SHA3-384",
"content" : "1244f326a9b0b165b27b0061f1fcdf2580e3b64681cc3f09df3afd9a4526ab5491a20213a8fb9edcc671fbae8b51a010"
},
{
"alg" : "SHA3-512",
"content" : "e6020e5b9adbe61f1c53e575ab0c51b9eef7dbea3dbe21f970607002ed0373b322c893433fd429b04acde5eb58e1d9ca356a0ae9b6c485d239174f642082cb7a"
}
],
"purl" : "pkg:maven/com.example/cyclonedx-test-dep@1?type=jar",
"type" : "library",
"bom-ref" : "pkg:maven/com.example/cyclonedx-test-dep@1?type=jar"
}
],
"dependencies" : [
{
"ref" : "pkg:maven/com.example/[email protected]?type=jar",
"dependsOn" : [
"pkg:maven/com.example/cyclonedx-test-dep@1?type=jar"
]
},
{
"ref" : "pkg:maven/com.example/cyclonedx-test-dep@1?type=jar",
"dependsOn" : [ ]
}
]
}
Loading

0 comments on commit 2971708

Please sign in to comment.