Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: improve cyclonedx-mvn bom file check to include files with cus… #524

Merged
merged 15 commits into from
Nov 24, 2023
Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions scripts/dev_scripts/integration_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ echo "apache/maven: Analyzing with PURL and repository path without dependency r
echo -e "----------------------------------------------------------------------------------\n"
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/purl/maven/maven.json
JSON_RESULT=$WORKSPACE/output/reports/maven/apache/maven/maven.json
$RUN_MACARON analyze -purl pkg:maven/apache/maven -rp https://github.com/apache/maven -b master -d 6767f2500f1d005924ccff27f04350c253858a84 --skip-deps || log_fail
$RUN_MACARON analyze -purl pkg:maven/apache/maven -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail

python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail

Expand All @@ -166,7 +166,7 @@ JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/maven/maven.json
JSON_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/maven.json
DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_apache_maven.json
DEP_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/dependencies.json
$RUN_MACARON analyze -rp https://github.com/apache/maven -b master -d 6767f2500f1d005924ccff27f04350c253858a84 || log_fail
$RUN_MACARON analyze -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b || log_fail

python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail

Expand All @@ -179,7 +179,7 @@ SBOM_FILE=$WORKSPACE/tests/dependency_analyzer/cyclonedx/resources/apache_maven_
DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/apache_maven_with_sbom_provided.json
DEP_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/dependencies.json

$RUN_MACARON analyze -rp https://github.com/apache/maven -b master -d 6767f2500f1d005924ccff27f04350c253858a84 -sbom "$SBOM_FILE" || log_fail
$RUN_MACARON analyze -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b -sbom "$SBOM_FILE" || log_fail

python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail

Expand Down Expand Up @@ -303,7 +303,7 @@ echo "Test using the default template file."
echo -e "----------------------------------------------------------------------------------\n"
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/maven/maven.json
JSON_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/maven.json
$RUN_MACARON analyze -rp https://github.com/apache/maven --skip-deps -b master -d 6767f2500f1d005924ccff27f04350c253858a84 -g $WORKSPACE/src/macaron/output_reporter/templates/macaron.html || log_fail
$RUN_MACARON analyze -rp https://github.com/apache/maven --skip-deps -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b -g $WORKSPACE/src/macaron/output_reporter/templates/macaron.html || log_fail

python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail

Expand Down Expand Up @@ -352,7 +352,7 @@ JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/maven/maven.json
JSON_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/maven.json
DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_apache_maven.json
DEP_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/dependencies.json
$RUN_MACARON -lr $WORKSPACE/output/git_repos/github_com analyze -rp apache/maven -b master -d 6767f2500f1d005924ccff27f04350c253858a84 || log_fail
$RUN_MACARON -lr $WORKSPACE/output/git_repos/github_com analyze -rp apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b || log_fail

python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail
python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
Expand Down Expand Up @@ -388,7 +388,7 @@ declare -a COMPARE_FILES=(
"mockito.json"
)

$RUN_MACARON -lr $WORKSPACE/output/git_repos/github_com/ analyze -rp apache/maven -b master -d 6767f2500f1d005924ccff27f04350c253858a84 --skip-deps || log_fail
$RUN_MACARON -lr $WORKSPACE/output/git_repos/github_com/ analyze -rp apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail
for i in "${COMPARE_FILES[@]}"
do
python $COMPARE_JSON_OUT $JSON_RESULT_DIR/$i $JSON_EXPECT_DIR/$i || log_fail
Expand All @@ -404,7 +404,7 @@ git clone $WORKSPACE/output/git_repos/github_com/apache/maven $WORKSPACE/output/
JSON_EXPECTED=$WORKSPACE/output/reports/local_repos/maven/maven.json
HTML_EXPECTED=$WORKSPACE/output/reports/local_repos/maven/maven.html

$RUN_MACARON -lr $WORKSPACE/output/git_repos/local_repos/ analyze -rp test_repo -b master -d 6767f2500f1d005924ccff27f04350c253858a84 --skip-deps || log_fail
$RUN_MACARON -lr $WORKSPACE/output/git_repos/local_repos/ analyze -rp test_repo -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail

# We don't compare the report content because the remote_path fields in the reports are undeterministic when running
# this test locally and running it in the GitHub Actions runner. We only check if the reports are generated as
Expand Down
6 changes: 3 additions & 3 deletions scripts/dev_scripts/integration_tests_docker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ echo -e "-----------------------------------------------------------------------
JSON_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/maven.json
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/maven/maven.json

$RUN_MACARON_SCRIPT -lr $WORKSPACE/output/git_repos/github_com analyze -r apache/maven -b master -d 6767f2500f1d005924ccff27f04350c253858a84 --skip-deps || log_fail
$RUN_MACARON_SCRIPT -lr $WORKSPACE/output/git_repos/github_com analyze -r apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail
python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail

echo -e "\n----------------------------------------------------------------------------------"
Expand Down Expand Up @@ -87,7 +87,7 @@ SBOM_FILE=$WORKSPACE/tests/dependency_analyzer/cyclonedx/resources/apache_maven_
DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/apache_maven_with_sbom_provided.json
DEP_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/dependencies.json

$RUN_MACARON_SCRIPT analyze -rp https://github.com/apache/maven -b master -d 6767f2500f1d005924ccff27f04350c253858a84 -sbom $SBOM_FILE || log_fail
$RUN_MACARON_SCRIPT analyze -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b -sbom $SBOM_FILE || log_fail

python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail

Expand All @@ -96,7 +96,7 @@ echo "apache/maven: Analyzing with PURL and repository path without dependency r
echo -e "----------------------------------------------------------------------------------\n"
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/purl/maven/maven.json
JSON_RESULT=$WORKSPACE/output/reports/maven/apache/maven/maven.json
$RUN_MACARON_SCRIPT analyze -purl pkg:maven/apache/maven -rp https://github.com/apache/maven -b master -d 6767f2500f1d005924ccff27f04350c253858a84 --skip-deps || log_fail
$RUN_MACARON_SCRIPT analyze -purl pkg:maven/apache/maven -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail

python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail

Expand Down
2 changes: 1 addition & 1 deletion src/macaron/dependency_analyzer/cyclonedx.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def deserialize_bom_json(file_path: Path) -> dict:
# TODO: use the official CycloneDX library to validate and deserialize BOM files
# once merged: https://github.com/CycloneDX/cyclonedx-python-lib/pull/290
if not os.path.exists(file_path):
raise CycloneDXParserError(f"Unable to locate the BOM file: {str(file_path)}.")
raise CycloneDXParserError(f"Unable to locate any BOM files at: {str(file_path.parent)}.")

with open(file_path, encoding="utf8") as file:
try:
Expand Down
22 changes: 21 additions & 1 deletion src/macaron/dependency_analyzer/cyclonedx_mvn.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,14 +64,34 @@ def collect_dependencies(self, dir_path: str) -> dict[str, DependencyInfo]:
"""
# Load the top level file separately as it has different content.
top_path = Path(os.path.join(dir_path, "target", self.file_name))
top_path_altered = False
if not os.path.exists(top_path):
# If the expected bom file does not exist, allow other named .json files instead.
possible_paths = glob.glob(os.path.join(dir_path, "target", "*.json"))
if possible_paths:
if len(possible_paths) > 1:
logger.debug("Too many JSON SBOM files found. Expected: 1, Found: %s", len(possible_paths))
return {}
top_path = Path(possible_paths[0])
tromai marked this conversation as resolved.
Show resolved Hide resolved
top_path_altered = True

# Collect all the dependency files recursively.
child_paths = [
Path(path)
for path in glob.glob(os.path.join(dir_path, "**", "target", self.file_name), recursive=True)
for path in glob.glob(
os.path.join(dir_path, "**", "target", "*.json" if top_path_altered else self.file_name), recursive=True
Copy link
Member

@tromai tromai Nov 7, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What would happen if there is multiple *.json files in the "children" taget directories, given that there exist a custom-named SBOM file in the top path?

)
if Path(path) != top_path
]

# Ensure recursively found SBOMs are at most one per directory.
child_paths_set = set()
Copy link
Member

@tromai tromai Nov 24, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This set contains the parent directory names of all children SBOM files. I think we could re-name it to something like child_sbom_dir_names

for path in child_paths:
child_paths_set.add(path.parent)
if len(child_paths_set) != len(child_paths):
logger.debug("Only one JSON SBOM file is permitted per child directory.")
return {}

# Check if the root BOM has been analyzed before as a child BOM.
self.visited_deps.update(child_paths)
if top_path in self.visited_deps:
Expand Down
4 changes: 2 additions & 2 deletions tests/dependency_analyzer/configurations/maven_config.yaml
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved.
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.

target:
id: apache/maven
branch: master
digest: 6767f2500f1d005924ccff27f04350c253858a84
digest: 3fc399318edef0d5ba593723a24fff64291d6f9b
path: https://github.com/apache/maven.git
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
{
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
{
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
{
"bomFormat" : "CycloneDX",
"specVersion" : "1.4",
"serialNumber" : "urn:uuid:53576e41-735f-4da4-9249-7f63234ebd94",
"version" : 1,
"metadata" : {
"timestamp" : "2023-10-23T00:57:55Z",
"tools" : [
{
"vendor" : "OWASP Foundation",
"name" : "CycloneDX Maven plugin",
"version" : "2.6.2",
"hashes" : [
{
"alg" : "MD5",
"content" : "ff29fc50797fce0b33058a6b2b283f64"
},
{
"alg" : "SHA-1",
"content" : "597e59ebf21c3b8bfb1faeb622569df324eca956"
},
{
"alg" : "SHA-256",
"content" : "3cf9130fcac45a7beb6df2ae9c3fc9c062d1fddd0731d6a302968586f0aa586e"
},
{
"alg" : "SHA-384",
"content" : "8111a6788c959305af23daecbc79defd4478c1e274cba65bfe860e09b30cd9fe29822d5d3d3eea608e4926a9418f92e3"
},
{
"alg" : "SHA-512",
"content" : "2bea87b7bcd70897bf46a28a806b6064a6708d0a45e884e1ceddc25f97ca7bdf4ed190f30d9a28cc9416b6c66176d518c5876fd25bc06bdcb00d39367215e56e"
},
{
"alg" : "SHA3-256",
"content" : "f0f7b771749955e7898665c2fff8f4f2cd734d9cbe4d29883292db772f1be00e"
},
{
"alg" : "SHA3-384",
"content" : "a87d4c18bac4d48a46c0b8611ab92934e457fcd55bd4d39dbc9c4e5044d2736d3bda991c43d67b0987eddcf4c88510ff"
},
{
"alg" : "SHA3-512",
"content" : "90c38f168600787fc90b7e37e743b386b7296bceb10152190de6e30e0f251da3e01698d1b1e11ad84f207532b5a0743aac105f3c5006ff4607d21f30c9ea779f"
}
]
}
],
"component" : {
"group" : "com.example",
"name" : "cyclonedx-test",
"version" : "1.0-SNAPSHOT",
"licenses" : [ ],
"purl" : "pkg:maven/com.example/[email protected]?type=jar",
"type" : "library",
"bom-ref" : "pkg:maven/com.example/[email protected]?type=jar"
}
},
"components" : [
{
"group" : "com.example",
"name" : "cyclonedx-test-dep",
"version" : "1",
"scope" : "optional",
"hashes" : [
{
"alg" : "MD5",
"content" : "c7b63da4c25c163825cca671e7899fbe"
},
{
"alg" : "SHA-1",
"content" : "5aa25ee1bf1ffd60b76f16fe0a8edd76f870958c"
},
{
"alg" : "SHA-256",
"content" : "c38cef49f7676227c1d4cf98e59b96f7a6bf33704d10314d83d682acd2b47d10"
},
{
"alg" : "SHA-384",
"content" : "7afa5feaa7d3a4ca4ecba7d4bd1b093e75be2ee2a25eefbc5fd90eb8b9a4712fa1a720265765a28d858fc64412dbed2b"
},
{
"alg" : "SHA-512",
"content" : "bf69097c4c0d165e5521a918ee79c1e5e211e9e74410d48042994c4c6cf5788cf4d62129e7c0d7a22294835178398c91c31929ce6861068c71ea14059f6f6e56"
},
{
"alg" : "SHA3-256",
"content" : "ba7656644f127c4b10d53c777aee2ed023ac3caf7f420ecb4ca48a909d775a17"
},
{
"alg" : "SHA3-384",
"content" : "1244f326a9b0b165b27b0061f1fcdf2580e3b64681cc3f09df3afd9a4526ab5491a20213a8fb9edcc671fbae8b51a010"
},
{
"alg" : "SHA3-512",
"content" : "e6020e5b9adbe61f1c53e575ab0c51b9eef7dbea3dbe21f970607002ed0373b322c893433fd429b04acde5eb58e1d9ca356a0ae9b6c485d239174f642082cb7a"
}
],
"purl" : "pkg:maven/com.example/cyclonedx-test-dep@1?type=jar",
"type" : "library",
"bom-ref" : "pkg:maven/com.example/cyclonedx-test-dep@1?type=jar"
}
],
"dependencies" : [
{
"ref" : "pkg:maven/com.example/[email protected]?type=jar",
"dependsOn" : [
"pkg:maven/com.example/cyclonedx-test-dep@1?type=jar"
]
},
{
"ref" : "pkg:maven/com.example/cyclonedx-test-dep@1?type=jar",
"dependsOn" : [ ]
}
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
{
"bomFormat" : "CycloneDX",
"specVersion" : "1.4",
"serialNumber" : "urn:uuid:53576e41-735f-4da4-9249-7f63234ebd94",
"version" : 1,
"metadata" : {
"timestamp" : "2023-10-23T00:57:55Z",
"tools" : [
{
"vendor" : "OWASP Foundation",
"name" : "CycloneDX Maven plugin",
"version" : "2.6.2",
"hashes" : [
{
"alg" : "MD5",
"content" : "ff29fc50797fce0b33058a6b2b283f64"
},
{
"alg" : "SHA-1",
"content" : "597e59ebf21c3b8bfb1faeb622569df324eca956"
},
{
"alg" : "SHA-256",
"content" : "3cf9130fcac45a7beb6df2ae9c3fc9c062d1fddd0731d6a302968586f0aa586e"
},
{
"alg" : "SHA-384",
"content" : "8111a6788c959305af23daecbc79defd4478c1e274cba65bfe860e09b30cd9fe29822d5d3d3eea608e4926a9418f92e3"
},
{
"alg" : "SHA-512",
"content" : "2bea87b7bcd70897bf46a28a806b6064a6708d0a45e884e1ceddc25f97ca7bdf4ed190f30d9a28cc9416b6c66176d518c5876fd25bc06bdcb00d39367215e56e"
},
{
"alg" : "SHA3-256",
"content" : "f0f7b771749955e7898665c2fff8f4f2cd734d9cbe4d29883292db772f1be00e"
},
{
"alg" : "SHA3-384",
"content" : "a87d4c18bac4d48a46c0b8611ab92934e457fcd55bd4d39dbc9c4e5044d2736d3bda991c43d67b0987eddcf4c88510ff"
},
{
"alg" : "SHA3-512",
"content" : "90c38f168600787fc90b7e37e743b386b7296bceb10152190de6e30e0f251da3e01698d1b1e11ad84f207532b5a0743aac105f3c5006ff4607d21f30c9ea779f"
}
]
}
],
"component" : {
"group" : "com.example",
"name" : "cyclonedx-test",
"version" : "1.0-SNAPSHOT",
"licenses" : [ ],
"purl" : "pkg:maven/com.example/[email protected]?type=jar",
"type" : "library",
"bom-ref" : "pkg:maven/com.example/[email protected]?type=jar"
}
},
"components" : [
{
"group" : "com.example",
"name" : "cyclonedx-test-dep",
"version" : "1",
"scope" : "optional",
"hashes" : [
{
"alg" : "MD5",
"content" : "c7b63da4c25c163825cca671e7899fbe"
},
{
"alg" : "SHA-1",
"content" : "5aa25ee1bf1ffd60b76f16fe0a8edd76f870958c"
},
{
"alg" : "SHA-256",
"content" : "c38cef49f7676227c1d4cf98e59b96f7a6bf33704d10314d83d682acd2b47d10"
},
{
"alg" : "SHA-384",
"content" : "7afa5feaa7d3a4ca4ecba7d4bd1b093e75be2ee2a25eefbc5fd90eb8b9a4712fa1a720265765a28d858fc64412dbed2b"
},
{
"alg" : "SHA-512",
"content" : "bf69097c4c0d165e5521a918ee79c1e5e211e9e74410d48042994c4c6cf5788cf4d62129e7c0d7a22294835178398c91c31929ce6861068c71ea14059f6f6e56"
},
{
"alg" : "SHA3-256",
"content" : "ba7656644f127c4b10d53c777aee2ed023ac3caf7f420ecb4ca48a909d775a17"
},
{
"alg" : "SHA3-384",
"content" : "1244f326a9b0b165b27b0061f1fcdf2580e3b64681cc3f09df3afd9a4526ab5491a20213a8fb9edcc671fbae8b51a010"
},
{
"alg" : "SHA3-512",
"content" : "e6020e5b9adbe61f1c53e575ab0c51b9eef7dbea3dbe21f970607002ed0373b322c893433fd429b04acde5eb58e1d9ca356a0ae9b6c485d239174f642082cb7a"
}
],
"purl" : "pkg:maven/com.example/cyclonedx-test-dep@1?type=jar",
"type" : "library",
"bom-ref" : "pkg:maven/com.example/cyclonedx-test-dep@1?type=jar"
}
],
"dependencies" : [
{
"ref" : "pkg:maven/com.example/[email protected]?type=jar",
"dependsOn" : [
"pkg:maven/com.example/cyclonedx-test-dep@1?type=jar"
]
},
{
"ref" : "pkg:maven/com.example/cyclonedx-test-dep@1?type=jar",
"dependsOn" : [ ]
}
]
}
Loading
Loading