diff --git a/.github/workflows/build-pipeline.yml b/.github/workflows/build-pipeline.yml index 6f7363e..0887168 100755 --- a/.github/workflows/build-pipeline.yml +++ b/.github/workflows/build-pipeline.yml @@ -24,10 +24,10 @@ jobs: - uses: actions/setup-java@v2 with: distribution: 'adopt' - java-version: '8.0.232' + java-version: '11.0.6' - uses: gradle/gradle-build-action@v1 with: - gradle-version: 4.0.1 + gradle-version: 8.0.1 - name: Install Python 3 uses: actions/setup-python@v2 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index d9b2627..3c7d9a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.8.0] +### Added +- **User contribution** + - Add the "fileName" and "key" properties to the granuleFile object from buildS3GranuleFile function +### Changed +- **PODAAC-5877** + - Support java 11 + - SonarQube and Jacoco report +### Deprecated +### Removed +### Fixed +### Security +- Snyk: Upgrade com.amazonaws:aws-java-sdk-s3: 1.12.565 -> 1.12.661 + ## [v1.7.0] - 2022-12-12 ### Added - **PODAAC-4308** @@ -157,4 +171,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -### Security \ No newline at end of file +### Security diff --git a/README.md b/README.md index 769340d..76f7825 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,17 @@ ## Installation -To build the Lambda code: - +To build the Lambda code, Refer to following Confluence page: +https://wiki.jpl.nasa.gov/pages/viewpage.action?spaceKey=PD&title=SonarQube%2C+Jacoco+and+Java+17+upgrade ```shell +* Build with sonarQube and Jacoco report +mvn clean verify sonar:sonar \ + -Dsonar.projectKey=cnm2cma-opensource \ + -Dsonar.projectName='cnm2cma-opensource' \ + -Dsonar.host.url=http://localhost:9000 \ + -Dsonar.token=sqp_6dc05b1aa1f622b45112927d2a0510f209776860 + +* Makde sure using java 11 and gradle 8.3 mvn clean dependency:copy-dependencies gradle build ``` diff --git a/build.gradle b/build.gradle index 5e9673b..6aafd3d 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,6 @@ apply plugin: 'java' -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = 1.11 +targetCompatibility = 1.11 dependencies { implementation fileTree(dir: 'target/dependency/', include: '*.jar') @@ -13,7 +13,7 @@ task buildZip(type: Zip) { into('lib') { from configurations.runtimeClasspath } - archiveName 'cnmToGranule.zip' + archiveFileName.set('cnmToGranule.zip') } build.dependsOn buildZip diff --git a/pom.xml b/pom.xml index 721cf13..d8637c1 100755 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ gov.nasa.cumulus cnm-to-granule - 1.7.0 + 1.8.0-rc.1 jar cnm-to-granule @@ -18,11 +18,21 @@ clojars.org https://repo.clojars.org + + central + Maven Central + https://repo1.maven.org/maven2 + + true + + + false + + - com.google.code.gson @@ -32,12 +42,12 @@ gov.nasa.earthdata cumulus-message-adapter - 1.3.9 + 2.0.0 com.amazonaws aws-java-sdk-s3 - 1.12.215 + 1.12.661 com.fasterxml.jackson.core @@ -53,7 +63,7 @@ com.amazonaws aws-lambda-java-core - 1.2.1 + 1.2.3 junit @@ -65,7 +75,78 @@ com.fasterxml.jackson.core jackson-databind - 2.13.2.2 + 2.13.4.2 + + + commons-logging + commons-logging + 1.2 + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.0 + + 11 + 11 + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + + + package + + shade + + + + + org.wordinator.xml2docx.MakeDocx + + true + + + + + + + + + + org.jacoco + jacoco-maven-plugin + 0.8.7 + + + prepare-agent + + prepare-agent + + + + report + + report + + + + XML + + + + + + + org.sonarsource.scanner.maven + sonar-maven-plugin + 3.10.0.2594 + + + diff --git a/src/main/java/gov/nasa/cumulus/CnmToGranuleHandler.java b/src/main/java/gov/nasa/cumulus/CnmToGranuleHandler.java index 8e9ba7a..ea737c7 100644 --- a/src/main/java/gov/nasa/cumulus/CnmToGranuleHandler.java +++ b/src/main/java/gov/nasa/cumulus/CnmToGranuleHandler.java @@ -151,7 +151,8 @@ public String PerformFunction(String input, Context context) throws Exception { String uri = StringUtils.trim(cnmFile.get("uri").getAsString()); if (StringUtils.beginsWithIgnoreCase(uri, "s3://")) { granuleFile = buildS3GranuleFile(cnmFile); - } else if (StringUtils.beginsWithIgnoreCase(uri, "https://")) { + } else if (StringUtils.beginsWithIgnoreCase(uri, "https://") || + StringUtils.beginsWithIgnoreCase(uri, "http://")) { granuleFile = buildHttpsGranuleFile(cnmFile); } else if (StringUtils.beginsWithIgnoreCase(uri, "sftp://")) { granuleFile = buildSftpGranuleFile(cnmFile); @@ -207,6 +208,10 @@ public JsonObject buildS3GranuleFile(JsonObject cnmFile) { granuleFile.addProperty("checksum", cnmFile.get("checksum").getAsString()); } granuleFile.addProperty("type", cnmFile.get("type").getAsString()); + + // Add the "fileName" and "key" properties as listed in the later version of the cumulus granule file schema + granuleFile.addProperty("fileName", cnmFile.get("name").getAsString()); + granuleFile.addProperty("key", url_path + '/' + cnmFile.get("name").getAsString()); return granuleFile; } @@ -214,6 +219,7 @@ public JsonObject buildHttpsGranuleFile(JsonObject cnmFile){ String uri = StringUtils.trim(cnmFile.get("uri").getAsString()); AdapterLogger.LogInfo(this.className + " uri: " + uri); String path = uri.replace("https://", ""); + path = path.replace("http://", ""); // Work-a-around to working with http links also //find the path by getting character from (first / plus 1) to lastIndex of / String url_path = path.substring(path.indexOf("/") + 1, path.lastIndexOf("/")); diff --git a/src/test/java/gov/nasa/cumulus/CnmToGranuleHandlerTest.java b/src/test/java/gov/nasa/cumulus/CnmToGranuleHandlerTest.java index f2e731c..224f27f 100644 --- a/src/test/java/gov/nasa/cumulus/CnmToGranuleHandlerTest.java +++ b/src/test/java/gov/nasa/cumulus/CnmToGranuleHandlerTest.java @@ -147,4 +147,20 @@ public void testBuildSftpGranuleFile() throws Exception { assert(expectedJson.equals(outputJson.getAsJsonObject("output"))); } + + public void testBuildHttpGranuleFile() throws Exception { + ClassLoader classLoader = getClass().getClassLoader(); + File inputJsonFile = new File(classLoader.getResource("http_input.json").getFile()); + File expectedJsonFile = new File(classLoader.getResource("http_output.json").getFile()); + + String input = new String(Files.readAllBytes(inputJsonFile.toPath())); + String expected = new String(Files.readAllBytes(expectedJsonFile.toPath())); + JsonObject expectedJson = new JsonParser().parse(expected).getAsJsonObject(); + + CnmToGranuleHandler cnmToGranuleHandler = new CnmToGranuleHandler(); + String output = cnmToGranuleHandler.PerformFunction(input, null); + JsonObject outputJson = new JsonParser().parse(output).getAsJsonObject(); + + assert(expectedJson.equals(outputJson.getAsJsonObject("output"))); + } } diff --git a/src/test/resources/http_input.json b/src/test/resources/http_input.json new file mode 100644 index 0000000..40d8208 --- /dev/null +++ b/src/test/resources/http_input.json @@ -0,0 +1,88 @@ +{ + "input": { + "collection": "HLSL30", + "identifier": "2ffc5609-6ab4-4f58-b422-4d71ff43b269", + "duplicationid": "HLS.L30.T11VNK.2020259T185742.v2.0", + "version": "1.4", + "submissionTime": "2021-10-29T19:47:06Z", + "product": { + "name": "HLS.L30.T11VNK.2020259T185742.v2.0", + "dataVersion": "2.0", + "id": "HLS.L30.T11VNK.2020259T185742.v2.0", + "files": [ + { + "name": "HLS.L30.T11VNK.2020259T185742.v2.0_stac.json", + "size": 6430, + "checksum": "d5241ce4d88c52b45a44c48c8553f79dd8a883b87fc4da8fad4cd9506783e1e7aaedb6724a58e4f1cab27df17f0261c8b2d376ae2193d892bed02204b9d637f3", + "checksumType": "SHA512", + "uri": "http://e4ftl01.cr.usgs.gov/L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0/HLS.L30.T11VNK.2020259T185742.v2.0_stac.json", + "type": "metadata" + }, + { + "name": "HLS.L30.T11VNK.2020259T185742.v2.0.VZA.tif", + "size": 798505, + "checksum": "b110a6c6375e85887b760d027be47172b8e14576bde2213155627de26169d8ac045bda05432c6495d62a795fb2bdba17d246710dba104a3d76f1cf727272dbbd", + "checksumType": "SHA512", + "uri": "http://e4ftl01.cr.usgs.gov/L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0/HLS.L30.T11VNK.2020259T185742.v2.0.VZA.tif", + "type": "data" + }, + { + "name": "HLS.L30.T11VNK.2020259T185742.v2.0.SZA.tif", + "size": 561155, + "checksum": "88955f6139b6d5cce674e78d19d72ca6861e1e59bdd764863e34442f6232084b7209cab3c159fcfc059986632a6ebb684a9338290950bceb8da273c047b6d678", + "checksumType": "SHA512", + "uri": "http://e4ftl01.cr.usgs.gov/L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0/HLS.L30.T11VNK.2020259T185742.v2.0.SZA.tif", + "type": "data" + }, + { + "name": "HLS.L30.T11VNK.2020259T185742.v2.0.jpg", + "size": 194484, + "checksum": "9db4cb08b333b8a97b859d89f0f99500541816a70a0017d89db59b9c9dd8684e92fad200cb501a95139f0a510d11b82ea54329a218597f24d0dca98eaa66f6f3", + "checksumType": "SHA512", + "uri": "http://e4ftl01.cr.usgs.gov/L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0/HLS.L30.T11VNK.2020259T185742.v2.0.jpg", + "type": "browse" + } + ] + } + }, + "config": { + "collection": { + "files": [ + { + "regex": ".*.h5$", + "sampleFileName": "HLSL30_product_0001-of-0050.h5", + "type": "data", + "bucket": "protected" + }, + { + "regex": ".*.iso.xml$", + "sampleFileName": "HLSL30_product_0001-of-0019.iso.xml", + "type": "metadata", + "bucket": "protected" + }, + { + "regex": ".*.cmr.json$", + "sampleFileName": "HLSL30_product_0001-of-0019.cmr.json", + "type": "metadata", + "bucket": "public" + } + ], + "name": "HLSL30", + "granuleIdExtraction": "^(.*)((\\.cmr\\.json)|(\\.h5)|(\\.h5\\.mp))$", + "granuleId": "^.*$", + "dataType": "HLSL30", + "provider_path": "HLSL30/", + "version": "2.0", + "updatedAt": 1552434051881, + "duplicateHandling": "replace", + "sampleFileName": "HLSL30_product_0001-of-0050.h5", + "createdAt": 1552434051881, + "meta": { + "required-files": [ + ".*.h5$", + ".*.iso.xml$" + ] + } + } + } +} \ No newline at end of file diff --git a/src/test/resources/http_output.json b/src/test/resources/http_output.json new file mode 100644 index 0000000..abf0655 --- /dev/null +++ b/src/test/resources/http_output.json @@ -0,0 +1,43 @@ +{ + "granules": [ + { + "granuleId": "HLS.L30.T11VNK.2020259T185742.v2.0", + "version": "2.0", + "dataType": "HLSL30", + "files": [ + { + "name": "HLS.L30.T11VNK.2020259T185742.v2.0_stac.json", + "path": "L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0", + "size": 6430, + "checksumType": "SHA512", + "checksum": "d5241ce4d88c52b45a44c48c8553f79dd8a883b87fc4da8fad4cd9506783e1e7aaedb6724a58e4f1cab27df17f0261c8b2d376ae2193d892bed02204b9d637f3", + "type": "metadata" + }, + { + "name": "HLS.L30.T11VNK.2020259T185742.v2.0.VZA.tif", + "path": "L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0", + "size": 798505, + "checksumType": "SHA512", + "checksum": "b110a6c6375e85887b760d027be47172b8e14576bde2213155627de26169d8ac045bda05432c6495d62a795fb2bdba17d246710dba104a3d76f1cf727272dbbd", + "type": "data" + }, + { + "name": "HLS.L30.T11VNK.2020259T185742.v2.0.SZA.tif", + "path": "L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0", + "size": 561155, + "checksumType": "SHA512", + "checksum": "88955f6139b6d5cce674e78d19d72ca6861e1e59bdd764863e34442f6232084b7209cab3c159fcfc059986632a6ebb684a9338290950bceb8da273c047b6d678", + "type": "data" + }, + { + "name": "HLS.L30.T11VNK.2020259T185742.v2.0.jpg", + "path": "L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0", + "size": 194484, + "checksumType": "SHA512", + "checksum": "9db4cb08b333b8a97b859d89f0f99500541816a70a0017d89db59b9c9dd8684e92fad200cb501a95139f0a510d11b82ea54329a218597f24d0dca98eaa66f6f3", + "type": "browse" + } + ] + } + ] +} \ No newline at end of file