Skip to content

Commit

Permalink
Use SupportsPrefixOperations for Remove OrphanFile Procedure on Spark…
Browse files Browse the repository at this point in the history
… 3.5, improve naming
  • Loading branch information
Ismail Simsek committed Jan 8, 2025
1 parent 74664ad commit bdd982c
Showing 1 changed file with 16 additions and 14 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -843,41 +843,41 @@ public void testCompareToFileList() throws IOException {
.withColumnRenamed("filePath", "file_path")
.withColumnRenamed("lastModified", "last_modified");

DeleteOrphanFiles.Result result1 =
DeleteOrphanFiles.Result deletedOrphanFiles1 =
actions
.deleteOrphanFiles(table)
.compareToFileList(compareToFileList)
.deleteWith(s -> {})
.execute();
assertThat(result1.orphanFileLocations())
assertThat(deletedOrphanFiles1.orphanFileLocations())
.as("Default olderThan interval should be safe")
.isEmpty();

DeleteOrphanFiles.Result result2 =
DeleteOrphanFiles.Result deletedOrphanFiles2 =
actions
.deleteOrphanFiles(table)
.compareToFileList(compareToFileList)
.olderThan(System.currentTimeMillis())
.deleteWith(s -> {})
.execute();
assertThat(result2.orphanFileLocations())
assertThat(deletedOrphanFiles2.orphanFileLocations())
.as("Action should find 1 file")
.isEqualTo(invalidFilePaths);
assertThat(fs.exists(new Path(invalidFilePaths.get(0))))
.as("Invalid file should be present")
.isTrue();

DeleteOrphanFilesSparkAction deleteOrphanFilesSparkAction =
DeleteOrphanFilesSparkAction deleteOrphanFilesSparkAction3 =
actions
.deleteOrphanFiles(table)
.compareToFileList(compareToFileList)
.olderThan(System.currentTimeMillis());
assertThatDatasetsAreEqualIgnoringOrder(
deleteOrphanFilesSparkAction.listWithPrefix(),
deleteOrphanFilesSparkAction.listWithoutPrefix());
deleteOrphanFilesSparkAction3.listWithPrefix(),
deleteOrphanFilesSparkAction3.listWithoutPrefix());

DeleteOrphanFiles.Result result3 = deleteOrphanFilesSparkAction.execute();
assertThat(result3.orphanFileLocations())
DeleteOrphanFiles.Result deletedOrphanFiles3 = deleteOrphanFilesSparkAction3.execute();
assertThat(deletedOrphanFiles3.orphanFileLocations())
.as("Action should delete 1 file")
.isEqualTo(invalidFilePaths);
assertThat(fs.exists(new Path(invalidFilePaths.get(0))))
Expand All @@ -902,17 +902,19 @@ public void testCompareToFileList() throws IOException {
.withColumnRenamed("filePath", "file_path")
.withColumnRenamed("lastModified", "last_modified");

deleteOrphanFilesSparkAction =
DeleteOrphanFilesSparkAction deleteOrphanFilesSparkAction4 =
actions
.deleteOrphanFiles(table)
.compareToFileList(compareToFileListWithOutsideLocation)
.deleteWith(s -> {});
assertThatDatasetsAreEqualIgnoringOrder(
deleteOrphanFilesSparkAction.listWithPrefix(),
deleteOrphanFilesSparkAction.listWithoutPrefix());
deleteOrphanFilesSparkAction4.listWithPrefix(),
deleteOrphanFilesSparkAction4.listWithoutPrefix());

DeleteOrphanFiles.Result result4 = deleteOrphanFilesSparkAction.execute();
assertThat(result4.orphanFileLocations()).as("Action should find nothing").isEmpty();
DeleteOrphanFiles.Result deletedOrphanFiles4 = deleteOrphanFilesSparkAction4.execute();
assertThat(deletedOrphanFiles4.orphanFileLocations())
.as("Action should find nothing")
.isEmpty();
}

protected long waitUntilAfter(long timestampMillis) {
Expand Down

0 comments on commit bdd982c

Please sign in to comment.