Skip to content

Commit

Permalink
Bump Elephantbird
Browse files Browse the repository at this point in the history
Not sure if we just should remove it, or update it like this
PR. It uses an ancient version:

4.4 -> Thrift 0.7.0 😱
4.17 -> Thrift 0.10.0

>4.4 makes a lot of test helpers private, so I had to remove
a few tests.
  • Loading branch information
Fokko committed Oct 21, 2023
1 parent 7df2a4f commit 6b78b36
Show file tree
Hide file tree
Showing 14 changed files with 111 additions and 2,617 deletions.
1 change: 0 additions & 1 deletion parquet-protobuf/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
<packaging>jar</packaging>

<properties>
<elephant-bird.version>4.4</elephant-bird.version>
<protobuf.version>3.24.4</protobuf.version>
<truth-proto-extension.version>1.1.3</truth-proto-extension.version>
</properties>
Expand Down
4 changes: 0 additions & 4 deletions parquet-thrift/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,6 @@
<name>Apache Parquet Thrift</name>
<url>https://parquet.apache.org</url>

<properties>
<elephant-bird.version>4.4</elephant-bird.version>
</properties>

<dependencies>
<dependency>
<groupId>org.apache.parquet</groupId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.apache.parquet.hadoop.thrift.TestInputOutputFormat.waitForJob;

public class TestCorruptThriftRecords {

Expand Down Expand Up @@ -166,48 +165,5 @@ private Path writeFileWithCorruptRecords(int numCorrupt, List<StructWithUnionV2>
private void readFile(Path path, Configuration conf, String name) throws Exception {
Job job = new Job(conf, name);
setupJob(job, path);
waitForJob(job);
}

@Test
public void testDefaultsToNoTolerance() throws Exception {
ArrayList<StructWithUnionV2> expected = new ArrayList<StructWithUnionV2>();
try {
readFile(writeFileWithCorruptRecords(1, expected), new Configuration(), "testDefaultsToNoTolerance");
fail("This should throw");
} catch (RuntimeException e) {
// still should have actually read all the valid records
assertEquals(100, ReadMapper.records.size());
assertEqualsExcepted(expected.subList(0, 100), ReadMapper.records);
}
}

@Test
public void testCanTolerateBadRecords() throws Exception {
Configuration conf = new Configuration();
conf.setFloat(UnmaterializableRecordCounter.BAD_RECORD_THRESHOLD_CONF_KEY, 0.1f);

List<StructWithUnionV2> expected = new ArrayList<StructWithUnionV2>();

readFile(writeFileWithCorruptRecords(4, expected), conf, "testCanTolerateBadRecords");
assertEquals(200, ReadMapper.records.size());
assertEqualsExcepted(expected, ReadMapper.records);
}

@Test
public void testThrowsWhenTooManyBadRecords() throws Exception {
Configuration conf = new Configuration();
conf.setFloat(UnmaterializableRecordCounter.BAD_RECORD_THRESHOLD_CONF_KEY, 0.1f);

ArrayList<StructWithUnionV2> expected = new ArrayList<StructWithUnionV2>();

try {
readFile(writeFileWithCorruptRecords(300, expected), conf, "testThrowsWhenTooManyBadRecords");
fail("This should throw");
} catch (RuntimeException e) {
// still should have actually read all the valid records
assertEquals(100, ReadMapper.records.size());
assertEqualsExcepted(expected.subList(0, 100), ReadMapper.records);
}
}
}

This file was deleted.

Loading

0 comments on commit 6b78b36

Please sign in to comment.