renamed scrub -> sanitize; added comments

crowemi · May 17, 2023 · da8bb0b · da8bb0b
1 parent 39a35d4
commit da8bb0b
Showing 1 changed file with 4 additions and 3 deletions.
diff --git a/target_s3/formats/format_parquet.py b/target_s3/formats/format_parquet.py
@@ -154,7 +154,7 @@ def validate_list(value, fields):
 
         return value
 
-    def clean(self, value):
+    def sanitize(self, value):
         if isinstance(value, dict) and not value:
             # pyarrow can't process empty struct
             return None
@@ -174,14 +174,15 @@ def create_dataframe(self) -> Table:
                 schema = dict()
                 input = {
                     f: [
-                        self.validate(schema, self.clean(f), row.get(f))
+                        self.validate(schema, self.sanitize(f), row.get(f))
                         for row in self.records
                     ]
                     for f in fields
                 }
             else:
                 input = {
-                    f: [self.clean(row.get(f)) for row in self.records] for f in fields
+                    f: [self.sanitize(row.get(f)) for row in self.records]
+                    for f in fields
                 }
 
             ret = Table.from_pydict(mapping=input)