[DAT-1492] Make bigtable keys deterministic (#145)

Signed-off-by: Khor Shu Heng <[email protected]> Co-authored-by: Khor Shu Heng <[email protected]>
feast-dev · May 18, 2022 · fef7c57 · fef7c57
1 parent fcb80a0
commit fef7c57
Showing 1 changed file with 4 additions and 4 deletions.
diff --git a/spark/ingestion/src/main/scala/feast/ingestion/stores/bigtable/BigTableSinkRelation.scala b/spark/ingestion/src/main/scala/feast/ingestion/stores/bigtable/BigTableSinkRelation.scala
@@ -109,13 +109,13 @@ class BigTableSinkRelation(
 
     val featureColumns = featureFields.map(f => data(f.name))
 
-    val entityColumns   = config.entityColumns.map(c => data(c).cast(StringType))
-    val schema          = serializer.convertSchema(StructType(featureFields))
-    val schemaReference = serializer.schemaReference(schema)
+    val sortedEntityColumns = config.entityColumns.sorted.map(c => data(c).cast(StringType))
+    val schema              = serializer.convertSchema(StructType(featureFields))
+    val schemaReference     = serializer.schemaReference(schema)
 
     data
       .select(
-        joinEntityKey(struct(entityColumns: _*)).alias("key"),
+        joinEntityKey(struct(sortedEntityColumns: _*)).alias("key"),
         serializer.serializeData(schema)(struct(featureColumns: _*)).alias("value"),
         col(config.timestampColumn).alias("ts")
       )