fix extract schema (#1067)

aryn-ai · Dec 9, 2024 · 0981618 · 0981618
1 parent 4b69ea9
commit 0981618
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/lib/sycamore/sycamore/connectors/elasticsearch/elasticsearch_writer.py b/lib/sycamore/sycamore/connectors/elasticsearch/elasticsearch_writer.py
@@ -6,7 +6,7 @@
 
 from sycamore.data.document import Document
 from sycamore.connectors.base_writer import BaseDBWriter
-from sycamore.connectors.common import flatten_data, check_dictionary_compatibility
+from sycamore.connectors.common import flatten_data, check_dictionary_compatibility, drop_types
 
 if typing.TYPE_CHECKING:
     from elasticsearch import Elasticsearch
@@ -90,7 +90,7 @@ def bulk_action_generator():
                     yield {
                         "_index": target_params.index_name,
                         "_id": r.doc_id,
-                        "properties": r.properties,
+                        "properties": drop_types(r.properties),
                         "embedding": r.embedding,
                         "parent_id": r.parent_id,
                     }

diff --git a/lib/sycamore/sycamore/transforms/extract_schema.py b/lib/sycamore/sycamore/transforms/extract_schema.py
@@ -174,7 +174,8 @@ def extract_properties(self, document: Document) -> Document:
             answer = extract_json(payload)
         except (json.JSONDecodeError, AttributeError):
             answer = entities
-
+        if answer == "None":
+            answer = {}
         if "entity" in document.properties:
             document.properties["entity"].update(answer)
         else: