From affd2bc928489fe946b14d6f9331d465342c33ea Mon Sep 17 00:00:00 2001 From: john681611 Date: Mon, 6 Nov 2023 16:56:17 +0000 Subject: [PATCH] Added: Explorer backend endpoint --- application/database/db.py | 84 +++++++++++++++++++----- application/tests/db_test.py | 120 ++++++++++++++++++++++++++++++----- application/web/web_main.py | 11 ++++ 3 files changed, 184 insertions(+), 31 deletions(-) diff --git a/application/database/db.py b/application/database/db.py index f5edf344e..ae007d224 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -8,6 +8,7 @@ UniqueIdProperty, Relationship, RelationshipTo, + RelationshipFrom, ArrayProperty, StructuredRel, db, @@ -204,9 +205,21 @@ class NeoDocument(StructuredNode): related = Relationship("NeoDocument", "RELATED", model=RelatedRel) @classmethod - def to_cre_def(self, node): + def to_cre_def(self, node, parse_links=True): raise Exception(f"Shouldn't be parsing a NeoDocument") + @classmethod + def get_links(self, links_dict): + links = [] + for key in links_dict: + links.extend( + [ + cre_defs.Link(c.to_cre_def(c, parse_links=False), key) + for c in links_dict[key] + ] + ) + return links + class NeoNode(NeoDocument): doctype = StringProperty() @@ -214,7 +227,7 @@ class NeoNode(NeoDocument): hyperlink = StringProperty() @classmethod - def to_cre_def(self, node): + def to_cre_def(self, node, parse_links=True): raise Exception(f"Shouldn't be parsing a NeoNode") @@ -224,10 +237,9 @@ class NeoStandard(NeoNode): section_id = StringProperty() @classmethod - def to_cre_def(self, node) -> cre_defs.Standard: + def to_cre_def(self, node, parse_links=True) -> cre_defs.Standard: return cre_defs.Standard( name=node.name, - id=node.document_id, description=node.description, tags=node.tags, hyperlink=node.hyperlink, @@ -235,6 +247,13 @@ def to_cre_def(self, node) -> cre_defs.Standard: section=node.section, sectionID=node.section_id, subsection=node.subsection, + links=self.get_links( + { + "Related": node.related, + } + ) + if parse_links + else [], ) @@ -242,10 +261,9 @@ class NeoTool(NeoStandard): tooltype = StringProperty(required=True) @classmethod - def to_cre_def(self, node) -> cre_defs.Tool: + def to_cre_def(self, node, parse_links=True) -> cre_defs.Tool: return cre_defs.Tool( name=node.name, - id=node.document_id, description=node.description, tags=node.tags, hyperlink=node.hyperlink, @@ -253,35 +271,59 @@ def to_cre_def(self, node) -> cre_defs.Tool: section=node.section, sectionID=node.section_id, subsection=node.subsection, + links=self.get_links( + { + "Related": node.related, + } + ) + if parse_links + else [], ) class NeoCode(NeoNode): @classmethod - def to_cre_def(self, node) -> cre_defs.Code: + def to_cre_def(self, node, parse_links=True) -> cre_defs.Code: return cre_defs.Code( name=node.name, - id=node.document_id, description=node.description, tags=node.tags, hyperlink=node.hyperlink, version=node.version, + links=self.get_links( + { + "Related": node.related, + } + ) + if parse_links + else [], ) class NeoCRE(NeoDocument): # type: ignore external_id = StringProperty() contains = RelationshipTo("NeoCRE", "CONTAINS", model=ContainsRel) + contained_in = RelationshipFrom("NeoCRE", "CONTAINS", model=ContainsRel) linked = RelationshipTo("NeoStandard", "LINKED_TO", model=LinkedToRel) same_as = RelationshipTo("NeoStandard", "SAME", model=SameRel) @classmethod - def to_cre_def(self, node) -> cre_defs.CRE: + def to_cre_def(self, node, parse_links=True) -> cre_defs.CRE: return cre_defs.CRE( name=node.name, - id=node.document_id, + id=node.external_id, description=node.description, tags=node.tags, + links=self.get_links( + { + "Contains": [*node.contains, *node.contained_in], + "Linked To": node.linked, + "Same as": node.same_as, + "Related": node.related, + } + ) + if parse_links + else [], ) @@ -340,6 +382,7 @@ def add_cre(self, dbcre: CRE): "description": dbcre.description, "links": [], # dbcre.links, "tags": [dbcre.tags] if isinstance(dbcre.tags, str) else dbcre.tags, + "external_id": dbcre.external_id, } ) @@ -477,19 +520,19 @@ def format_segment(seg: StructuredRel, nodes): ][0] return { - "start": NEO_DB.parse_node(start_node), - "end": NEO_DB.parse_node(end_node), + "start": NEO_DB.parse_node_no_links(start_node), + "end": NEO_DB.parse_node_no_links(end_node), "relationship": relation_map[type(seg)], } def format_path_record(rec): return { - "start": NEO_DB.parse_node(rec.start_node), - "end": NEO_DB.parse_node(rec.end_node), + "start": NEO_DB.parse_node_no_links(rec.start_node), + "end": NEO_DB.parse_node_no_links(rec.end_node), "path": [format_segment(seg, rec.nodes) for seg in rec.relationships], } - return [NEO_DB.parse_node(rec) for rec in base_standard], [ + return [NEO_DB.parse_node_no_links(rec) for rec in base_standard], [ format_path_record(rec[0]) for rec in (path_records + path_records_all) ] @@ -502,10 +545,18 @@ def standards(self) -> List[str]: results.extend(x) return list(set(results)) + @classmethod + def everything(self) -> List[str]: + return [NEO_DB.parse_node(rec) for rec in NeoDocument.nodes.all()] + @staticmethod def parse_node(node: NeoDocument) -> cre_defs.Document: return node.to_cre_def(node) + @staticmethod + def parse_node_no_links(node: NeoDocument) -> cre_defs.Document: + return node.to_cre_def(node, parse_links=False) + class CRE_Graph: graph: nx.Graph = None @@ -1183,6 +1234,9 @@ def export(self, dir: str = None, dry_run: bool = False) -> List[cre_defs.Docume return list(docs.values()) + def all_nodes_flat(self) -> List[cre_defs.Document]: + return self.neo_db.everything() + def add_cre(self, cre: cre_defs.CRE) -> CRE: entry: CRE query: sqla.Query = self.session.query(CRE).filter( diff --git a/application/tests/db_test.py b/application/tests/db_test.py index 6439e4f5b..03c82dccb 100644 --- a/application/tests/db_test.py +++ b/application/tests/db_test.py @@ -1577,32 +1577,37 @@ def test_gap_analysis_dump_to_cache(self, gap_mock): def test_neo_db_parse_node_code(self): name = "name" - id = "id" description = "description" tags = "tags" version = "version" hyperlink = "version" expected = defs.Code( name=name, - id=id, description=description, tags=tags, version=version, hyperlink=hyperlink, + links=[ + defs.Link( + defs.CRE(id="123", description="gcCD2", name="gcC2"), "Related" + ) + ], ) graph_node = db.NeoCode( name=name, - document_id=id, description=description, tags=tags, version=version, hyperlink=hyperlink, + related=[ + db.NeoCRE(external_id="123", description="gcCD2", name="gcC2"), + ], ) - self.assertEqual(db.NEO_DB.parse_node(graph_node), expected) + + self.assertEqual(db.NEO_DB.parse_node(graph_node).todict(), expected.todict()) def test_neo_db_parse_node_standard(self): name = "name" - id = "id" description = "description" tags = "tags" version = "version" @@ -1612,7 +1617,6 @@ def test_neo_db_parse_node_standard(self): hyperlink = "version" expected = defs.Standard( name=name, - id=id, description=description, tags=tags, version=version, @@ -1620,10 +1624,14 @@ def test_neo_db_parse_node_standard(self): sectionID=sectionID, subsection=subsection, hyperlink=hyperlink, + links=[ + defs.Link( + defs.CRE(id="123", description="gcCD2", name="gcC2"), "Related" + ) + ], ) graph_node = db.NeoStandard( name=name, - document_id=id, description=description, tags=tags, version=version, @@ -1631,12 +1639,14 @@ def test_neo_db_parse_node_standard(self): section_id=sectionID, subsection=subsection, hyperlink=hyperlink, + related=[ + db.NeoCRE(external_id="123", description="gcCD2", name="gcC2"), + ], ) - self.assertEqual(db.NEO_DB.parse_node(graph_node), expected) + self.assertEqual(db.NEO_DB.parse_node(graph_node).todict(), expected.todict()) def test_neo_db_parse_node_tool(self): name = "name" - id = "id" description = "description" tags = "tags" version = "version" @@ -1646,7 +1656,6 @@ def test_neo_db_parse_node_tool(self): hyperlink = "version" expected = defs.Tool( name=name, - id=id, description=description, tags=tags, version=version, @@ -1654,10 +1663,14 @@ def test_neo_db_parse_node_tool(self): sectionID=sectionID, subsection=subsection, hyperlink=hyperlink, + links=[ + defs.Link( + defs.CRE(id="123", description="gcCD2", name="gcC2"), "Related" + ) + ], ) graph_node = db.NeoTool( name=name, - document_id=id, description=description, tags=tags, version=version, @@ -1665,27 +1678,102 @@ def test_neo_db_parse_node_tool(self): section_id=sectionID, subsection=subsection, hyperlink=hyperlink, + related=[ + db.NeoCRE(external_id="123", description="gcCD2", name="gcC2"), + ], ) - self.assertEqual(db.NEO_DB.parse_node(graph_node), expected) + self.assertEqual(db.NEO_DB.parse_node(graph_node).todict(), expected.todict()) def test_neo_db_parse_node_cre(self): name = "name" - id = "id" description = "description" tags = "tags" + external_id = "abc" expected = defs.CRE( name=name, - id=id, description=description, + id=external_id, tags=tags, + links=[ + defs.Link( + defs.CRE(id="123", description="gcCD2", name="gcC2"), "Contains" + ), + defs.Link( + defs.CRE(id="123", description="gcCD3", name="gcC3"), "Contains" + ), + defs.Link( + defs.Standard( + hyperlink="gc3", + name="gcS2", + section="gc1", + subsection="gc2", + version="gc1.1.1", + ), + "Linked To", + ), + ], + ) + graph_node = db.NeoCRE( + name=name, + description=description, + tags=tags, + external_id=external_id, + contained_in=[], + contains=[ + db.NeoCRE(external_id="123", description="gcCD2", name="gcC2"), + db.NeoCRE(external_id="123", description="gcCD3", name="gcC3"), + ], + linked=[ + db.NeoStandard( + hyperlink="gc3", + name="gcS2", + section="gc1", + subsection="gc2", + version="gc1.1.1", + ) + ], + same_as=[], + related=[], + ) + + parsed = db.NEO_DB.parse_node(graph_node) + self.maxDiff = None + self.assertEqual(parsed.todict(), expected.todict()) + + def test_neo_db_parse_node_no_links_cre(self): + name = "name" + description = "description" + tags = "tags" + external_id = "abc" + expected = defs.CRE( + name=name, description=description, id=external_id, tags=tags, links=[] ) graph_node = db.NeoCRE( name=name, - document_id=id, description=description, tags=tags, + external_id=external_id, + contained_in=[], + contains=[ + db.NeoCRE(external_id="123", description="gcCD2", name="gcC2"), + db.NeoCRE(external_id="123", description="gcCD3", name="gcC3"), + ], + linked=[ + db.NeoStandard( + hyperlink="gc3", + name="gcS2", + section="gc1", + subsection="gc2", + version="gc1.1.1", + ) + ], + same_as=[], + related=[], ) - self.assertEqual(db.NEO_DB.parse_node(graph_node), expected) + + parsed = db.NEO_DB.parse_node_no_links(graph_node) + self.maxDiff = None + self.assertEqual(parsed.todict(), expected.todict()) def test_neo_db_parse_node_Document(self): name = "name" diff --git a/application/web/web_main.py b/application/web/web_main.py index 7d8be9483..7f1c8f26b 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -632,5 +632,16 @@ def logout(): return redirect("/") +@app.route("/rest/v1/everything", methods=["GET"]) +def everything() -> Any: + database = db.Node_collection() + documents = database.all_nodes_flat() + if documents: + res = [doc.todict() for doc in documents] + result = {"data": res} + return jsonify(result) + abort(404) + + if __name__ == "__main__": app.run(use_reloader=False, debug=False)