From 7cd8fb50d36f8fb67e36c56c496bb69c074cb50e Mon Sep 17 00:00:00 2001 From: Aleksandr Rozhkov Date: Fri, 8 Dec 2023 13:58:44 +0300 Subject: [PATCH] Added SecondChance algorithm to allow you to retry importing nodes that could not be imported because they did not find their parents. This may be the case if we have a node type of Object which has its own parent node type of Variable. For example, this could be a HasHistoricalConfiguration reference type. Also, the status about how many nodes we have imported shows numbers that include nodes that were not imported due to some error. --- backends/open62541/src/import.c | 141 +++++++++++++++++++++----- include/NodesetLoader/NodesetLoader.h | 11 +- src/InternalRefService.c | 9 ++ src/Nodeset.c | 18 ++++ src/Nodeset.h | 5 + src/NodesetLoader.c | 10 ++ 6 files changed, 165 insertions(+), 29 deletions(-) diff --git a/backends/open62541/src/import.c b/backends/open62541/src/import.c index 77abeb6a..e3820941 100644 --- a/backends/open62541/src/import.c +++ b/backends/open62541/src/import.c @@ -16,6 +16,7 @@ #include "conversion.h" #include "NodesetLoader/NodesetLoader.h" #include "RefServiceImpl.h" +#include "nodes/NodeContainer.h" #include @@ -115,7 +116,7 @@ static UA_NodeId getParentId(const NL_Node *node, UA_NodeId *parentRefId) return parentId; } -static void +static UA_StatusCode handleObjectNode(const NL_ObjectNode *node, UA_NodeId *id, const UA_NodeId *parentId, const UA_NodeId *parentReferenceId, const UA_LocalizedText *lt, const UA_QualifiedName *qn, @@ -134,13 +135,13 @@ handleObjectNode(const NL_ObjectNode *node, UA_NodeId *id, // addNode_begin is used, otherwise all mandatory childs from type are // instantiated - UA_Server_addNode_begin(server, UA_NODECLASS_OBJECT, *id, *parentId, + return UA_Server_addNode_begin(server, UA_NODECLASS_OBJECT, *id, *parentId, *parentReferenceId, *qn, typeDefId, &oAttr, &UA_TYPES[UA_TYPES_OBJECTATTRIBUTES], node->extension, NULL); } -static void +static UA_StatusCode handleViewNode(const NL_ViewNode *node, UA_NodeId *id, const UA_NodeId *parentId, const UA_NodeId *parentReferenceId, const UA_LocalizedText *lt, const UA_QualifiedName *qn, const UA_LocalizedText *description, @@ -151,11 +152,11 @@ handleViewNode(const NL_ViewNode *node, UA_NodeId *id, const UA_NodeId *parentId attr.description = *description; attr.eventNotifier = (UA_Byte)atoi(node->eventNotifier); attr.containsNoLoops = isValTrue(node->containsNoLoops); - UA_Server_addViewNode(server, *id, *parentId, *parentReferenceId, *qn, attr, + return UA_Server_addViewNode(server, *id, *parentId, *parentReferenceId, *qn, attr, node->extension, NULL); } -static void +static UA_StatusCode handleMethodNode(const NL_MethodNode *node, UA_NodeId *id, const UA_NodeId *parentId, const UA_NodeId *parentReferenceId, const UA_LocalizedText *lt, const UA_QualifiedName *qn, @@ -167,7 +168,7 @@ handleMethodNode(const NL_MethodNode *node, UA_NodeId *id, attr.displayName = *lt; attr.description = *description; - UA_Server_addMethodNode(server, *id, *parentId, *parentReferenceId, *qn, + return UA_Server_addMethodNode(server, *id, *parentId, *parentReferenceId, *qn, attr, NULL, 0, NULL, 0, NULL, node->extension, NULL); } @@ -198,7 +199,7 @@ static size_t getArrayDimensions(const char *s, UA_UInt32 **dims) return arrSize; } -static void handleVariableNode(const NL_VariableNode *node, UA_NodeId *id, +static UA_StatusCode handleVariableNode(const NL_VariableNode *node, UA_NodeId *id, const UA_NodeId *parentId, const UA_NodeId *parentReferenceId, const UA_LocalizedText *lt, @@ -278,19 +279,20 @@ static void handleVariableNode(const NL_VariableNode *node, UA_NodeId *id, } //value is copied by open62541 - UA_Server_addNode_begin(ServerContext_getServerObject(serverContext), UA_NODECLASS_VARIABLE, *id, *parentId, + UA_StatusCode Status = UA_Server_addNode_begin(ServerContext_getServerObject(serverContext), UA_NODECLASS_VARIABLE, *id, *parentId, *parentReferenceId, *qn, typeDefId, &attr, &UA_TYPES[UA_TYPES_VARIABLEATTRIBUTES], node->extension, NULL); //cannot call addNode finish, otherwise the nodes for e.g. range will be instantiated twice //UA_Server_addNode_finish(server, *id); UA_Variant_clear(&attr.value); - + RawData_delete(data); UA_free(attr.arrayDimensions); + return Status; } -static void handleObjectTypeNode(const NL_ObjectTypeNode *node, UA_NodeId *id, +static UA_StatusCode handleObjectTypeNode(const NL_ObjectTypeNode *node, UA_NodeId *id, const UA_NodeId *parentId, const UA_NodeId *parentReferenceId, const UA_LocalizedText *lt, @@ -303,11 +305,11 @@ static void handleObjectTypeNode(const NL_ObjectTypeNode *node, UA_NodeId *id, oAttr.isAbstract = isValTrue(node->isAbstract); oAttr.description = *description; - UA_Server_addObjectTypeNode(server, *id, *parentId, *parentReferenceId, *qn, + return UA_Server_addObjectTypeNode(server, *id, *parentId, *parentReferenceId, *qn, oAttr, node->extension, NULL); } -static void handleReferenceTypeNode(const NL_ReferenceTypeNode *node, +static UA_StatusCode handleReferenceTypeNode(const NL_ReferenceTypeNode *node, UA_NodeId *id, const UA_NodeId *parentId, const UA_NodeId *parentReferenceId, const UA_LocalizedText *lt, @@ -322,11 +324,11 @@ static void handleReferenceTypeNode(const NL_ReferenceTypeNode *node, attr.inverseName = UA_LOCALIZEDTEXT(node->inverseName.locale, node->inverseName.text); - UA_Server_addReferenceTypeNode(server, *id, *parentId, *parentReferenceId, + return UA_Server_addReferenceTypeNode(server, *id, *parentId, *parentReferenceId, *qn, attr, node->extension, NULL); } -static void handleVariableTypeNode(const NL_VariableTypeNode *node, UA_NodeId *id, +static UA_StatusCode handleVariableTypeNode(const NL_VariableTypeNode *node, UA_NodeId *id, const UA_NodeId *parentId, const UA_NodeId *parentReferenceId, const UA_LocalizedText *lt, @@ -351,13 +353,13 @@ static void handleVariableTypeNode(const NL_VariableTypeNode *node, UA_NodeId *i } } - UA_Server_addNode_begin(server, UA_NODECLASS_VARIABLETYPE, *id, *parentId, + return UA_Server_addNode_begin(server, UA_NODECLASS_VARIABLETYPE, *id, *parentId, *parentReferenceId, *qn, UA_NODEID_NULL, &attr, &UA_TYPES[UA_TYPES_VARIABLETYPEATTRIBUTES], node->extension, NULL); } -static void handleDataTypeNode(const NL_DataTypeNode *node, UA_NodeId *id, +static UA_StatusCode handleDataTypeNode(const NL_DataTypeNode *node, UA_NodeId *id, const UA_NodeId *parentId, const UA_NodeId *parentReferenceId, const UA_LocalizedText *lt, @@ -370,11 +372,11 @@ static void handleDataTypeNode(const NL_DataTypeNode *node, UA_NodeId *id, attr.description = *description; attr.isAbstract = isValTrue(node->isAbstract); - UA_Server_addDataTypeNode(server, *id, *parentId, *parentReferenceId, *qn, + return UA_Server_addDataTypeNode(server, *id, *parentId, *parentReferenceId, *qn, attr, node->extension, NULL); } -static void addNodeImpl(ServerContext *serverContext, const NL_Node *node) +static void addNodeImpl(ServerContext *serverContext, NL_Node *node) { UA_NodeId id = node->id; UA_NodeId parentReferenceId = UA_NODEID_NULL; @@ -389,43 +391,43 @@ static void addNodeImpl(ServerContext *serverContext, const NL_Node *node) switch (node->nodeClass) { case NODECLASS_OBJECT: - handleObjectNode((const NL_ObjectNode *)node, &id, &parentId, + node->processingResult = handleObjectNode((const NL_ObjectNode *)node, &id, &parentId, &parentReferenceId, <, &qn, &description, ServerContext_getServerObject(serverContext)); break; case NODECLASS_METHOD: - handleMethodNode((const NL_MethodNode *)node, &id, &parentId, + node->processingResult = handleMethodNode((const NL_MethodNode *)node, &id, &parentId, &parentReferenceId, <, &qn, &description, ServerContext_getServerObject(serverContext)); break; case NODECLASS_OBJECTTYPE: - handleObjectTypeNode((const NL_ObjectTypeNode *)node, &id, &parentId, + node->processingResult = handleObjectTypeNode((const NL_ObjectTypeNode *)node, &id, &parentId, &parentReferenceId, <, &qn, &description, ServerContext_getServerObject(serverContext)); break; case NODECLASS_REFERENCETYPE: - handleReferenceTypeNode((const NL_ReferenceTypeNode *)node, &id, + node->processingResult = handleReferenceTypeNode((const NL_ReferenceTypeNode *)node, &id, &parentId, &parentReferenceId, <, &qn, &description, ServerContext_getServerObject(serverContext)); break; case NODECLASS_VARIABLETYPE: - handleVariableTypeNode((const NL_VariableTypeNode *)node, &id, &parentId, + node->processingResult = handleVariableTypeNode((const NL_VariableTypeNode *)node, &id, &parentId, &parentReferenceId, <, &qn, &description, ServerContext_getServerObject(serverContext)); break; case NODECLASS_VARIABLE: - handleVariableNode((const NL_VariableNode *)node, &id, &parentId, + node->processingResult = handleVariableNode((const NL_VariableNode *)node, &id, &parentId, &parentReferenceId, <, &qn, &description, serverContext); break; case NODECLASS_DATATYPE: - handleDataTypeNode((const NL_DataTypeNode *)node, &id, &parentId, + node->processingResult = handleDataTypeNode((const NL_DataTypeNode *)node, &id, &parentId, &parentReferenceId, <, &qn, &description, ServerContext_getServerObject(serverContext)); break; case NODECLASS_VIEW: - handleViewNode((const NL_ViewNode *)node, &id, &parentId, + node->processingResult = handleViewNode((const NL_ViewNode *)node, &id, &parentId, &parentReferenceId, <, &qn, &description, ServerContext_getServerObject(serverContext)); break; } @@ -541,6 +543,56 @@ static void addNonHierachicalRefs(UA_Server *server, NL_Node *node) } } +static size_t secondChanceAddNodes(ServerContext *serverContext, + NodeContainer **badStatusNodes, + const NodesetLoader_Logger *logger) +{ + const size_t attemptsNum = 10; + + size_t numberOfAllAddedNodes = 0; + // We will only add nodes with failed parent nodes in at most Num attempts. + // This gives us the opportunity to add a chain like Object(child) -> + // Variable(parent). + // For example, this reference is used in the HasHistoricalConfiguration + // reference type. + size_t attempt = 1; + while (attempt != attemptsNum && (*badStatusNodes)->size != 0) + { + NodeContainer *local_badStatusNodes = + NodeContainer_new((*badStatusNodes)->size, false); + size_t counterOfAdddedNodesForOneAttempt = 0; + for (size_t counter = 0; counter < (*badStatusNodes)->size; counter++) + { + // Try to add nodes only with the BADPARENTNODEIDINVALID status. + if ((*badStatusNodes)->nodes[counter]->processingResult == + UA_STATUSCODE_BADPARENTNODEIDINVALID) + { + addNodeImpl(serverContext, + (*badStatusNodes) + ->nodes[counter]); // Import to server again + if (UA_StatusCode_isBad( + (*badStatusNodes)->nodes[counter]->processingResult)) + { + NodeContainer_add(local_badStatusNodes, + (*badStatusNodes)->nodes[counter]); + } + else + { + counterOfAdddedNodesForOneAttempt++; + } + } + } + numberOfAllAddedNodes += counterOfAdddedNodesForOneAttempt; + logger->log(logger->context, NODESETLOADER_LOGLEVEL_DEBUG, + "attempt (%zu), imported nodes: %zu", attempt, + counterOfAdddedNodesForOneAttempt); + NodeContainer_delete((*badStatusNodes)); + (*badStatusNodes) = local_badStatusNodes; + attempt++; + } + return numberOfAllAddedNodes; +} + static void addNodes(NodesetLoader *loader, ServerContext *serverContext, NodesetLoader_Logger *logger) { @@ -548,21 +600,54 @@ static void addNodes(NodesetLoader *loader, ServerContext *serverContext, NODECLASS_REFERENCETYPE, NODECLASS_DATATYPE, NODECLASS_OBJECTTYPE, NODECLASS_VARIABLETYPE, NODECLASS_OBJECT, NODECLASS_METHOD, NODECLASS_VARIABLE, NODECLASS_VIEW}; + const size_t containerInitialSize = 100; + + // If we have a problem adding nodes to the server, let's add references + // to these nodes to the container. + NodeContainer* badStatusNodes = NodeContainer_new(containerInitialSize, false); + // Since every cycle we add one node class we need to save + // counter of previous badStatusNodes because badStatusNodes + // will always be adding new bad nodes to one list, and we have to calculate + // the real number of bad status nodes on every single cycle. + size_t previous_loop_badStatusNodes_size = 0; for (size_t i = 0; i < NL_NODECLASS_COUNT; i++) { const NL_NodeClass classToImport = order[i]; size_t cnt = - NodesetLoader_forEachNode(loader, classToImport, serverContext, + NodesetLoader_forEachNode_withBadStatusNodes(loader, classToImport, + serverContext, + badStatusNodes, (NodesetLoader_forEachNode_Func)addNodeImpl); if (classToImport == NODECLASS_DATATYPE) { importDataTypes(loader, ServerContext_getServerObject(serverContext)); } + + // Now we can see the nodes that could not be added and can calculate + // and show the actual nodes added. logger->log(logger->context, NODESETLOADER_LOGLEVEL_DEBUG, - "imported %ss: %zu", NL_NODECLASS_NAME[classToImport], cnt); + "imported %ss: %zu", NL_NODECLASS_NAME[classToImport], + cnt - (badStatusNodes->size - previous_loop_badStatusNodes_size)); + previous_loop_badStatusNodes_size = badStatusNodes->size; } + // second chance algorithm + if (badStatusNodes->size != 0) + { + logger->log(logger->context, NODESETLOADER_LOGLEVEL_WARNING, + "Couldn't import: %zu. Let's try importing with an invalid " + "parent node several times.", + badStatusNodes->size); + size_t numberOfAllAddedNodes = + secondChanceAddNodes(serverContext, &badStatusNodes, logger); + logger->log(logger->context, NODESETLOADER_LOGLEVEL_WARNING, + "imported after attempts: %zu", numberOfAllAddedNodes); + } + + // Delete only reference and container. Not NL_Nodes objects. + NodeContainer_delete(badStatusNodes); + for (size_t i = 0; i < NL_NODECLASS_COUNT; i++) { const NL_NodeClass classToImport = order[i]; diff --git a/include/NodesetLoader/NodesetLoader.h b/include/NodesetLoader/NodesetLoader.h index 4e70e5e9..322b5f1b 100644 --- a/include/NodesetLoader/NodesetLoader.h +++ b/include/NodesetLoader/NodesetLoader.h @@ -87,7 +87,8 @@ typedef struct NL_LocalizedText NL_LocalizedText; NL_Reference *hierachicalRefs; \ NL_Reference *nonHierachicalRefs; \ NL_Reference *unknownRefs; \ - void *extension; + void *extension; \ + UA_StatusCode processingResult; #define NL_NODE_INSTANCE_ATTRIBUTES UA_NodeId parentNodeId; @@ -256,6 +257,8 @@ typedef struct NL_FileContext NL_FileContext; struct NodesetLoader; typedef struct NodesetLoader NodesetLoader; +struct NodeContainer; + LOADER_EXPORT NodesetLoader *NodesetLoader_new(NodesetLoader_Logger *logger, struct NL_ReferenceService *refService); LOADER_EXPORT bool NodesetLoader_importFile(NodesetLoader *loader, @@ -268,6 +271,12 @@ typedef void (*NodesetLoader_forEachNode_Func)(void *context, NL_Node *node); LOADER_EXPORT size_t NodesetLoader_forEachNode(NodesetLoader *loader, NL_NodeClass nodeClass, void *context, NodesetLoader_forEachNode_Func fn); +LOADER_EXPORT size_t +NodesetLoader_forEachNode_withBadStatusNodes(NodesetLoader *loader, + NL_NodeClass nodeClass, + void *context, + struct NodeContainer *badStatusNodes, + NodesetLoader_forEachNode_Func fn); LOADER_EXPORT bool NodesetLoader_isInstanceNode (const NL_Node *baseNode); #ifdef __cplusplus } diff --git a/src/InternalRefService.c b/src/InternalRefService.c index 2682b7ac..1ae77938 100644 --- a/src/InternalRefService.c +++ b/src/InternalRefService.c @@ -31,6 +31,7 @@ NL_ReferenceTypeNode hierachicalRefs[MAX_HIERACHICAL_REFS] = { NULL, NULL, NULL, + UA_STATUSCODE_GOOD, {NULL, NULL}, NULL, }, @@ -44,6 +45,7 @@ NL_ReferenceTypeNode hierachicalRefs[MAX_HIERACHICAL_REFS] = { NULL, NULL, NULL, + UA_STATUSCODE_GOOD, {NULL, NULL}, NULL}, {NODECLASS_REFERENCETYPE, @@ -56,6 +58,7 @@ NL_ReferenceTypeNode hierachicalRefs[MAX_HIERACHICAL_REFS] = { NULL, NULL, NULL, + UA_STATUSCODE_GOOD, {NULL, NULL}, NULL}, {NODECLASS_REFERENCETYPE, @@ -68,6 +71,7 @@ NL_ReferenceTypeNode hierachicalRefs[MAX_HIERACHICAL_REFS] = { NULL, NULL, NULL, + UA_STATUSCODE_GOOD, {NULL, NULL}, NULL}, {NODECLASS_REFERENCETYPE, @@ -80,6 +84,7 @@ NL_ReferenceTypeNode hierachicalRefs[MAX_HIERACHICAL_REFS] = { NULL, NULL, NULL, + UA_STATUSCODE_GOOD, {NULL, NULL}, NULL}, {NODECLASS_REFERENCETYPE, @@ -92,6 +97,7 @@ NL_ReferenceTypeNode hierachicalRefs[MAX_HIERACHICAL_REFS] = { NULL, NULL, NULL, + UA_STATUSCODE_GOOD, {NULL, NULL}, NULL}, {NODECLASS_REFERENCETYPE, @@ -104,6 +110,7 @@ NL_ReferenceTypeNode hierachicalRefs[MAX_HIERACHICAL_REFS] = { NULL, NULL, NULL, + UA_STATUSCODE_GOOD, {NULL, NULL}, NULL}, {NODECLASS_REFERENCETYPE, @@ -116,6 +123,7 @@ NL_ReferenceTypeNode hierachicalRefs[MAX_HIERACHICAL_REFS] = { NULL, NULL, NULL, + UA_STATUSCODE_GOOD, {NULL, NULL}, NULL}, {NODECLASS_REFERENCETYPE, @@ -128,6 +136,7 @@ NL_ReferenceTypeNode hierachicalRefs[MAX_HIERACHICAL_REFS] = { NULL, NULL, NULL, + UA_STATUSCODE_GOOD, {NULL, NULL}, NULL}, }; diff --git a/src/Nodeset.c b/src/Nodeset.c index f531e271..a6fe30f7 100644 --- a/src/Nodeset.c +++ b/src/Nodeset.c @@ -641,3 +641,21 @@ size_t Nodeset_forEachNode(Nodeset *nodeset, NL_NodeClass nodeClass, } return c->size; } + +size_t Nodeset_forEachNode_withBadStatusNodes(Nodeset *nodeset, + NL_NodeClass nodeClass, + void *context, + NodeContainer *badStatusNodes, + NodesetLoader_forEachNode_Func fn) +{ + NodeContainer *c = nodeset->nodes[nodeClass]; + for (NL_Node **node = c->nodes; node != c->nodes + c->size; node++) + { + fn(context, *node); + if(UA_StatusCode_isBad(((NL_Node*)*node)->processingResult)) + { + NodeContainer_add(badStatusNodes, *node); + } + } + return c->size;; +} diff --git a/src/Nodeset.h b/src/Nodeset.h index bc67733f..c3db20c7 100644 --- a/src/Nodeset.h +++ b/src/Nodeset.h @@ -72,4 +72,9 @@ const NL_BiDirectionalReference * Nodeset_getBiDirectionalRefs(const Nodeset *nodeset); size_t Nodeset_forEachNode(Nodeset *nodeset, NL_NodeClass nodeClass, void *context, NodesetLoader_forEachNode_Func fn); +size_t Nodeset_forEachNode_withBadStatusNodes(Nodeset *nodeset, + NL_NodeClass nodeClass, + void *context, + struct NodeContainer *badStatusNodes, + NodesetLoader_forEachNode_Func fn); #endif diff --git a/src/NodesetLoader.c b/src/NodesetLoader.c index 4e80577b..2558e4cb 100644 --- a/src/NodesetLoader.c +++ b/src/NodesetLoader.c @@ -572,3 +572,13 @@ size_t NodesetLoader_forEachNode(NodesetLoader *loader, NL_NodeClass nodeClass, { return Nodeset_forEachNode(loader->nodeset, nodeClass, context, fn); } + +size_t +NodesetLoader_forEachNode_withBadStatusNodes(NodesetLoader *loader, + NL_NodeClass nodeClass, + void *context, + struct NodeContainer *badStatusNodes, + NodesetLoader_forEachNode_Func fn) +{ + return Nodeset_forEachNode_withBadStatusNodes(loader->nodeset, nodeClass, context, badStatusNodes, fn); +}