Skip to content

Commit

Permalink
Added SecondChance algorithm to allow you to retry importing nodes th…
Browse files Browse the repository at this point in the history 
…at could not be imported because they did not find their parents. This may be the case if we have a node type of Object which has its own parent node type of Variable. For example, this could be a HasHistoricalConfiguration reference type. Also, the status about how many nodes we have imported shows numbers that include nodes that were not imported due to some error.
  • Loading branch information
@xydan83
xydan83 committed Dec 8, 2023
1 parent fd60349 commit 7cd8fb5
Show file tree
Hide file tree
Showing 6 changed files with 165 additions and 29 deletions.
141 changes: 113 additions & 28 deletions backends/open62541/src/import.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include "conversion.h"
#include "NodesetLoader/NodesetLoader.h"
#include "RefServiceImpl.h"
#include "nodes/NodeContainer.h"

#include <assert.h>

Expand Down Expand Up @@ -115,7 +116,7 @@ static UA_NodeId getParentId(const NL_Node *node, UA_NodeId *parentRefId)
return parentId;
}

static void
static UA_StatusCode
handleObjectNode(const NL_ObjectNode *node, UA_NodeId *id,
const UA_NodeId *parentId, const UA_NodeId *parentReferenceId,
const UA_LocalizedText *lt, const UA_QualifiedName *qn,
Expand All @@ -134,13 +135,13 @@ handleObjectNode(const NL_ObjectNode *node, UA_NodeId *id,

// addNode_begin is used, otherwise all mandatory childs from type are
// instantiated
UA_Server_addNode_begin(server, UA_NODECLASS_OBJECT, *id, *parentId,
return UA_Server_addNode_begin(server, UA_NODECLASS_OBJECT, *id, *parentId,
*parentReferenceId, *qn, typeDefId, &oAttr,
&UA_TYPES[UA_TYPES_OBJECTATTRIBUTES],
node->extension, NULL);
}

static void
static UA_StatusCode
handleViewNode(const NL_ViewNode *node, UA_NodeId *id, const UA_NodeId *parentId,
const UA_NodeId *parentReferenceId, const UA_LocalizedText *lt,
const UA_QualifiedName *qn, const UA_LocalizedText *description,
Expand All @@ -151,11 +152,11 @@ handleViewNode(const NL_ViewNode *node, UA_NodeId *id, const UA_NodeId *parentId
attr.description = *description;
attr.eventNotifier = (UA_Byte)atoi(node->eventNotifier);
attr.containsNoLoops = isValTrue(node->containsNoLoops);
UA_Server_addViewNode(server, *id, *parentId, *parentReferenceId, *qn, attr,
return UA_Server_addViewNode(server, *id, *parentId, *parentReferenceId, *qn, attr,
node->extension, NULL);
}

static void
static UA_StatusCode
handleMethodNode(const NL_MethodNode *node, UA_NodeId *id,
const UA_NodeId *parentId, const UA_NodeId *parentReferenceId,
const UA_LocalizedText *lt, const UA_QualifiedName *qn,
Expand All @@ -167,7 +168,7 @@ handleMethodNode(const NL_MethodNode *node, UA_NodeId *id,
attr.displayName = *lt;
attr.description = *description;

UA_Server_addMethodNode(server, *id, *parentId, *parentReferenceId, *qn,
return UA_Server_addMethodNode(server, *id, *parentId, *parentReferenceId, *qn,
attr, NULL, 0, NULL, 0, NULL, node->extension,
NULL);
}
Expand Down Expand Up @@ -198,7 +199,7 @@ static size_t getArrayDimensions(const char *s, UA_UInt32 **dims)
return arrSize;
}

static void handleVariableNode(const NL_VariableNode *node, UA_NodeId *id,
static UA_StatusCode handleVariableNode(const NL_VariableNode *node, UA_NodeId *id,
const UA_NodeId *parentId,
const UA_NodeId *parentReferenceId,
const UA_LocalizedText *lt,
Expand Down Expand Up @@ -278,19 +279,20 @@ static void handleVariableNode(const NL_VariableNode *node, UA_NodeId *id,
}

//value is copied by open62541
UA_Server_addNode_begin(ServerContext_getServerObject(serverContext), UA_NODECLASS_VARIABLE, *id, *parentId,
UA_StatusCode Status = UA_Server_addNode_begin(ServerContext_getServerObject(serverContext), UA_NODECLASS_VARIABLE, *id, *parentId,
*parentReferenceId, *qn, typeDefId, &attr,
&UA_TYPES[UA_TYPES_VARIABLEATTRIBUTES],
node->extension, NULL);
//cannot call addNode finish, otherwise the nodes for e.g. range will be instantiated twice
//UA_Server_addNode_finish(server, *id);
UA_Variant_clear(&attr.value);

RawData_delete(data);
UA_free(attr.arrayDimensions);
return Status;
}

static void handleObjectTypeNode(const NL_ObjectTypeNode *node, UA_NodeId *id,
static UA_StatusCode handleObjectTypeNode(const NL_ObjectTypeNode *node, UA_NodeId *id,
const UA_NodeId *parentId,
const UA_NodeId *parentReferenceId,
const UA_LocalizedText *lt,
Expand All @@ -303,11 +305,11 @@ static void handleObjectTypeNode(const NL_ObjectTypeNode *node, UA_NodeId *id,
oAttr.isAbstract = isValTrue(node->isAbstract);
oAttr.description = *description;

UA_Server_addObjectTypeNode(server, *id, *parentId, *parentReferenceId, *qn,
return UA_Server_addObjectTypeNode(server, *id, *parentId, *parentReferenceId, *qn,
oAttr, node->extension, NULL);
}

static void handleReferenceTypeNode(const NL_ReferenceTypeNode *node,
static UA_StatusCode handleReferenceTypeNode(const NL_ReferenceTypeNode *node,
UA_NodeId *id, const UA_NodeId *parentId,
const UA_NodeId *parentReferenceId,
const UA_LocalizedText *lt,
Expand All @@ -322,11 +324,11 @@ static void handleReferenceTypeNode(const NL_ReferenceTypeNode *node,
attr.inverseName =
UA_LOCALIZEDTEXT(node->inverseName.locale, node->inverseName.text);

UA_Server_addReferenceTypeNode(server, *id, *parentId, *parentReferenceId,
return UA_Server_addReferenceTypeNode(server, *id, *parentId, *parentReferenceId,
*qn, attr, node->extension, NULL);
}

static void handleVariableTypeNode(const NL_VariableTypeNode *node, UA_NodeId *id,
static UA_StatusCode handleVariableTypeNode(const NL_VariableTypeNode *node, UA_NodeId *id,
const UA_NodeId *parentId,
const UA_NodeId *parentReferenceId,
const UA_LocalizedText *lt,
Expand All @@ -351,13 +353,13 @@ static void handleVariableTypeNode(const NL_VariableTypeNode *node, UA_NodeId *i
}
}

UA_Server_addNode_begin(server, UA_NODECLASS_VARIABLETYPE, *id, *parentId,
return UA_Server_addNode_begin(server, UA_NODECLASS_VARIABLETYPE, *id, *parentId,
*parentReferenceId, *qn, UA_NODEID_NULL, &attr,
&UA_TYPES[UA_TYPES_VARIABLETYPEATTRIBUTES],
node->extension, NULL);
}

static void handleDataTypeNode(const NL_DataTypeNode *node, UA_NodeId *id,
static UA_StatusCode handleDataTypeNode(const NL_DataTypeNode *node, UA_NodeId *id,
const UA_NodeId *parentId,
const UA_NodeId *parentReferenceId,
const UA_LocalizedText *lt,
Expand All @@ -370,11 +372,11 @@ static void handleDataTypeNode(const NL_DataTypeNode *node, UA_NodeId *id,
attr.description = *description;
attr.isAbstract = isValTrue(node->isAbstract);

UA_Server_addDataTypeNode(server, *id, *parentId, *parentReferenceId, *qn,
return UA_Server_addDataTypeNode(server, *id, *parentId, *parentReferenceId, *qn,
attr, node->extension, NULL);
}

static void addNodeImpl(ServerContext *serverContext, const NL_Node *node)
static void addNodeImpl(ServerContext *serverContext, NL_Node *node)
{
UA_NodeId id = node->id;
UA_NodeId parentReferenceId = UA_NODEID_NULL;
Expand All @@ -389,43 +391,43 @@ static void addNodeImpl(ServerContext *serverContext, const NL_Node *node)
switch (node->nodeClass)
{
case NODECLASS_OBJECT:
handleObjectNode((const NL_ObjectNode *)node, &id, &parentId,
node->processingResult = handleObjectNode((const NL_ObjectNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description, ServerContext_getServerObject(serverContext));
break;

case NODECLASS_METHOD:
handleMethodNode((const NL_MethodNode *)node, &id, &parentId,
node->processingResult = handleMethodNode((const NL_MethodNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description, ServerContext_getServerObject(serverContext));
break;

case NODECLASS_OBJECTTYPE:
handleObjectTypeNode((const NL_ObjectTypeNode *)node, &id, &parentId,
node->processingResult = handleObjectTypeNode((const NL_ObjectTypeNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description,
ServerContext_getServerObject(serverContext));
break;

case NODECLASS_REFERENCETYPE:
handleReferenceTypeNode((const NL_ReferenceTypeNode *)node, &id,
node->processingResult = handleReferenceTypeNode((const NL_ReferenceTypeNode *)node, &id,
&parentId, &parentReferenceId, &lt, &qn,
&description, ServerContext_getServerObject(serverContext));
break;

case NODECLASS_VARIABLETYPE:
handleVariableTypeNode((const NL_VariableTypeNode *)node, &id, &parentId,
node->processingResult = handleVariableTypeNode((const NL_VariableTypeNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description,
ServerContext_getServerObject(serverContext));
break;

case NODECLASS_VARIABLE:
handleVariableNode((const NL_VariableNode *)node, &id, &parentId,
node->processingResult = handleVariableNode((const NL_VariableNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description, serverContext);
break;
case NODECLASS_DATATYPE:
handleDataTypeNode((const NL_DataTypeNode *)node, &id, &parentId,
node->processingResult = handleDataTypeNode((const NL_DataTypeNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description, ServerContext_getServerObject(serverContext));
break;
case NODECLASS_VIEW:
handleViewNode((const NL_ViewNode *)node, &id, &parentId,
node->processingResult = handleViewNode((const NL_ViewNode *)node, &id, &parentId,
&parentReferenceId, &lt, &qn, &description, ServerContext_getServerObject(serverContext));
break;
}
Expand Down Expand Up @@ -541,28 +543,111 @@ static void addNonHierachicalRefs(UA_Server *server, NL_Node *node)
}
}

static size_t secondChanceAddNodes(ServerContext *serverContext,
NodeContainer **badStatusNodes,
const NodesetLoader_Logger *logger)
{
const size_t attemptsNum = 10;

size_t numberOfAllAddedNodes = 0;
// We will only add nodes with failed parent nodes in at most Num attempts.
// This gives us the opportunity to add a chain like Object(child) ->
// Variable(parent).
// For example, this reference is used in the HasHistoricalConfiguration
// reference type.
size_t attempt = 1;
while (attempt != attemptsNum && (*badStatusNodes)->size != 0)
{
NodeContainer *local_badStatusNodes =
NodeContainer_new((*badStatusNodes)->size, false);
size_t counterOfAdddedNodesForOneAttempt = 0;
for (size_t counter = 0; counter < (*badStatusNodes)->size; counter++)
{
// Try to add nodes only with the BADPARENTNODEIDINVALID status.
if ((*badStatusNodes)->nodes[counter]->processingResult ==
UA_STATUSCODE_BADPARENTNODEIDINVALID)
{
addNodeImpl(serverContext,
(*badStatusNodes)
->nodes[counter]); // Import to server again
if (UA_StatusCode_isBad(
(*badStatusNodes)->nodes[counter]->processingResult))
{
NodeContainer_add(local_badStatusNodes,
(*badStatusNodes)->nodes[counter]);
}
else
{
counterOfAdddedNodesForOneAttempt++;
}
}
}
numberOfAllAddedNodes += counterOfAdddedNodesForOneAttempt;
logger->log(logger->context, NODESETLOADER_LOGLEVEL_DEBUG,
"attempt (%zu), imported nodes: %zu", attempt,
counterOfAdddedNodesForOneAttempt);
NodeContainer_delete((*badStatusNodes));
(*badStatusNodes) = local_badStatusNodes;
attempt++;
}
return numberOfAllAddedNodes;
}

static void addNodes(NodesetLoader *loader, ServerContext *serverContext,
NodesetLoader_Logger *logger)
{
const NL_NodeClass order[NL_NODECLASS_COUNT] = {
NODECLASS_REFERENCETYPE, NODECLASS_DATATYPE, NODECLASS_OBJECTTYPE,
NODECLASS_VARIABLETYPE, NODECLASS_OBJECT, NODECLASS_METHOD,
NODECLASS_VARIABLE, NODECLASS_VIEW};
const size_t containerInitialSize = 100;


// If we have a problem adding nodes to the server, let's add references
// to these nodes to the container.
NodeContainer* badStatusNodes = NodeContainer_new(containerInitialSize, false);
// Since every cycle we add one node class we need to save
// counter of previous badStatusNodes because badStatusNodes
// will always be adding new bad nodes to one list, and we have to calculate
// the real number of bad status nodes on every single cycle.
size_t previous_loop_badStatusNodes_size = 0;
for (size_t i = 0; i < NL_NODECLASS_COUNT; i++)
{
const NL_NodeClass classToImport = order[i];
size_t cnt =
NodesetLoader_forEachNode(loader, classToImport, serverContext,
NodesetLoader_forEachNode_withBadStatusNodes(loader, classToImport,
serverContext,
badStatusNodes,
(NodesetLoader_forEachNode_Func)addNodeImpl);
if (classToImport == NODECLASS_DATATYPE)
{
importDataTypes(loader, ServerContext_getServerObject(serverContext));
}

// Now we can see the nodes that could not be added and can calculate
// and show the actual nodes added.
logger->log(logger->context, NODESETLOADER_LOGLEVEL_DEBUG,
"imported %ss: %zu", NL_NODECLASS_NAME[classToImport], cnt);
"imported %ss: %zu", NL_NODECLASS_NAME[classToImport],
cnt - (badStatusNodes->size - previous_loop_badStatusNodes_size));
previous_loop_badStatusNodes_size = badStatusNodes->size;
}

// second chance algorithm
if (badStatusNodes->size != 0)
{
logger->log(logger->context, NODESETLOADER_LOGLEVEL_WARNING,
"Couldn't import: %zu. Let's try importing with an invalid "
"parent node several times.",
badStatusNodes->size);
size_t numberOfAllAddedNodes =
secondChanceAddNodes(serverContext, &badStatusNodes, logger);
logger->log(logger->context, NODESETLOADER_LOGLEVEL_WARNING,
"imported after attempts: %zu", numberOfAllAddedNodes);
}

// Delete only reference and container. Not NL_Nodes objects.
NodeContainer_delete(badStatusNodes);

for (size_t i = 0; i < NL_NODECLASS_COUNT; i++)
{
const NL_NodeClass classToImport = order[i];
Expand Down
11 changes: 10 additions & 1 deletion include/NodesetLoader/NodesetLoader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,8 @@ typedef struct NL_LocalizedText NL_LocalizedText;
NL_Reference *hierachicalRefs; \
NL_Reference *nonHierachicalRefs; \
NL_Reference *unknownRefs; \
void *extension;
void *extension; \
UA_StatusCode processingResult;

#define NL_NODE_INSTANCE_ATTRIBUTES UA_NodeId parentNodeId;

Expand Down Expand Up @@ -256,6 +257,8 @@ typedef struct NL_FileContext NL_FileContext;
struct NodesetLoader;
typedef struct NodesetLoader NodesetLoader;

struct NodeContainer;

LOADER_EXPORT NodesetLoader *NodesetLoader_new(NodesetLoader_Logger *logger,
struct NL_ReferenceService *refService);
LOADER_EXPORT bool NodesetLoader_importFile(NodesetLoader *loader,
Expand All @@ -268,6 +271,12 @@ typedef void (*NodesetLoader_forEachNode_Func)(void *context, NL_Node *node);
LOADER_EXPORT size_t
NodesetLoader_forEachNode(NodesetLoader *loader, NL_NodeClass nodeClass,
void *context, NodesetLoader_forEachNode_Func fn);
LOADER_EXPORT size_t
NodesetLoader_forEachNode_withBadStatusNodes(NodesetLoader *loader,
NL_NodeClass nodeClass,
void *context,
struct NodeContainer *badStatusNodes,
NodesetLoader_forEachNode_Func fn);
LOADER_EXPORT bool NodesetLoader_isInstanceNode (const NL_Node *baseNode);
#ifdef __cplusplus
}
Expand Down
Loading

0 comments on commit 7cd8fb5

Please sign in to comment.