From e15c9233d5cf7eab21595a3d33aec3a6cf948235 Mon Sep 17 00:00:00 2001 From: Anjali-NEC Date: Tue, 13 Jun 2023 12:16:40 +0000 Subject: [PATCH 01/10] Updated code --- CMakeLists.txt | 2 +- src/app/contextBroker/contextBroker.cpp | 4 + src/lib/alarmMgr/AlarmManager.cpp | 75 +++++++++++++++++++ src/lib/alarmMgr/AlarmManager.h | 8 ++ src/lib/common/globals.cpp | 1 + src/lib/common/limits.h | 2 + src/lib/logSummary/logSummary.cpp | 1 + src/lib/ngsiNotify/QueueNotifier.cpp | 19 ++++- src/lib/ngsiNotify/QueueNotifier.h | 2 +- .../default_service_notif_queue.test | 14 ++-- 10 files changed, 118 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 694bbe0f8a..57ebc85d53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -221,7 +221,7 @@ SET (BOOST_MT # See http://mongoc.org/libmongoc/current/tutorial.html#cmake # This find_package() command provides the mongo::mongoc_static used in # SET for common static libs. We use 1.23.1 as reference version. -find_package (mongoc-1.0 1.23.1 EXACT) +find_package (mongoc-1.0 1.23.2 EXACT) # Static libs common to contextBroker and unitTest binaries SET (COMMON_STATIC_LIBS diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index 476ed64980..41adf748e4 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -159,6 +159,7 @@ bool mtenant; char allowedOrigin[64]; int maxAge; long dbTimeout; +//long thresholdMaxSize; long httpTimeout; long mqttTimeout; int dbPoolSize; @@ -253,6 +254,7 @@ int mqttMaxAge; #define REQ_POOL_SIZE "size of thread pool for incoming connections" #define IN_REQ_PAYLOAD_MAX_SIZE_DESC "maximum size (in bytes) of the payload of incoming requests" #define OUT_REQ_MSG_MAX_SIZE_DESC "maximum size (in bytes) of outgoing forward and notification request messages" +#define THRESHOLD_MAX_SIZE_DESC "Alarm for notification queue overpassing a given threshold" #define SIMULATED_NOTIF_DESC "simulate notifications instead of actual sending them (only for testing)" #define STAT_COUNTERS "enable request/notification counters statistics" #define STAT_SEM_WAIT "enable semaphore waiting time statistics" @@ -331,6 +333,8 @@ PaArgument paArgs[] = { "-inReqPayloadMaxSize", &inReqPayloadMaxSize, "IN_REQ_PAYLOAD_MAX_SIZE", PaULong, PaOpt, DEFAULT_IN_REQ_PAYLOAD_MAX_SIZE, 0, PaNL, IN_REQ_PAYLOAD_MAX_SIZE_DESC }, { "-outReqMsgMaxSize", &outReqMsgMaxSize, "OUT_REQ_MSG_MAX_SIZE", PaULong, PaOpt, DEFAULT_OUT_REQ_MSG_MAX_SIZE, 0, PaNL, OUT_REQ_MSG_MAX_SIZE_DESC }, + //{ "-thresholdMaxSize", &thresholdMaxSize, "THRESHOLD_MAX_SIZE", PaULong, PaOpt, 0, + // 0, PaNL, THRESHOLD_MAX_SIZE_DESC }, { "-notificationMode", ¬ificationMode, "NOTIF_MODE", PaString, PaOpt, _i "transient", PaNL, PaNL, NOTIFICATION_MODE_DESC }, { "-notifFlowControl", ¬ifFlowControl, "NOTIF_FLOW_CONTROL", PaString, PaOpt, _i "", PaNL, PaNL, FLOW_CONTROL_DESC }, diff --git a/src/lib/alarmMgr/AlarmManager.cpp b/src/lib/alarmMgr/AlarmManager.cpp index da84e66b79..4e9b857a4b 100644 --- a/src/lib/alarmMgr/AlarmManager.cpp +++ b/src/lib/alarmMgr/AlarmManager.cpp @@ -58,6 +58,8 @@ AlarmManager::AlarmManager() badInputResets(0), notificationErrors(0), notificationErrorResets(0), + notificationQueues(0), + notificationQueueResets(0), forwardingErrors(0), forwardingErrorResets(0), mqttConnectionErrors(0), @@ -66,6 +68,7 @@ AlarmManager::AlarmManager() dbErrorResets(0), dbOk(true), notificationErrorLogAlways(false), + notificationQueueLogAlways(false), forwardingErrorLogAlways(false), mqttConnectionErrorLogAlways(false), badInputLogAlways(false), @@ -82,6 +85,7 @@ AlarmManager::AlarmManager() int AlarmManager::init(bool logAlreadyRaisedAlarms) { notificationErrorLogAlways = logAlreadyRaisedAlarms; + notificationQueueLogAlways = logAlreadyRaisedAlarms; badInputLogAlways = logAlreadyRaisedAlarms; dbErrorLogAlways = logAlreadyRaisedAlarms; mqttConnectionErrorLogAlways = logAlreadyRaisedAlarms; @@ -246,6 +250,21 @@ void AlarmManager::notificationErrorGet(int64_t* active, int64_t* raised, int64_ } +/* **************************************************************************** +* +* AlarmManager::notificationErrorGet - +* +* NOTE +* To read values, no semaphore is used. +*/ +void AlarmManager::notificationQueueGet(int64_t* active, int64_t* raised, int64_t* released) +{ + *active = notificationQ.size(); + *raised = notificationQueues; + *released = notificationQueueResets; +} + + /* **************************************************************************** * @@ -587,6 +606,62 @@ bool AlarmManager::forwardingErrorReset(const std::string& url) return true; } +/*notification Queue*/ +bool AlarmManager::notificationQueue(const std::string& service, const std::string& details) +{ + semTake(); + + std::map::iterator iter = notificationQ.find(details); + + if (iter != notificationQ.end()) // Already exists - add to the 'url-specific' counter + { + iter->second += 1; + + if (notificationQueueLogAlways) + { + LM_W(("Repeated notificationQueue %s: %s", service.c_str(), details.c_str())); + } + else + { + // even if repeat alarms is off, this is a relevant event in debug level + LM_T(LmtCPrForwardRequestPayload, ("Repeated notificationQueue %s: %s", service.c_str(), details.c_str())); + } + + semGive(); + return false; + } + + ++notificationQueues; + + notificationQ[details] = 1; + semGive(); + + LM_W(("Raising alarm notificationQueue %s: %s", service.c_str(), details.c_str())); + + return true; +} + + + +/*notificationReset*/ +bool AlarmManager::notificationQueuesResets(const std::string& details) +{ + semTake(); + + if (notificationQ.find(details) == notificationQ.end()) // Doesn't exist + { + semGive(); + return false; + } + + notificationQ.erase(details); + ++notificationQueues; + semGive(); + + LM_W(("Releasing alarm notificationQueue %s", details.c_str())); + + return true; +} /* **************************************************************************** diff --git a/src/lib/alarmMgr/AlarmManager.h b/src/lib/alarmMgr/AlarmManager.h index 685eba17ed..b05010dc91 100644 --- a/src/lib/alarmMgr/AlarmManager.h +++ b/src/lib/alarmMgr/AlarmManager.h @@ -44,6 +44,8 @@ class AlarmManager int64_t badInputResets; int64_t notificationErrors; int64_t notificationErrorResets; + int64_t notificationQueues; + int64_t notificationQueueResets; int64_t forwardingErrors; int64_t forwardingErrorResets; int64_t mqttConnectionErrors; @@ -53,11 +55,13 @@ class AlarmManager bool dbOk; std::map notificationV; + std::map notificationQ; std::map forwardingErrorV; std::map badInputV; std::map mqttConnectionErrorV; bool notificationErrorLogAlways; + bool notificationQueueLogAlways; bool forwardingErrorLogAlways; bool mqttConnectionErrorLogAlways; bool badInputLogAlways; @@ -80,6 +84,9 @@ class AlarmManager bool notificationError(const std::string& url, const std::string& details); bool notificationErrorReset(const std::string& url); + bool notificationQueuesResets(const std::string& details); + bool notificationQueue(const std::string& url, const std::string& details); + bool forwardingError(const std::string& url, const std::string& details); bool forwardingErrorReset(const std::string& url); @@ -93,6 +100,7 @@ class AlarmManager void dbErrorsGet(bool* active, int64_t* raised, int64_t* released); void badInputGet(int64_t* active, int64_t* raised, int64_t* released); void notificationErrorGet(int64_t* active, int64_t* raised, int64_t* released); + void notificationQueueGet(int64_t* active, int64_t* raised, int64_t* released); void forwardingErrorGet(int64_t* active, int64_t* raised, int64_t* released); void mqttConnectionErrorGet(int64_t* active, int64_t* raised, int64_t* released); diff --git a/src/lib/common/globals.cpp b/src/lib/common/globals.cpp index 511dbd9d0d..c379efb252 100644 --- a/src/lib/common/globals.cpp +++ b/src/lib/common/globals.cpp @@ -57,6 +57,7 @@ bool notifQueueStatistics = false; bool checkIdv1 = false; unsigned long long inReqPayloadMaxSize = DEFAULT_IN_REQ_PAYLOAD_MAX_SIZE; unsigned long long outReqMsgMaxSize = DEFAULT_OUT_REQ_MSG_MAX_SIZE; +unsigned long long thresholdMaxSize = DEFAULT_THRESHOLD_MAX_SIZE; diff --git a/src/lib/common/limits.h b/src/lib/common/limits.h index 39295ec64d..b43511f057 100644 --- a/src/lib/common/limits.h +++ b/src/lib/common/limits.h @@ -112,6 +112,8 @@ #define DEFAULT_OUT_REQ_MSG_MAX_SIZE (8 * 1024 * 1024) // 8 MB default max size of any outgoing request message (see CLI -outReqMsgMaxSize) +#define DEFAULT_THRESHOLD_MAX_SIZE 5 + /* **************************************************************************** * diff --git a/src/lib/logSummary/logSummary.cpp b/src/lib/logSummary/logSummary.cpp index d566f122c0..165c00d034 100644 --- a/src/lib/logSummary/logSummary.cpp +++ b/src/lib/logSummary/logSummary.cpp @@ -114,6 +114,7 @@ static void* logSummary(void* vP) alarmMgr.dbErrorsGet(&deActive, &deRaised, &deReleased); alarmMgr.badInputGet(&biActive, &biRaised, &biReleased); alarmMgr.notificationErrorGet(&neActive, &neRaised, &neReleased); + alarmMgr.notificationQueueGet(&neActive, &neRaised, &neReleased); alarmMgr.forwardingErrorGet(&fwdActive, &fwdRaised, &fwdReleased); alarmMgr.mqttConnectionErrorGet(&mceActive, &mceRaised, &mceReleased); diff --git a/src/lib/ngsiNotify/QueueNotifier.cpp b/src/lib/ngsiNotify/QueueNotifier.cpp index 71a8e47ae2..4e36b14edf 100644 --- a/src/lib/ngsiNotify/QueueNotifier.cpp +++ b/src/lib/ngsiNotify/QueueNotifier.cpp @@ -31,7 +31,7 @@ #include "ngsiNotify/QueueStatistics.h" #include "ngsiNotify/QueueNotifier.h" - +#include "common/globals.h" /* **************************************************************************** @@ -153,6 +153,8 @@ void QueueNotifier::sendNotifyContextRequest // Early return if some problem occurred with params building // Nothing is added to the queue in this case + // + if (paramsP == NULL) { return; @@ -165,6 +167,7 @@ void QueueNotifier::sendNotifyContextRequest std::map::iterator iter = serviceSq.find(nsf.tenant); std::string queueName; +// std::string out; if (iter != serviceSq.end()) { queueName = nsf.tenant; @@ -176,7 +179,21 @@ void QueueNotifier::sendNotifyContextRequest sq = &defaultSq; } + std::string details; + extern int thresholdMaxSize; + + details = "Notification queue exceed the thresholdMaxSize limit"; bool enqueued = sq->try_push(paramsP); + + if (QueueStatistics::getOut() <= thresholdMaxSize) + { + alarmMgr.notificationQueue(service, "notification queue is full: " + details); + } + else + { + alarmMgr.notificationQueuesResets(service); + } + if (!enqueued) { QueueStatistics::incReject(1); diff --git a/src/lib/ngsiNotify/QueueNotifier.h b/src/lib/ngsiNotify/QueueNotifier.h index bb8f1e7f20..bee60baaf4 100644 --- a/src/lib/ngsiNotify/QueueNotifier.h +++ b/src/lib/ngsiNotify/QueueNotifier.h @@ -66,7 +66,7 @@ class QueueNotifier : public Notifier const std::vector& attrsFilter, bool blacklist, bool covered, - const std::vector& metadataFilter); + const std::vector& metadataFilter); int start(); size_t queueSize(const std::string& service); void release(); diff --git a/test/functionalTest/cases/3843_per_service_notif_queue/default_service_notif_queue.test b/test/functionalTest/cases/3843_per_service_notif_queue/default_service_notif_queue.test index f0ee2ff1bf..8c6c41f3a9 100644 --- a/test/functionalTest/cases/3843_per_service_notif_queue/default_service_notif_queue.test +++ b/test/functionalTest/cases/3843_per_service_notif_queue/default_service_notif_queue.test @@ -60,7 +60,7 @@ payload='{ }, "notification": { "http": { - "url": "http://localhost:'$LISTENER_PORT'/waitForever" + "url": "http://localhost:'$LISTENER_PORT'/notify" } } }' @@ -83,7 +83,7 @@ payload='{ }, "notification": { "http": { - "url": "http://localhost:'$LISTENER_PORT'/waitForever" + "url": "http://localhost:'$LISTENER_PORT'/notify" } } }' @@ -106,7 +106,7 @@ payload='{ }, "notification": { "http": { - "url": "http://localhost:'$LISTENER_PORT'/waitForever" + "url": "http://localhost:'$LISTENER_PORT'/notify" } } }' @@ -125,7 +125,7 @@ payload='{ "type": "Number" } }' -for i in {1..3} +for i in {1..50} do echo $i orionCurl --url '/v2/entities?options=upsert,forcedUpdate' --payload "$payload" --tenant serv1 > /dev/null @@ -136,7 +136,7 @@ echo echo "05. Create/update entity in serv2 4 times, causing 2 ERRORs" echo "===========================================================" -for i in {1..4} +for i in {1..470} do echo $i orionCurl --url '/v2/entities?options=upsert,forcedUpdate' --payload "$payload" --tenant serv2 > /dev/null @@ -147,7 +147,7 @@ echo echo "06. Create/update entity in serv3 5 times, causing 5 ERRORs" echo "===========================================================" -for i in {1..5} +for i in {1..500} do echo $i orionCurl --url '/v2/entities?options=upsert,forcedUpdate' --payload "$payload" --tenant serv3 > /dev/null @@ -158,7 +158,7 @@ echo echo "07. Grep log for ERROR, getting 7 in the default service" echo "========================================================" -grep ERROR /tmp/contextBroker.log | grep "Runtime Error" | awk -F 'Runtime Error ' '{print $2}' +grep ERROR /tmp/contextBroker.log | grep "WARN" echo echo From e6f9388806385dbf637a2c8da1ce53c6ed3c2fd5 Mon Sep 17 00:00:00 2001 From: Anjali-NEC Date: Wed, 14 Jun 2023 19:32:54 +0000 Subject: [PATCH 02/10] Updated code --- src/app/contextBroker/contextBroker.cpp | 5 +--- src/lib/alarmMgr/AlarmManager.cpp | 8 +++--- src/lib/common/limits.h | 2 +- src/lib/ngsiNotify/QueueNotifier.cpp | 25 +++++++++++-------- .../default_service_notif_queue.test | 6 ++--- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index dadce98181..ba48358efa 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -159,7 +159,6 @@ bool mtenant; char allowedOrigin[64]; int maxAge; long dbTimeout; -//long thresholdMaxSize; long httpTimeout; long mqttTimeout; int dbPoolSize; @@ -277,6 +276,7 @@ bool logDeprecate; #define INSECURE_NOTIF_DESC "allow HTTPS notifications to peers which certificate cannot be authenticated with known CA certificates" #define NGSIV1_AUTOCAST_DESC "automatic cast for number, booleans and dates in NGSIv1 update/create attribute operations" #define MQTT_MAX_AGE_DESC "max time (in minutes) that an unused MQTT connection is kept, default: 60" +//#define MAX_THRESHOLD_DESC "max time (in minutes) that an unused MQTT connection is kept, default: 60" #define LOG_DEPRECATE_DESC "log deprecation usages as warnings" @@ -338,9 +338,6 @@ PaArgument paArgs[] = { "-inReqPayloadMaxSize", &inReqPayloadMaxSize, "IN_REQ_PAYLOAD_MAX_SIZE", PaULong, PaOpt, DEFAULT_IN_REQ_PAYLOAD_MAX_SIZE, 0, PaNL, IN_REQ_PAYLOAD_MAX_SIZE_DESC }, { "-outReqMsgMaxSize", &outReqMsgMaxSize, "OUT_REQ_MSG_MAX_SIZE", PaULong, PaOpt, DEFAULT_OUT_REQ_MSG_MAX_SIZE, 0, PaNL, OUT_REQ_MSG_MAX_SIZE_DESC }, - //{ "-thresholdMaxSize", &thresholdMaxSize, "THRESHOLD_MAX_SIZE", PaULong, PaOpt, 0, - // 0, PaNL, THRESHOLD_MAX_SIZE_DESC }, - { "-notificationMode", ¬ificationMode, "NOTIF_MODE", PaString, PaOpt, _i "transient", PaNL, PaNL, NOTIFICATION_MODE_DESC }, { "-notifFlowControl", ¬ifFlowControl, "NOTIF_FLOW_CONTROL", PaString, PaOpt, _i "", PaNL, PaNL, FLOW_CONTROL_DESC }, { "-simulatedNotification", &simulatedNotification, "DROP_NOTIF", PaBool, PaOpt, false, false, true, SIMULATED_NOTIF_DESC }, diff --git a/src/lib/alarmMgr/AlarmManager.cpp b/src/lib/alarmMgr/AlarmManager.cpp index 4e9b857a4b..2ff428ccf0 100644 --- a/src/lib/alarmMgr/AlarmManager.cpp +++ b/src/lib/alarmMgr/AlarmManager.cpp @@ -619,12 +619,12 @@ bool AlarmManager::notificationQueue(const std::string& service, const std::stri if (notificationQueueLogAlways) { - LM_W(("Repeated notificationQueue %s: %s", service.c_str(), details.c_str())); + LM_W(("Repeated NotificationQueue %s: %s", service.c_str(), details.c_str())); } else { // even if repeat alarms is off, this is a relevant event in debug level - LM_T(LmtCPrForwardRequestPayload, ("Repeated notificationQueue %s: %s", service.c_str(), details.c_str())); + LM_T(LmtCPrForwardRequestPayload, ("Repeated NotificationQueue %s: %s", service.c_str(), details.c_str())); } semGive(); @@ -636,7 +636,7 @@ bool AlarmManager::notificationQueue(const std::string& service, const std::stri notificationQ[details] = 1; semGive(); - LM_W(("Raising alarm notificationQueue %s: %s", service.c_str(), details.c_str())); + LM_W(("Raising alarm NotificationQueue %s: %s", service.c_str(), details.c_str())); return true; } @@ -658,7 +658,7 @@ bool AlarmManager::notificationQueuesResets(const std::string& details) ++notificationQueues; semGive(); - LM_W(("Releasing alarm notificationQueue %s", details.c_str())); + LM_W(("Releasing alarm NotificationQueue %s:", details.c_str())); return true; } diff --git a/src/lib/common/limits.h b/src/lib/common/limits.h index b43511f057..e7c0c8e80d 100644 --- a/src/lib/common/limits.h +++ b/src/lib/common/limits.h @@ -112,7 +112,7 @@ #define DEFAULT_OUT_REQ_MSG_MAX_SIZE (8 * 1024 * 1024) // 8 MB default max size of any outgoing request message (see CLI -outReqMsgMaxSize) -#define DEFAULT_THRESHOLD_MAX_SIZE 5 +#define DEFAULT_THRESHOLD_MAX_SIZE 3 /* **************************************************************************** diff --git a/src/lib/ngsiNotify/QueueNotifier.cpp b/src/lib/ngsiNotify/QueueNotifier.cpp index 4e36b14edf..9b0bccc92a 100644 --- a/src/lib/ngsiNotify/QueueNotifier.cpp +++ b/src/lib/ngsiNotify/QueueNotifier.cpp @@ -167,7 +167,7 @@ void QueueNotifier::sendNotifyContextRequest std::map::iterator iter = serviceSq.find(nsf.tenant); std::string queueName; -// std::string out; + if (iter != serviceSq.end()) { queueName = nsf.tenant; @@ -179,19 +179,22 @@ void QueueNotifier::sendNotifyContextRequest sq = &defaultSq; } - std::string details; - extern int thresholdMaxSize; - - details = "Notification queue exceed the thresholdMaxSize limit"; bool enqueued = sq->try_push(paramsP); - if (QueueStatistics::getOut() <= thresholdMaxSize) - { - alarmMgr.notificationQueue(service, "notification queue is full: " + details); - } - else + if (enqueued) { - alarmMgr.notificationQueuesResets(service); + std::string details; + extern int thresholdMaxSize; + details = "(Max threshold limit: 3)"; + + if (QueueStatistics::getIn() >= thresholdMaxSize) + { + alarmMgr.notificationQueue(service, "notification queue reached the threshold " + details); + } + else + { + alarmMgr.notificationQueuesResets(service); + } } if (!enqueued) diff --git a/test/functionalTest/cases/3843_per_service_notif_queue/default_service_notif_queue.test b/test/functionalTest/cases/3843_per_service_notif_queue/default_service_notif_queue.test index 10946c5b66..0308f1a3ff 100644 --- a/test/functionalTest/cases/3843_per_service_notif_queue/default_service_notif_queue.test +++ b/test/functionalTest/cases/3843_per_service_notif_queue/default_service_notif_queue.test @@ -125,7 +125,7 @@ payload='{ "type": "Number" } }' -for i in {1..50} +for i in {1..3} do echo $i orionCurl --url '/v2/entities?options=upsert,forcedUpdate' --payload "$payload" --tenant serv1 > /dev/null @@ -136,7 +136,7 @@ echo echo "05. Create/update entity in serv2 4 times, causing 2 ERRORs" echo "===========================================================" -for i in {1..470} +for i in {1..4} do echo $i orionCurl --url '/v2/entities?options=upsert,forcedUpdate' --payload "$payload" --tenant serv2 > /dev/null @@ -147,7 +147,7 @@ echo echo "06. Create/update entity in serv3 5 times, causing 5 ERRORs" echo "===========================================================" -for i in {1..500} +for i in {1..5} do echo $i orionCurl --url '/v2/entities?options=upsert,forcedUpdate' --payload "$payload" --tenant serv3 > /dev/null From 4a02af98724773d9fa19aec45d62d57d4894a6e0 Mon Sep 17 00:00:00 2001 From: Anjali-NEC Date: Wed, 14 Jun 2023 20:09:14 +0000 Subject: [PATCH 03/10] Updated code --- src/app/contextBroker/contextBroker.cpp | 3 +- src/lib/alarmMgr/AlarmManager.cpp | 20 +- src/lib/ngsiNotify/QueueNotifier.cpp | 5 +- src/lib/ngsiNotify/QueueNotifier.h | 2 +- .../default_service_notif_queue.test | 8 +- ...n_queue_overpassing_a_given_threshold.test | 230 ++++++++++++++++++ 6 files changed, 253 insertions(+), 15 deletions(-) create mode 100644 test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold.test diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index ba48358efa..25908db8e3 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -256,7 +256,7 @@ bool logDeprecate; #define REQ_POOL_SIZE "size of thread pool for incoming connections" #define IN_REQ_PAYLOAD_MAX_SIZE_DESC "maximum size (in bytes) of the payload of incoming requests" #define OUT_REQ_MSG_MAX_SIZE_DESC "maximum size (in bytes) of outgoing forward and notification request messages" -#define THRESHOLD_MAX_SIZE_DESC "Alarm for notification queue overpassing a given threshold" +#define THRESHOLD_MAX_SIZE_DESC "maximum threshold limit for notificationQueue" #define SIMULATED_NOTIF_DESC "simulate notifications instead of actual sending them (only for testing)" #define STAT_COUNTERS "enable request/notification counters statistics" #define STAT_SEM_WAIT "enable semaphore waiting time statistics" @@ -335,7 +335,6 @@ PaArgument paArgs[] = { "-connectionMemory", &connectionMemory, "CONN_MEMORY", PaUInt, PaOpt, 64, 0, 1024, CONN_MEMORY_DESC }, { "-maxConnections", &maxConnections, "MAX_CONN", PaUInt, PaOpt, 1020, 1, PaNL, MAX_CONN_DESC }, { "-reqPoolSize", &reqPoolSize, "TRQ_POOL_SIZE", PaUInt, PaOpt, 0, 0, 1024, REQ_POOL_SIZE }, - { "-inReqPayloadMaxSize", &inReqPayloadMaxSize, "IN_REQ_PAYLOAD_MAX_SIZE", PaULong, PaOpt, DEFAULT_IN_REQ_PAYLOAD_MAX_SIZE, 0, PaNL, IN_REQ_PAYLOAD_MAX_SIZE_DESC }, { "-outReqMsgMaxSize", &outReqMsgMaxSize, "OUT_REQ_MSG_MAX_SIZE", PaULong, PaOpt, DEFAULT_OUT_REQ_MSG_MAX_SIZE, 0, PaNL, OUT_REQ_MSG_MAX_SIZE_DESC }, { "-notificationMode", ¬ificationMode, "NOTIF_MODE", PaString, PaOpt, _i "transient", PaNL, PaNL, NOTIFICATION_MODE_DESC }, diff --git a/src/lib/alarmMgr/AlarmManager.cpp b/src/lib/alarmMgr/AlarmManager.cpp index 2ff428ccf0..efa1507d8e 100644 --- a/src/lib/alarmMgr/AlarmManager.cpp +++ b/src/lib/alarmMgr/AlarmManager.cpp @@ -252,7 +252,7 @@ void AlarmManager::notificationErrorGet(int64_t* active, int64_t* raised, int64_ /* **************************************************************************** * -* AlarmManager::notificationErrorGet - +* AlarmManager::notificationQueueGet - * * NOTE * To read values, no semaphore is used. @@ -606,14 +606,21 @@ bool AlarmManager::forwardingErrorReset(const std::string& url) return true; } -/*notification Queue*/ + + +/* **************************************************************************** +* +* AlarmManager::notificationQueue - +* +* Returns false if no effective alarm transition occurs, otherwise, true is returned. +*/ bool AlarmManager::notificationQueue(const std::string& service, const std::string& details) { semTake(); std::map::iterator iter = notificationQ.find(details); - if (iter != notificationQ.end()) // Already exists - add to the 'url-specific' counter + if (iter != notificationQ.end()) { iter->second += 1; @@ -643,7 +650,12 @@ bool AlarmManager::notificationQueue(const std::string& service, const std::stri -/*notificationReset*/ +/* **************************************************************************** +* +* AlarmManager::notificationQueuesResets - +* +* Returns false if no effective alarm transition occurs, otherwise, true is returned. +*/ bool AlarmManager::notificationQueuesResets(const std::string& details) { semTake(); diff --git a/src/lib/ngsiNotify/QueueNotifier.cpp b/src/lib/ngsiNotify/QueueNotifier.cpp index 9b0bccc92a..e5d42a9c25 100644 --- a/src/lib/ngsiNotify/QueueNotifier.cpp +++ b/src/lib/ngsiNotify/QueueNotifier.cpp @@ -31,7 +31,7 @@ #include "ngsiNotify/QueueStatistics.h" #include "ngsiNotify/QueueNotifier.h" -#include "common/globals.h" + /* **************************************************************************** @@ -153,8 +153,6 @@ void QueueNotifier::sendNotifyContextRequest // Early return if some problem occurred with params building // Nothing is added to the queue in this case - // - if (paramsP == NULL) { return; @@ -167,7 +165,6 @@ void QueueNotifier::sendNotifyContextRequest std::map::iterator iter = serviceSq.find(nsf.tenant); std::string queueName; - if (iter != serviceSq.end()) { queueName = nsf.tenant; diff --git a/src/lib/ngsiNotify/QueueNotifier.h b/src/lib/ngsiNotify/QueueNotifier.h index bee60baaf4..bb8f1e7f20 100644 --- a/src/lib/ngsiNotify/QueueNotifier.h +++ b/src/lib/ngsiNotify/QueueNotifier.h @@ -66,7 +66,7 @@ class QueueNotifier : public Notifier const std::vector& attrsFilter, bool blacklist, bool covered, - const std::vector& metadataFilter); + const std::vector& metadataFilter); int start(); size_t queueSize(const std::string& service); void release(); diff --git a/test/functionalTest/cases/3843_per_service_notif_queue/default_service_notif_queue.test b/test/functionalTest/cases/3843_per_service_notif_queue/default_service_notif_queue.test index 0308f1a3ff..551899ccb4 100644 --- a/test/functionalTest/cases/3843_per_service_notif_queue/default_service_notif_queue.test +++ b/test/functionalTest/cases/3843_per_service_notif_queue/default_service_notif_queue.test @@ -60,7 +60,7 @@ payload='{ }, "notification": { "http": { - "url": "http://localhost:'$LISTENER_PORT'/notify" + "url": "http://localhost:'$LISTENER_PORT'/waitForever" } } }' @@ -83,7 +83,7 @@ payload='{ }, "notification": { "http": { - "url": "http://localhost:'$LISTENER_PORT'/notify" + "url": "http://localhost:'$LISTENER_PORT'/waitForever" } } }' @@ -106,7 +106,7 @@ payload='{ }, "notification": { "http": { - "url": "http://localhost:'$LISTENER_PORT'/notify" + "url": "http://localhost:'$LISTENER_PORT'/waitForever" } } }' @@ -158,7 +158,7 @@ echo echo "07. Grep log for ERROR, getting 7 in the default service" echo "========================================================" -grep ERROR /tmp/contextBroker.log | grep "WARN" +grep ERROR /tmp/contextBroker.log | grep "Runtime Error" | awk -F 'Runtime Error ' '{print $2}' echo echo diff --git a/test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold.test b/test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold.test new file mode 100644 index 0000000000..572801f225 --- /dev/null +++ b/test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold.test @@ -0,0 +1,230 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +alarm for notification queue overpassing a given threshold + +--SHELL-INIT-- +dbInit ${CB_DB_NAME} serv1 +dbInit ${CB_DB_NAME} serv2 +dbInit ${CB_DB_NAME} serv3 +brokerStart CB 0 IPv4 -multiservice -notificationMode threadpool:3:2 +accumulatorStart --pretty-print localhost $LISTENER_PORT + +--SHELL-- + +# +# 01. Subscribe serv1 to the accumulator endpoint that never responses +# 02. Subscribe serv2 to the accumulator endpoint that never responses +# 03. Subscribe serv3 to the accumulator endpoint that never responses +# 04. Create/update entity in serv1 3 times +# 05. Create/update entity in serv2 4 times +# 06. Create/update entity in serv3 5 times +# 07. Grep log for notificationQueue alarm +# + + +echo '01. Subscribe serv1 to the accumulator endpoint that never responses' +echo '====================================================================' +payload='{ + "description": "serv1 subscriptions", + "subject": { + "entities": [ + { + "id": "E", + "type": "T" + } + ] + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/waitForever" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" --tenant serv1 +echo +echo + + +echo '02. Subscribe serv2 to the accumulator endpoint that never responses' +echo '====================================================================' +payload='{ + "description": "serv2 subscriptions", + "subject": { + "entities": [ + { + "id": "E", + "type": "T" + } + ] + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" --tenant serv2 +echo +echo + + +echo '03. Subscribe serv3 to the accumulator endpoint that never responses' +echo '====================================================================' +payload='{ + "description": "serv3 subscriptions", + "subject": { + "entities": [ + { + "id": "E", + "type": "T" + } + ] + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/notify" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" --tenant serv3 +echo +echo + + +echo "04. Create/update entity in serv1 3 times" +echo "=========================================" +payload='{ + "id": "E", + "type": "T", + "A": { + "value": 1, + "type": "Number" + } +}' +for i in {1..3} +do + echo $i + orionCurl --url '/v2/entities?options=upsert,forcedUpdate' --payload "$payload" --tenant serv1 > /dev/null +done +echo +echo + + +echo "05. Create/update entity in serv2 4 times" +echo "=========================================" +for i in {1..4} +do + echo $i + orionCurl --url '/v2/entities?options=upsert,forcedUpdate' --payload "$payload" --tenant serv2 > /dev/null +done +echo +echo + + +echo "06. Create/update entity in serv3 5 times" +echo "=========================================" +for i in {1..5} +do + echo $i + orionCurl --url '/v2/entities?options=upsert,forcedUpdate' --payload "$payload" --tenant serv3 > /dev/null +done +echo +echo + + +echo "07. Grep log for notificationQueue alarm" +echo "========================================" +egrep 'Raising alarm|Releasing alarm' /tmp/contextBroker.log | awk -F'|' '{ print $10 }' | awk -F= '{ print $2 }' +echo +echo + + +accumulatorStop $LISTENER_PORT + + +--REGEXPECT-- +01. Subscribe serv1 to the accumulator endpoint that never responses +==================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Subscribe serv2 to the accumulator endpoint that never responses +==================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03. Subscribe serv3 to the accumulator endpoint that never responses +==================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +04. Create/update entity in serv1 3 times +========================================= +1 +2 +3 + + +05. Create/update entity in serv2 4 times +========================================= +1 +2 +3 +4 + + +06. Create/update entity in serv3 5 times +========================================= +1 +2 +3 +4 +5 + + +07. Grep log for notificationQueue alarm +======================================== +Raising alarm NotificationQueue serv2: notification queue reached the threshold (Max threshold limit: 3) + + +--TEARDOWN-- +brokerStop CB +dbDrop ${CB_DB_NAME} serv1 +dbDrop ${CB_DB_NAME} serv2 +dbDrop ${CB_DB_NAME} serv3 From 406d42349a3154e046b8f7596e55ae3d94ca7c4f Mon Sep 17 00:00:00 2001 From: Anjali-NEC Date: Wed, 14 Jun 2023 20:12:06 +0000 Subject: [PATCH 04/10] Updated file --- CHANGES_NEXT_RELEASE | 1 + CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 528af55d4c..0c9dc37103 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1 +1,2 @@ +- Fix: Alarm for notification queue overpassing a given threshold (#4113) - Fix: logDeprecate not working correctly (`geo:json` wrongly considered as deprecated) diff --git a/CMakeLists.txt b/CMakeLists.txt index 57ebc85d53..694bbe0f8a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -221,7 +221,7 @@ SET (BOOST_MT # See http://mongoc.org/libmongoc/current/tutorial.html#cmake # This find_package() command provides the mongo::mongoc_static used in # SET for common static libs. We use 1.23.1 as reference version. -find_package (mongoc-1.0 1.23.2 EXACT) +find_package (mongoc-1.0 1.23.1 EXACT) # Static libs common to contextBroker and unitTest binaries SET (COMMON_STATIC_LIBS From 51a71bbd8113d91b0c1858d8f1f1aada294a61be Mon Sep 17 00:00:00 2001 From: Anjali-NEC Date: Wed, 5 Jul 2023 04:17:34 +0000 Subject: [PATCH 05/10] Updated code as per comment --- CHANGES_NEXT_RELEASE | 2 +- doc/manuals/admin/logs.md | 1 + src/app/contextBroker/contextBroker.cpp | 15 +++++++++++++-- src/lib/common/globals.cpp | 1 - src/lib/common/limits.h | 2 -- src/lib/ngsiNotify/QueueNotifier.cpp | 14 ++++++++------ ...ation_queue_overpassing_a_given_threshold.test | 8 ++++---- 7 files changed, 27 insertions(+), 16 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 0c9dc37103..f1c73a9f48 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,2 @@ -- Fix: Alarm for notification queue overpassing a given threshold (#4113) +- Fix: alarm for notification queue overpassing a given threshold (#4113) - Fix: logDeprecate not working correctly (`geo:json` wrongly considered as deprecated) diff --git a/doc/manuals/admin/logs.md b/doc/manuals/admin/logs.md index 192e835dbe..6553757bdd 100644 --- a/doc/manuals/admin/logs.md +++ b/doc/manuals/admin/logs.md @@ -233,6 +233,7 @@ Alarm conditions: | 5 | WARNING | The following WARN text appears in the 'msg' field: "Raising alarm NotificationError ``: ``". | The following WARN text appears in the 'msg' field: "Releasing alarm NotificationError ``", where `` is the same one that triggered the alarm. Orion prints this trace when it successfully sent a notification to that URL. | Notification Failure. The ``text contains the detailed information. | Orion is trying to send the HTTP notification to a given receiver and some problem has occurred. It could be due to a problem with the network connectivity or on the receiver, e.g. the receiver is down. In the second case, the owner of the receiver of the notification should be reported. No specific action has to be performed at Orion Context Broker service. | 6 | WARNING | The following WARN text appears in the 'msg' field: "Raising alarm ForwardingError ``": ``". | The following WARN text appears in the 'msg' field: "Releasing alarm ForwardingError ``", where `` is the same one that triggered the alarm. Orion prints this trace when it successfully interact with ContextProvider to that URL.| Forwarding Error. The ``text contains the detailed information. | Orion is trying to interact with ContextProvider and some problem has occurred. It may be due to context provider response for forwarded query or update is empty. No specific action has to be performed at Orion Context Broker service. | 7 | WARNING | The following WARN text appears in the 'msg' field: "Raising alarm MqttConnectionError ``": ``". | The following WARN text appears in the 'msg' field: "Releasing alarm MqttConnectionError ``", where `` is the same one that triggered the alarm. Orion prints this trace when it successfully interact with ContextProvider to that URL.| Error connection to MQTT broker. The ``text contains the detailed information. | Orion is trying to connecto to an MQTT broker (associated to a subscription) and some problem has occurred. It may be due to several reasons: MQTT broker in unreachable, user/pass is wrong, etc. No specific action has to be performed at Orion Context Broker service, but maybe in the MQTT broker configuration or in the associated subscription. +| 8 | WARNING | The following WARN text appears in the 'msg' field: "Raising alarm NotificaitonQueue ``": ``". | The following WARN text appears in the 'msg' field: "Releasing alarm NotificaitonQueue ``", where `` is the same one that triggered the alarm. Orion prints this trace when notification queue goes back below the threshold. | The notification queue associated to the service (or ``" "default" for default queue) has overpassed the alarm threshold. The ``" text described the particular threshold. | No specific action has to be performed at Orion Context Broker service, but the update flow causing the notification on that service (or default queue) should be lowered in order to reduce pressure on queue. Another possible problem is due to malfunctioning notification receivers, if they are slow processing notifications and responding to Orion. By default, Orion only traces the origin (i.e. raising) and end (i.e. releasing) of an alarm, e.g: diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index 25908db8e3..59ff306550 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -206,6 +206,7 @@ unsigned long fcMaxInterval; int mqttMaxAge; bool logDeprecate; +long int maxthreshold; @@ -256,7 +257,6 @@ bool logDeprecate; #define REQ_POOL_SIZE "size of thread pool for incoming connections" #define IN_REQ_PAYLOAD_MAX_SIZE_DESC "maximum size (in bytes) of the payload of incoming requests" #define OUT_REQ_MSG_MAX_SIZE_DESC "maximum size (in bytes) of outgoing forward and notification request messages" -#define THRESHOLD_MAX_SIZE_DESC "maximum threshold limit for notificationQueue" #define SIMULATED_NOTIF_DESC "simulate notifications instead of actual sending them (only for testing)" #define STAT_COUNTERS "enable request/notification counters statistics" #define STAT_SEM_WAIT "enable semaphore waiting time statistics" @@ -276,8 +276,8 @@ bool logDeprecate; #define INSECURE_NOTIF_DESC "allow HTTPS notifications to peers which certificate cannot be authenticated with known CA certificates" #define NGSIV1_AUTOCAST_DESC "automatic cast for number, booleans and dates in NGSIv1 update/create attribute operations" #define MQTT_MAX_AGE_DESC "max time (in minutes) that an unused MQTT connection is kept, default: 60" -//#define MAX_THRESHOLD_DESC "max time (in minutes) that an unused MQTT connection is kept, default: 60" #define LOG_DEPRECATE_DESC "log deprecation usages as warnings" +#define MAX_THRESHOLD_DESC "maximum threshold for notification queue, default: 80%" @@ -368,6 +368,8 @@ PaArgument paArgs[] = { "-logDeprecate", &logDeprecate, "LOG_DEPRECATE", PaBool, PaOpt, false, false, true, LOG_DEPRECATE_DESC }, + { "-maxthreshold", &maxthreshold, "MAX_THRESHOLD", PaInt, PaOpt, 0, PaNL, PaNL, MAX_THRESHOLD_DESC }, + PA_END_OF_ARGS }; @@ -658,6 +660,15 @@ static void contextBrokerInit(void) QueueNotifier* pQNotifier = new QueueNotifier(notificationQueueSize, notificationThreadNum, serviceV, serviceQueueSizeV, serviceNumThreadV); int rc = pQNotifier->start(); + if (maxthreshold > 0) + { + maxthreshold=notificationQueueSize*maxthreshold/100; + } + else + { + maxthreshold=notificationQueueSize*80; + } + if (rc != 0) { LM_X(1,("Runtime Error starting notification queue workers (%d)", rc)); diff --git a/src/lib/common/globals.cpp b/src/lib/common/globals.cpp index c379efb252..511dbd9d0d 100644 --- a/src/lib/common/globals.cpp +++ b/src/lib/common/globals.cpp @@ -57,7 +57,6 @@ bool notifQueueStatistics = false; bool checkIdv1 = false; unsigned long long inReqPayloadMaxSize = DEFAULT_IN_REQ_PAYLOAD_MAX_SIZE; unsigned long long outReqMsgMaxSize = DEFAULT_OUT_REQ_MSG_MAX_SIZE; -unsigned long long thresholdMaxSize = DEFAULT_THRESHOLD_MAX_SIZE; diff --git a/src/lib/common/limits.h b/src/lib/common/limits.h index e7c0c8e80d..39295ec64d 100644 --- a/src/lib/common/limits.h +++ b/src/lib/common/limits.h @@ -112,8 +112,6 @@ #define DEFAULT_OUT_REQ_MSG_MAX_SIZE (8 * 1024 * 1024) // 8 MB default max size of any outgoing request message (see CLI -outReqMsgMaxSize) -#define DEFAULT_THRESHOLD_MAX_SIZE 3 - /* **************************************************************************** * diff --git a/src/lib/ngsiNotify/QueueNotifier.cpp b/src/lib/ngsiNotify/QueueNotifier.cpp index e5d42a9c25..1d2a88677f 100644 --- a/src/lib/ngsiNotify/QueueNotifier.cpp +++ b/src/lib/ngsiNotify/QueueNotifier.cpp @@ -180,17 +180,19 @@ void QueueNotifier::sendNotifyContextRequest if (enqueued) { - std::string details; - extern int thresholdMaxSize; - details = "(Max threshold limit: 3)"; - if (QueueStatistics::getIn() >= thresholdMaxSize) + extern long int maxthreshold; + extern int notificationQueueSize; + + std::string details = ("notification queue reached maximum threshold"); + + if (maxthreshold >= notificationQueueSize) { - alarmMgr.notificationQueue(service, "notification queue reached the threshold " + details); + alarmMgr.notificationQueue(queueName.c_str(), details); } else { - alarmMgr.notificationQueuesResets(service); + alarmMgr.notificationQueuesResets(queueName.c_str()); } } diff --git a/test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold.test b/test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold.test index 572801f225..8f31934fbb 100644 --- a/test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold.test +++ b/test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold.test @@ -27,7 +27,7 @@ alarm for notification queue overpassing a given threshold dbInit ${CB_DB_NAME} serv1 dbInit ${CB_DB_NAME} serv2 dbInit ${CB_DB_NAME} serv3 -brokerStart CB 0 IPv4 -multiservice -notificationMode threadpool:3:2 +brokerStart CB 0 IPv4 -multiservice -maxthreshold 100 -notificationMode threadpool:3:2,serv1:3:2,serv2:3:2,serv3:3:2 accumulatorStart --pretty-print localhost $LISTENER_PORT --SHELL-- @@ -80,7 +80,7 @@ payload='{ }, "notification": { "http": { - "url": "http://localhost:'$LISTENER_PORT'/notify" + "url": "http://localhost:'$LISTENER_PORT'/waitForever" } } }' @@ -103,7 +103,7 @@ payload='{ }, "notification": { "http": { - "url": "http://localhost:'$LISTENER_PORT'/notify" + "url": "http://localhost:'$LISTENER_PORT'/waitForever" } } }' @@ -220,7 +220,7 @@ Content-Length: 0 07. Grep log for notificationQueue alarm ======================================== -Raising alarm NotificationQueue serv2: notification queue reached the threshold (Max threshold limit: 3) +Raising alarm NotificationQueue serv1: notification queue reached maximum threshold --TEARDOWN-- From 1ebccb65531531a8dc0856d8de499b6a34d5a733 Mon Sep 17 00:00:00 2001 From: Anjali-NEC Date: Thu, 20 Jul 2023 17:22:31 +0000 Subject: [PATCH 06/10] Updated code --- doc/manuals/admin/cli.md | 2 ++ src/app/contextBroker/contextBroker.cpp | 15 ++------- src/lib/ngsiNotify/QueueNotifier.cpp | 32 +++++++++++-------- .../0000_cli/bool_option_with_value.test | 1 + .../cases/0000_cli/command_line_options.test | 1 + .../tracelevel_without_logLevel_as_DEBUG.test | 1 + .../cases/3658_env_vars/env_vars.test | 1 + ...n_queue_overpassing_a_given_threshold.test | 2 +- 8 files changed, 29 insertions(+), 26 deletions(-) diff --git a/doc/manuals/admin/cli.md b/doc/manuals/admin/cli.md index a133939397..57e95e0f25 100644 --- a/doc/manuals/admin/cli.md +++ b/doc/manuals/admin/cli.md @@ -185,6 +185,7 @@ The list of available options is the following: to the `-k` or `--insecure` parameteres of the curl command. - **-mqttMaxAge**. Max time (in minutes) that an unused MQTT connection is kept. Default: 60 - **-logDeprecate**. Log deprecation usages as warnings. More information in [this section of the documentation](../deprecated.md#log-deprecation-warnings). Default is: false. It can be changed after Orion startup with the [log admin REST API](management_api.md#log-configs-and-trace-levels), with the `deprecated` field +- **-notifAlarmThreshold**. Maximum threshold for notification queue alarms, as a percentage of the maximum queue size, default 0 (meaning no queue alarms are used) ## Configuration using environment variables @@ -263,3 +264,4 @@ Two facts have to be taken into account: | ORION_NGSIV1_AUTOCAST | ngsiv1Autocast | | ORION_MQTT_MAX_AGE | mqttMaxAge | | ORION_LOG_DEPRECATE | logDeprecate | +| ORION_NOTIF_ALARM_THRESHOLD | notifAlarmThreshold | diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index 59ff306550..52e2800530 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -206,7 +206,7 @@ unsigned long fcMaxInterval; int mqttMaxAge; bool logDeprecate; -long int maxthreshold; +long unsigned int notifAlarmThreshold; @@ -277,7 +277,7 @@ long int maxthreshold; #define NGSIV1_AUTOCAST_DESC "automatic cast for number, booleans and dates in NGSIv1 update/create attribute operations" #define MQTT_MAX_AGE_DESC "max time (in minutes) that an unused MQTT connection is kept, default: 60" #define LOG_DEPRECATE_DESC "log deprecation usages as warnings" -#define MAX_THRESHOLD_DESC "maximum threshold for notification queue, default: 80%" +#define NOTIF_ALARM_THRESHOLD_DESC "maximum threshold for notification queue alarms, as a percentage of the maximum queue size, default 0 (meaning no queue alarms are used)" @@ -368,7 +368,7 @@ PaArgument paArgs[] = { "-logDeprecate", &logDeprecate, "LOG_DEPRECATE", PaBool, PaOpt, false, false, true, LOG_DEPRECATE_DESC }, - { "-maxthreshold", &maxthreshold, "MAX_THRESHOLD", PaInt, PaOpt, 0, PaNL, PaNL, MAX_THRESHOLD_DESC }, + { "-notifAlarmThreshold", ¬ifAlarmThreshold, "NOTIF_ALARM_THRESHOLD", PaInt, PaOpt, 0, PaNL, PaNL, NOTIF_ALARM_THRESHOLD_DESC }, PA_END_OF_ARGS }; @@ -660,15 +660,6 @@ static void contextBrokerInit(void) QueueNotifier* pQNotifier = new QueueNotifier(notificationQueueSize, notificationThreadNum, serviceV, serviceQueueSizeV, serviceNumThreadV); int rc = pQNotifier->start(); - if (maxthreshold > 0) - { - maxthreshold=notificationQueueSize*maxthreshold/100; - } - else - { - maxthreshold=notificationQueueSize*80; - } - if (rc != 0) { LM_X(1,("Runtime Error starting notification queue workers (%d)", rc)); diff --git a/src/lib/ngsiNotify/QueueNotifier.cpp b/src/lib/ngsiNotify/QueueNotifier.cpp index 1d2a88677f..049f1d948a 100644 --- a/src/lib/ngsiNotify/QueueNotifier.cpp +++ b/src/lib/ngsiNotify/QueueNotifier.cpp @@ -178,15 +178,30 @@ void QueueNotifier::sendNotifyContextRequest bool enqueued = sq->try_push(paramsP); - if (enqueued) + if (!enqueued) { + QueueStatistics::incReject(1); + LM_E(("Runtime Error (%s notification queue is full)", queueName.c_str())); + delete paramsP; - extern long int maxthreshold; - extern int notificationQueueSize; + return; + } + extern long unsigned int notifAlarmThreshold; + + if (notifAlarmThreshold != 0) + { std::string details = ("notification queue reached maximum threshold"); - if (maxthreshold >= notificationQueueSize) + if (notifAlarmThreshold > 0) + { + notifAlarmThreshold=queueSize(service)*notifAlarmThreshold/100; + } + if (notifAlarmThreshold > 100) + { + LM_X(1, ("Fatal Error (notifAlarmThreshol value is greater than 100%)")); + } + if (notifAlarmThreshold >= queueSize(service)) { alarmMgr.notificationQueue(queueName.c_str(), details); } @@ -196,14 +211,5 @@ void QueueNotifier::sendNotifyContextRequest } } - if (!enqueued) - { - QueueStatistics::incReject(1); - LM_E(("Runtime Error (%s notification queue is full)", queueName.c_str())); - delete paramsP; - - return; - } - QueueStatistics::incIn(1); } diff --git a/test/functionalTest/cases/0000_cli/bool_option_with_value.test b/test/functionalTest/cases/0000_cli/bool_option_with_value.test index dbdc0d45b7..021d81530b 100644 --- a/test/functionalTest/cases/0000_cli/bool_option_with_value.test +++ b/test/functionalTest/cases/0000_cli/bool_option_with_value.test @@ -103,5 +103,6 @@ Usage: contextBroker [option '-U' (extended usage)] [option '-ngsiv1Autocast' (automatic cast for number, booleans and dates in NGSIv1 update/create attribute operations)] [option '-mqttMaxAge' ] [option '-logDeprecate' (log deprecation usages as warnings)] + [option '-notifAlarmThreshold' ] --TEARDOWN-- diff --git a/test/functionalTest/cases/0000_cli/command_line_options.test b/test/functionalTest/cases/0000_cli/command_line_options.test index 8e5b9b1ecb..48282dd937 100644 --- a/test/functionalTest/cases/0000_cli/command_line_options.test +++ b/test/functionalTest/cases/0000_cli/command_line_options.test @@ -92,5 +92,6 @@ Usage: contextBroker [option '-U' (extended usage)] [option '-ngsiv1Autocast' (automatic cast for number, booleans and dates in NGSIv1 update/create attribute operations)] [option '-mqttMaxAge' ] [option '-logDeprecate' (log deprecation usages as warnings)] + [option '-notifAlarmThreshold' ] --TEARDOWN-- diff --git a/test/functionalTest/cases/0000_cli/tracelevel_without_logLevel_as_DEBUG.test b/test/functionalTest/cases/0000_cli/tracelevel_without_logLevel_as_DEBUG.test index 2ae2c7ad60..847d74af82 100644 --- a/test/functionalTest/cases/0000_cli/tracelevel_without_logLevel_as_DEBUG.test +++ b/test/functionalTest/cases/0000_cli/tracelevel_without_logLevel_as_DEBUG.test @@ -93,5 +93,6 @@ Usage: contextBroker [option '-U' (extended usage)] [option '-ngsiv1Autocast' (automatic cast for number, booleans and dates in NGSIv1 update/create attribute operations)] [option '-mqttMaxAge' ] [option '-logDeprecate' (log deprecation usages as warnings)] + [option '-notifAlarmThreshold' ] --TEARDOWN-- diff --git a/test/functionalTest/cases/3658_env_vars/env_vars.test b/test/functionalTest/cases/3658_env_vars/env_vars.test index bd68c35e98..f08dc9dbe2 100644 --- a/test/functionalTest/cases/3658_env_vars/env_vars.test +++ b/test/functionalTest/cases/3658_env_vars/env_vars.test @@ -138,6 +138,7 @@ Extended Usage: contextBroker [option '-U' (extended usage)] [option '-ngsiv1Autocast' (automatic cast for number, booleans and dates in NGSI] ORION_NGSIV1_AUTOCAST FALSE /FALSE/ [option '-mqttMaxAge' Date: Fri, 11 Aug 2023 13:00:35 +0000 Subject: [PATCH 07/10] Updated code as per comments --- CHANGES_NEXT_RELEASE | 2 +- CMakeLists.txt | 2 +- doc/manuals/admin/logs.md | 2 +- src/app/contextBroker/contextBroker.cpp | 11 ++- src/lib/alarmMgr/AlarmManager.cpp | 6 +- src/lib/ngsiNotify/QueueNotifier.cpp | 16 ++-- ...n_queue_overpassing_a_given_threshold.test | 86 ++++++++++++++++--- 7 files changed, 95 insertions(+), 30 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index f1c73a9f48..7d7438b8ed 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,2 @@ -- Fix: alarm for notification queue overpassing a given threshold (#4113) +- Add: alarm for notification queue overpassing a given threshold (new CLI `-notifAlarmThreshold` / env var `ORION_NOTIF_ALARM_THRESHOLD`) (#4113) - Fix: logDeprecate not working correctly (`geo:json` wrongly considered as deprecated) diff --git a/CMakeLists.txt b/CMakeLists.txt index 694bbe0f8a..57ebc85d53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -221,7 +221,7 @@ SET (BOOST_MT # See http://mongoc.org/libmongoc/current/tutorial.html#cmake # This find_package() command provides the mongo::mongoc_static used in # SET for common static libs. We use 1.23.1 as reference version. -find_package (mongoc-1.0 1.23.1 EXACT) +find_package (mongoc-1.0 1.23.2 EXACT) # Static libs common to contextBroker and unitTest binaries SET (COMMON_STATIC_LIBS diff --git a/doc/manuals/admin/logs.md b/doc/manuals/admin/logs.md index 6553757bdd..5fd63ef899 100644 --- a/doc/manuals/admin/logs.md +++ b/doc/manuals/admin/logs.md @@ -233,7 +233,7 @@ Alarm conditions: | 5 | WARNING | The following WARN text appears in the 'msg' field: "Raising alarm NotificationError ``: ``". | The following WARN text appears in the 'msg' field: "Releasing alarm NotificationError ``", where `` is the same one that triggered the alarm. Orion prints this trace when it successfully sent a notification to that URL. | Notification Failure. The ``text contains the detailed information. | Orion is trying to send the HTTP notification to a given receiver and some problem has occurred. It could be due to a problem with the network connectivity or on the receiver, e.g. the receiver is down. In the second case, the owner of the receiver of the notification should be reported. No specific action has to be performed at Orion Context Broker service. | 6 | WARNING | The following WARN text appears in the 'msg' field: "Raising alarm ForwardingError ``": ``". | The following WARN text appears in the 'msg' field: "Releasing alarm ForwardingError ``", where `` is the same one that triggered the alarm. Orion prints this trace when it successfully interact with ContextProvider to that URL.| Forwarding Error. The ``text contains the detailed information. | Orion is trying to interact with ContextProvider and some problem has occurred. It may be due to context provider response for forwarded query or update is empty. No specific action has to be performed at Orion Context Broker service. | 7 | WARNING | The following WARN text appears in the 'msg' field: "Raising alarm MqttConnectionError ``": ``". | The following WARN text appears in the 'msg' field: "Releasing alarm MqttConnectionError ``", where `` is the same one that triggered the alarm. Orion prints this trace when it successfully interact with ContextProvider to that URL.| Error connection to MQTT broker. The ``text contains the detailed information. | Orion is trying to connecto to an MQTT broker (associated to a subscription) and some problem has occurred. It may be due to several reasons: MQTT broker in unreachable, user/pass is wrong, etc. No specific action has to be performed at Orion Context Broker service, but maybe in the MQTT broker configuration or in the associated subscription. -| 8 | WARNING | The following WARN text appears in the 'msg' field: "Raising alarm NotificaitonQueue ``": ``". | The following WARN text appears in the 'msg' field: "Releasing alarm NotificaitonQueue ``", where `` is the same one that triggered the alarm. Orion prints this trace when notification queue goes back below the threshold. | The notification queue associated to the service (or ``" "default" for default queue) has overpassed the alarm threshold. The ``" text described the particular threshold. | No specific action has to be performed at Orion Context Broker service, but the update flow causing the notification on that service (or default queue) should be lowered in order to reduce pressure on queue. Another possible problem is due to malfunctioning notification receivers, if they are slow processing notifications and responding to Orion. +| 8 | WARNING | The following WARN text appears in the 'msg' field: "Raising alarm NotificationQueue ``: ``". | The following WARN text appears in the 'msg' field: "Releasing alarm NotificationQueue ``", where `` is the same one that triggered the alarm. Orion prints this trace when notification queue goes back below the threshold. | The notification queue associated to the service (or "default" for default queue) has overpassed the alarm threshold. The `` text described the particular threshold. | No specific action has to be performed at Orion Context Broker service, but the update flow causing the notification on that service (or default queue) should be lowered in order to reduce pressure on queue. Another possible problem is due to malfunctioning notification receivers, if they are slow processing notifications and responding to Orion. By default, Orion only traces the origin (i.e. raising) and end (i.e. releasing) of an alarm, e.g: diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index 52e2800530..18ddc5dd1b 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -206,7 +206,7 @@ unsigned long fcMaxInterval; int mqttMaxAge; bool logDeprecate; -long unsigned int notifAlarmThreshold; +int notifAlarmThreshold; @@ -672,6 +672,15 @@ static void contextBrokerInit(void) pNotifier = new Notifier(); } + if (notifAlarmThreshold < 0) + { + LM_X(1, ("Fatal Error (notifAlarmThreshold negative value not allowed)")); + } + if (notifAlarmThreshold > 100) + { + LM_X(1, ("Fatal Error (notifAlarmThreshold value is greater than 100)")); + } + /* Set notifier object (singleton) */ setNotifier(pNotifier); diff --git a/src/lib/alarmMgr/AlarmManager.cpp b/src/lib/alarmMgr/AlarmManager.cpp index efa1507d8e..aa8a8b5595 100644 --- a/src/lib/alarmMgr/AlarmManager.cpp +++ b/src/lib/alarmMgr/AlarmManager.cpp @@ -618,7 +618,7 @@ bool AlarmManager::notificationQueue(const std::string& service, const std::stri { semTake(); - std::map::iterator iter = notificationQ.find(details); + std::map::iterator iter = notificationQ.find(service); if (iter != notificationQ.end()) { @@ -640,7 +640,7 @@ bool AlarmManager::notificationQueue(const std::string& service, const std::stri ++notificationQueues; - notificationQ[details] = 1; + notificationQ[service] = 1; semGive(); LM_W(("Raising alarm NotificationQueue %s: %s", service.c_str(), details.c_str())); @@ -670,7 +670,7 @@ bool AlarmManager::notificationQueuesResets(const std::string& details) ++notificationQueues; semGive(); - LM_W(("Releasing alarm NotificationQueue %s:", details.c_str())); + LM_W(("Releasing alarm NotificationQueue %s", details.c_str())); return true; } diff --git a/src/lib/ngsiNotify/QueueNotifier.cpp b/src/lib/ngsiNotify/QueueNotifier.cpp index 049f1d948a..b402369f42 100644 --- a/src/lib/ngsiNotify/QueueNotifier.cpp +++ b/src/lib/ngsiNotify/QueueNotifier.cpp @@ -187,23 +187,17 @@ void QueueNotifier::sendNotifyContextRequest return; } - extern long unsigned int notifAlarmThreshold; + extern int notifAlarmThreshold; if (notifAlarmThreshold != 0) { std::string details = ("notification queue reached maximum threshold"); - if (notifAlarmThreshold > 0) - { - notifAlarmThreshold=queueSize(service)*notifAlarmThreshold/100; - } - if (notifAlarmThreshold > 100) - { - LM_X(1, ("Fatal Error (notifAlarmThreshol value is greater than 100%)")); - } - if (notifAlarmThreshold >= queueSize(service)) + long unsigned int threshold = queueSize(service)*notifAlarmThreshold/100; + + if (threshold >= queueSize(service)) { - alarmMgr.notificationQueue(queueName.c_str(), details); + alarmMgr.notificationQueue(queueName.c_str(), details.c_str()); } else { diff --git a/test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold.test b/test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold.test index f85127f46c..bee3e80c97 100644 --- a/test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold.test +++ b/test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold.test @@ -36,10 +36,12 @@ accumulatorStart --pretty-print localhost $LISTENER_PORT # 01. Subscribe serv1 to the accumulator endpoint that never responses # 02. Subscribe serv2 to the accumulator endpoint that never responses # 03. Subscribe serv3 to the accumulator endpoint that never responses -# 04. Create/update entity in serv1 3 times -# 05. Create/update entity in serv2 4 times -# 06. Create/update entity in serv3 5 times -# 07. Grep log for notificationQueue alarm +# 04. Subscribe serv4 to the accumulator endpoint that never responses +# 05. Create/update entity in serv1 3 times +# 06. Create/update entity in serv2 4 times +# 07. Create/update entity in serv3 5 times +# 08. Create/update entity in serv4 5 times (default queue) +# 09. Grep log for notificationQueue alarm # @@ -112,7 +114,30 @@ echo echo -echo "04. Create/update entity in serv1 3 times" +echo '04. Subscribe serv3 to the accumulator endpoint that never responses' +echo '====================================================================' +payload='{ + "description": "serv4 subscriptions", + "subject": { + "entities": [ + { + "id": "E", + "type": "T" + } + ] + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/waitForever" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" --tenant serv4 +echo +echo + + +echo "05. Create/update entity in serv1 3 times" echo "=========================================" payload='{ "id": "E", @@ -131,7 +156,7 @@ echo echo -echo "05. Create/update entity in serv2 4 times" +echo "06. Create/update entity in serv2 4 times" echo "=========================================" for i in {1..4} do @@ -142,7 +167,7 @@ echo echo -echo "06. Create/update entity in serv3 5 times" +echo "07. Create/update entity in serv3 5 times" echo "=========================================" for i in {1..5} do @@ -153,7 +178,18 @@ echo echo -echo "07. Grep log for notificationQueue alarm" +echo "08. Create/update entity in serv4 5 times (default queue)" +echo "=========================================================" +for i in {1..5} +do + echo $i + orionCurl --url '/v2/entities?options=upsert,forcedUpdate' --payload "$payload" --tenant serv4 > /dev/null +done +echo +echo + + +echo "09. Grep log for notificationQueue alarm" echo "========================================" egrep 'Raising alarm|Releasing alarm' /tmp/contextBroker.log | awk -F'|' '{ print $10 }' | awk -F= '{ print $2 }' echo @@ -194,14 +230,24 @@ Content-Length: 0 -04. Create/update entity in serv1 3 times +04. Subscribe serv3 to the accumulator endpoint that never responses +==================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +05. Create/update entity in serv1 3 times ========================================= 1 2 3 -05. Create/update entity in serv2 4 times +06. Create/update entity in serv2 4 times ========================================= 1 2 @@ -209,7 +255,7 @@ Content-Length: 0 4 -06. Create/update entity in serv3 5 times +07. Create/update entity in serv3 5 times ========================================= 1 2 @@ -218,9 +264,25 @@ Content-Length: 0 5 -07. Grep log for notificationQueue alarm +08. Create/update entity in serv4 5 times (default queue) +========================================================= +1 +2 +3 +4 +5 + + +09. Grep log for notificationQueue alarm ======================================== Raising alarm NotificationQueue serv1: notification queue reached maximum threshold +Releasing alarm NotificationQueue serv1 +Raising alarm NotificationQueue serv2: notification queue reached maximum threshold +Releasing alarm NotificationQueue serv2 +Raising alarm NotificationQueue serv3: notification queue reached maximum threshold +Releasing alarm NotificationQueue serv3 +Raising alarm NotificationQueue default: notification queue reached maximum threshold +Releasing alarm NotificationQueue default --TEARDOWN-- From c8b1fa050f6fc8d2fe27c1b6edafbbd0b9238339 Mon Sep 17 00:00:00 2001 From: Anjali-NEC Date: Fri, 11 Aug 2023 15:34:20 +0000 Subject: [PATCH 08/10] modified src/app/contextBroker/contextBroker.cpp --- src/app/contextBroker/contextBroker.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/app/contextBroker/contextBroker.cpp b/src/app/contextBroker/contextBroker.cpp index 9626a03801..18ddc5dd1b 100644 --- a/src/app/contextBroker/contextBroker.cpp +++ b/src/app/contextBroker/contextBroker.cpp @@ -277,10 +277,7 @@ int notifAlarmThreshold; #define NGSIV1_AUTOCAST_DESC "automatic cast for number, booleans and dates in NGSIv1 update/create attribute operations" #define MQTT_MAX_AGE_DESC "max time (in minutes) that an unused MQTT connection is kept, default: 60" #define LOG_DEPRECATE_DESC "log deprecation usages as warnings" -<<<<<<< HEAD #define NOTIF_ALARM_THRESHOLD_DESC "maximum threshold for notification queue alarms, as a percentage of the maximum queue size, default 0 (meaning no queue alarms are used)" -======= ->>>>>>> upstream/master @@ -371,11 +368,8 @@ PaArgument paArgs[] = { "-logDeprecate", &logDeprecate, "LOG_DEPRECATE", PaBool, PaOpt, false, false, true, LOG_DEPRECATE_DESC }, -<<<<<<< HEAD { "-notifAlarmThreshold", ¬ifAlarmThreshold, "NOTIF_ALARM_THRESHOLD", PaInt, PaOpt, 0, PaNL, PaNL, NOTIF_ALARM_THRESHOLD_DESC }, -======= ->>>>>>> upstream/master PA_END_OF_ARGS }; From f44ccc7915f1366bd48df216696701a06a2b7b1a Mon Sep 17 00:00:00 2001 From: Anjali-NEC Date: Fri, 11 Aug 2023 15:40:55 +0000 Subject: [PATCH 09/10] Removed extra lines --- CMakeLists.txt | 2 +- doc/manuals/admin/cli.md | 6 ------ .../cases/0000_cli/bool_option_with_value.test | 3 --- .../functionalTest/cases/0000_cli/command_line_options.test | 3 --- .../0000_cli/tracelevel_without_logLevel_as_DEBUG.test | 3 --- test/functionalTest/cases/3658_env_vars/env_vars.test | 3 --- 6 files changed, 1 insertion(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 57ebc85d53..694bbe0f8a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -221,7 +221,7 @@ SET (BOOST_MT # See http://mongoc.org/libmongoc/current/tutorial.html#cmake # This find_package() command provides the mongo::mongoc_static used in # SET for common static libs. We use 1.23.1 as reference version. -find_package (mongoc-1.0 1.23.2 EXACT) +find_package (mongoc-1.0 1.23.1 EXACT) # Static libs common to contextBroker and unitTest binaries SET (COMMON_STATIC_LIBS diff --git a/doc/manuals/admin/cli.md b/doc/manuals/admin/cli.md index de84c886ee..57e95e0f25 100644 --- a/doc/manuals/admin/cli.md +++ b/doc/manuals/admin/cli.md @@ -185,10 +185,7 @@ The list of available options is the following: to the `-k` or `--insecure` parameteres of the curl command. - **-mqttMaxAge**. Max time (in minutes) that an unused MQTT connection is kept. Default: 60 - **-logDeprecate**. Log deprecation usages as warnings. More information in [this section of the documentation](../deprecated.md#log-deprecation-warnings). Default is: false. It can be changed after Orion startup with the [log admin REST API](management_api.md#log-configs-and-trace-levels), with the `deprecated` field -<<<<<<< HEAD - **-notifAlarmThreshold**. Maximum threshold for notification queue alarms, as a percentage of the maximum queue size, default 0 (meaning no queue alarms are used) -======= ->>>>>>> upstream/master ## Configuration using environment variables @@ -267,7 +264,4 @@ Two facts have to be taken into account: | ORION_NGSIV1_AUTOCAST | ngsiv1Autocast | | ORION_MQTT_MAX_AGE | mqttMaxAge | | ORION_LOG_DEPRECATE | logDeprecate | -<<<<<<< HEAD | ORION_NOTIF_ALARM_THRESHOLD | notifAlarmThreshold | -======= ->>>>>>> upstream/master diff --git a/test/functionalTest/cases/0000_cli/bool_option_with_value.test b/test/functionalTest/cases/0000_cli/bool_option_with_value.test index 028af17689..021d81530b 100644 --- a/test/functionalTest/cases/0000_cli/bool_option_with_value.test +++ b/test/functionalTest/cases/0000_cli/bool_option_with_value.test @@ -103,9 +103,6 @@ Usage: contextBroker [option '-U' (extended usage)] [option '-ngsiv1Autocast' (automatic cast for number, booleans and dates in NGSIv1 update/create attribute operations)] [option '-mqttMaxAge' ] [option '-logDeprecate' (log deprecation usages as warnings)] -<<<<<<< HEAD [option '-notifAlarmThreshold' ] -======= ->>>>>>> upstream/master --TEARDOWN-- diff --git a/test/functionalTest/cases/0000_cli/command_line_options.test b/test/functionalTest/cases/0000_cli/command_line_options.test index 60a8baff08..48282dd937 100644 --- a/test/functionalTest/cases/0000_cli/command_line_options.test +++ b/test/functionalTest/cases/0000_cli/command_line_options.test @@ -92,9 +92,6 @@ Usage: contextBroker [option '-U' (extended usage)] [option '-ngsiv1Autocast' (automatic cast for number, booleans and dates in NGSIv1 update/create attribute operations)] [option '-mqttMaxAge' ] [option '-logDeprecate' (log deprecation usages as warnings)] -<<<<<<< HEAD [option '-notifAlarmThreshold' ] -======= ->>>>>>> upstream/master --TEARDOWN-- diff --git a/test/functionalTest/cases/0000_cli/tracelevel_without_logLevel_as_DEBUG.test b/test/functionalTest/cases/0000_cli/tracelevel_without_logLevel_as_DEBUG.test index 8bc168e294..847d74af82 100644 --- a/test/functionalTest/cases/0000_cli/tracelevel_without_logLevel_as_DEBUG.test +++ b/test/functionalTest/cases/0000_cli/tracelevel_without_logLevel_as_DEBUG.test @@ -93,9 +93,6 @@ Usage: contextBroker [option '-U' (extended usage)] [option '-ngsiv1Autocast' (automatic cast for number, booleans and dates in NGSIv1 update/create attribute operations)] [option '-mqttMaxAge' ] [option '-logDeprecate' (log deprecation usages as warnings)] -<<<<<<< HEAD [option '-notifAlarmThreshold' ] -======= ->>>>>>> upstream/master --TEARDOWN-- diff --git a/test/functionalTest/cases/3658_env_vars/env_vars.test b/test/functionalTest/cases/3658_env_vars/env_vars.test index 86d48a9d68..f08dc9dbe2 100644 --- a/test/functionalTest/cases/3658_env_vars/env_vars.test +++ b/test/functionalTest/cases/3658_env_vars/env_vars.test @@ -138,10 +138,7 @@ Extended Usage: contextBroker [option '-U' (extended usage)] [option '-ngsiv1Autocast' (automatic cast for number, booleans and dates in NGSI] ORION_NGSIV1_AUTOCAST FALSE /FALSE/ [option '-mqttMaxAge' >>>>>> upstream/master From 8e4ea933d2078dce4f7962d4fd9f04ac5785f0be Mon Sep 17 00:00:00 2001 From: Anjali-NEC Date: Tue, 5 Sep 2023 19:59:56 +0000 Subject: [PATCH 10/10] Added test case --- ...e_overpassing_a_given_threshold_relog.test | 240 ++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold_relog.test diff --git a/test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold_relog.test b/test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold_relog.test new file mode 100644 index 0000000000..dc37e8a61c --- /dev/null +++ b/test/functionalTest/cases/4113_alarm_for_notification_queue_overpassing_a_given_threshold/alarm_for_notification_queue_overpassing_a_given_threshold_relog.test @@ -0,0 +1,240 @@ +# Copyright 2023 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +alarm for notification queue overpassing a given threshold + +--SHELL-INIT-- +dbInit ${CB_DB_NAME} serv1 +dbInit ${CB_DB_NAME} serv2 +dbInit ${CB_DB_NAME} serv3 +brokerStart CB 0 IPv4 -multiservice -relogAlarms -notifAlarmThreshold 50 -notificationMode threadpool:6:2,serv1:4:2,serv2:2:2 +accumulatorStart --pretty-print localhost $LISTENER_PORT + +--SHELL-- + +# +# 01. Subscribe serv1 to the accumulator endpoint that never responses +# 02. Subscribe serv2 to the accumulator endpoint that never responses +# 03. Subscribe serv3 to the accumulator endpoint that never responses +# 04. Create/update entity in serv1 5 times (update #3 raises alarm, update #4 and #5 cause repeated log) +# 05. Create/update entity in serv2 3 times (update #2 raises alarm, update #3 cause repeated log) +# 06. Create/update entity in serv3 (default) 7 times (update #4 raises alarm, updates #5, #6 and #7 cause repeated log) +# 07. Grep log for notificationQueue alarm +# + + +echo '01. Subscribe serv1 to the accumulator endpoint that never responses' +echo '====================================================================' +payload='{ + "description": "serv1 subscriptions", + "subject": { + "entities": [ + { + "id": "E", + "type": "T" + } + ] + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/waitForever" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" --tenant serv1 +echo +echo + + +echo '02. Subscribe serv2 to the accumulator endpoint that never responses' +echo '====================================================================' +payload='{ + "description": "serv2 subscriptions", + "subject": { + "entities": [ + { + "id": "E", + "type": "T" + } + ] + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/waitForever" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" --tenant serv2 +echo +echo + + + +echo '03. Subscribe serv3 to the accumulator endpoint that never responses' +echo '====================================================================' +payload='{ + "description": "serv3 subscriptions", + "subject": { + "entities": [ + { + "id": "E", + "type": "T" + } + ] + }, + "notification": { + "http": { + "url": "http://localhost:'$LISTENER_PORT'/waitForever" + } + } +}' +orionCurl --url /v2/subscriptions --payload "$payload" --tenant serv3 +echo +echo + + + +echo "04. Create/update entity in serv1 5 times (update #3 raises alarm, update #4 and #5 cause repeated log)" +echo "=======================================================================================================" +payload='{ + "id": "E", + "type": "T", + "A": { + "value": 1, + "type": "Number" + } +}' +for i in {1..5} +do + echo $i + orionCurl --url '/v2/entities?options=upsert,forcedUpdate' --payload "$payload" --tenant serv1 > /dev/null +done +echo +echo + + +echo "05. Create/update entity in serv2 3 times (update #2 raises alarm, update #3 cause repeated log)" +echo "================================================================================================" +for i in {1..3} +do + echo $i + orionCurl --url '/v2/entities?options=upsert,forcedUpdate' --payload "$payload" --tenant serv2 > /dev/null +done +echo +echo + + +echo "06. Create/update entity in serv3 (default) 7 times (update #4 raises alarm, updates #5, #6 and #7 cause repeated log)" +echo "======================================================================================================================" +for i in {1..7} +do + echo $i + orionCurl --url '/v2/entities?options=upsert,forcedUpdate' --payload "$payload" --tenant serv3 > /dev/null +done +echo +echo + + +echo "07. Grep log for notificationQueue alarm" +echo "========================================" +egrep 'Raising alarm|Releasing alarm|Repeated' /tmp/contextBroker.log | awk -F'|' '{ print $10 }' | awk -F= '{ print $2 }' +echo +echo + + +accumulatorStop $LISTENER_PORT + + +--REGEXPECT-- +01. Subscribe serv1 to the accumulator endpoint that never responses +==================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +02. Subscribe serv2 to the accumulator endpoint that never responses +==================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +03. Subscribe serv3 to the accumulator endpoint that never responses +==================================================================== +HTTP/1.1 201 Created +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Location: /v2/subscriptions/REGEX([0-9a-f]{24}) +Content-Length: 0 + + + +04. Create/update entity in serv1 5 times (update #3 raises alarm, update #4 and #5 cause repeated log) +======================================================================================================= +1 +2 +3 +4 +5 + + +05. Create/update entity in serv2 3 times (update #2 raises alarm, update #3 cause repeated log) +================================================================================================ +1 +2 +3 + + +06. Create/update entity in serv3 (default) 7 times (update #4 raises alarm, updates #5, #6 and #7 cause repeated log) +====================================================================================================================== +1 +2 +3 +4 +5 +6 +7 + + +07. Grep log for notificationQueue alarm +======================================== +Raising alarm NotificationQueue serv1: notification queue reached maximum threshold +Repeated NotificationQueue serv1: notification queue reached maximum threshold +Raising alarm NotificationQueue serv2: notification queue reached maximum threshold +Repeated NotificationQueue serv2: notification queue reached maximum threshold +Raising alarm NotificationQueue default: notification queue reached maximum threshold +Repeated NotificationQueue default: notification queue reached maximum threshold + + +--TEARDOWN-- +brokerStop CB +dbDrop ${CB_DB_NAME} serv1 +dbDrop ${CB_DB_NAME} serv2 +dbDrop ${CB_DB_NAME} serv3