Plugin smart retry (#937)

* only retry failed plugins * consistent error messages
robotastic · Apr 5, 2024 · 87bc0f6 · 87bc0f6
1 parent 877d116
commit 87bc0f6
Show file tree

Hide file tree

Showing 5 changed files with 45 additions and 12 deletions.
diff --git a/trunk-recorder/call_concluder/call_concluder.cc b/trunk-recorder/call_concluder/call_concluder.cc
@@ -4,6 +4,7 @@
 #include <filesystem>
 namespace fs = std::filesystem;
 
+const int Call_Concluder::MAX_RETRY = 2;
 std::list<std::future<Call_Data_t>> Call_Concluder::call_data_workers = {};
 std::list<Call_Data_t> Call_Concluder::retry_call_list = {};
 

diff --git a/trunk-recorder/call_concluder/call_concluder.h b/trunk-recorder/call_concluder/call_concluder.h
@@ -18,9 +18,9 @@
 Call_Data_t upload_call_worker(Call_Data_t call_info);
 
 class Call_Concluder {
-  static const int MAX_RETRY = 2;
 
 public:
+  static const int MAX_RETRY;
   static std::list<Call_Data_t> retry_call_list;
   static std::list<std::future<Call_Data_t>> call_data_workers;
 

diff --git a/trunk-recorder/global_structs.h b/trunk-recorder/global_structs.h
@@ -136,6 +136,7 @@ struct Call_Data_t {
   time_t process_call_time;
   int retry_attempt;
 
+  std::vector<int> plugin_retry_list;
   nlohmann::ordered_json call_json;
 };
 

diff --git a/trunk-recorder/plugin_manager/plugin_manager.cc b/trunk-recorder/plugin_manager/plugin_manager.cc
@@ -168,19 +168,50 @@ int plugman_call_start(Call *call) {
   return error;
 }
 
-int plugman_call_end(Call_Data_t call_info) {
-  int total_error = 0;
-  for (std::vector<Plugin *>::iterator it = plugins.begin(); it != plugins.end(); it++) {
-    Plugin *plugin = *it;
-    if (plugin->state == PLUGIN_RUNNING) {
-      int plugin_error = plugin->api->call_end(call_info);
-      if (plugin_error) {
-        BOOST_LOG_TRIVIAL(error) << "Plugin Manager: call_end -  " << plugin->name << " failed";
+int plugman_call_end(Call_Data_t& call_info) {
+  std::vector<int> plugin_retry_list;
+
+  std::stringstream logstream;
+  logstream << "[" << call_info.short_name << "]\t\033[0;34m" << call_info.call_num << "C\033[0m\tTG: " << call_info.talkgroup_display << "\tFreq: " << format_freq(call_info.freq) << "\t";
+  std::string loghdr = logstream.str();
+
+  // On INITIAL, run call_end for all active plugins and note failues 
+  if (call_info.status == INITIAL)
+  {
+    for (std::vector<Plugin *>::iterator it = plugins.begin(); it != plugins.end(); it++) {
+      Plugin *plugin = *it;
+      if (plugin->state == PLUGIN_RUNNING) {
+        int plugin_error = plugin->api->call_end(call_info);
+        if (plugin_error) {
+          BOOST_LOG_TRIVIAL(error) << loghdr << "Plugin Manager: call_end -  " << plugin->name << " failed.";
+          int plugin_index = std::distance(plugins.begin(), it );
+          plugin_retry_list.push_back(plugin_index);
+        }
+      }
+    }
+  } 
+  // On RETRY, run call_end only for plugins reporting previous failue
+  else if (call_info.status == RETRY)
+  {
+    for (std::vector<int>::iterator it = call_info.plugin_retry_list.begin(); it != call_info.plugin_retry_list.end(); it++) {
+      Plugin *plugin = plugins[*it];
+      if (plugin->state == PLUGIN_RUNNING) {
+        BOOST_LOG_TRIVIAL(info) << loghdr << "Plugin Manager: call_end - retry (" << call_info.retry_attempt << "/" << Call_Concluder::MAX_RETRY << ") - " << plugin->name;
+        int plugin_error = plugin->api->call_end(call_info);
+        if (plugin_error) {
+          BOOST_LOG_TRIVIAL(error) << loghdr << "Plugin Manager: call_end - retry (" << call_info.retry_attempt << "/" << Call_Concluder::MAX_RETRY << ") - " << plugin->name << " failed.";
+          plugin_retry_list.push_back(*it);
+        }
       }
-      total_error = total_error + plugin_error;
     }
   }
-  return total_error;
+
+  if (plugin_retry_list.size() == 0) {
+    return 0;
+  } else {
+    call_info.plugin_retry_list = plugin_retry_list;
+    return 1;
+  }
 }
 
 int plugman_calls_active(std::vector<Call *> calls) {

diff --git a/trunk-recorder/plugin_manager/plugin_manager.h b/trunk-recorder/plugin_manager/plugin_manager.h
@@ -35,7 +35,7 @@ void plugman_audio_callback(Call *call, Recorder *recorder, int16_t *samples, in
 int plugman_signal(long unitId, const char *signaling_type, gr::blocks::SignalType sig_type, Call *call, System *system, Recorder *recorder);
 int plugman_trunk_message(std::vector<TrunkMessage> messages, System *system);
 int plugman_call_start(Call *call);
-int plugman_call_end(Call_Data_t call_info);
+int plugman_call_end(Call_Data_t& call_info);
 int plugman_calls_active(std::vector<Call *> calls);
 void plugman_setup_recorder(Recorder *recorder);
 void plugman_setup_system(System *system);