Monitor for exceptions and dropped events (#23)

* fix: Log exceptions from the queue (e.g. from jebml). * Add metrics and assert no exceptions or dropped events during tests.
jitsi · Dec 17, 2024 · 1b50721 · 1b50721
1 parent 5c16366
commit 1b50721
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 0 deletions.
diff --git a/src/main/kotlin/org/jitsi/recorder/MediaJsonMkaRecorder.kt b/src/main/kotlin/org/jitsi/recorder/MediaJsonMkaRecorder.kt
@@ -24,6 +24,7 @@ import org.jitsi.recorder.opus.GapTooLargeException
 import org.jitsi.recorder.opus.OpusPacket
 import org.jitsi.recorder.opus.PacketLossConcealmentInserter
 import org.jitsi.utils.logging2.Logger
+import org.jitsi.utils.queue.ErrorHandler
 import org.jitsi.utils.queue.PacketQueue
 import java.io.File
 import java.time.Clock
@@ -43,6 +44,17 @@ class MediaJsonMkaRecorder(directory: File, parentLogger: Logger) : MediaJsonRec
     val queue = EventQueue {
         handleEvent(it)
         true
+    }.also {
+        it.setErrorHandler(object : ErrorHandler {
+            override fun packetDropped() {
+                logger.warn("Dropped an event.")
+                RecorderMetrics.instance.queueEventsDropped.inc()
+            }
+            override fun packetHandlingFailed(t: Throwable) {
+                logger.error("Error handling event: ", t)
+                RecorderMetrics.instance.queueExceptions.inc()
+            }
+        })
     }
     private val trackRecorders = mutableMapOf<String, TrackRecorder>()
 

diff --git a/src/main/kotlin/org/jitsi/recorder/RecorderMetrics.kt b/src/main/kotlin/org/jitsi/recorder/RecorderMetrics.kt
@@ -32,6 +32,8 @@ class RecorderMetrics private constructor() : MetricsContainer(namespace = "jits
         "plc_milliseconds",
         "Duration of recorded packet concealment packets in milliseconds"
     )
+    val queueEventsDropped = registerCounter("queue_events_dropped", "Number of events dropped")
+    val queueExceptions = registerCounter("queue_exceptions", "Number of exceptions from the event queue")
 
     companion object {
         /**

diff --git a/src/test/kotlin/org/jitsi/recorder/KotestProjectConfig.kt b/src/test/kotlin/org/jitsi/recorder/KotestProjectConfig.kt
@@ -18,6 +18,7 @@
 package org.jitsi.recorder
 
 import io.kotest.core.config.AbstractProjectConfig
+import io.kotest.matchers.shouldBe
 import org.jitsi.metaconfig.MetaconfigSettings
 
 class KotestProjectConfig : AbstractProjectConfig() {
@@ -26,4 +27,8 @@ class KotestProjectConfig : AbstractProjectConfig() {
         // freely modify the config without affecting other tests executing afterwards).
         MetaconfigSettings.cacheEnabled = false
     }
+    override suspend fun afterProject() = super.afterProject().also {
+        RecorderMetrics.instance.queueExceptions.get() shouldBe 0
+        RecorderMetrics.instance.queueEventsDropped.get() shouldBe 0
+    }
 }