From 36a0b2defa840d116146439b9ae7c4e25f22aa35 Mon Sep 17 00:00:00 2001
From: Reinaldo Oliveira <reinaldo.oliveira@wildlifestudios.com>
Date: Wed, 20 Dec 2023 15:39:25 -0300
Subject: [PATCH] feat: add metrics about room termination

---
 .../operations/healthcontroller/executor.go   | 12 ++++++--
 .../operations/healthcontroller/metrics.go    | 30 +++++++++++++++++++
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/internal/core/operations/healthcontroller/executor.go b/internal/core/operations/healthcontroller/executor.go
index c93c8df98..6e6726ab4 100644
--- a/internal/core/operations/healthcontroller/executor.go
+++ b/internal/core/operations/healthcontroller/executor.go
@@ -92,7 +92,7 @@ func (ex *Executor) Execute(ctx context.Context, op *operation.Operation, defini
 		ex.tryEnsureCorrectRoomsOnStorage(ctx, op, logger, nonexistentGameRoomsIDs)
 	}
 
-	availableRooms, expiredRooms := ex.findAvailableAndExpiredRooms(ctx, op, existentGameRoomsInstancesMap)
+	availableRooms, expiredRooms := ex.findAvailableAndExpiredRooms(ctx, scheduler, op, existentGameRoomsInstancesMap)
 	reportCurrentNumberOfRooms(scheduler.Game, scheduler.Name, len(availableRooms))
 
 	if len(expiredRooms) > 0 {
@@ -215,7 +215,10 @@ func (ex *Executor) ensureDesiredAmountOfInstances(ctx context.Context, op *oper
 	return nil
 }
 
-func (ex *Executor) findAvailableAndExpiredRooms(ctx context.Context, op *operation.Operation, existentGameRoomsInstancesMap map[string]*game_room.Instance) (availableRoomsIDs, expiredRoomsIDs []string) {
+func (ex *Executor) findAvailableAndExpiredRooms(ctx context.Context, scheduler *entities.Scheduler, op *operation.Operation, existentGameRoomsInstancesMap map[string]*game_room.Instance) (availableRoomsIDs, expiredRoomsIDs []string) {
+	terminationTimedOutRooms := 0
+	terminatedRooms := 0
+
 	for gameRoomId, instance := range existentGameRoomsInstancesMap {
 		if instance.Status.Type == game_room.InstancePending {
 			availableRoomsIDs = append(availableRoomsIDs, gameRoomId)
@@ -234,8 +237,10 @@ func (ex *Executor) findAvailableAndExpiredRooms(ctx context.Context, op *operat
 			expiredRoomsIDs = append(expiredRoomsIDs, gameRoomId)
 		case ex.isRoomTerminatingExpired(room):
 			expiredRoomsIDs = append(expiredRoomsIDs, gameRoomId)
+			terminationTimedOutRooms += 1
 		case ex.isRoomStatus(room, game_room.GameStatusTerminated):
 			expiredRoomsIDs = append(expiredRoomsIDs, gameRoomId)
+			terminatedRooms += 1
 		case ex.isRoomStatus(room, game_room.GameStatusTerminating):
 			continue
 		case ex.isRoomStatus(room, game_room.GameStatusError):
@@ -245,6 +250,9 @@ func (ex *Executor) findAvailableAndExpiredRooms(ctx context.Context, op *operat
 		}
 	}
 
+	reportRoomsWithTerminationTimeout(scheduler.Game, scheduler.Name, terminationTimedOutRooms)
+	reportRoomsProperlyTerminated(scheduler.Game, scheduler.Name, terminatedRooms)
+
 	return availableRoomsIDs, expiredRoomsIDs
 }
 
diff --git a/internal/core/operations/healthcontroller/metrics.go b/internal/core/operations/healthcontroller/metrics.go
index e2770e81f..1c3908b8a 100644
--- a/internal/core/operations/healthcontroller/metrics.go
+++ b/internal/core/operations/healthcontroller/metrics.go
@@ -46,6 +46,28 @@ var (
 			monitoring.LabelScheduler,
 		},
 	})
+
+	roomsWithTerminationTimeout = monitoring.CreateGaugeMetric(&monitoring.MetricOpts{
+		Namespace: monitoring.Namespace,
+		Subsystem: monitoring.SubsystemWorker,
+		Name:      "rooms_with_termination_timeout",
+		Help:      "Number of rooms that extrapolate the graceful termination period",
+		Labels: []string{
+			monitoring.LabelGame,
+			monitoring.LabelScheduler,
+		},
+	})
+
+	roomsProperlyTerminated = monitoring.CreateGaugeMetric(&monitoring.MetricOpts{
+		Namespace: monitoring.Namespace,
+		Subsystem: monitoring.SubsystemWorker,
+		Name:      "rooms_properly_terminated",
+		Help:      "Number of rooms that were properly terminated",
+		Labels: []string{
+			monitoring.LabelGame,
+			monitoring.LabelScheduler,
+		},
+	})
 )
 
 func reportDesiredNumberOfRooms(game, scheduler string, desired int) {
@@ -55,3 +77,11 @@ func reportDesiredNumberOfRooms(game, scheduler string, desired int) {
 func reportCurrentNumberOfRooms(game, scheduler string, current int) {
 	currentNumberOfRoomsMetric.WithLabelValues(game, scheduler).Set(float64(current))
 }
+
+func reportRoomsWithTerminationTimeout(game, scheduler string, rooms int) {
+	roomsWithTerminationTimeout.WithLabelValues(game, scheduler).Set(float64(rooms))
+}
+
+func reportRoomsProperlyTerminated(game, scheduler string, rooms int) {
+	roomsProperlyTerminated.WithLabelValues(game, scheduler).Set(float64(rooms))
+}