feat: add metrics about room termination (#604)

topfreegames · Jan 2, 2024 · 30df220 · 30df220
1 parent 8326acd
commit 30df220
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 2 deletions.
diff --git a/internal/core/operations/healthcontroller/executor.go b/internal/core/operations/healthcontroller/executor.go
@@ -92,7 +92,7 @@ func (ex *Executor) Execute(ctx context.Context, op *operation.Operation, defini
 		ex.tryEnsureCorrectRoomsOnStorage(ctx, op, logger, nonexistentGameRoomsIDs)
 	}
 
-	availableRooms, expiredRooms := ex.findAvailableAndExpiredRooms(ctx, op, existentGameRoomsInstancesMap)
+	availableRooms, expiredRooms := ex.findAvailableAndExpiredRooms(ctx, scheduler, op, existentGameRoomsInstancesMap)
 	reportCurrentNumberOfRooms(scheduler.Game, scheduler.Name, len(availableRooms))
 
 	if len(expiredRooms) > 0 {
@@ -215,7 +215,10 @@ func (ex *Executor) ensureDesiredAmountOfInstances(ctx context.Context, op *oper
 	return nil
 }
 
-func (ex *Executor) findAvailableAndExpiredRooms(ctx context.Context, op *operation.Operation, existentGameRoomsInstancesMap map[string]*game_room.Instance) (availableRoomsIDs, expiredRoomsIDs []string) {
+func (ex *Executor) findAvailableAndExpiredRooms(ctx context.Context, scheduler *entities.Scheduler, op *operation.Operation, existentGameRoomsInstancesMap map[string]*game_room.Instance) (availableRoomsIDs, expiredRoomsIDs []string) {
+	terminationTimedOutRooms := 0
+	terminatedRooms := 0
+
 	for gameRoomId, instance := range existentGameRoomsInstancesMap {
 		if instance.Status.Type == game_room.InstancePending {
 			availableRoomsIDs = append(availableRoomsIDs, gameRoomId)
@@ -234,8 +237,10 @@ func (ex *Executor) findAvailableAndExpiredRooms(ctx context.Context, op *operat
 			expiredRoomsIDs = append(expiredRoomsIDs, gameRoomId)
 		case ex.isRoomTerminatingExpired(room):
 			expiredRoomsIDs = append(expiredRoomsIDs, gameRoomId)
+			terminationTimedOutRooms += 1
 		case ex.isRoomStatus(room, game_room.GameStatusTerminated):
 			expiredRoomsIDs = append(expiredRoomsIDs, gameRoomId)
+			terminatedRooms += 1
 		case ex.isRoomStatus(room, game_room.GameStatusTerminating):
 			continue
 		case ex.isRoomStatus(room, game_room.GameStatusError):
@@ -245,6 +250,9 @@ func (ex *Executor) findAvailableAndExpiredRooms(ctx context.Context, op *operat
 		}
 	}
 
+	reportRoomsWithTerminationTimeout(scheduler.Game, scheduler.Name, terminationTimedOutRooms)
+	reportRoomsProperlyTerminated(scheduler.Game, scheduler.Name, terminatedRooms)
+
 	return availableRoomsIDs, expiredRoomsIDs
 }
 

diff --git a/internal/core/operations/healthcontroller/metrics.go b/internal/core/operations/healthcontroller/metrics.go
@@ -46,6 +46,28 @@ var (
 			monitoring.LabelScheduler,
 		},
 	})
+
+	roomsWithTerminationTimeout = monitoring.CreateGaugeMetric(&monitoring.MetricOpts{
+		Namespace: monitoring.Namespace,
+		Subsystem: monitoring.SubsystemWorker,
+		Name:      "rooms_with_termination_timeout",
+		Help:      "Number of rooms that extrapolate the graceful termination period",
+		Labels: []string{
+			monitoring.LabelGame,
+			monitoring.LabelScheduler,
+		},
+	})
+
+	roomsProperlyTerminated = monitoring.CreateGaugeMetric(&monitoring.MetricOpts{
+		Namespace: monitoring.Namespace,
+		Subsystem: monitoring.SubsystemWorker,
+		Name:      "rooms_properly_terminated",
+		Help:      "Number of rooms that were properly terminated",
+		Labels: []string{
+			monitoring.LabelGame,
+			monitoring.LabelScheduler,
+		},
+	})
 )
 
 func reportDesiredNumberOfRooms(game, scheduler string, desired int) {
@@ -55,3 +77,11 @@ func reportDesiredNumberOfRooms(game, scheduler string, desired int) {
 func reportCurrentNumberOfRooms(game, scheduler string, current int) {
 	currentNumberOfRoomsMetric.WithLabelValues(game, scheduler).Set(float64(current))
 }
+
+func reportRoomsWithTerminationTimeout(game, scheduler string, rooms int) {
+	roomsWithTerminationTimeout.WithLabelValues(game, scheduler).Set(float64(rooms))
+}
+
+func reportRoomsProperlyTerminated(game, scheduler string, rooms int) {
+	roomsProperlyTerminated.WithLabelValues(game, scheduler).Set(float64(rooms))
+}