Remove expired terminating game room (#496)

* Remove expired terminating game room
topfreegames · Jul 20, 2022 · 05d1644 · 05d1644
1 parent 9278846
commit 05d1644
Show file tree

Hide file tree

Showing 3 changed files with 71 additions and 1 deletion.
diff --git a/internal/core/operations/healthcontroller/health_controller_executor.go b/internal/core/operations/healthcontroller/health_controller_executor.go
@@ -46,6 +46,7 @@ import (
 type Config struct {
 	RoomInitializationTimeout time.Duration
 	RoomPingTimeout           time.Duration
+	RoomDeletionTimeout       time.Duration
 }
 
 // SchedulerHealthControllerExecutor holds dependencies to execute SchedulerHealthControllerExecutor.
@@ -206,6 +207,8 @@ func (ex *SchedulerHealthControllerExecutor) findAvailableAndExpiredRooms(ctx co
 			expiredRoomsIDs = append(expiredRoomsIDs, gameRoomId)
 		case ex.isRoomPingExpired(room):
 			expiredRoomsIDs = append(expiredRoomsIDs, gameRoomId)
+		case ex.isRoomTerminatingExpired(room):
+			expiredRoomsIDs = append(expiredRoomsIDs, gameRoomId)
 		case ex.isRoomStatus(room, game_room.GameStatusTerminating):
 			continue
 		case ex.isRoomStatus(room, game_room.GameStatusError):
@@ -229,6 +232,11 @@ func (ex *SchedulerHealthControllerExecutor) isRoomPingExpired(room *game_room.G
 	return !ex.isRoomStatus(room, game_room.GameStatusPending) && timeDurationWithoutPing > ex.config.RoomPingTimeout
 }
 
+func (ex *SchedulerHealthControllerExecutor) isRoomTerminatingExpired(room *game_room.GameRoom) bool {
+	timeDurationWithoutPing := time.Since(room.LastPingAt)
+	return ex.isRoomStatus(room, game_room.GameStatusTerminating) && timeDurationWithoutPing > ex.config.RoomDeletionTimeout
+}
+
 func (ex *SchedulerHealthControllerExecutor) isRoomStatus(room *game_room.GameRoom, status game_room.GameRoomStatus) bool {
 	return room.Status == status
 }

diff --git a/internal/core/operations/healthcontroller/health_controller_executor_test.go b/internal/core/operations/healthcontroller/health_controller_executor_test.go
@@ -985,7 +985,7 @@ func TestSchedulerHealthController_Execute(t *testing.T) {
 			},
 		},
 		{
-			title: "room with status terminating, consider room ignored and enqueues new add rooms",
+			title: "room with status terminating, and considered valid, consider room ignored and enqueues new add rooms",
 			executionPlan: executionPlan{
 				planMocks: func(
 					roomStorage *mockports.MockRoomStorage,
@@ -1037,6 +1037,65 @@ func TestSchedulerHealthController_Execute(t *testing.T) {
 				},
 			},
 		},
+		{
+			title: "game room status terminating with terminating timeout found, considered expired, remove room operation enqueued",
+			executionPlan: executionPlan{
+				planMocks: func(
+					roomStorage *mockports.MockRoomStorage,
+					instanceStorage *ismock.MockGameRoomInstanceStorage,
+					schedulerStorage *mockports.MockSchedulerStorage,
+					operationManager *mockports.MockOperationManager,
+					autoscaler *mockports.MockAutoscaler,
+				) {
+					gameRoomIDs := []string{"existent-1", "existent-terminating-2"}
+					instances := []*game_room.Instance{
+						{
+							ID: "existent-1",
+							Status: game_room.InstanceStatus{
+								Type: game_room.InstanceReady,
+							},
+						}, {
+							ID: "existent-terminating-2",
+							Status: game_room.InstanceStatus{
+								Type: game_room.InstanceReady,
+							},
+						},
+					}
+					// load
+					roomStorage.EXPECT().GetAllRoomIDs(gomock.Any(), gomock.Any()).Return(gameRoomIDs, nil)
+					instanceStorage.EXPECT().GetAllInstances(gomock.Any(), gomock.Any()).Return(instances, nil)
+					schedulerStorage.EXPECT().GetScheduler(gomock.Any(), gomock.Any()).Return(genericSchedulerNoAutoscaling, nil)
+
+					// Find game room
+					gameRoom := &game_room.GameRoom{
+						ID:          gameRoomIDs[0],
+						SchedulerID: genericSchedulerNoAutoscaling.Name,
+						Status:      game_room.GameStatusReady,
+						LastPingAt:  time.Now(),
+					}
+					roomStorage.EXPECT().GetRoom(gomock.Any(), genericSchedulerNoAutoscaling.Name, gameRoomIDs[0]).Return(gameRoom, nil)
+
+					expiredCreatedAt := time.Now().Add(5 * -time.Minute)
+					gameRoomTerminating := &game_room.GameRoom{
+						ID:          gameRoomIDs[1],
+						SchedulerID: genericSchedulerNoAutoscaling.Name,
+						Status:      game_room.GameStatusTerminating,
+						LastPingAt:  time.Now().Add(5 * -time.Minute),
+						CreatedAt:   expiredCreatedAt,
+					}
+					roomStorage.EXPECT().GetRoom(gomock.Any(), genericSchedulerNoAutoscaling.Name, gameRoomIDs[1]).Return(gameRoomTerminating, nil)
+
+					op := operation.New(genericSchedulerNoAutoscaling.Name, genericDefinition.Name(), nil)
+					operationManager.EXPECT().AppendOperationEventToExecutionHistory(gomock.Any(), gomock.Any(), gomock.Any())
+					operationManager.EXPECT().CreatePriorityOperation(gomock.Any(), genericSchedulerNoAutoscaling.Name, &remove_rooms.RemoveRoomsDefinition{RoomsIDs: []string{gameRoomIDs[1]}}).Return(op, nil)
+
+					operationManager.EXPECT().AppendOperationEventToExecutionHistory(gomock.Any(), gomock.Any(), gomock.Any())
+					operationManager.EXPECT().CreatePriorityOperation(gomock.Any(), genericSchedulerNoAutoscaling.Name, &add_rooms.AddRoomsDefinition{Amount: 1}).Return(op, nil)
+
+					genericSchedulerNoAutoscaling.RoomsReplicas = 2
+				},
+			},
+		},
 	}
 
 	for _, testCase := range testCases {
@@ -1050,6 +1109,7 @@ func TestSchedulerHealthController_Execute(t *testing.T) {
 			config := healthcontroller.Config{
 				RoomPingTimeout:           2 * time.Minute,
 				RoomInitializationTimeout: 4 * time.Minute,
+				RoomDeletionTimeout:       4 * time.Minute,
 			}
 			executor := healthcontroller.NewExecutor(roomsStorage, instanceStorage, schedulerStorage, operationManager, autoscaler, config)
 

diff --git a/internal/service/config.go b/internal/service/config.go
@@ -59,10 +59,12 @@ func NewCreateSchedulerVersionConfig(c config.Config) newschedulerversion.Config
 func NewHealthControllerConfig(c config.Config) healthcontroller.Config {
 	initializationTimeout := time.Duration(c.GetInt(roomInitializationTimeoutMillisConfigPath)) * time.Millisecond
 	pingTimeout := time.Duration(c.GetInt(roomPingTimeoutMillisConfigPath)) * time.Millisecond
+	deletionTimeout := time.Duration(c.GetInt(roomDeletionTimeoutMillisConfigPath)) * time.Millisecond
 
 	config := healthcontroller.Config{
 		RoomInitializationTimeout: initializationTimeout,
 		RoomPingTimeout:           pingTimeout,
+		RoomDeletionTimeout:       deletionTimeout,
 	}
 
 	return config