diff --git a/.gitignore b/.gitignore index 288f04b38..94dfc6ed3 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ _testmain.go # IDE files .vscode .history +.idea # Python __pycache__ diff --git a/Dockerfile b/Dockerfile index 347770510..dd0f3c104 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,6 +42,8 @@ RUN cd /go/src/github.com/topfreegames/maestro && \ FROM debian:buster-slim +RUN apt update && apt install -y ca-certificates openssl + WORKDIR /app COPY --from=build-env /app/maestro /app/maestro diff --git a/api/room_handler.go b/api/room_handler.go index 0135e7bd1..297477c49 100644 --- a/api/room_handler.go +++ b/api/room_handler.go @@ -90,6 +90,7 @@ func (g *RoomPingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { g.App.RedisClient.Trace(ctx), g.App.DBClient.WithContext(ctx), kubernetesClient, + mr, room, fmt.Sprintf("ping%s", strings.Title(payload.Status)), "", payload.Metadata, @@ -184,6 +185,7 @@ func NewRoomEventHandler(a *App) *RoomEventHandler { // ServeHTTP method func (g *RoomEventHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ctx := r.Context() + mr := metricsReporterFromCtx(ctx) l := middleware.GetLogger(ctx) params := roomParamsFromContext(ctx) payload := roomEventPayloadFromCtx(ctx) @@ -216,6 +218,7 @@ func (g *RoomEventHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { g.App.RedisClient.Trace(ctx), g.App.DBClient.WithContext(ctx), kubernetesClient, + mr, room, "roomEvent", "", @@ -298,6 +301,7 @@ func (g *RoomStatusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { g.App.RedisClient.Trace(ctx), g.App.DBClient.WithContext(ctx), kubernetesClient, + mr, room, payload.Status, "", payload.Metadata, g.App.SchedulerCache, @@ -328,6 +332,7 @@ func NewRoomAddressHandler(a *App) *RoomAddressHandler { // ServerHTTP method func (h *RoomAddressHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ctx := r.Context() + mr := metricsReporterFromCtx(ctx) l := middleware.GetLogger(ctx) params := roomParamsFromContext(ctx) @@ -340,7 +345,7 @@ func (h *RoomAddressHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { room := models.NewRoom(params.Name, params.Scheduler) kubernetesClient := kubernetes.TryWithContext(h.App.KubernetesClient, ctx) - roomAddresses, err := h.App.RoomAddrGetter.Get(room, kubernetesClient, h.App.RedisClient.Trace(ctx)) + roomAddresses, err := h.App.RoomAddrGetter.Get(room, kubernetesClient, h.App.RedisClient.Trace(ctx), mr) if err != nil { status := http.StatusInternalServerError diff --git a/api/room_handler_test.go b/api/room_handler_test.go index 945456187..5d7b220e3 100644 --- a/api/room_handler_test.go +++ b/api/room_handler_test.go @@ -67,19 +67,24 @@ forwarders: createNamespace := func(name string, clientset kubernetes.Interface) error { return models.NewNamespace(name).Create(clientset) } - createPod := func(name, namespace string, clientset kubernetes.Interface) error { + createPod := func(name, namespace string, clientset kubernetes.Interface) (*models.Pod, error) { configYaml := &models.ConfigYAML{ Name: namespace, Game: "game", Image: "img", } - pod, err := models.NewPod(name, nil, configYaml, mockClientset, mockRedisClient) + MockPodNotFound(mockRedisClient, namespace, name) + pod, err := models.NewPod(name, nil, configYaml, mockClientset, mockRedisClient, mmr) if err != nil { - return err + return nil, err } - _, err = pod.Create(clientset) - return err + podv1, err := pod.Create(clientset) + if err != nil { + return nil, err + } + pod.Spec = podv1.Spec + return pod, nil } BeforeEach(func() { // Record HTTP responses. @@ -257,10 +262,12 @@ forwarders: Context("with eventforwarders", func() { var app *api.App + var pod *models.Pod game := "somegame" BeforeEach(func() { + var err error createNamespace(namespace, clientset) - err := createPod(roomName, namespace, clientset) + pod, err = createPod(roomName, namespace, clientset) Expect(err).NotTo(HaveOccurred()) app, err = api.NewApp("0.0.0.0", 9998, config, logger, false, "", mockDb, mockCtxWrapper, mockRedisClient, mockRedisTraceWrapper, clientset, metricsClientset) Expect(err).NotTo(HaveOccurred()) @@ -287,6 +294,12 @@ forwarders: scheduler.Game = game }) + jsonBytes, err := pod.MarshalToRedis() + Expect(err).NotTo(HaveOccurred()) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(namespace), roomName). + Return(redis.NewStringResult(string(jsonBytes), nil)) + mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient) mockRedisClient.EXPECT(). HGet("scheduler:schedulerName:rooms:roomName", "metadata"). @@ -315,6 +328,7 @@ forwarders: "metadata": map[string]interface{}{ "ipv6Label": "", "region": "us", + "ports": "[]", }, }, gomock.Any()) @@ -333,6 +347,13 @@ forwarders: }) request, _ = http.NewRequest("PUT", url, reader) + jsonBytes, err := pod.MarshalToRedis() + Expect(err).NotTo(HaveOccurred()) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(namespace), roomName). + Return(redis.NewStringResult(string(jsonBytes), nil)) + + mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient) MockLoadScheduler(namespace, mockDb). Do(func(scheduler *models.Scheduler, query string, modifier string) { @@ -363,6 +384,7 @@ forwarders: "metadata": map[string]interface{}{ "ipv6Label": "", "region": "us", + "ports": "[]", }, }, gomock.Any()) @@ -541,10 +563,12 @@ forwarders: Context("with eventforwarders", func() { // TODO map status from api to something standard var app *api.App + var pod *models.Pod game := "somegame" BeforeEach(func() { + var err error createNamespace(namespace, clientset) - err := createPod(roomName, namespace, clientset) + pod, err = createPod(roomName, namespace, clientset) Expect(err).NotTo(HaveOccurred()) app, err = api.NewApp("0.0.0.0", 9998, config, logger, false, "", mockDb, mockCtxWrapper, mockRedisClient, mockRedisTraceWrapper, clientset, metricsClientset) Expect(err).NotTo(HaveOccurred()) @@ -584,6 +608,13 @@ forwarders: }) request, _ = http.NewRequest("PUT", url, reader) + jsonBytes, err := pod.MarshalToRedis() + Expect(err).NotTo(HaveOccurred()) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(namespace), roomName). + Return(redis.NewStringResult(string(jsonBytes), nil)) + + mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient) mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) mockPipeline.EXPECT().HMSet(rKey, map[string]interface{}{ @@ -622,6 +653,12 @@ forwarders: }) request, _ = http.NewRequest("PUT", url, reader) + jsonBytes, err := pod.MarshalToRedis() + Expect(err).NotTo(HaveOccurred()) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(namespace), roomName). + Return(redis.NewStringResult(string(jsonBytes), nil)) + mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient) mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) mockPipeline.EXPECT().HMSet(rKey, map[string]interface{}{ @@ -649,6 +686,7 @@ forwarders: Expect(infos["metadata"]).To(BeEquivalentTo(map[string]interface{}{ "type": "sometype", "ipv6Label": "", + "ports": "[]", })) }) mockEventForwarder2.EXPECT().Forward(gomock.Any(), status, gomock.Any(), gomock.Any()).Do( @@ -658,6 +696,7 @@ forwarders: Expect(infos["metadata"]).To(BeEquivalentTo(map[string]interface{}{ "type": "sometype", "ipv6Label": "", + "ports": "[]", })) }) @@ -848,10 +887,12 @@ forwarders: Describe("POST /scheduler/{schedulerName}/rooms/{roomName}/roomevent", func() { url := "/scheduler/schedulerName/rooms/roomName/roomevent" var app *api.App + var pod *models.Pod BeforeEach(func() { + var err error createNamespace(namespace, clientset) - err := createPod("roomName", namespace, clientset) + pod, err = createPod("roomName", namespace, clientset) Expect(err).NotTo(HaveOccurred()) app, err = api.NewApp("0.0.0.0", 9998, config, logger, false, "", mockDb, mockCtxWrapper, mockRedisClient, mockRedisTraceWrapper, clientset, metricsClientset) Expect(err).NotTo(HaveOccurred()) @@ -898,6 +939,12 @@ forwarders: scheduler.Game = game }) + jsonBytes, err := pod.MarshalToRedis() + Expect(err).NotTo(HaveOccurred()) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(namespace), "roomName"). + Return(redis.NewStringResult(string(jsonBytes), nil)) + mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient) mockEventForwarder1.EXPECT().Forward(gomock.Any(), "roomEvent", gomock.Any(), gomock.Any()).Return( int32(500), "", errors.New("some error occurred"), @@ -906,7 +953,7 @@ forwarders: app.Router.ServeHTTP(recorder, request) Expect(recorder.Code).To(Equal(500)) var obj map[string]interface{} - err := json.Unmarshal([]byte(recorder.Body.String()), &obj) + err = json.Unmarshal([]byte(recorder.Body.String()), &obj) Expect(err).NotTo(HaveOccurred()) Expect(obj["code"]).To(Equal("MAE-000")) Expect(obj["error"]).To(Equal("Room event forward failed")) @@ -922,6 +969,13 @@ forwarders: "metadata": make(map[string]interface{}), }) request, _ = http.NewRequest("POST", url, reader) + + jsonBytes, err := pod.MarshalToRedis() + Expect(err).NotTo(HaveOccurred()) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(namespace), "roomName"). + Return(redis.NewStringResult(string(jsonBytes), nil)) + mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient) MockLoadScheduler("schedulerName", mockDb).Do(func(scheduler *models.Scheduler, query string, modifier string) { scheduler.YAML = yamlStr @@ -935,7 +989,7 @@ forwarders: app.Router.ServeHTTP(recorder, request) Expect(recorder.Code).To(Equal(500)) var obj map[string]interface{} - err := json.Unmarshal([]byte(recorder.Body.String()), &obj) + err = json.Unmarshal([]byte(recorder.Body.String()), &obj) Expect(err).NotTo(HaveOccurred()) Expect(obj["code"]).To(Equal("MAE-000")) Expect(obj["error"]).To(Equal("room event forward failed")) @@ -957,6 +1011,12 @@ forwarders: scheduler.Game = game }) + jsonBytes, err := pod.MarshalToRedis() + Expect(err).NotTo(HaveOccurred()) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(namespace), "roomName"). + Return(redis.NewStringResult(string(jsonBytes), nil)) + mockEventForwarder1.EXPECT().Forward(gomock.Any(), "roomEvent", gomock.Any(), gomock.Any()).Return( int32(200), "all went well", nil, ) @@ -964,7 +1024,7 @@ forwarders: app.Router.ServeHTTP(recorder, request) Expect(recorder.Code).To(Equal(200)) var obj map[string]interface{} - err := json.Unmarshal([]byte(recorder.Body.String()), &obj) + err = json.Unmarshal([]byte(recorder.Body.String()), &obj) Expect(err).NotTo(HaveOccurred()) Expect(obj["success"]).To(Equal(true)) Expect(obj["message"]).To(Equal("all went well")) @@ -1007,11 +1067,21 @@ forwarders: err := ns.Create(clientset) Expect(err).NotTo(HaveOccurred()) - pod, err := models.NewPod(name, nil, configYaml, mockClientset, mockRedisClient) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(namespace), name). + Return(redis.NewStringResult("", redis.Nil)) + + pod, err := models.NewPod(name, nil, configYaml, mockClientset, mockRedisClient, mmr) Expect(err).NotTo(HaveOccurred()) _, err = pod.Create(clientset) Expect(err).NotTo(HaveOccurred()) + jsonBytes, err := pod.MarshalToRedis() + Expect(err).NotTo(HaveOccurred()) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(namespace), "roomName"). + Return(redis.NewStringResult(string(jsonBytes), nil)) + mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient) url := fmt.Sprintf( "/scheduler/%s/rooms/%s/address", @@ -1036,13 +1106,21 @@ forwarders: err := ns.Create(clientset) Expect(err).NotTo(HaveOccurred()) - pod, err := models.NewPod(name, nil, configYaml, mockClientset, mockRedisClient) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(namespace), name). + Return(redis.NewStringResult("", redis.Nil)) + pod, err := models.NewPod(name, nil, configYaml, mockClientset, mockRedisClient, mmr) Expect(err).NotTo(HaveOccurred()) _, err = pod.Create(clientset) Expect(err).NotTo(HaveOccurred()) mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient) namespace := "unexisting-name" + + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey("unexisting-name"), name). + Return(redis.NewStringResult("", redis.Nil)) + url := fmt.Sprintf( "/scheduler/%s/rooms/%s/address", namespace, @@ -1058,7 +1136,7 @@ forwarders: err = json.Unmarshal(recorder.Body.Bytes(), &obj) Expect(err).NotTo(HaveOccurred()) Expect(obj).To(HaveKeyWithValue("code", "MAE-000")) - Expect(obj).To(HaveKeyWithValue("description", "pods \"roomName\" not found")) + Expect(obj).To(HaveKeyWithValue("description", "pod \"roomName\" not found on redis podMap")) Expect(obj).To(HaveKeyWithValue("error", "Address handler error")) Expect(obj).To(HaveKeyWithValue("success", false)) }) diff --git a/api/scheduler_handler.go b/api/scheduler_handler.go index bbc1a940e..7890f9b4a 100644 --- a/api/scheduler_handler.go +++ b/api/scheduler_handler.go @@ -583,12 +583,14 @@ func (g *SchedulerScaleHandler) ServeHTTP(w http.ResponseWriter, r *http.Request db := g.App.DBClient.WithContext(r.Context()) err := controller.ScaleScheduler( + r.Context(), logger, g.App.RoomManager, mr, db, - g.App.RedisClient.Trace(r.Context()), + g.App.RedisClient, g.App.KubernetesClient, + g.App.Config, g.App.Config.GetInt("scaleUpTimeoutSeconds"), g.App.Config.GetInt("scaleDownTimeoutSeconds"), scaleParams.ScaleUp, scaleParams.ScaleDown, scaleParams.Replicas, params.SchedulerName, diff --git a/api/scheduler_handler_test.go b/api/scheduler_handler_test.go index bbe6482c2..2c1a9aa50 100644 --- a/api/scheduler_handler_test.go +++ b/api/scheduler_handler_test.go @@ -32,6 +32,7 @@ import ( "github.com/topfreegames/maestro/controller" "github.com/topfreegames/maestro/login" "github.com/topfreegames/maestro/models" + "github.com/topfreegames/maestro/testing" "k8s.io/api/core/v1" ) @@ -309,24 +310,11 @@ var _ = Describe("Scheduler Handler", func() { Context("when all services are healthy", func() { It("returns a status code of 201 and success body", func() { mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient) - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(10) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(10) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey("scheduler-name"), gomock.Any()).Times(10) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey("scheduler-name", "creating"), gomock.Any()).Times(10) - mockPipeline.EXPECT().Exec().Times(10) + + testing.MockScaleUp(mockPipeline, mockRedisClient, "scheduler-name", 10) MockInsertScheduler(mockDb, nil) MockUpdateScheduler(mockDb, nil, nil) - mockRedisClient.EXPECT(). - Get(models.GlobalPortsPoolKey). - Return(goredis.NewStringResult(workerPortRange, nil)). - Times(10) - var configYaml1 models.ConfigYAML err := yaml.Unmarshal([]byte(yamlString), &configYaml1) Expect(err).NotTo(HaveOccurred()) @@ -401,28 +389,13 @@ autoscaling: Expect(err).NotTo(HaveOccurred()) mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient).Times(2) - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(2) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(2) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey("scheduler-name-1"), gomock.Any()) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey("scheduler-name-2"), gomock.Any()) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey("scheduler-name-1", "creating"), gomock.Any()) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey("scheduler-name-2", "creating"), gomock.Any()) - mockPipeline.EXPECT().Exec().Times(2) - + testing.MockScaleUp(mockPipeline, mockRedisClient, "scheduler-name-1", 1) MockInsertScheduler(mockDb, nil) MockUpdateScheduler(mockDb, nil, nil) - mockRedisClient.EXPECT().Get(models.GlobalPortsPoolKey). - Return(goredis.NewStringResult(workerPortRange, nil)) + testing.MockScaleUp(mockPipeline, mockRedisClient, "scheduler-name-2", 1) MockInsertScheduler(mockDb, nil) MockUpdateScheduler(mockDb, nil, nil) - mockRedisClient.EXPECT().Get(models.GlobalPortsPoolKey). - Return(goredis.NewStringResult(workerPortRange, nil)) var configYaml1 models.ConfigYAML err = yaml.Unmarshal([]byte(yamlString), &configYaml1) @@ -507,6 +480,16 @@ autoscaling: Context("when postgres is down", func() { It("returns status code of 500 if database is unavailable", func() { mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient) + + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(2) + mockPipeline.EXPECT(). + HLen(gomock.Any()). + Return(redis.NewIntResult(0, nil)). + Times(2) + mockPipeline.EXPECT().Exec().Times(2) + + //testing.MockListPods(mockPipeline, mockRedisClient, "scheduler-name", []string{}, nil) + MockInsertScheduler(mockDb, errors.New("sql: database is closed")) mockDb.EXPECT().Exec("DELETE FROM schedulers WHERE name = ?", gomock.Any()) @@ -530,6 +513,13 @@ autoscaling: mockDb.EXPECT().Exec("DELETE FROM schedulers WHERE name = ?", gomock.Any()) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(2) + mockPipeline.EXPECT(). + HLen(gomock.Any()). + Return(redis.NewIntResult(0, nil)). + Times(2) + mockPipeline.EXPECT().Exec().Times(2) + app.Router.ServeHTTP(recorder, request) Expect(recorder.Code).To(Equal(http.StatusUnprocessableEntity)) var obj map[string]interface{} @@ -564,6 +554,13 @@ autoscaling: }) mockDb.EXPECT().Exec("DELETE FROM schedulers WHERE name = ?", "schedulerName") + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(2) + mockPipeline.EXPECT(). + HLen(gomock.Any()). + Return(redis.NewIntResult(0, nil)). + Times(2) + mockPipeline.EXPECT().Exec().Times(2) + app.Router.ServeHTTP(recorder, request) Expect(recorder.Code).To(Equal(200)) Expect(recorder.Body.String()).To(Equal(`{"success": true}`)) @@ -692,6 +689,10 @@ autoscaling: AnyTimes() MockOperationManager(opManager, timeoutDur, mockRedisClient, mockPipeline) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml.Name), gomock.Any()). + Return(redis.NewStringResult("", redis.Nil)). + Times(len(pods.Items)) scheduler1.Version = "v2.0" for _, pod := range pods.Items { Expect(pod.ObjectMeta.Labels["heritage"]).To(Equal("maestro")) @@ -717,6 +718,10 @@ autoscaling: configLockKey := models.GetSchedulerConfigLockKey(config.GetString("watcher.lockKey"), scheduler1.Name) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(configYaml.Name)).Return(redis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + // Get config lock MockRedisLock(mockRedisClient, configLockKey, lockTimeoutMs, true, nil) @@ -791,23 +796,7 @@ autoscaling: Expect(err).NotTo(HaveOccurred()) mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient).Times(2) - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT(). - ZAdd(models.GetRoomPingRedisKey("scheduler-name"), gomock.Any()). - Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT(). - SAdd(models.GetRoomStatusSetRedisKey("scheduler-name", "creating"), gomock.Any()). - Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT(). - Exec(). - Times(configYaml1.AutoScaling.Min) - + testing.MockScaleUp(mockPipeline, mockRedisClient, configYaml1.Name, configYaml1.AutoScaling.Min) MockInsertScheduler(mockDb, nil) MockUpdateScheduler(mockDb, nil, nil) @@ -1037,6 +1026,10 @@ autoscaling: "description": models.OpManagerRollingUpdate, }, nil)).AnyTimes() + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml.Name), gomock.Any()). + Return(redis.NewStringResult("", redis.Nil)). + Times(len(pods.Items)) scheduler1.Version = "v2.0" for _, pod := range pods.Items { Expect(pod.ObjectMeta.Labels["heritage"]).To(Equal("maestro")) @@ -1074,6 +1067,10 @@ autoscaling: configLockKey := models.GetSchedulerConfigLockKey(config.GetString("watcher.lockKey"), scheduler1.Name) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(configYaml.Name)).Return(redis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + // Get config lock MockRedisLock(mockRedisClient, configLockKey, lockTimeoutMs, true, nil) @@ -1545,18 +1542,7 @@ game: game-name scheduler.YAML = yamlStr }) mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient) - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ) - mockPipeline.EXPECT(). - ZAdd(models.GetRoomPingRedisKey(schedulerName), gomock.Any()) - mockPipeline.EXPECT(). - SAdd(models.GetRoomStatusSetRedisKey(schedulerName, "creating"), gomock.Any()) - mockPipeline.EXPECT().Exec() + testing.MockScaleUp(mockPipeline, mockRedisClient, schedulerName,1) err = MockSetScallingAmount( mockRedisClient, @@ -1604,27 +1590,19 @@ game: game-name Do(func(scheduler *models.Scheduler, query string, modifier string) { scheduler.YAML = yamlStr }) - mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient) + + downscalingLockKey := models.GetSchedulerDownScalingLockKey(config.GetString("watcher.lockKey"), schedulerName) + MockRedisLock(mockRedisClient, downscalingLockKey, 0, true, nil) + MockReturnRedisLock(mockRedisClient, downscalingLockKey, nil) + + mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient).AnyTimes() mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) mockPipeline.EXPECT(). SPop(models.GetRoomStatusSetRedisKey(schedulerName, models.StatusReady)). - Return(redis.NewStringCmd("room-id")) + Return(redis.NewStringResult("room-id", nil)) mockPipeline.EXPECT().Exec() - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - room := models.NewRoom("room-id", schedulerName) - for _, status := range allStatus { - mockPipeline.EXPECT(). - SRem(models.GetRoomStatusSetRedisKey(room.SchedulerName, status), gomock.Any()) - mockPipeline.EXPECT(). - ZRem(models.GetLastStatusRedisKey(room.SchedulerName, status), gomock.Any()) - } - for _, mt := range allMetrics { - mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(room.SchedulerName, mt), gomock.Any()) - } - mockPipeline.EXPECT().ZRem(models.GetRoomPingRedisKey(room.SchedulerName), gomock.Any()) - mockPipeline.EXPECT().Del(gomock.Any()) - mockPipeline.EXPECT().Exec() + MockPodNotFound(mockRedisClient, schedulerName, "room-id") port := 5000 pod := &v1.Pod{} @@ -1667,17 +1645,14 @@ game: game-name scheduler.YAML = yamlStr }) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT(). + HLen(models.GetPodMapRedisKey(configYaml1.Name)). + Return(redis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient) - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(replicas) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(replicas) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey("scheduler-name"), gomock.Any()).Times(replicas) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey("scheduler-name", "creating"), gomock.Any()).Times(replicas) - mockPipeline.EXPECT().Exec().Times(replicas) + MockScaleUp(mockPipeline, mockRedisClient, configYaml1.Name, replicas) err = MockSetScallingAmount( mockRedisClient, @@ -1736,17 +1711,14 @@ game: game-name scheduler.YAML = yamlStr }) - mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient).Times(2) - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(replicasBefore) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(replicasBefore) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey("scheduler-name"), gomock.Any()).Times(replicasBefore) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey("scheduler-name", "creating"), gomock.Any()).Times(replicasBefore) - mockPipeline.EXPECT().Exec().Times(replicasBefore) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT(). + HLen(models.GetPodMapRedisKey(schedulerName)). + Return(redis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + + mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient).AnyTimes() + MockScaleUp(mockPipeline, mockRedisClient, schedulerName, replicasBefore) err = MockSetScallingAmount( mockRedisClient, @@ -1778,34 +1750,28 @@ game: game-name scheduler.YAML = yamlStr }) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT(). + HLen(models.GetPodMapRedisKey(schedulerName)). + Return(redis.NewIntResult(int64(replicasBefore), nil)) + mockPipeline.EXPECT().Exec() + names, err := controller.GetPodNames(replicasBefore-replicasAfter, schedulerName, clientset) Expect(err).NotTo(HaveOccurred()) + downscalingLockKey := models.GetSchedulerDownScalingLockKey(config.GetString("watcher.lockKey"), schedulerName) + MockRedisLock(mockRedisClient, downscalingLockKey, 0, true, nil) + MockReturnRedisLock(mockRedisClient, downscalingLockKey, nil) + readyKey := models.GetRoomStatusSetRedisKey(schedulerName, models.StatusReady) mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) for _, name := range names { mockPipeline.EXPECT().SPop(readyKey).Return(redis.NewStringResult(name, nil)) + MockPodNotFound(mockRedisClient, schedulerName, name) } mockPipeline.EXPECT().Exec() - for _, name := range names { - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - room := models.NewRoom(name, schedulerName) - for _, status := range allStatus { - mockPipeline.EXPECT(). - SRem(models.GetRoomStatusSetRedisKey(room.SchedulerName, status), room.GetRoomRedisKey()) - mockPipeline.EXPECT(). - ZRem(models.GetLastStatusRedisKey(room.SchedulerName, status), room.ID) - } - for _, mt := range allMetrics { - mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(room.SchedulerName, mt), room.ID) - } - mockPipeline.EXPECT().ZRem(models.GetRoomPingRedisKey(schedulerName), room.ID) - mockPipeline.EXPECT().Del(room.GetRoomRedisKey()) - mockPipeline.EXPECT().Exec() - } - url = fmt.Sprintf("http://%s/scheduler/%s", app.Address, schedulerName) request, err = http.NewRequest("POST", url, reader) Expect(err).NotTo(HaveOccurred()) @@ -1842,22 +1808,20 @@ game: game-name request, err := http.NewRequest("POST", url, reader) Expect(err).NotTo(HaveOccurred()) + mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient).AnyTimes() + MockLoadScheduler(schedulerName, mockDb). Do(func(scheduler *models.Scheduler, query string, modifier string) { - scheduler.YAML = yamlStr - }) + scheduler.YAML = yamlStr + }) - mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient).Times(2) - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(replicasBefore) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(replicasBefore) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey("scheduler-name"), gomock.Any()).Times(replicasBefore) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey("scheduler-name", "creating"), gomock.Any()).Times(replicasBefore) - mockPipeline.EXPECT().Exec().Times(replicasBefore) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT(). + HLen(models.GetPodMapRedisKey(schedulerName)). + Return(redis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + + MockScaleUp(mockPipeline, mockRedisClient, schedulerName, replicasBefore) err = MockSetScallingAmount( mockRedisClient, @@ -1889,34 +1853,29 @@ game: game-name scheduler.YAML = yamlStr }) + + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT(). + HLen(models.GetPodMapRedisKey(schedulerName)). + Return(redis.NewIntResult(int64(replicasBefore), nil)) + mockPipeline.EXPECT().Exec() + names, err := controller.GetPodNames(replicasBefore-replicasAfter, schedulerName, clientset) Expect(err).NotTo(HaveOccurred()) + downscalingLockKey := models.GetSchedulerDownScalingLockKey(config.GetString("watcher.lockKey"), schedulerName) + MockRedisLock(mockRedisClient, downscalingLockKey, 0, true, nil) + MockReturnRedisLock(mockRedisClient, downscalingLockKey, nil) + readyKey := models.GetRoomStatusSetRedisKey(schedulerName, models.StatusReady) mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) for _, name := range names { mockPipeline.EXPECT().SPop(readyKey).Return(redis.NewStringResult(name, nil)) + MockPodNotFound(mockRedisClient, schedulerName, name) } mockPipeline.EXPECT().Exec() - for _, name := range names { - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - room := models.NewRoom(name, schedulerName) - for _, status := range allStatus { - mockPipeline.EXPECT(). - SRem(models.GetRoomStatusSetRedisKey(room.SchedulerName, status), room.GetRoomRedisKey()) - mockPipeline.EXPECT(). - ZRem(models.GetLastStatusRedisKey(room.SchedulerName, status), room.ID) - } - for _, mt := range allMetrics { - mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(room.SchedulerName, mt), room.ID) - } - mockPipeline.EXPECT().ZRem(models.GetRoomPingRedisKey(schedulerName), room.ID) - mockPipeline.EXPECT().Del(room.GetRoomRedisKey()) - mockPipeline.EXPECT().Exec() - } - url = fmt.Sprintf("http://%s/scheduler/%s", app.Address, schedulerName) request, err = http.NewRequest("POST", url, reader) Expect(err).NotTo(HaveOccurred()) @@ -1943,7 +1902,6 @@ game: game-name }) It("should return 400 if scaleup and scaledown are both specified", func() { - mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient) body := map[string]interface{}{"scaledown": 1, "scaleup": 1} bts, _ := json.Marshal(body) reader := strings.NewReader(string(bts)) @@ -2135,6 +2093,10 @@ game: game-name AnyTimes() MockOperationManager(opManager, timeoutDur, mockRedisClient, mockPipeline) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml.Name), gomock.Any()). + Return(redis.NewStringResult("", redis.Nil)). + Times(len(pods.Items)) scheduler1.Version = "v2.0" for _, pod := range pods.Items { Expect(pod.ObjectMeta.Labels["heritage"]).To(Equal("maestro")) @@ -2159,6 +2121,10 @@ game: game-name configLockKey := models.GetSchedulerConfigLockKey(config.GetString("watcher.lockKey"), scheduler1.Name) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(scheduler1.Name)).Return(redis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + // Get config lock MockRedisLock(mockRedisClient, configLockKey, lockTimeoutMs, true, nil) @@ -2221,6 +2187,10 @@ game: game-name opManager = models.NewOperationManager(configYaml.Name, mockRedisClient, logger) MockOperationManager(opManager, timeoutDur, mockRedisClient, mockPipeline) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml.Name), gomock.Any()). + Return(redis.NewStringResult("", redis.Nil)). + Times(len(pods.Items)) scheduler1.Version = "v2.0" for _, pod := range pods.Items { Expect(pod.ObjectMeta.Labels["heritage"]).To(Equal("maestro")) @@ -2245,6 +2215,10 @@ game: game-name configLockKey := models.GetSchedulerConfigLockKey(config.GetString("watcher.lockKey"), scheduler1.Name) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(scheduler1.Name)).Return(redis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + // Get config lock MockRedisLock(mockRedisClient, configLockKey, lockTimeoutMs, true, nil) @@ -2587,6 +2561,10 @@ game: game-name opManager = models.NewOperationManager(configYaml1.Name, mockRedisClient, logger) MockOperationManager(opManager, timeoutDur, mockRedisClient, mockPipeline) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml.Name), gomock.Any()). + Return(redis.NewStringResult("", redis.Nil)). + Times(len(pods.Items)) scheduler1.Version = "v2.0" for _, pod := range pods.Items { Expect(pod.ObjectMeta.Labels["heritage"]).To(Equal("maestro")) @@ -2611,6 +2589,10 @@ game: game-name configLockKey := models.GetSchedulerConfigLockKey(config.GetString("watcher.lockKey"), scheduler1.Name) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(scheduler1.Name)).Return(redis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + // Get config lock MockRedisLock(mockRedisClient, configLockKey, lockTimeoutMs, true, nil) @@ -2698,6 +2680,10 @@ game: game-name opManager = models.NewOperationManager(configYaml1.Name, mockRedisClient, logger) MockOperationManager(opManager, timeoutDur, mockRedisClient, mockPipeline) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml1.Name), gomock.Any()). + Return(redis.NewStringResult("", redis.Nil)). + Times(len(pods.Items)) scheduler1.Version = "v2.0" for _, pod := range pods.Items { Expect(pod.ObjectMeta.Labels["heritage"]).To(Equal("maestro")) @@ -2725,6 +2711,10 @@ game: game-name configLockKey := models.GetSchedulerConfigLockKey(config.GetString("watcher.lockKey"), scheduler1.Name) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(scheduler1.Name)).Return(redis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + // Get config lock MockRedisLock(mockRedisClient, configLockKey, lockTimeoutMs, true, nil) @@ -2789,6 +2779,10 @@ game: game-name "description": models.OpManagerRollingUpdate, }, nil)).AnyTimes() + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml.Name), gomock.Any()). + Return(redis.NewStringResult("", redis.Nil)). + Times(len(pods.Items)) scheduler1.Version = "v2.0" for _, pod := range pods.Items { Expect(pod.ObjectMeta.Labels["heritage"]).To(Equal("maestro")) @@ -2827,6 +2821,10 @@ game: game-name calls := NewCalls() + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(scheduler1.Name)).Return(redis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + configLockKey := models.GetSchedulerConfigLockKey(config.GetString("watcher.lockKey"), scheduler1.Name) // Get config lock @@ -2958,27 +2956,7 @@ game: game-name Expect(err).NotTo(HaveOccurred()) mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient) - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT(). - ZAdd(models.GetRoomPingRedisKey("scheduler-name"), gomock.Any()). - Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT(). - SAdd(models.GetRoomStatusSetRedisKey("scheduler-name", "creating"), gomock.Any()). - Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT(). - Exec(). - Times(configYaml1.AutoScaling.Min) - - mockRedisClient.EXPECT(). - Get(models.GlobalPortsPoolKey). - Return(goredis.NewStringResult(workerPortRange, nil)). - Times(configYaml1.AutoScaling.Min) + MockScaleUp(mockPipeline, mockRedisClient, configYaml1.Name, configYaml1.AutoScaling.Min) MockInsertScheduler(mockDb, nil) MockUpdateScheduler(mockDb, nil, nil) diff --git a/api/scheduler_operation_handler_test.go b/api/scheduler_operation_handler_test.go index b341246a1..fce5de9c1 100644 --- a/api/scheduler_operation_handler_test.go +++ b/api/scheduler_operation_handler_test.go @@ -99,6 +99,10 @@ var _ = Describe("SchedulerOperationHandler", func() { // Mock getting invalid rooms from redis to track progress MockGetInvalidRooms(mockRedisClient, mockPipeline, schedulerName, 1, 2, nil) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(schedulerName)).Return(goredis.NewIntResult(2, nil)) + mockPipeline.EXPECT().Exec() + // Create half of the pods in version v1.0 and half in v2.0 createPod("pod1", schedulerName, "v1.0") createPod("pod2", schedulerName, "v2.0") @@ -209,6 +213,10 @@ var _ = Describe("SchedulerOperationHandler", func() { createPod("pod1", schedulerName, "v1.0") createPod("pod2", schedulerName, "v2.0") + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(schedulerName)).Return(goredis.NewIntResult(2, nil)) + mockPipeline.EXPECT().Exec() + // Mock getting invalid rooms from redis to track progress MockGetInvalidRooms(mockRedisClient, mockPipeline, schedulerName, 1, 2, nil) diff --git a/cmd/root.go b/cmd/root.go index 4c98ed318..cd5d9b80d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -82,7 +82,9 @@ func InitConfig() { } func launchPProf() { - fmt.Println("Starting PProf HTTP server") - config.SetDefault("pprof.address", "localhost:6060") - professor.Launch(config.GetString("pprof.address")) + if pprof { + fmt.Println("Starting PProf HTTP server") + config.SetDefault("pprof.address", "localhost:6060") + professor.Launch(config.GetString("pprof.address")) + } } diff --git a/config/local.yaml b/config/local.yaml index 71b37248b..1df95fce2 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -27,7 +27,9 @@ watcher: lockKey: "maestro-lock-key" lockTimeoutMs: 180000 gracefulShutdownTimeout: 300 + maxScaleUpAmount: 300 maxSurge: 25 + goroutinePoolSize: 100 cache: defaultExpiration: 5m cleanupInterval: 10m diff --git a/controller/controller.go b/controller/controller.go index c60754506..b2c1ecca2 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -11,6 +11,7 @@ import ( "context" "errors" "fmt" + "math/rand" "strconv" "strings" "time" @@ -133,7 +134,7 @@ func CreateScheduler( } logger.Info("creating pods of new scheduler") - err = ScaleUp(logger, roomManager, mr, db, redisClient, clientset, scheduler, configYAML.AutoScaling.Min, timeoutSec, true) + err = ScaleUp(logger, roomManager, mr, db, redisClient, clientset, scheduler, configYAML.AutoScaling.Min, timeoutSec, true, nil) if err != nil { logger.WithError(err).Error("error scaling up scheduler, deleting it") if scheduler.LastScaleOpAt == int64(0) { @@ -374,16 +375,22 @@ func ScaleUp( scheduler *models.Scheduler, amount, timeoutSec int, initalOp bool, + config *viper.Viper, ) error { + scaleUpLimit := 0 + if config != nil { + scaleUpLimit = config.GetInt("watcher.maxScaleUpAmount") + } l := logger.WithFields(logrus.Fields{ - "source": "scaleUp", - "scheduler": scheduler.Name, - "amount": amount, + "source": "scaleUp", + "scheduler": scheduler.Name, + "amount": amount, + "scaleUpLimit": scaleUpLimit, }) configYAML, _ := models.NewConfigYAML(scheduler.YAML) - existPendingPods, err := pendingPods(clientset, scheduler.Name, mr) + existPendingPods, err := pendingPods(clientset, redisClient, scheduler.Name, mr) if err != nil { return err } @@ -406,6 +413,12 @@ func ScaleUp( return err } + // SAFETY - hard cap on scale up amount in order to not break etcd + if config != nil && amount > config.GetInt("watcher.maxScaleUpAmount") { + l.Warnf("amount to scale up is higher than limit") + amount = config.GetInt("watcher.maxScaleUpAmount") + } + pods := make([]*v1.Pod, amount) var creationErr error willTimeoutAt := time.Now().Add(time.Duration(timeoutSec) * time.Second) @@ -432,9 +445,11 @@ func ScaleUp( return errors.New("timeout scaling up scheduler") } + rand.Seed(time.Now().UnixNano()) err = waitForPods( willTimeoutAt.Sub(time.Now()), clientset, + redisClient, configYAML.Name, pods, l, @@ -449,11 +464,12 @@ func ScaleUp( // ScaleDown scales down the number of rooms func ScaleDown( + ctx context.Context, logger logrus.FieldLogger, roomManager models.RoomManager, mr *models.MixedMetricsReporter, db pginterfaces.DB, - redisClient redisinterfaces.RedisClient, + redisClient *redis.Client, clientset kubernetes.Interface, scheduler *models.Scheduler, amount, timeoutSec int, @@ -464,15 +480,14 @@ func ScaleDown( "amount": amount, }) - // check if scheduler update is in place - + redisClientWithContext := redisClient.Trace(ctx) willTimeoutAt := time.Now().Add(time.Duration(timeoutSec) * time.Second) l.Debug("accessing redis") //TODO: check redis version and use SPopN if version is >= 3.2, since SPopN is O(1) roomSet := make(map[*goredis.StringCmd]bool) readyKey := models.GetRoomStatusSetRedisKey(scheduler.Name, models.StatusReady) - pipe := redisClient.TxPipeline() + pipe := redisClientWithContext.TxPipeline() configYAML, err := models.NewConfigYAML(scheduler.YAML) if err != nil { return err @@ -482,7 +497,7 @@ func ScaleDown( logger, mr, db, - redisClient, + redisClientWithContext, scheduler, configYAML.AutoScaling.Max, configYAML.AutoScaling.Min, @@ -498,6 +513,7 @@ func ScaleDown( // SPop returns a random room name that can already selected roomSet[pipe.SPop(readyKey)] = true } + l.Debugf("popped %d ready rooms to scale down", amount) err = mr.WithSegment(models.SegmentPipeExec, func() error { var err error _, err = pipe.Exec() @@ -526,17 +542,10 @@ func ScaleDown( var deletionErr error for _, roomName := range idleRooms { - err := roomManager.Delete(logger, mr, clientset, redisClient, configYAML, roomName, reportersConstants.ReasonScaleDown) + err := roomManager.Delete(logger, mr, clientset, redisClientWithContext, configYAML, roomName, reportersConstants.ReasonScaleDown) if err != nil && !strings.Contains(err.Error(), "not found") { logger.WithField("roomName", roomName).WithError(err).Error("error deleting room") deletionErr = err - } else { - room := models.NewRoom(roomName, scheduler.Name) - err = room.ClearAll(redisClient, mr) - if err != nil { - logger.WithField("roomName", roomName).WithError(err).Error("error removing room info from redis") - return err - } } } @@ -555,14 +564,33 @@ func ScaleDown( return errors.New("timeout scaling down scheduler") case <-ticker.C: for _, name := range idleRooms { + var pod *models.Pod err := mr.WithSegment(models.SegmentPod, func() error { var err error - _, err = clientset.CoreV1().Pods(scheduler.Name).Get(name, metav1.GetOptions{}) + pod, err = models.GetPodFromRedis(redisClientWithContext, mr, name, scheduler.Name) return err }) - if err == nil { - exit = false - } else if !strings.Contains(err.Error(), "not found") { + if err == nil && pod != nil { + if pod.IsTerminating { + logger.WithField("pod", pod.Name).Debugf("pod is terminating") + exit = false + continue + } + + logger.WithField("pod", pod.Name).Debugf("pod still exists, deleting again") + err := roomManager.Delete(logger, mr, clientset, redisClientWithContext, configYAML, pod.Name, reportersConstants.ReasonScaleDown) + if err != nil && !strings.Contains(err.Error(), "not found") { + logger.WithField("roomName", pod.Name).WithError(err).Error("error deleting room") + deletionErr = err + exit = false + } else if err != nil && strings.Contains(err.Error(), "not found") { + logger.WithField("pod", pod.Name).Debugf("pod already deleted") + } + + if err == nil { + exit = false + } + } else if pod != nil { l.WithError(err).Error("scale down pod error") exit = false } @@ -954,12 +982,14 @@ func UpdateSchedulerMin( // ScaleScheduler scale up or down, depending on what parameters are passed func ScaleScheduler( + ctx context.Context, logger logrus.FieldLogger, roomManager models.RoomManager, mr *models.MixedMetricsReporter, db pginterfaces.DB, - redisClient redisinterfaces.RedisClient, + redisClient *redis.Client, clientset kubernetes.Interface, + config *viper.Viper, timeoutScaleup, timeoutScaledown int, amountUp, amountDown, replicas uint, schedulerName string, @@ -970,6 +1000,8 @@ func ScaleScheduler( return errors.New("invalid scale parameter: can't handle more than one parameter") } + redisClientWithContext := redisClient.Trace(ctx) + scheduler := models.NewScheduler(schedulerName, "", "") err = scheduler.Load(db) if err != nil { @@ -988,16 +1020,41 @@ func ScaleScheduler( roomManager, mr, db, - redisClient, + redisClientWithContext, clientset, scheduler, int(amountUp), timeoutScaleup, false, + nil, ) } else if amountDown > 0 { logger.Infof("manually scaling down scheduler %s in %d GRUs", schedulerName, amountDown) + + // lock so autoscaler doesn't recreate rooms deleted + downscalingLockKey := models.GetSchedulerDownScalingLockKey(config.GetString("watcher.lockKey"), scheduler.Name) + downscalingLock, _, err := AcquireLock( + ctx, + logger, + redisClient, + config, + nil, + downscalingLockKey, + scheduler.Name, + ) + defer ReleaseLock( + logger, + redisClient, + downscalingLock, + scheduler.Name, + ) + if err != nil { + logger.WithError(err).Error("not able to acquire downScalingLock. Not scaling down") + return err + } + err = ScaleDown( + ctx, logger, roomManager, mr, @@ -1011,12 +1068,12 @@ func ScaleScheduler( } else { logger.Infof("manually scaling scheduler %s to %d GRUs", schedulerName, replicas) // get list of actual pods - pods, err := ListCurrentPods(mr, clientset, schedulerName) + podCount, err := models.GetPodCountFromRedis(redisClientWithContext, mr, schedulerName) if err != nil { return err } - nPods := uint(len(pods.Items)) + nPods := uint(podCount) logger.Debugf("current number of pods: %d", nPods) if replicas > nPods { @@ -1025,15 +1082,39 @@ func ScaleScheduler( roomManager, mr, db, - redisClient, + redisClientWithContext, clientset, scheduler, int(replicas-nPods), timeoutScaleup, false, + nil, ) } else if replicas < nPods { + // lock so autoscaler doesn't recreate rooms deleted + downscalingLockKey := models.GetSchedulerDownScalingLockKey(config.GetString("watcher.lockKey"), scheduler.Name) + downscalingLock, _, err := AcquireLock( + ctx, + logger, + redisClient, + config, + nil, + downscalingLockKey, + scheduler.Name, + ) + defer ReleaseLock( + logger, + redisClient, + downscalingLock, + scheduler.Name, + ) + if err != nil { + logger.WithError(err).Error("not able to acquire downScalingLock. Not scaling down") + return err + } + err = ScaleDown( + ctx, logger, roomManager, mr, @@ -1119,6 +1200,7 @@ func SetRoomStatus( cachedScheduler.ConfigYAML.AutoScaling.Up.Delta, config.GetInt("scaleUpTimeoutSeconds"), false, + nil, ) if err != nil { log.WithError(err).Error(err) diff --git a/controller/controller_suite_test.go b/controller/controller_suite_test.go index ea8e44514..b9ff293ae 100644 --- a/controller/controller_suite_test.go +++ b/controller/controller_suite_test.go @@ -11,7 +11,6 @@ package controller_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "testing" "time" @@ -31,22 +30,22 @@ import ( ) var ( - hook *test.Hook - logger *logrus.Logger - metricsClientset *metricsFake.Clientset - mockCtrl *gomock.Controller - config *viper.Viper - mockDb *pgmocks.MockDB - mockPipeline *redismocks.MockPipeliner - mockRedisClient *redismocks.MockRedisClient - mockRedisTraceWrapper *redismocks.MockTraceWrapper - mockClock *clockmocks.MockClock - mockPortChooser *mocks.MockPortChooser - redisClient *redis.Client - mr *models.MixedMetricsReporter - schedulerCache *models.SchedulerCache - err error - allStatus = []string{ + hook *test.Hook + logger *logrus.Logger + metricsClientset *metricsFake.Clientset + mockCtrl *gomock.Controller + config *viper.Viper + mockDb *pgmocks.MockDB + mockPipeline *redismocks.MockPipeliner + mockRedisClient *redismocks.MockRedisClient + mockRedisTraceWrapper *redismocks.MockTraceWrapper + mockClock *clockmocks.MockClock + mockPortChooser *mocks.MockPortChooser + redisClient *redis.Client + mr *models.MixedMetricsReporter + schedulerCache *models.SchedulerCache + err error + allStatus = []string{ models.StatusCreating, models.StatusReady, models.StatusOccupied, diff --git a/controller/controller_test.go b/controller/controller_test.go index b835134e0..0772a421b 100644 --- a/controller/controller_test.go +++ b/controller/controller_test.go @@ -20,16 +20,15 @@ import ( "time" goredis "github.com/go-redis/redis" - mt "github.com/topfreegames/maestro/testing" - yaml "gopkg.in/yaml.v2" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/golang/mock/gomock" "github.com/topfreegames/extensions/clock" "github.com/topfreegames/extensions/pg" "github.com/topfreegames/maestro/controller" "github.com/topfreegames/maestro/models" + mt "github.com/topfreegames/maestro/testing" + yaml "gopkg.in/yaml.v2" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/client-go/kubernetes/fake" ) @@ -335,26 +334,11 @@ var _ = Describe("Controller", func() { Describe("CreateScheduler", func() { It("should succeed", func() { - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT(). - ZAdd(models.GetRoomPingRedisKey(configYaml1.Name), gomock.Any()). - Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT(). - SAdd(models.GetRoomStatusSetRedisKey(configYaml1.Name, "creating"), gomock.Any()). - Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT().Exec().Times(configYaml1.AutoScaling.Min) + mt.MockScaleUp(mockPipeline, mockRedisClient, configYaml1.Name, configYaml1.AutoScaling.Min) mt.MockInsertScheduler(mockDb, nil) mt.MockUpdateScheduler(mockDb, nil, nil) - mt.MockGetPortsFromPool(&configYaml1, mockRedisClient, mockPortChooser, workerPortRange, portStart, portEnd, 0) - err = mt.MockSetScallingAmount( mockRedisClient, mockPipeline, @@ -426,22 +410,11 @@ cmd: err := yaml.Unmarshal([]byte(yaml1), &configYaml1) Expect(err).NotTo(HaveOccurred()) - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml1.Name), gomock.Any()).Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml1.Name, "creating"), gomock.Any()).Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT().Exec().Times(configYaml1.AutoScaling.Min) + mt.MockScaleUp(mockPipeline, mockRedisClient, configYaml1.Name, configYaml1.AutoScaling.Min) mt.MockInsertScheduler(mockDb, nil) mt.MockUpdateScheduler(mockDb, nil, nil) - mt.MockGetPortsFromPool(&configYaml1, mockRedisClient, mockPortChooser, workerPortRange, portStart, portEnd, 0) - err = mt.MockSetScallingAmount( mockRedisClient, mockPipeline, @@ -518,19 +491,7 @@ portRange: err := yaml.Unmarshal([]byte(yaml1), &configYaml1) Expect(err).NotTo(HaveOccurred()) - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }).Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT(). - ZAdd(models.GetRoomPingRedisKey(configYaml1.Name), gomock.Any()). - Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT(). - SAdd(models.GetRoomStatusSetRedisKey(configYaml1.Name, "creating"), gomock.Any()). - Times(configYaml1.AutoScaling.Min) - mockPipeline.EXPECT().Exec().Times(configYaml1.AutoScaling.Min) + mt.MockScaleUp(mockPipeline, mockRedisClient, configYaml1.Name, configYaml1.AutoScaling.Min) mt.MockInsertScheduler(mockDb, nil) mt.MockUpdateScheduler(mockDb, nil, nil) @@ -542,11 +503,6 @@ portRange: Get(models.GlobalPortsPoolKey). Return(goredis.NewStringResult(workerPortRange, nil)) - schedulerPortStart := configYaml1.PortRange.Start - schedulerPortEnd := configYaml1.PortRange.End - mt.MockGetPortsFromPool(&configYaml1, mockRedisClient, mockPortChooser, - workerPortRange, schedulerPortStart, schedulerPortEnd, 0) - err = mt.MockSetScallingAmount( mockRedisClient, mockPipeline, @@ -622,6 +578,13 @@ portRange: err := yaml.Unmarshal([]byte(yamlStr), &configYaml1) Expect(err).NotTo(HaveOccurred()) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(2) + mockPipeline.EXPECT(). + HLen(models.GetPodMapRedisKey(configYaml1.Name)). + Return(goredis.NewIntResult(0, nil)). + Times(2) + mockPipeline.EXPECT().Exec().Times(2) + mockDb.EXPECT().Query(gomock.Any(), `SELECT name FROM schedulers`) mockDb.EXPECT().Query(gomock.Any(), `SELECT * FROM schedulers WHERE name IN (?)`, gomock.Any()) mockRedisClient.EXPECT().Get(models.GlobalPortsPoolKey).Return(goredis.NewStringResult(workerPortRange, nil)) @@ -763,6 +726,13 @@ portRange: err := yaml.Unmarshal([]byte(yaml1), &configYaml1) Expect(err).NotTo(HaveOccurred()) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(2) + mockPipeline.EXPECT(). + HLen(models.GetPodMapRedisKey(configYaml1.Name)). + Return(goredis.NewIntResult(0, nil)). + Times(2) + mockPipeline.EXPECT().Exec().Times(2) + mt.MockInsertScheduler(mockDb, errDB) mockDb.EXPECT().Exec("DELETE FROM schedulers WHERE name = ?", configYaml1.Name) @@ -798,6 +768,8 @@ portRange: ) Expect(err).NotTo(HaveOccurred()) + mt.MockListPods(mockPipeline, mockRedisClient, configYaml1.Name, []string{}, nil) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( func(schedulerName string, statusInfo map[string]interface{}) { @@ -807,9 +779,17 @@ portRange: ) mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml1.Name), gomock.Any()) mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml1.Name, "creating"), gomock.Any()) - mockPipeline.EXPECT().Exec().Return([]goredis.Cmder{}, errors.New("some error in redis")) + errorExec := mockPipeline.EXPECT().Exec().Return([]goredis.Cmder{}, errors.New("some error in redis")) + mockDb.EXPECT().Exec("DELETE FROM schedulers WHERE name = ?", configYaml1.Name) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(2) + mockPipeline.EXPECT(). + HLen(models.GetPodMapRedisKey(configYaml1.Name)). + Return(goredis.NewIntResult(0, nil)). + Times(2) + mockPipeline.EXPECT().Exec().Times(2).After(errorExec) + err = controller.CreateScheduler(logger, roomManager, mr, mockDb, mockRedisClient, clientset, &configYaml1, timeoutSec) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("some error in redis")) @@ -881,6 +861,13 @@ portRange: Do(func(scheduler *models.Scheduler, query string, modifier string) { scheduler.YAML = yaml1 }) + + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(2) + mockPipeline.EXPECT(). + HLen(models.GetPodMapRedisKey(configYaml1.Name)). + Return(goredis.NewIntResult(0, nil)).Times(2) + mockPipeline.EXPECT().Exec().Times(2) + mockDb.EXPECT().Exec("DELETE FROM schedulers WHERE name = ?", configYaml1.Name) terminationLockKey := models.GetSchedulerTerminationLockKey(config.GetString("watcher.lockKey"), configYaml1.Name) @@ -930,6 +917,12 @@ portRange: scheduler.YAML = yaml1 }) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(2) + mockPipeline.EXPECT(). + HLen(models.GetPodMapRedisKey(configYaml1.Name)). + Return(goredis.NewIntResult(0, nil)).Times(2) + mockPipeline.EXPECT().Exec().Times(2) + terminationLockKey := models.GetSchedulerTerminationLockKey(config.GetString("watcher.lockKey"), configYaml1.Name) // Get redis lock mt.MockRedisLock(mockRedisClient, terminationLockKey, lockTimeoutMs, true, nil) @@ -979,6 +972,12 @@ portRange: scheduler.YAML = yaml1 }) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(2) + mockPipeline.EXPECT(). + HLen(models.GetPodMapRedisKey(configYaml1.Name)). + Return(goredis.NewIntResult(0, nil)).Times(2) + mockPipeline.EXPECT().Exec().Times(2) + terminationLockKey := models.GetSchedulerTerminationLockKey(config.GetString("watcher.lockKey"), configYaml1.Name) // Get redis lock mt.MockRedisLock(mockRedisClient, terminationLockKey, lockTimeoutMs, true, nil) @@ -1035,6 +1034,12 @@ cmd: scheduler.YAML = yaml1 }) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(1) + mockPipeline.EXPECT(). + HLen(models.GetPodMapRedisKey(configYaml1.Name)). + Return(goredis.NewIntResult(0, nil)).Times(1) + mockPipeline.EXPECT().Exec().Times(1) + terminationLockKey := models.GetSchedulerTerminationLockKey(config.GetString("watcher.lockKey"), configYaml1.Name) // Get redis lock mt.MockRedisLock(mockRedisClient, terminationLockKey, lockTimeoutMs, true, nil) @@ -1233,13 +1238,17 @@ cmd: namespace := models.NewNamespace(scheduler.Name) err := namespace.Create(clientset) Expect(err).NotTo(HaveOccurred()) + for _, roomName := range expectedRooms { - pod, err := models.NewPod(roomName, nil, configYaml, clientset, mockRedisClient) + mt.MockPodNotFound(mockRedisClient, configYaml.Name, roomName) + pod, err := models.NewPod(roomName, nil, configYaml, clientset, mockRedisClient, mr) Expect(err).NotTo(HaveOccurred()) _, err = pod.Create(clientset) Expect(err).NotTo(HaveOccurred()) + } - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + for _, roomName := range expectedRooms { + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(2) for _, st := range allStatus { mockPipeline.EXPECT().SRem(models.GetRoomStatusSetRedisKey(scheduler.Name, st), gomock.Any()) mockPipeline.EXPECT().ZRem(models.GetLastStatusRedisKey(scheduler.Name, st), roomName) @@ -1249,7 +1258,8 @@ cmd: mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(scheduler.Name, mt), gomock.Any()) } mockPipeline.EXPECT().Del(gomock.Any()) - mockPipeline.EXPECT().Exec() + mockPipeline.EXPECT().HDel(models.GetPodMapRedisKey(configYaml.Name), roomName) + mockPipeline.EXPECT().Exec().Times(2) } err = controller.DeleteUnavailableRooms(logger, roomManager, mr, mockRedisClient, clientset, scheduler, expectedRooms, "deletion_reason") @@ -1272,14 +1282,15 @@ cmd: err := namespace.Create(clientset) Expect(err).NotTo(HaveOccurred()) for _, roomName := range expectedRooms { - pod, err := models.NewPod(roomName, nil, configYaml, clientset, mockRedisClient) + mt.MockPodNotFound(mockRedisClient, scheduler.Name, roomName) + pod, err := models.NewPod(roomName, nil, configYaml, clientset, mockRedisClient, mr) Expect(err).NotTo(HaveOccurred()) _, err = pod.Create(clientset) Expect(err).NotTo(HaveOccurred()) } for _, name := range expectedRooms { - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(2) room := models.NewRoom(name, scheduler.Name) for _, status := range allStatus { mockPipeline.EXPECT(). @@ -1292,7 +1303,8 @@ cmd: } mockPipeline.EXPECT().ZRem(models.GetRoomPingRedisKey(scheduler.Name), room.ID) mockPipeline.EXPECT().Del(room.GetRoomRedisKey()) - mockPipeline.EXPECT().Exec() + mockPipeline.EXPECT().HDel(models.GetPodMapRedisKey(scheduler.Name), room.ID) + mockPipeline.EXPECT().Exec().Times(2) } err = controller.DeleteUnavailableRooms(logger, roomManager, mr, mockRedisClient, clientset, scheduler, expectedRooms, "deletion_reason") @@ -1308,7 +1320,7 @@ cmd: Expect(err).NotTo(HaveOccurred()) for _, name := range expectedRooms { - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(2) room := models.NewRoom(name, scheduler.Name) for _, status := range allStatus { mockPipeline.EXPECT(). @@ -1321,7 +1333,8 @@ cmd: } mockPipeline.EXPECT().ZRem(models.GetRoomPingRedisKey(scheduler.Name), room.ID) mockPipeline.EXPECT().Del(room.GetRoomRedisKey()) - mockPipeline.EXPECT().Exec() + mockPipeline.EXPECT().HDel(models.GetPodMapRedisKey(scheduler.Name), room.ID) + mockPipeline.EXPECT().Exec().Times(2) } err = controller.DeleteUnavailableRooms(logger, roomManager, mr, mockRedisClient, clientset, scheduler, expectedRooms, "deletion_reason") @@ -1337,21 +1350,14 @@ cmd: err := namespace.Create(clientset) Expect(err).NotTo(HaveOccurred()) for _, roomName := range expectedRooms { - pod, err := models.NewPod(roomName, nil, configYaml, clientset, mockRedisClient) + mt.MockPodNotFound(mockRedisClient, scheduler.Name, roomName) + pod, err := models.NewPod(roomName, nil, configYaml, clientset, mockRedisClient, mr) Expect(err).NotTo(HaveOccurred()) _, err = pod.Create(clientset) Expect(err).NotTo(HaveOccurred()) mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - for _, st := range allStatus { - mockPipeline.EXPECT().SRem(models.GetRoomStatusSetRedisKey(scheduler.Name, st), gomock.Any()) - mockPipeline.EXPECT().ZRem(models.GetLastStatusRedisKey(scheduler.Name, st), roomName) - } - mockPipeline.EXPECT().ZRem(models.GetRoomPingRedisKey(scheduler.Name), roomName) - for _, mt := range allMetrics { - mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(scheduler.Name, mt), gomock.Any()) - } - mockPipeline.EXPECT().Del(gomock.Any()) + mockPipeline.EXPECT().HDel(models.GetPodMapRedisKey(scheduler.Name), roomName) mockPipeline.EXPECT().Exec().Return(nil, errors.New("redis error")) } @@ -1361,6 +1367,17 @@ cmd: }) Describe("ScaleUp", func() { + var maxScaleUpAmount int + + BeforeEach(func() { + maxScaleUpAmount = config.GetInt("watcher.maxScaleUpAmount") + config.Set("watcher.maxScaleUpAmount", 100) + }) + + AfterEach(func() { + config.Set("watcher.maxScaleUpAmount", maxScaleUpAmount) + }) + It("should fail and return error if error creating pods and initial op", func() { amount := 5 var configYaml1 models.ConfigYAML @@ -1368,16 +1385,14 @@ cmd: Expect(err).NotTo(HaveOccurred()) scheduler := models.NewScheduler(configYaml1.Name, configYaml1.Game, yaml1) - err = mt.MockSetScallingAmount( + firstExec := mt.MockSetScallingAmountAndReturnExec( mockRedisClient, mockPipeline, - mockDb, - clientset, &configYaml1, 0, - yaml1, ) - Expect(err).NotTo(HaveOccurred()) + + secondExec := mt.MockListPods(mockPipeline, mockRedisClient, configYaml1.Name, []string{}, nil).After(firstExec) mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( @@ -1388,9 +1403,12 @@ cmd: ) mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml1.Name), gomock.Any()) mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml1.Name, "creating"), gomock.Any()) - mockPipeline.EXPECT().Exec().Return([]goredis.Cmder{}, errors.New("some error in redis")) - err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, true) + mockPipeline.EXPECT().Exec(). + Return([]goredis.Cmder{}, errors.New("some error in redis")). + After(secondExec) + + err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, true, config) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("some error in redis")) @@ -1406,25 +1424,7 @@ cmd: Expect(err).NotTo(HaveOccurred()) scheduler := models.NewScheduler(configYaml1.Name, configYaml1.Game, yaml1) - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(amount) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(amount) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml1.Name), gomock.Any()).Times(amount) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml1.Name, "creating"), gomock.Any()).Times(amount) - mockPipeline.EXPECT().Exec().Times(amount) - - mockRedisClient.EXPECT(). - Get(models.GlobalPortsPoolKey). - Return(goredis.NewStringResult(workerPortRange, nil)). - Times(amount) - mockPortChooser.EXPECT(). - Choose(portStart, portEnd, 2). - Return([]int{5000, 5001}). - Times(amount) + mt.MockScaleUp(mockPipeline, mockRedisClient, configYaml1.Name, amount) err = mt.MockSetScallingAmount( mockRedisClient, @@ -1438,7 +1438,7 @@ cmd: Expect(err).NotTo(HaveOccurred()) start := time.Now().UnixNano() - err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, true) + err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, true, config) elapsed := time.Now().UnixNano() - start Expect(err).NotTo(HaveOccurred()) Expect(elapsed).To(BeNumerically(">=", 100*time.Millisecond)) @@ -1455,26 +1455,14 @@ cmd: Expect(err).NotTo(HaveOccurred()) scheduler := models.NewScheduler(configYaml1.Name, configYaml1.Game, yaml1) - mockRedisClient.EXPECT(). - Get(models.GlobalPortsPoolKey). - Return(goredis.NewStringResult(workerPortRange, nil)).Times(amount - 1) - nPorts := len(configYaml1.Ports) - ports := make([]int, nPorts) - for i := 0; i < nPorts; i++ { - ports[i] = portStart + i - } - mockPortChooser.EXPECT().Choose(portStart, portEnd, nPorts).Return(ports).Times(amount - 1) + firstExec := mt.MockListPods(mockPipeline, mockRedisClient, configYaml1.Name, []string{}, nil) - err = mt.MockSetScallingAmount( + secondExec := mt.MockSetScallingAmountAndReturnExec( mockRedisClient, mockPipeline, - mockDb, - clientset, &configYaml1, 0, - yaml1, - ) - Expect(err).NotTo(HaveOccurred()) + ).After(firstExec) mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(amount) mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( @@ -1485,10 +1473,13 @@ cmd: ).Times(amount) mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml1.Name), gomock.Any()).Times(amount) mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml1.Name, "creating"), gomock.Any()).Times(amount) - mockPipeline.EXPECT().Exec().Return([]goredis.Cmder{}, errors.New("some error in redis")) - mockPipeline.EXPECT().Exec().Times(amount - 1) + thirdExec := mockPipeline.EXPECT().Exec().Return([]goredis.Cmder{}, errors.New("some error in redis")).After(secondExec) + mockPipeline.EXPECT().Exec().Times(amount - 1).After(thirdExec) - err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, false) + mt.MockAnyRunningPod(mockRedisClient, configYaml1.Name, (amount-1)*2) + + + err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, false, config) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("some error in redis")) @@ -1504,10 +1495,12 @@ cmd: Expect(err).NotTo(HaveOccurred()) scheduler := models.NewScheduler(configYaml1.Name, configYaml1.Game, yaml1) + mt.MockListPods(mockPipeline, mockRedisClient, configYaml1.Name, []string{}, nil) + mt.MockAnyRunningPod(mockRedisClient, configYaml1.Name, amount) mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(amount) mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) + Expect(statusInfo["status"]).To(Equal("creating")) Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) }, ).Times(amount) @@ -1515,15 +1508,6 @@ cmd: mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml1.Name, "creating"), gomock.Any()).Times(amount) mockPipeline.EXPECT().Exec().Times(amount) - mockRedisClient.EXPECT(). - Get(models.GlobalPortsPoolKey). - Return(goredis.NewStringResult(workerPortRange, nil)). - Times(amount) - mockPortChooser.EXPECT(). - Choose(portStart, portEnd, 2). - Return([]int{5000, 5001}). - Times(amount) - err = mt.MockSetScallingAmount( mockRedisClient, mockPipeline, @@ -1536,7 +1520,7 @@ cmd: Expect(err).NotTo(HaveOccurred()) timeoutSec = 0 - err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, true) + err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, true, config) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("timeout scaling up scheduler")) }) @@ -1548,20 +1532,26 @@ cmd: Expect(err).NotTo(HaveOccurred()) scheduler := models.NewScheduler(configYaml1.Name, configYaml1.Game, yaml1) - pod := &v1.Pod{} - pod.Name = "room-0" - pod.Status.Phase = v1.PodPending - _, err = clientset.CoreV1().Pods(scheduler.Name).Create(pod) - Expect(err).NotTo(HaveOccurred()) - - for i := 1; i < amount; i++ { - pod := &v1.Pod{} + pods := make(map[string]string, amount) + for i := 0; i < amount; i++ { + pod := &models.Pod{} pod.Name = fmt.Sprintf("room-%d", i) - _, err := clientset.CoreV1().Pods(scheduler.Name).Create(pod) + pod.Status.Phase = v1.PodRunning + if i == 0 { + pod.Status.Phase = v1.PodPending + } + jsonBytes, err := pod.MarshalToRedis() Expect(err).NotTo(HaveOccurred()) + pods[pod.Name] = string(jsonBytes) } - err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, true) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT(). + HGetAll(models.GetPodMapRedisKey(configYaml1.Name)). + Return(goredis.NewStringStringMapResult(pods, nil)) + mockPipeline.EXPECT().Exec() + + err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, true, config) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("there are pending pods, check if there are enough CPU and memory to allocate new rooms")) }) @@ -1575,27 +1565,7 @@ cmd: Expect(err).NotTo(HaveOccurred()) scheduler := models.NewScheduler(configYaml1.Name, configYaml1.Game, yamlWithLimit) - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(configYaml1.AutoScaling.Max - currentRooms) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(configYaml1.AutoScaling.Max - currentRooms) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml1.Name), gomock.Any()). - Times(configYaml1.AutoScaling.Max - currentRooms) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml1.Name, "creating"), gomock.Any()). - Times(configYaml1.AutoScaling.Max - currentRooms) - mockPipeline.EXPECT().Exec().Times(configYaml1.AutoScaling.Max - currentRooms) - - mockRedisClient.EXPECT(). - Get(models.GlobalPortsPoolKey). - Return(goredis.NewStringResult(workerPortRange, nil)). - Times(configYaml1.AutoScaling.Max - currentRooms) - mockPortChooser.EXPECT(). - Choose(portStart, portEnd, 2). - Return([]int{5000, 5001}). - Times(configYaml1.AutoScaling.Max - currentRooms) + mt.MockScaleUp(mockPipeline, mockRedisClient, configYaml1.Name, configYaml1.AutoScaling.Max - currentRooms) mt.MockSetScallingAmount(mockRedisClient, mockPipeline, mockDb, clientset, &configYaml1, currentRooms, yamlWithLimit) for i := 0; i < currentRooms; i++ { @@ -1605,7 +1575,7 @@ cmd: Expect(err).NotTo(HaveOccurred()) } - err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, true) + err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, true, config) Expect(err).ToNot(HaveOccurred()) pods, err := clientset.CoreV1().Pods(configYaml1.Name).List(metav1.ListOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1622,6 +1592,8 @@ cmd: Expect(err).NotTo(HaveOccurred()) scheduler := models.NewScheduler(configYaml1.Name, configYaml1.Game, yamlWithLimit) + mt.MockListPods(mockPipeline, mockRedisClient, configYaml1.Name, []string{}, nil) + mt.MockSetScallingAmount(mockRedisClient, mockPipeline, mockDb, clientset, &configYaml1, currentRooms, yamlWithLimit) for i := 0; i < currentRooms; i++ { pod := &v1.Pod{} @@ -1630,7 +1602,7 @@ cmd: Expect(err).NotTo(HaveOccurred()) } - err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, true) + err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, true, config) Expect(err).ToNot(HaveOccurred()) pods, err := clientset.CoreV1().Pods(configYaml1.Name).List(metav1.ListOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1646,6 +1618,8 @@ cmd: Expect(err).NotTo(HaveOccurred()) scheduler := models.NewScheduler(configYaml1.Name, configYaml1.Game, yamlWithLimit) + mt.MockListPods(mockPipeline, mockRedisClient, configYaml1.Name, []string{}, nil) + mt.MockSetScallingAmount(mockRedisClient, mockPipeline, mockDb, clientset, &configYaml1, currentRooms, yamlWithLimit) for i := 0; i < currentRooms; i++ { pod := &v1.Pod{} @@ -1654,7 +1628,7 @@ cmd: Expect(err).NotTo(HaveOccurred()) } - err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, true) + err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, true, config) Expect(err).ToNot(HaveOccurred()) pods, err := clientset.CoreV1().Pods(configYaml1.Name).List(metav1.ListOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1668,25 +1642,7 @@ cmd: Expect(err).NotTo(HaveOccurred()) scheduler := models.NewScheduler(configYaml1.Name, configYaml1.Game, yaml2) - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(amount) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(amount) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml1.Name), gomock.Any()).Times(amount) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml1.Name, "creating"), gomock.Any()).Times(amount) - mockPipeline.EXPECT().Exec().Times(amount) - - mockRedisClient.EXPECT(). - Get(models.GlobalPortsPoolKey). - Return(goredis.NewStringResult(workerPortRange, nil)). - Times(amount) - mockPortChooser.EXPECT(). - Choose(portStart, portEnd, 2). - Return([]int{5000, 5001}). - Times(amount * 2) + mt.MockScaleUp(mockPipeline, mockRedisClient, configYaml1.Name, amount) err = mt.MockSetScallingAmount( mockRedisClient, @@ -1699,7 +1655,7 @@ cmd: ) Expect(err).NotTo(HaveOccurred()) - err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, true) + err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, true, config) Expect(err).NotTo(HaveOccurred()) pods, err := clientset.CoreV1().Pods(configYaml1.Name).List(metav1.ListOptions{}) @@ -1742,21 +1698,7 @@ portRange: Expect(err).NotTo(HaveOccurred()) scheduler := models.NewScheduler(configYaml.Name, configYaml.Game, yamlStr) - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(amount) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(amount) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml.Name), gomock.Any()).Times(amount) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml.Name, "creating"), gomock.Any()).Times(amount) - mockPipeline.EXPECT().Exec().Times(amount) - - schedulerPortStart := configYaml.PortRange.Start - schedulerPortEnd := configYaml.PortRange.End - mt.MockGetPortsFromPool(&configYaml, mockRedisClient, mockPortChooser, - workerPortRange, schedulerPortStart, schedulerPortEnd, 0) + mt.MockScaleUp(mockPipeline, mockRedisClient, configYaml1.Name, amount) err = mt.MockSetScallingAmount( mockRedisClient, @@ -1769,7 +1711,7 @@ portRange: ) Expect(err).NotTo(HaveOccurred()) - err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, true) + err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, amount, timeoutSec, true, config) Expect(err).NotTo(HaveOccurred()) pods, err := clientset.CoreV1().Pods(configYaml.Name).List(metav1.ListOptions{}) @@ -1779,6 +1721,18 @@ portRange: }) Describe("ScaleDown", func() { + + var maxScaleUpAmount int + + BeforeEach(func() { + maxScaleUpAmount = config.GetInt("watcher.maxScaleUpAmount") + config.Set("watcher.maxScaleUpAmount", 100) + }) + + AfterEach(func() { + config.Set("watcher.maxScaleUpAmount", maxScaleUpAmount) + }) + It("should succeed in scaling down", func() { var configYaml1 models.ConfigYAML err := yaml.Unmarshal([]byte(yaml1), &configYaml1) @@ -1787,32 +1741,7 @@ portRange: // ScaleUp scaleUpAmount := 5 - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(scaleUpAmount) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(scaleUpAmount) - mockPipeline.EXPECT(). - ZAdd(models.GetRoomPingRedisKey(configYaml1.Name), gomock.Any()). - Times(scaleUpAmount) - mockPipeline.EXPECT(). - SAdd(models.GetRoomStatusSetRedisKey(configYaml1.Name, "creating"), gomock.Any()). - Times(scaleUpAmount) - mockPipeline.EXPECT().Exec().Times(scaleUpAmount) - - for i := 0; i < scaleUpAmount; i++ { - mockRedisClient.EXPECT(). - Get(models.GlobalPortsPoolKey). - Return(goredis.NewStringResult(workerPortRange, nil)) - nPorts := len(configYaml1.Ports) - ports := make([]int, nPorts) - for i := 0; i < nPorts; i++ { - ports[i] = portStart + i - } - mockPortChooser.EXPECT().Choose(portStart, portEnd, nPorts).Return(ports) - } + mt.MockScaleUp(mockPipeline, mockRedisClient, configYaml1.Name, scaleUpAmount) err = mt.MockSetScallingAmount( mockRedisClient, @@ -1825,7 +1754,8 @@ portRange: ) Expect(err).NotTo(HaveOccurred()) - err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, scaleUpAmount, timeoutSec, true) + err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, scaleUpAmount, timeoutSec, true, config) + Expect(err).NotTo(HaveOccurred()) // ScaleDown scaleDownAmount := 2 @@ -1836,27 +1766,11 @@ portRange: mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) for _, name := range names { mockPipeline.EXPECT().SPop(readyKey).Return(goredis.NewStringResult(name, nil)) + mt.MockPodNotFound(mockRedisClient, configYaml1.Name, name) } mockPipeline.EXPECT().Exec() - for _, name := range names { - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - room := models.NewRoom(name, scheduler.Name) - for _, status := range allStatus { - mockPipeline.EXPECT(). - SRem(models.GetRoomStatusSetRedisKey(room.SchedulerName, status), room.GetRoomRedisKey()) - mockPipeline.EXPECT(). - ZRem(models.GetLastStatusRedisKey(room.SchedulerName, status), room.ID) - } - for _, mt := range allMetrics { - mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(scheduler.Name, mt), gomock.Any()) - } - mockPipeline.EXPECT().ZRem(models.GetRoomPingRedisKey(scheduler.Name), room.ID) - mockPipeline.EXPECT().Del(room.GetRoomRedisKey()) - mockPipeline.EXPECT().Exec() - } - err = mt.MockSetScallingAmount( mockRedisClient, mockPipeline, @@ -1869,7 +1783,7 @@ portRange: Expect(err).NotTo(HaveOccurred()) timeoutSec = 300 - err = controller.ScaleDown(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, scaleDownAmount, timeoutSec) + err = controller.ScaleDown(context.Background(), logger, roomManager, mr, mockDb, redisClient, clientset, scheduler, scaleDownAmount, timeoutSec) Expect(err).NotTo(HaveOccurred()) pods, err := clientset.CoreV1().Pods(scheduler.Name).List(metav1.ListOptions{ FieldSelector: fields.Everything().String(), @@ -1912,29 +1826,7 @@ portRange: // ScaleUp scaleUpAmount := 5 - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(scaleUpAmount) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(scaleUpAmount) - mockPipeline.EXPECT(). - ZAdd(models.GetRoomPingRedisKey(configYaml.Name), gomock.Any()). - Times(scaleUpAmount) - mockPipeline.EXPECT(). - SAdd(models.GetRoomStatusSetRedisKey(configYaml.Name, "creating"), gomock.Any()). - Times(scaleUpAmount) - mockPipeline.EXPECT().Exec().Times(scaleUpAmount) - - for i := 0; i < scaleUpAmount; i++ { - nPorts := len(configYaml.Ports) - ports := make([]int, nPorts) - for i := 0; i < nPorts; i++ { - ports[i] = configYaml.PortRange.Start + i - } - mockPortChooser.EXPECT().Choose(configYaml.PortRange.Start, configYaml.PortRange.End, nPorts).Return(ports) - } + mt.MockScaleUp(mockPipeline, mockRedisClient, configYaml1.Name, scaleUpAmount) err = mt.MockSetScallingAmount( mockRedisClient, @@ -1948,7 +1840,7 @@ portRange: Expect(err).NotTo(HaveOccurred()) err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, - scheduler, scaleUpAmount, timeoutSec, true) + scheduler, scaleUpAmount, timeoutSec, true, config) // ScaleDown scaleDownAmount := 2 @@ -1959,27 +1851,11 @@ portRange: mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) for _, name := range names { mockPipeline.EXPECT().SPop(readyKey).Return(goredis.NewStringResult(name, nil)) + mt.MockPodNotFound(mockRedisClient, configYaml.Name, name) } mockPipeline.EXPECT().Exec() - for _, name := range names { - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - room := models.NewRoom(name, scheduler.Name) - for _, status := range allStatus { - mockPipeline.EXPECT(). - SRem(models.GetRoomStatusSetRedisKey(room.SchedulerName, status), room.GetRoomRedisKey()) - mockPipeline.EXPECT(). - ZRem(models.GetLastStatusRedisKey(room.SchedulerName, status), room.ID) - } - for _, mt := range allMetrics { - mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(scheduler.Name, mt), gomock.Any()) - } - mockPipeline.EXPECT().ZRem(models.GetRoomPingRedisKey(scheduler.Name), room.ID) - mockPipeline.EXPECT().Del(room.GetRoomRedisKey()) - mockPipeline.EXPECT().Exec() - } - err = mt.MockSetScallingAmount( mockRedisClient, mockPipeline, @@ -1992,7 +1868,7 @@ portRange: Expect(err).NotTo(HaveOccurred()) timeoutSec = 300 - err = controller.ScaleDown(logger, roomManager, mr, mockDb, mockRedisClient, clientset, + err = controller.ScaleDown(context.Background(), logger, roomManager, mr, mockDb, redisClient, clientset, scheduler, scaleDownAmount, timeoutSec) Expect(err).NotTo(HaveOccurred()) pods, err := clientset.CoreV1().Pods(scheduler.Name).List(metav1.ListOptions{ @@ -2002,101 +1878,6 @@ portRange: Expect(pods.Items).To(HaveLen(scaleUpAmount - scaleDownAmount)) }) - It("should return error if redis fails to clear room statuses", func() { - var configYaml1 models.ConfigYAML - err := yaml.Unmarshal([]byte(yaml1), &configYaml1) - Expect(err).NotTo(HaveOccurred()) - scheduler := models.NewScheduler(configYaml1.Name, configYaml1.Game, yaml1) - - // ScaleUp - scaleUpAmount := 5 - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(scaleUpAmount) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(scaleUpAmount) - mockPipeline.EXPECT(). - ZAdd(models.GetRoomPingRedisKey(configYaml1.Name), gomock.Any()). - Times(scaleUpAmount) - mockPipeline.EXPECT(). - SAdd(models.GetRoomStatusSetRedisKey(configYaml1.Name, "creating"), gomock.Any()). - Times(scaleUpAmount) - mockPipeline.EXPECT().Exec().Times(scaleUpAmount) - - for i := 0; i < scaleUpAmount; i++ { - mockRedisClient.EXPECT(). - Get(models.GlobalPortsPoolKey). - Return(goredis.NewStringResult(workerPortRange, nil)) - nPorts := len(configYaml1.Ports) - ports := make([]int, nPorts) - for i := 0; i < nPorts; i++ { - ports[i] = portStart + i - } - mockPortChooser.EXPECT().Choose(portStart, portEnd, nPorts).Return(ports) - } - - err = mt.MockSetScallingAmount( - mockRedisClient, - mockPipeline, - mockDb, - clientset, - &configYaml1, - 0, - yamlWithLimit, - ) - Expect(err).NotTo(HaveOccurred()) - - err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, - scheduler, scaleUpAmount, timeoutSec, true) - - // ScaleDown - scaleDownAmount := 2 - names, err := controller.GetPodNames(scaleDownAmount, scheduler.Name, clientset) - Expect(err).NotTo(HaveOccurred()) - - readyKey := models.GetRoomStatusSetRedisKey(configYaml1.Name, models.StatusReady) - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - for _, name := range names { - mockPipeline.EXPECT().SPop(readyKey).Return(goredis.NewStringResult(name, nil)) - } - mockPipeline.EXPECT().Exec() - - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - for range allStatus { - mockPipeline.EXPECT(). - SRem(gomock.Any(), gomock.Any()) - mockPipeline.EXPECT(). - ZRem(gomock.Any(), gomock.Any()) - } - for _, mt := range allMetrics { - mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(scheduler.Name, mt), gomock.Any()) - } - mockPipeline.EXPECT().ZRem(models.GetRoomPingRedisKey(scheduler.Name), gomock.Any()) - mockPipeline.EXPECT().Del(gomock.Any()) - mockPipeline.EXPECT().Exec().Return([]goredis.Cmder{}, errors.New("some error in redis")) - - err = mt.MockSetScallingAmount( - mockRedisClient, - mockPipeline, - mockDb, - clientset, - &configYaml1, - scaleUpAmount, - yamlWithLimit, - ) - Expect(err).NotTo(HaveOccurred()) - - err = controller.ScaleDown(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, scaleDownAmount, timeoutSec) - Expect(err).To(HaveOccurred()) - pods, err := clientset.CoreV1().Pods(scheduler.Name).List(metav1.ListOptions{ - FieldSelector: fields.Everything().String(), - }) - Expect(err).NotTo(HaveOccurred()) - Expect(pods.Items).To(HaveLen(scaleUpAmount - 1)) - }) - It("should not return error if delete non existing pod", func() { var configYaml1 models.ConfigYAML err := yaml.Unmarshal([]byte(yaml1), &configYaml1) @@ -2121,7 +1902,7 @@ portRange: Expect(err).NotTo(HaveOccurred()) timeoutSec = 300 - err = controller.ScaleDown(logger, roomManager, mr, mockDb, mockRedisClient, clientset, + err = controller.ScaleDown(context.Background(), logger, roomManager, mr, mockDb, redisClient, clientset, scheduler, scaleDownAmount, timeoutSec) Expect(err).NotTo(HaveOccurred()) }) @@ -2134,32 +1915,7 @@ portRange: // ScaleUp scaleUpAmount := 5 - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(scaleUpAmount) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(scaleUpAmount) - mockPipeline.EXPECT(). - ZAdd(models.GetRoomPingRedisKey(configYaml1.Name), gomock.Any()). - Times(scaleUpAmount) - mockPipeline.EXPECT(). - SAdd(models.GetRoomStatusSetRedisKey(configYaml1.Name, "creating"), gomock.Any()). - Times(scaleUpAmount) - mockPipeline.EXPECT().Exec().Times(scaleUpAmount) - - for i := 0; i < scaleUpAmount; i++ { - mockRedisClient.EXPECT(). - Get(models.GlobalPortsPoolKey). - Return(goredis.NewStringResult(workerPortRange, nil)) - nPorts := len(configYaml1.Ports) - ports := make([]int, nPorts) - for i := 0; i < nPorts; i++ { - ports[i] = portStart + i - } - mockPortChooser.EXPECT().Choose(portStart, portEnd, nPorts).Return(ports) - } + mt.MockScaleUp(mockPipeline, mockRedisClient, configYaml1.Name, scaleUpAmount) err = mt.MockSetScallingAmount( mockRedisClient, @@ -2173,7 +1929,7 @@ portRange: Expect(err).NotTo(HaveOccurred()) err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, - scheduler, scaleUpAmount, timeoutSec, true) + scheduler, scaleUpAmount, timeoutSec, true, config) // ScaleDown scaleDownAmount := 2 @@ -2187,23 +1943,6 @@ portRange: } mockPipeline.EXPECT().Exec() - for _, name := range names { - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - room := models.NewRoom(name, scheduler.Name) - for _, status := range allStatus { - mockPipeline.EXPECT(). - SRem(models.GetRoomStatusSetRedisKey(room.SchedulerName, status), room.GetRoomRedisKey()) - mockPipeline.EXPECT(). - ZRem(models.GetLastStatusRedisKey(room.SchedulerName, status), room.ID) - } - for _, mt := range allMetrics { - mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(scheduler.Name, mt), gomock.Any()) - } - mockPipeline.EXPECT().ZRem(models.GetRoomPingRedisKey(scheduler.Name), room.ID) - mockPipeline.EXPECT().Del(room.GetRoomRedisKey()) - mockPipeline.EXPECT().Exec() - } - err = mt.MockSetScallingAmount( mockRedisClient, mockPipeline, @@ -2216,7 +1955,7 @@ portRange: Expect(err).NotTo(HaveOccurred()) timeoutSec = 0 - err = controller.ScaleDown(logger, roomManager, mr, mockDb, mockRedisClient, clientset, + err = controller.ScaleDown(context.Background(), logger, roomManager, mr, mockDb, redisClient, clientset, scheduler, scaleDownAmount, timeoutSec) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("timeout scaling down scheduler")) @@ -3629,6 +3368,10 @@ cmd: mt.MockCreateRoomsAnyTimes(mockRedisClient, mockPipeline, &configYaml2, len(pods.Items)) mt.MockGetPortsFromPool(&configYaml2, mockRedisClient, mockPortChooser, workerPortRange, portStart, portEnd, 0) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml2.Name), gomock.Any()). + Return(goredis.NewStringResult("", goredis.Nil)). + Times(len(pods.Items)) scheduler1.Version = "v2.0" for _, pod := range pods.Items { Expect(pod.ObjectMeta.Labels["heritage"]).To(Equal("maestro")) @@ -3640,6 +3383,10 @@ cmd: } scheduler1.Version = "v1.0" + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(scheduler1.Name)).Return(goredis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + calls := mt.NewCalls() mt.MockSaveSchedulerFlow( @@ -3778,6 +3525,10 @@ portRange: mt.MockCreateRoomsAnyTimes(mockRedisClient, mockPipeline, &configYaml2, len(pods.Items)) mt.MockGetPortsFromPool(&configYaml2, mockRedisClient, mockPortChooser, workerPortRange, configYaml2.PortRange.Start, configYaml2.PortRange.End, 0) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml2.Name), gomock.Any()). + Return(goredis.NewStringResult("", goredis.Nil)). + Times(len(pods.Items)) scheduler1.Version = "v2.0" for _, pod := range pods.Items { Expect(pod.ObjectMeta.Labels["heritage"]).To(Equal("maestro")) @@ -3789,6 +3540,10 @@ portRange: } scheduler1.Version = "v1.0" + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(scheduler1.Name)).Return(goredis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + // check other scheduler ports mt.MockSelectSchedulerNames(mockDb, []string{}, nil) mt.MockSelectConfigYamls(mockDb, []models.Scheduler{}, nil) @@ -3971,6 +3726,11 @@ cmd: mt.MockCreateRoomsAnyTimes(mockRedisClient, mockPipeline, &configYaml2, len(pods.Items)) mt.MockGetPortsFromPool(&configYaml2, mockRedisClient, mockPortChooser, workerPortRange, portStart, portEnd, 0) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml2.Name), gomock.Any()). + Return(goredis.NewStringResult("", goredis.Nil)). + Times(len(pods.Items)) + scheduler1.Version = "v2.0" for _, pod := range pods.Items { Expect(pod.ObjectMeta.Labels["heritage"]).To(Equal("maestro")) @@ -3982,6 +3742,10 @@ cmd: } scheduler1.Version = "v1.0" + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(scheduler1.Name)).Return(goredis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + calls := mt.NewCalls() mt.MockSaveSchedulerFlow( @@ -4158,6 +3922,11 @@ portRange: mt.MockCreateRoomsAnyTimes(mockRedisClient, mockPipeline, &configYaml2, len(pods.Items)) mt.MockGetPortsFromPool(&configYaml2, mockRedisClient, mockPortChooser, workerPortRange, configYaml2.PortRange.Start, configYaml2.PortRange.End, 0) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml2.Name), gomock.Any()). + Return(goredis.NewStringResult("", goredis.Nil)). + Times(len(pods.Items)) + scheduler1.Version = "v2.0" for _, pod := range pods.Items { Expect(pod.ObjectMeta.Labels["heritage"]).To(Equal("maestro")) @@ -4169,6 +3938,10 @@ portRange: } scheduler1.Version = "v1.0" + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(scheduler1.Name)).Return(goredis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + // check other scheduler ports mt.MockSelectSchedulerNames(mockDb, []string{}, nil) mt.MockSelectConfigYamls(mockDb, []models.Scheduler{}, nil) @@ -4506,6 +4279,10 @@ cmd: }) It("should return error if timeout when creating rooms", func() { + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(scheduler1.Name)).Return(goredis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + calls := mt.NewCalls() mt.MockSaveSchedulerFlow( @@ -4562,6 +4339,10 @@ cmd: }) It("should return error if timeout when deleting rooms", func() { + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(scheduler1.Name)).Return(goredis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + calls := mt.NewCalls() mt.MockSaveSchedulerFlow( @@ -4632,6 +4413,10 @@ cmd: mt.MockCreateRoomsAnyTimes(mockRedisClient, mockPipeline, &configYaml2, len(pods.Items)) mt.MockGetPortsFromPool(&configYaml2, mockRedisClient, mockPortChooser, workerPortRange, portStart, portEnd, 0) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml2.Name), gomock.Any()). + Return(goredis.NewStringResult("", goredis.Nil)). + Times(len(pods.Items)) scheduler1.Version = "v2.0" for _, pod := range pods.Items { err = roomManager.Delete(logger, mr, clientset, mockRedisClient, &configYaml2, pod.Name, "deletion_reason") @@ -4641,6 +4426,10 @@ cmd: } scheduler1.Version = "v1.0" + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(scheduler1.Name)).Return(goredis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + calls := mt.NewCalls() mt.MockSaveSchedulerFlow( @@ -4917,7 +4706,8 @@ containers: Expect(err).NotTo(HaveOccurred()) for _, roomName := range expectedRooms { - pod, err := models.NewPod(roomName, nil, configYaml, clientset, mockRedisClient) + mt.MockPodNotFound(mockRedisClient, configYaml.Name, roomName) + pod, err := models.NewPod(roomName, nil, configYaml, clientset, mockRedisClient, mr) Expect(err).NotTo(HaveOccurred()) _, err = pod.Create(clientset) Expect(err).NotTo(HaveOccurred()) @@ -4925,6 +4715,11 @@ containers: for _, roomName := range expectedRooms { room := models.NewRoom(roomName, scheduler.Name) + + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HDel(models.GetPodMapRedisKey(configYaml.Name), room.ID) + mockPipeline.EXPECT().Exec() + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) for _, status := range allStatus { mockPipeline.EXPECT().SRem(models.GetRoomStatusSetRedisKey(scheduler.Name, status), room.GetRoomRedisKey()) @@ -4952,7 +4747,8 @@ containers: err := namespace.Create(clientset) Expect(err).NotTo(HaveOccurred()) for _, roomName := range expectedRooms { - pod, err := models.NewPod(roomName, nil, configYaml, clientset, mockRedisClient) + mt.MockPodNotFound(mockRedisClient, configYaml.Name, roomName) + pod, err := models.NewPod(roomName, nil, configYaml, clientset, mockRedisClient, mr) Expect(err).NotTo(HaveOccurred()) _, err = pod.Create(clientset) Expect(err).NotTo(HaveOccurred()) @@ -4972,7 +4768,8 @@ containers: Expect(err).NotTo(HaveOccurred()) for _, roomName := range expectedRooms { - pod, err := models.NewPod(roomName, nil, configYaml, clientset, mockRedisClient) + mt.MockPodNotFound(mockRedisClient, configYaml.Name, roomName) + pod, err := models.NewPod(roomName, nil, configYaml, clientset, mockRedisClient, mr) Expect(err).NotTo(HaveOccurred()) _, err = pod.Create(clientset) Expect(err).NotTo(HaveOccurred()) @@ -4980,15 +4777,7 @@ containers: for _, roomName := range expectedRooms { room := models.NewRoom(roomName, scheduler.Name) mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - for _, status := range allStatus { - mockPipeline.EXPECT().SRem(models.GetRoomStatusSetRedisKey(scheduler.Name, status), room.GetRoomRedisKey()) - mockPipeline.EXPECT().ZRem(models.GetLastStatusRedisKey(scheduler.Name, status), roomName) - } - for _, mt := range allMetrics { - mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(scheduler.Name, mt), gomock.Any()) - } - mockPipeline.EXPECT().ZRem(models.GetRoomPingRedisKey(scheduler.Name), roomName) - mockPipeline.EXPECT().Del(room.GetRoomRedisKey()) + mockPipeline.EXPECT().HDel(models.GetPodMapRedisKey(configYaml.Name), room.ID) mockPipeline.EXPECT().Exec().Return(nil, errors.New("redis error")) } @@ -5004,6 +4793,11 @@ containers: for _, roomName := range expectedRooms { room := models.NewRoom(roomName, scheduler.Name) + + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HDel(models.GetPodMapRedisKey(scheduler.Name), room.ID) + mockPipeline.EXPECT().Exec() + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) for _, status := range allStatus { mockPipeline.EXPECT().SRem(models.GetRoomStatusSetRedisKey(scheduler.Name, status), room.GetRoomRedisKey()) @@ -5050,6 +4844,10 @@ containers: mt.MockCreateRoomsAnyTimes(mockRedisClient, mockPipeline, &configYaml1, len(pods.Items)) mt.MockGetPortsFromPool(&configYaml1, mockRedisClient, mockPortChooser, workerPortRange, portStart, portEnd, 0) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml1.Name), gomock.Any()). + Return(goredis.NewStringResult("", goredis.Nil)). + Times(len(pods.Items)) scheduler1.Version = "v2.0" configYaml1.Image = imageParams.Image for _, pod := range pods.Items { @@ -5060,6 +4858,10 @@ containers: } scheduler1.Version = "v1.0" + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(scheduler1.Name)).Return(goredis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + calls := mt.NewCalls() // Select current scheduler yaml @@ -5310,6 +5112,10 @@ containers: mt.MockCreateRoomsAnyTimes(mockRedisClient, mockPipeline, &configYaml1, len(pods.Items)) mt.MockGetPortsFromPool(&configYaml1, mockRedisClient, mockPortChooser, workerPortRange, portStart, portEnd, 4) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml1.Name), gomock.Any()). + Return(goredis.NewStringResult("", goredis.Nil)). + Times(len(pods.Items)) scheduler1.Version = "v2.0" configYaml1.Image = imageParams.Image for _, pod := range pods.Items { @@ -5320,6 +5126,11 @@ containers: } scheduler1.Version = "v1.0" + + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(scheduler1.Name)).Return(goredis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + calls := mt.NewCalls() // Select current scheduler yaml @@ -5745,15 +5556,28 @@ containers: }) Describe("ScaleScheduler", func() { + var maxScaleUpAmount int + + BeforeEach(func() { + maxScaleUpAmount = config.GetInt("watcher.maxScaleUpAmount") + config.Set("watcher.maxScaleUpAmount", 100) + }) + + AfterEach(func() { + config.Set("watcher.maxScaleUpAmount", maxScaleUpAmount) + }) + It("should return error if more than one parameter is set", func() { var amountUp, amountDown, replicas uint = 1, 1, 1 err := controller.ScaleScheduler( + context.Background(), logger, roomManager, mr, mockDb, - mockRedisClient, + redisClient, clientset, + config, 60, 60, amountUp, amountDown, replicas, configYaml1.Name, @@ -5768,12 +5592,14 @@ containers: Return(pg.NewTestResult(errors.New("some error in db"), 0), errors.New("some error in db")) err := controller.ScaleScheduler( + context.Background(), logger, roomManager, mr, mockDb, - mockRedisClient, + redisClient, clientset, + config, 60, 60, amountUp, amountDown, replicas, configYaml1.Name, @@ -5790,12 +5616,14 @@ containers: }) err := controller.ScaleScheduler( + context.Background(), logger, roomManager, mr, mockDb, - mockRedisClient, + redisClient, clientset, + config, 60, 60, amountUp, amountDown, replicas, configYaml1.Name, @@ -5812,28 +5640,7 @@ containers: *scheduler = *models.NewScheduler(configYaml1.Name, configYaml1.Game, yaml1) }) - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(int(amountUp)) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(int(amountUp)) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml1.Name), gomock.Any()).Times(int(amountUp)) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml1.Name, "creating"), gomock.Any()).Times(int(amountUp)) - mockPipeline.EXPECT().Exec().Times(int(amountUp)) - - for i := 0; i < int(amountUp); i++ { - mockRedisClient.EXPECT(). - Get(models.GlobalPortsPoolKey). - Return(goredis.NewStringResult(workerPortRange, nil)) - nPorts := len(configYaml1.Ports) - ports := make([]int, nPorts) - for i := 0; i < nPorts; i++ { - ports[i] = portStart + i - } - mockPortChooser.EXPECT().Choose(portStart, portEnd, nPorts).Return(ports) - } + mt.MockScaleUp(mockPipeline, mockRedisClient, configYaml1.Name, int(amountUp)) err := mt.MockSetScallingAmount( mockRedisClient, @@ -5847,12 +5654,14 @@ containers: Expect(err).NotTo(HaveOccurred()) err = controller.ScaleScheduler( + context.Background(), logger, roomManager, mr, mockDb, - mockRedisClient, + redisClient, clientset, + config, 60, 60, amountUp, amountDown, replicas, configYaml1.Name, @@ -5871,28 +5680,7 @@ containers: // ScaleUp scaleUpAmount := 6 - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(scaleUpAmount) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(scaleUpAmount) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml1.Name), gomock.Any()).Times(scaleUpAmount) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml1.Name, "creating"), gomock.Any()).Times(scaleUpAmount) - mockPipeline.EXPECT().Exec().Times(scaleUpAmount) - - for i := 0; i < scaleUpAmount; i++ { - mockRedisClient.EXPECT(). - Get(models.GlobalPortsPoolKey). - Return(goredis.NewStringResult(workerPortRange, nil)) - nPorts := len(configYaml1.Ports) - ports := make([]int, nPorts) - for i := 0; i < nPorts; i++ { - ports[i] = portStart + i - } - mockPortChooser.EXPECT().Choose(portStart, portEnd, nPorts).Return(ports) - } + mt.MockScaleUp(mockPipeline, mockRedisClient, configYaml1.Name, scaleUpAmount) err := mt.MockSetScallingAmount( mockRedisClient, @@ -5905,7 +5693,12 @@ containers: ) Expect(err).NotTo(HaveOccurred()) - err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, scaleUpAmount, timeoutSec, true) + err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, scaleUpAmount, timeoutSec, true, config) + Expect(err).NotTo(HaveOccurred()) + + downscalingLockKey := models.GetSchedulerDownScalingLockKey(config.GetString("watcher.lockKey"), configYaml1.Name) + mt.MockRedisLock(mockRedisClient, downscalingLockKey, 0, true, nil) + mt.MockReturnRedisLock(mockRedisClient, downscalingLockKey, nil) // ScaleDown mt.MockLoadScheduler(configYaml1.Name, mockDb). @@ -5921,27 +5714,10 @@ containers: mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) for _, name := range names { mockPipeline.EXPECT().SPop(readyKey).Return(goredis.NewStringResult(name, nil)) - + mt.MockPodNotFound(mockRedisClient, configYaml1.Name, name) } mockPipeline.EXPECT().Exec() - for _, name := range names { - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - room := models.NewRoom(name, scheduler.Name) - for _, status := range allStatus { - mockPipeline.EXPECT(). - SRem(models.GetRoomStatusSetRedisKey(room.SchedulerName, status), room.GetRoomRedisKey()) - mockPipeline.EXPECT(). - ZRem(models.GetLastStatusRedisKey(room.SchedulerName, status), room.ID) - } - for _, mt := range allMetrics { - mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(scheduler.Name, mt), gomock.Any()) - } - mockPipeline.EXPECT().ZRem(models.GetRoomPingRedisKey(scheduler.Name), room.ID) - mockPipeline.EXPECT().Del(room.GetRoomRedisKey()) - mockPipeline.EXPECT().Exec() - } - err = mt.MockSetScallingAmount( mockRedisClient, mockPipeline, @@ -5954,12 +5730,14 @@ containers: Expect(err).NotTo(HaveOccurred()) err = controller.ScaleScheduler( + context.Background(), logger, roomManager, mr, mockDb, - mockRedisClient, + redisClient, clientset, + config, 60, 60, amountUp, amountDown, replicas, configYaml1.Name, @@ -5978,28 +5756,13 @@ containers: *scheduler = *models.NewScheduler(configYaml1.Name, configYaml1.Game, yaml1) }) - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(int(replicas)) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(int(replicas)) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml1.Name), gomock.Any()).Times(int(replicas)) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml1.Name, "creating"), gomock.Any()).Times(int(replicas)) - mockPipeline.EXPECT().Exec().Times(int(replicas)) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT(). + HLen(models.GetPodMapRedisKey(configYaml1.Name)). + Return(goredis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() - for i := 0; i < int(replicas); i++ { - mockRedisClient.EXPECT(). - Get(models.GlobalPortsPoolKey). - Return(goredis.NewStringResult(workerPortRange, nil)) - nPorts := len(configYaml1.Ports) - ports := make([]int, nPorts) - for i := 0; i < nPorts; i++ { - ports[i] = portStart + i - } - mockPortChooser.EXPECT().Choose(portStart, portEnd, nPorts).Return(ports) - } + mt.MockScaleUp(mockPipeline, mockRedisClient, configYaml1.Name, int(replicas)) err := mt.MockSetScallingAmount( mockRedisClient, @@ -6013,12 +5776,14 @@ containers: Expect(err).NotTo(HaveOccurred()) err = controller.ScaleScheduler( + context.Background(), logger, roomManager, mr, mockDb, - mockRedisClient, + redisClient, clientset, + config, 60, 60, amountUp, amountDown, replicas, configYaml1.Name, @@ -6036,30 +5801,7 @@ containers: // ScaleUp scaleUpAmount := 5 - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(scaleUpAmount) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(scaleUpAmount) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml1.Name), gomock.Any()).Times(scaleUpAmount) - mockPipeline.EXPECT(). - SAdd(models.GetRoomStatusSetRedisKey(configYaml1.Name, "creating"), gomock.Any()). - Times(scaleUpAmount) - mockPipeline.EXPECT().Exec().Times(scaleUpAmount) - - for i := 0; i < scaleUpAmount; i++ { - mockRedisClient.EXPECT(). - Get(models.GlobalPortsPoolKey). - Return(goredis.NewStringResult(workerPortRange, nil)) - nPorts := len(configYaml1.Ports) - ports := make([]int, nPorts) - for i := 0; i < nPorts; i++ { - ports[i] = portStart + i - } - mockPortChooser.EXPECT().Choose(portStart, portEnd, nPorts).Return(ports) - } + mt.MockScaleUp(mockPipeline, mockRedisClient, configYaml1.Name, scaleUpAmount) err := mt.MockSetScallingAmount( mockRedisClient, @@ -6073,7 +5815,8 @@ containers: Expect(err).NotTo(HaveOccurred()) err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, - scheduler, scaleUpAmount, timeoutSec, true) + scheduler, scaleUpAmount, timeoutSec, true, config) + Expect(err).NotTo(HaveOccurred()) // ScaleDown mt.MockLoadScheduler(configYaml1.Name, mockDb). @@ -6081,6 +5824,16 @@ containers: *scheduler = *models.NewScheduler(configYaml1.Name, configYaml1.Game, yaml1) }) + downscalingLockKey := models.GetSchedulerDownScalingLockKey(config.GetString("watcher.lockKey"), configYaml1.Name) + mt.MockRedisLock(mockRedisClient, downscalingLockKey, 0, true, nil) + mt.MockReturnRedisLock(mockRedisClient, downscalingLockKey, nil) + + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT(). + HLen(models.GetPodMapRedisKey(configYaml1.Name)). + Return(goredis.NewIntResult(int64(scaleUpAmount), nil)) + mockPipeline.EXPECT().Exec() + scaleDownAmount := scaleUpAmount - int(replicas) names, err := controller.GetPodNames(scaleDownAmount, scheduler.Name, clientset) Expect(err).NotTo(HaveOccurred()) @@ -6089,26 +5842,10 @@ containers: mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) for _, name := range names { mockPipeline.EXPECT().SPop(readyKey).Return(goredis.NewStringResult(name, nil)) + mt.MockPodNotFound(mockRedisClient, configYaml1.Name, name) } mockPipeline.EXPECT().Exec() - for _, name := range names { - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - room := models.NewRoom(name, scheduler.Name) - for _, status := range allStatus { - mockPipeline.EXPECT(). - SRem(models.GetRoomStatusSetRedisKey(room.SchedulerName, status), room.GetRoomRedisKey()) - mockPipeline.EXPECT(). - ZRem(models.GetLastStatusRedisKey(room.SchedulerName, status), room.ID) - } - for _, mt := range allMetrics { - mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(scheduler.Name, mt), gomock.Any()) - } - mockPipeline.EXPECT().ZRem(models.GetRoomPingRedisKey(scheduler.Name), room.ID) - mockPipeline.EXPECT().Del(room.GetRoomRedisKey()) - mockPipeline.EXPECT().Exec() - } - err = mt.MockSetScallingAmount( mockRedisClient, mockPipeline, @@ -6121,12 +5858,14 @@ containers: Expect(err).NotTo(HaveOccurred()) err = controller.ScaleScheduler( + context.Background(), logger, roomManager, mr, mockDb, - mockRedisClient, + redisClient, clientset, + config, 60, 60, amountUp, amountDown, replicas, configYaml1.Name, @@ -6234,25 +5973,7 @@ containers: // ScaleUp scaleUpAmount := configYaml1.AutoScaling.Up.Delta - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(scaleUpAmount) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(scaleUpAmount) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml1.Name), gomock.Any()).Times(scaleUpAmount) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml1.Name, "creating"), gomock.Any()).Times(scaleUpAmount) - mockPipeline.EXPECT().Exec().Times(scaleUpAmount) - - mockRedisClient.EXPECT(). - Get(models.GlobalPortsPoolKey). - Return(goredis.NewStringResult(workerPortRange, nil)). - Times(scaleUpAmount) - mockPortChooser.EXPECT(). - Choose(portStart, portEnd, 2). - Return([]int{5000, 5001}). - Times(scaleUpAmount) + mt.MockScaleUp(mockPipeline, mockRedisClient, configYaml1.Name, scaleUpAmount) mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) diff --git a/controller/utils.go b/controller/utils.go index d535ade14..5744dde0d 100644 --- a/controller/utils.go +++ b/controller/utils.go @@ -12,8 +12,8 @@ import ( "errors" "fmt" "math" + "math/rand" "strings" - "sync" "time" "github.com/sirupsen/logrus" @@ -21,7 +21,7 @@ import ( "github.com/topfreegames/maestro/models" "github.com/topfreegames/maestro/reporters" yaml "gopkg.in/yaml.v2" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" @@ -48,12 +48,13 @@ func SegmentAndReplacePods( redisClient redisinterfaces.RedisClient, willTimeoutAt time.Time, configYAML *models.ConfigYAML, - pods []v1.Pod, + pods []*models.Pod, scheduler *models.Scheduler, operationManager *models.OperationManager, - maxSurge int, + maxSurge, goroutinePoolSize int, clock clockinterfaces.Clock, ) (timeoutErr, cancelErr, err error) { + rand.Seed(time.Now().UnixNano()) schedulerName := scheduler.Name l := logger.WithFields(logrus.Fields{ "source": "SegmentAndReplacePods", @@ -80,6 +81,7 @@ func SegmentAndReplacePods( scheduler, operationManager, clock, + goroutinePoolSize, ) if timedout { @@ -113,21 +115,84 @@ func replacePodsAndWait( redisClient redisinterfaces.RedisClient, willTimeoutAt time.Time, configYAML *models.ConfigYAML, - podsChunk []v1.Pod, + podsChunk []*models.Pod, scheduler *models.Scheduler, operationManager *models.OperationManager, clock clockinterfaces.Clock, + goroutinePoolSize int, ) (timedout, canceled bool, err error) { - timedout = false - canceled = false - var wg sync.WaitGroup - var mutex = &sync.Mutex{} + logger.Debug("starting to replace pods with new ones") + stop := make(chan struct{}, 1) + defer close(stop) // Dont leak goroutines + finishedReplace := make(chan struct{}) - // create a chunk of pods (chunkSize = maxSurge) and remove a chunk of old ones - wg.Add(len(podsChunk)) + timedoutChan := make(chan bool) + canceledChan := make(chan bool) + errChan := make(chan error) + + pods := make(chan *models.Pod, len(podsChunk)) for _, pod := range podsChunk { - go func(pod v1.Pod) { - defer wg.Done() + pods <- pod + } + + logger.Infof("starting %d in-memory workers to replace %d pods", goroutinePoolSize, len(podsChunk)) + for i := 0; i < goroutinePoolSize; i++ { + go replacePodWorker( + logger, + roomManager, + mr, + clientset, + db, + redisClient, + willTimeoutAt, + configYAML, + scheduler, + operationManager, + clock, + pods, + stop, finishedReplace, + timedoutChan, canceledChan, errChan, + ) + } + + select { + case err = <-errChan: + return false, false, err + + case canceled = <-canceledChan: + return false, canceled, nil + + case timedout = <-timedoutChan: + return timedout, false, nil + + case <-finishedReplace: + logger.Debug("all pods were successfully replaced") + break + } + + return timedout, canceled, err +} + +func replacePodWorker( + logger logrus.FieldLogger, + roomManager models.RoomManager, + mr *models.MixedMetricsReporter, + clientset kubernetes.Interface, + db pginterfaces.DB, + redisClient redisinterfaces.RedisClient, + willTimeoutAt time.Time, + configYAML *models.ConfigYAML, + scheduler *models.Scheduler, + operationManager *models.OperationManager, + clock clockinterfaces.Clock, + pods <-chan *models.Pod, + stop, finishedReplace chan struct{}, + timedoutChan, canceledChan chan<- bool, + errChan chan<- error, +) { + for { + select { + case pod := <-pods: localTimedout, localCanceled, localErr := createNewRemoveOldPod( logger, roomManager, @@ -139,32 +204,34 @@ func replacePodsAndWait( configYAML, scheduler, operationManager, - mutex, pod, clock, ) - // if a routine is timedout or canceled, - // rolling update should stop - if localTimedout { - mutex.Lock() - timedout = localTimedout - mutex.Unlock() + + if localErr != nil { + errChan <- localErr + return } + if localCanceled { - mutex.Lock() - canceled = localCanceled - mutex.Unlock() + canceledChan <- localCanceled + return } - if localErr != nil { - mutex.Lock() - err = localErr - mutex.Unlock() + + if localTimedout { + timedoutChan <- localTimedout + return } - }(pod) - } - wg.Wait() - return timedout, canceled, err + logger.Infof("pods remaining to replace: %d", len(pods)) + if len(pods) == 0 { + finishedReplace <- struct{}{} + return + } + case <-stop: + return + } + } } func createNewRemoveOldPod( @@ -178,8 +245,7 @@ func createNewRemoveOldPod( configYAML *models.ConfigYAML, scheduler *models.Scheduler, operationManager *models.OperationManager, - mutex *sync.Mutex, - pod v1.Pod, + pod *models.Pod, clock clockinterfaces.Clock, ) (timedout, canceled bool, err error) { logger.Debug("creating pod") @@ -196,46 +262,87 @@ func createNewRemoveOldPod( // wait for new pod to be created timeout := willTimeoutAt.Sub(clock.Now()) timedout, canceled, err = waitCreatingPods( - logger, clientset, timeout, configYAML.Name, + logger, clientset, redisClient, timeout, configYAML.Name, []v1.Pod{*newPod}, operationManager, mr) if timedout || canceled || err != nil { logger.Errorf("error waiting for pod to be created") return timedout, canceled, err } - // delete old pod - logger.Debugf("deleting pod %s", pod.GetName()) - err = DeletePodAndRoom(logger, roomManager, mr, clientset, redisClient, - configYAML, pod.GetName(), reportersConstants.ReasonUpdate) + timedout, canceled, err = DeletePodsAndWait( + logger, + roomManager, + mr, + clientset, + redisClient, + willTimeoutAt, + configYAML, + scheduler, + operationManager, + []*models.Pod{pod}, + clock, + ) + if err != nil && !strings.Contains(err.Error(), "redis") { - logger.WithError(err).Errorf("error deleting pod %s", pod.GetName()) return false, false, nil } - // wait for old pods to be deleted - // we assume that maxSurge == maxUnavailable as we can't set maxUnavailable yet - // so for every pod created in a chunk one is deleted right after it - timeout = willTimeoutAt.Sub(clock.Now()) - timedout, canceled = waitTerminatingPods( - logger, clientset, timeout, configYAML.Name, - []v1.Pod{pod}, operationManager, mr) if timedout || canceled { - logger.Errorf("error waiting for pod %s to be deleted", pod.GetName()) return timedout, canceled, nil } // Remove invalid rooms redis keys if in a rolling update operation // in order to track progress correctly if operationManager != nil { - err = models.RemoveInvalidRooms(redisClient, mr, configYAML.Name, []string{pod.GetName()}) + err = models.RemoveInvalidRooms(redisClient, mr, configYAML.Name, []string{pod.Name}) if err != nil { - logger.WithError(err).Warnf("error removing room %s from invalidRooms redis key during rolling update", pod.GetName()) + logger.WithError(err).Warnf("error removing room %s from invalidRooms redis key during rolling update", pod.Name) } } return false, false, nil } +// DeletePodsAndWait deletes a list of pods +func DeletePodsAndWait( + logger logrus.FieldLogger, + roomManager models.RoomManager, + mr *models.MixedMetricsReporter, + clientset kubernetes.Interface, + redisClient redisinterfaces.RedisClient, + willTimeoutAt time.Time, + configYAML *models.ConfigYAML, + scheduler *models.Scheduler, + operationManager *models.OperationManager, + pods []*models.Pod, + clock clockinterfaces.Clock, +) (timedout, canceled bool, err error) { + + for _, pod := range pods { + logger.Debugf("deleting pod %s", pod.Name) + err = DeletePodAndRoom(logger, roomManager, mr, clientset, redisClient, + configYAML, pod.Name, reportersConstants.ReasonUpdate) + if err != nil && !strings.Contains(err.Error(), "redis") { + logger.WithError(err).Errorf("error deleting pod %s", pod.Name) + return false, false, nil + } + } + + // wait for old pods to be deleted + // we assume that maxSurge == maxUnavailable as we can't set maxUnavailable yet + // so for every pod created in a chunk one is deleted right after it + timeout := willTimeoutAt.Sub(clock.Now()) + timedout, canceled = waitTerminatingPods( + logger, clientset, redisClient, timeout, configYAML.Name, + pods, operationManager, mr) + if timedout || canceled || err != nil { + logger.Error("error waiting for pods to be deleted") + return timedout, canceled, nil + } + + return false, false, nil +} + // DBRollback perform a rollback on a scheduler config in the database func DBRollback( ctx context.Context, @@ -291,9 +398,10 @@ func DBRollback( func waitTerminatingPods( l logrus.FieldLogger, clientset kubernetes.Interface, + redisClient redisinterfaces.RedisClient, timeout time.Duration, namespace string, - deletedPods []v1.Pod, + deletedPods []*models.Pod, operationManager *models.OperationManager, mr *models.MixedMetricsReporter, ) (timedout, wasCanceled bool) { @@ -320,28 +428,27 @@ func waitTerminatingPods( } for _, pod := range deletedPods { - err := mr.WithSegment(models.SegmentPod, func() error { - var err error - _, err = clientset.CoreV1().Pods(namespace).Get( - pod.GetName(), getOptions, - ) - return err - }) - - if err == nil || !strings.Contains(err.Error(), "not found") { - logger.WithField("pod", pod.GetName()).Debugf("pod still exists, deleting again") - err = mr.WithSegment(models.SegmentPod, func() error { - return clientset.CoreV1().Pods(namespace).Delete(pod.GetName(), deleteOptions) - }) + p, err := models.GetPodFromRedis(redisClient, mr, pod.Name, namespace) + if err != nil { + logger. + WithError(err). + WithField("pod", pod.Name). + Info("error getting pod") exit = false break } - if err != nil && !strings.Contains(err.Error(), "not found") { - logger. - WithError(err). - WithField("pod", pod.GetName()). - Info("error getting pod") + if p != nil { + if pod.IsTerminating { + logger.WithField("pod", pod.Name).Debugf("pod is terminating") + exit = false + break + } + + logger.WithField("pod", pod.Name).Debugf("pod still exists, deleting again") + err = mr.WithSegment(models.SegmentPod, func() error { + return clientset.CoreV1().Pods(namespace).Delete(pod.Name, deleteOptions) + }) exit = false break } @@ -363,6 +470,7 @@ func waitTerminatingPods( func waitCreatingPods( l logrus.FieldLogger, clientset kubernetes.Interface, + redisClient redisinterfaces.RedisClient, timeout time.Duration, namespace string, createdPods []v1.Pod, @@ -379,6 +487,12 @@ func waitCreatingPods( ticker := time.NewTicker(1 * time.Second) defer ticker.Stop() + var retryNo []int + for range createdPods { + retryNo = append(retryNo, 0) + } + backoffStart := time.Duration(1 * time.Second) + for { exit := true select { @@ -389,27 +503,33 @@ func waitCreatingPods( return false, true, nil } - for _, pod := range createdPods { - var createdPod *v1.Pod - err := mr.WithSegment(models.SegmentPod, func() error { - var err error - createdPod, err = clientset.CoreV1().Pods(namespace).Get( - pod.GetName(), getOptions, - ) - return err - }) - - if err != nil && strings.Contains(err.Error(), "not found") { + for i, pod := range createdPods { + createdPod, err := models.GetPodFromRedis(redisClient, mr, pod.GetName(), namespace) + if err != nil { + logger. + WithError(err). + WithField("pod", pod.GetName()). + Error("error getting pod") + exit = false + break + } + + if createdPod == nil { + // apply exponential backoff + retryNo[i]++ + backoff := exponentialBackoff(backoffStart, retryNo[i]) + exit = false logger. WithError(err). WithField("pod", pod.GetName()). - Error("error creating pod, recreating...") + Errorf("error creating pod, recreating in %s (retry %d)", backoff, retryNo[i]) pod.ResourceVersion = "" err = mr.WithSegment(models.SegmentPod, func() error { var err error _, err = clientset.CoreV1().Pods(namespace).Create(&pod) + time.Sleep(backoff) return err }) if err != nil { @@ -419,10 +539,11 @@ func waitCreatingPods( Errorf("error recreating pod") } break + } else { + retryNo[i] = 0 } - if len(createdPod.Status.Phase) == 0 { - //HACK! Trying to detect if we are running unit tests + if models.IsUnitTest(createdPod) { break } @@ -435,8 +556,29 @@ func waitCreatingPods( break } - if !models.IsPodReady(createdPod) { - logger.WithField("pod", createdPod.GetName()).Debug("pod not ready yet, waiting...") + if createdPod.Status.Phase != v1.PodRunning { + isPending, reason, message := models.PodPending(createdPod) + if isPending && strings.Contains(message, models.PodNotFitsHostPorts) { + l.WithFields(logrus.Fields{ + "pod": createdPod.Name, + "reason": reason, + "message": message, + }).Error("pod's host port is not available in any node of the pool, watcher will delete it soon") + continue + } else { + l.WithFields(logrus.Fields{ + "pod": createdPod.Name, + "pending": isPending, + "reason": reason, + "message": message, + }).Debug("pod is not running yet") + exit = false + break + } + } + + if !models.IsPodReady(createdPod) || !models.IsRoomReadyOrOccupied(logger, redisClient, namespace, createdPod.Name) { + logger.WithField("pod", createdPod.Name).Debug("pod not ready yet, waiting...") err = models.ValidatePodWaitingState(createdPod) if err != nil { @@ -475,7 +617,7 @@ func DeletePodAndRoom( var pod *models.Pod err := mr.WithSegment(models.SegmentPod, func() error { var err error - pod, err = models.NewPod(name, nil, configYaml, clientset, redisClient) + pod, err = models.NewPod(name, nil, configYaml, clientset, redisClient, mr) return err }) if err != nil { @@ -484,11 +626,11 @@ func DeletePodAndRoom( err = roomManager.Delete(logger, mr, clientset, redisClient, configYaml, pod.Name, reportersConstants.ReasonUpdate) - if err != nil { + if err != nil && !strings.Contains(err.Error(), "not found") { logger. WithField("roomName", pod.Name). WithError(err). - Error("error removing room info from redis") + Error("error removing pod from kube") return err } @@ -505,15 +647,15 @@ func DeletePodAndRoom( return nil } -func segmentPods(pods []v1.Pod, maxSurge int) [][]v1.Pod { +func segmentPods(pods []*models.Pod, maxSurge int) [][]*models.Pod { if pods == nil || len(pods) == 0 { - return make([][]v1.Pod, 0) + return make([][]*models.Pod, 0) } totalLength := len(pods) chunkLength := chunkLength(pods, maxSurge) chunks := nChunks(pods, chunkLength) - podChunks := make([][]v1.Pod, chunks) + podChunks := make([][]*models.Pod, chunks) for i := range podChunks { start := i * chunkLength @@ -528,20 +670,20 @@ func segmentPods(pods []v1.Pod, maxSurge int) [][]v1.Pod { return podChunks } -func chunkLength(pods []v1.Pod, maxSurge int) int { +func chunkLength(pods []*models.Pod, maxSurge int) int { denominator := 100.0 / float64(maxSurge) lenPods := float64(len(pods)) return int(math.Ceil(lenPods / denominator)) } -func nChunks(pods []v1.Pod, chunkLength int) int { +func nChunks(pods []*models.Pod, chunkLength int) int { return int(math.Ceil(float64(len(pods)) / float64(chunkLength))) } -func names(pods []v1.Pod) []string { +func names(pods []*models.Pod) []string { names := make([]string, len(pods)) for i, pod := range pods { - names[i] = pod.GetName() + names[i] = pod.Name } return names } @@ -549,6 +691,7 @@ func names(pods []v1.Pod) []string { func waitForPods( timeout time.Duration, clientset kubernetes.Interface, + redisClient redisinterfaces.RedisClient, namespace string, pods []*v1.Pod, l logrus.FieldLogger, @@ -559,6 +702,12 @@ func waitForPods( ticker := time.NewTicker(500 * time.Millisecond) defer ticker.Stop() + var retryNo []int + for range pods { + retryNo = append(retryNo, 0) + } + backoffStart := time.Duration(500 * time.Millisecond) + for { exit := true select { @@ -569,19 +718,21 @@ func waitForPods( case <-ticker.C: for i := range pods { if pods[i] != nil { - var pod *v1.Pod - err := mr.WithSegment(models.SegmentPod, func() error { - var err error - pod, err = clientset.CoreV1().Pods(namespace).Get(pods[i].GetName(), metav1.GetOptions{}) - return err - }) - if err != nil { + var pod *models.Pod + pod, err := models.GetPodFromRedis(redisClient, mr, pods[i].GetName(), namespace) + if err != nil || pod == nil { + // apply exponential backoff + retryNo[i]++ + backoff := exponentialBackoff(backoffStart, retryNo[i]) + //The pod does not exist (not even on Pending or ContainerCreating state), so create again exit = false - l.WithError(err).Infof("error creating pod %s, recreating...", pods[i].GetName()) + l.WithError(err).Infof("error creating pod %s, recreating in %s (retry %d)", pods[i].GetName(), backoff, retryNo[i]) + pods[i].ResourceVersion = "" err = mr.WithSegment(models.SegmentPod, func() error { _, err = clientset.CoreV1().Pods(namespace).Create(pods[i]) + time.Sleep(backoff) return err }) if err != nil { @@ -592,6 +743,8 @@ func waitForPods( break } + retryNo[i] = 0 + if pod.Status.Phase != v1.PodRunning { isPending, reason, message := models.PodPending(pod) if isPending && strings.Contains(message, models.PodNotFitsHostPorts) { @@ -602,7 +755,7 @@ func waitForPods( continue } else { l.WithFields(logrus.Fields{ - "pod": pod.GetName(), + "pod": pod.Name, "pending": isPending, "reason": reason, "message": message, @@ -631,24 +784,21 @@ func waitForPods( func pendingPods( clientset kubernetes.Interface, + redisClient redisinterfaces.RedisClient, namespace string, mr *models.MixedMetricsReporter, ) (bool, error) { - listOptions := metav1.ListOptions{ - LabelSelector: labels.Set{}.AsSelector().String(), - FieldSelector: fields.Everything().String(), - } - var pods *v1.PodList + var pods map[string]*models.Pod err := mr.WithSegment(models.SegmentPod, func() error { var err error - pods, err = clientset.CoreV1().Pods(namespace).List(listOptions) + pods, err = models.GetPodMapFromRedis(redisClient, mr, namespace) return err }) if err != nil { return false, maestroErrors.NewKubernetesError("error when listing pods", err) } - for _, pod := range pods.Items { + for _, pod := range pods { if pod.Status.Phase == v1.PodPending { return true, nil } @@ -1190,7 +1340,7 @@ func deleteSchedulerHelper( configYAML, _ := models.NewConfigYAML(scheduler.YAML) // Delete pods and wait for graceful termination before deleting the namespace err = mr.WithSegment(models.SegmentPod, func() error { - return namespace.DeletePods(clientset, redisClient, scheduler) + return namespace.DeletePods(clientset, redisClient, mr, scheduler) }) if err != nil { logger.WithError(err).Error("failed to delete namespace pods") @@ -1208,15 +1358,15 @@ func deleteSchedulerHelper( case <-timeoutPods.C: return errors.New("timeout deleting scheduler pods") case <-ticker.C: - var pods *v1.PodList + var podCount int listErr := mr.WithSegment(models.SegmentPod, func() error { var err error - pods, err = clientset.CoreV1().Pods(scheduler.Name).List(metav1.ListOptions{}) + podCount, err = models.GetPodCountFromRedis(redisClient, mr, scheduler.Name) return err }) if listErr != nil { logger.WithError(listErr).Error("error listing pods") - } else if len(pods.Items) == 0 { + } else if podCount == 0 { exit = true } logger.Debug("deleting scheduler pods") @@ -1269,3 +1419,11 @@ func deleteSchedulerHelper( return nil } + +func exponentialBackoff(backoffStart time.Duration, retryNo int) time.Duration { + min := 1 + max := int(math.Pow(2, float64(retryNo))) + k := rand.Intn(max-min) + min + + return time.Duration(k * int(backoffStart)) +} diff --git a/dev-room/Dockerfile b/dev-room/Dockerfile index 76704e71c..2c7752ed8 100644 --- a/dev-room/Dockerfile +++ b/dev-room/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3 +FROM python:3.7 WORKDIR /usr/src/app ADD . . diff --git a/docker-compose.yaml b/docker-compose.yaml index 6fe77309d..38f826b74 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -9,6 +9,7 @@ services: environment: - POSTGRES_DB=maestro - POSTGRES_USER=maestro + - POSTGRES_HOST_AUTH_METHOD=trust redis: image: redis:3.2-alpine ports: diff --git a/eventforwarder/eventforwarder_suite_test.go b/eventforwarder/eventforwarder_suite_test.go index 5c3768153..8b05c5ad1 100644 --- a/eventforwarder/eventforwarder_suite_test.go +++ b/eventforwarder/eventforwarder_suite_test.go @@ -1,6 +1,7 @@ package eventforwarder_test import ( + "github.com/go-redis/redis" "time" "github.com/btcsuite/btcutil/base58" @@ -24,6 +25,7 @@ import ( redismocks "github.com/topfreegames/extensions/redis/mocks" eventforwardermock "github.com/topfreegames/maestro/eventforwarder/mock" reportermock "github.com/topfreegames/maestro/reporters/mocks" + mtesting "github.com/topfreegames/maestro/testing" ) func TestEventforwarder(t *testing.T) { @@ -43,6 +45,7 @@ var ( mockReporter *reportermock.MockReporter room *models.Room clientset *fake.Clientset + mmr *models.MixedMetricsReporter cache *models.SchedulerCache metadata map[string]interface{} schedulerName = "scheduler" @@ -108,9 +111,22 @@ var _ = BeforeEach(func() { {Name: "port", HostPort: hostPort}, }}, } - _, err = clientset.CoreV1().Pods(schedulerName).Create(pod) + podv1, err := clientset.CoreV1().Pods(schedulerName).Create(pod) Expect(err).NotTo(HaveOccurred()) + var podModel models.Pod + podModel.Name = pod.Name + podModel.Namespace = pod.Namespace + podModel.Spec = pod.Spec + podModel.Status = podv1.Status + + jsonBytes, err := podModel.MarshalToRedis() + Expect(err).NotTo(HaveOccurred()) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(podModel.Namespace), podModel.Name). + Return(redis.NewStringResult(string(jsonBytes), nil)). + AnyTimes() + node := &v1.Node{} node.SetLabels(nodeLabels) node.Status.Addresses = []v1.NodeAddress{ @@ -121,4 +137,8 @@ var _ = BeforeEach(func() { room = models.NewRoom(roomName, schedulerName) roomAddrGetter = models.NewRoomAddressesFromHostPort(logger, ipv6KubernetesLabelKey, false, 0) + + fakeReporter := mtesting.FakeMetricsReporter{} + mmr = models.NewMixedMetricsReporter() + mmr.AddReporter(fakeReporter) }) diff --git a/eventforwarder/forward.go b/eventforwarder/forward.go index 9e4c0d274..9dd806d87 100644 --- a/eventforwarder/forward.go +++ b/eventforwarder/forward.go @@ -56,6 +56,7 @@ func ForwardRoomEvent( redis redisinterfaces.RedisClient, db pginterfaces.DB, kubernetesClient kubernetes.Interface, + mr *models.MixedMetricsReporter, room *models.Room, status string, eventType string, @@ -115,23 +116,25 @@ func ForwardRoomEvent( "game": cachedScheduler.Scheduler.Game, } if eventType != PingTimeoutEvent && eventType != OccupiedTimeoutEvent { - infos, err = room.GetRoomInfos(redis, db, kubernetesClient, schedulerCache, cachedScheduler.Scheduler, addrGetter) - metadata["ipv6Label"] = infos["ipv6Label"] + infos, err = room.GetRoomInfos(redis, db, kubernetesClient, schedulerCache, cachedScheduler.Scheduler, addrGetter, mr) if err != nil { l.WithError(err).Error("error getting room info from redis") return nil, err } - reportIpv6Status(infos, logger) - delete(infos, "ipv6Label") - } else { // fill host and port with zero values when pingTimeout or occupiedTimeout event so it won't break the GRPCForwarder infos["host"] = "" infos["port"] = int32(0) } - infos["metadata"] = metadata + if infos["metadata"] == nil { + infos["metadata"] = make(map[string]interface{}, len(metadata)) + } + + for key, info := range metadata { + infos["metadata"].(map[string]interface{})[key] = info + } eventWasForwarded = true return ForwardEventToForwarders(ctx, enabledForwarders, status, infos, l) @@ -316,7 +319,9 @@ func reportIpv6Status( reportersConstants.TagStatus: "success", } - if infos["ipv6Label"] == nil || infos["ipv6Label"].(string) == "" { + if infos["metadata"] == nil || + infos["metadata"].(map[string]interface{})["ipv6Label"] == nil || + infos["metadata"].(map[string]interface{})["ipv6Label"].(string) == "" { status[reportersConstants.TagStatus] = "failed" } diff --git a/eventforwarder/forward_test.go b/eventforwarder/forward_test.go index e27da24eb..9c90e7e92 100644 --- a/eventforwarder/forward_test.go +++ b/eventforwarder/forward_test.go @@ -3,6 +3,7 @@ package eventforwarder_test import ( "context" "errors" + "fmt" "github.com/golang/mock/gomock" . "github.com/topfreegames/maestro/eventforwarder" @@ -29,6 +30,7 @@ var _ = Describe("Forward", func() { "game": gameName, "metadata": map[string]interface{}{ "ipv6Label": ipv6Label, + "ports": fmt.Sprintf(`[{"name":"port","port":%d,"protocol":""}]`, hostPort), }, }, metadata, @@ -55,6 +57,7 @@ var _ = Describe("Forward", func() { mockRedisClient, mockDB, clientset, + mmr, room, models.StatusReady, "", @@ -100,6 +103,7 @@ var _ = Describe("Forward", func() { mockRedisClient, mockDB, clientset, + mmr, room, models.StatusReady, PingTimeoutEvent, @@ -142,6 +146,7 @@ var _ = Describe("Forward", func() { mockRedisClient, mockDB, clientset, + mmr, room, models.StatusTerminated, OccupiedTimeoutEvent, @@ -170,6 +175,7 @@ var _ = Describe("Forward", func() { "game": gameName, "metadata": map[string]interface{}{ "ipv6Label": "", + "ports": fmt.Sprintf(`[{"name":"port","port":%d,"protocol":""}]`, hostPort), }, }, metadata, @@ -197,6 +203,7 @@ var _ = Describe("Forward", func() { mockRedisClient, mockDB, clientset, + mmr, room, models.StatusReady, "", @@ -223,6 +230,7 @@ var _ = Describe("Forward", func() { "game": gameName, "metadata": map[string]interface{}{ "ipv6Label": ipv6Label, + "ports": fmt.Sprintf(`[{"name":"port","port":%d,"protocol":""}]`, hostPort), }, }, metadata, @@ -237,6 +245,7 @@ var _ = Describe("Forward", func() { mockRedisClient, mockDB, clientset, + mmr, room, models.StatusReady, "", @@ -266,6 +275,7 @@ game: game mockRedisClient, mockDB, clientset, + mmr, room, models.StatusReady, "", diff --git a/manifests/scheduler-dev-room.yaml b/manifests/scheduler-dev-room.yaml new file mode 100644 index 000000000..479b40cc6 --- /dev/null +++ b/manifests/scheduler-dev-room.yaml @@ -0,0 +1,54 @@ +--- +name: test-1 +game: game +occupiedTimeout: 1000 +shutdownTimeout: 100 +autoscaling: + min: 2 + up: + cooldown: 5 + delta: 1 + trigger: + usage: 70 + time: 60 + threshold: 80 + down: + cooldown: 5 + delta: 1 + trigger: + usage: 50 + time: 60 + threshold: 80 +containers: + - name: game + image: maestro-dev-room:latest + imagePullPolicy: Never + requests: + cpu: 10m + memory: 40Mi + limits: + cpu: 20m + memory: 60Mi + ports: + - containerPort: 8080 + protocol: TCP + name: tcp + env: + - name: MAESTRO_HOST_PORT + value: 192.168.64.1:8080 + valueFrom: + secretKeyRef: + name: "" + key: "" + - name: POLLING_INTERVAL_IN_SECONDS + value: "20" + valueFrom: + secretKeyRef: + name: "" + key: "" + - name: PING_INTERVAL_IN_SECONDS + value: "10" + valueFrom: + secretKeyRef: + name: "" + key: "" diff --git a/metadata/version.go b/metadata/version.go index d934c1725..3b6092c5a 100644 --- a/metadata/version.go +++ b/metadata/version.go @@ -8,7 +8,7 @@ package metadata //Version of Maestro -var Version = "9.1.1" +var Version = "9.4.1" //KubeVersion is the desired Kubernetes version var KubeVersion = "v1.13.9" diff --git a/migrations/migrations.go b/migrations/migrations.go index 346cec38f..d8266f1e9 100644 --- a/migrations/migrations.go +++ b/migrations/migrations.go @@ -5,6 +5,8 @@ // migrations/0003-CreateSchedulerVersionsTable.sql // migrations/0004-AlterSchedulerTableAddVersion.sql // migrations/0005-AlterVersionType.sql +// migrations/0006-AddStatusColumnToVersion.sql +// migrations/0007-AddRollbackVersionColumnToVersion.sql // DO NOT EDIT! package migrations @@ -87,7 +89,7 @@ func migrations0001CreateschedulertableSql() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "migrations/0001-CreateSchedulerTable.sql", size: 716, mode: os.FileMode(420), modTime: time.Unix(1555594840, 0)} + info := bindataFileInfo{name: "migrations/0001-CreateSchedulerTable.sql", size: 716, mode: os.FileMode(420), modTime: time.Unix(1563978252, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -107,7 +109,7 @@ func migrations0002CreateusertableSql() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "migrations/0002-CreateUserTable.sql", size: 658, mode: os.FileMode(420), modTime: time.Unix(1555594840, 0)} + info := bindataFileInfo{name: "migrations/0002-CreateUserTable.sql", size: 658, mode: os.FileMode(420), modTime: time.Unix(1563978252, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -127,7 +129,7 @@ func migrations0003CreateschedulerversionstableSql() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "migrations/0003-CreateSchedulerVersionsTable.sql", size: 664, mode: os.FileMode(420), modTime: time.Unix(1555594840, 0)} + info := bindataFileInfo{name: "migrations/0003-CreateSchedulerVersionsTable.sql", size: 664, mode: os.FileMode(420), modTime: time.Unix(1563978252, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -147,7 +149,7 @@ func migrations0004AlterschedulertableaddversionSql() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "migrations/0004-AlterSchedulerTableAddVersion.sql", size: 243, mode: os.FileMode(420), modTime: time.Unix(1555594840, 0)} + info := bindataFileInfo{name: "migrations/0004-AlterSchedulerTableAddVersion.sql", size: 243, mode: os.FileMode(420), modTime: time.Unix(1563978252, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -167,7 +169,47 @@ func migrations0005AlterversiontypeSql() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "migrations/0005-AlterVersionType.sql", size: 333, mode: os.FileMode(420), modTime: time.Unix(1555594840, 0)} + info := bindataFileInfo{name: "migrations/0005-AlterVersionType.sql", size: 333, mode: os.FileMode(420), modTime: time.Unix(1563978252, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _migrations0006AddstatuscolumntoversionSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x44\x8e\xc1\x4a\xc3\x40\x10\x86\xef\x79\x8a\x39\xea\x21\x5d\x2d\x14\xa4\x8a\x18\x6d\x15\x21\x5e\x24\xf7\xb0\xdd\x4c\x77\x17\x93\x9d\x65\x66\xb6\xc1\x47\xf2\x35\x7c\x32\x09\x56\x3c\xfe\x3f\xdf\xff\xf3\xd5\x35\x4c\x16\x45\x99\xaa\xba\x86\xa0\x9a\x65\x6b\x8c\x8f\x1a\xca\x61\xe5\x68\x32\x4a\xf9\xc8\x88\xde\x4e\x28\xe6\x1f\x5d\xe8\x36\x3a\x4c\x82\x03\x94\x34\x20\x83\x06\x84\xb7\xd7\x0e\xc6\xdf\x7a\xfb\x77\xb8\x35\x66\x9e\xe7\x15\x65\x4c\x42\x85\x1d\xae\x88\xbd\x39\x53\x62\xa6\xa8\xf5\x39\x2c\x8b\x27\xca\x9f\x1c\x7d\x50\xf8\xfe\x82\xf5\xd5\xf5\x0d\x74\x94\xe1\x99\x11\xe1\x65\x71\x80\xbb\x83\x75\x1f\x98\x86\x07\x3d\x7a\x47\x8b\xe3\x7d\x55\x35\x6d\xb7\x7f\x87\xae\x79\x6c\xf7\x20\x2e\xe0\x50\x46\xe4\xfe\x84\x2c\x91\x92\x40\xb3\xdb\x01\xd3\x38\xc6\xe4\xfb\x92\x07\xab\xd8\x8b\x5a\x2d\x02\x27\xcb\x2e\x58\xbe\x58\x6f\x36\x97\xb7\xd5\x4f\x00\x00\x00\xff\xff\x72\xed\xcd\x7a\x0e\x01\x00\x00") + +func migrations0006AddstatuscolumntoversionSqlBytes() ([]byte, error) { + return bindataRead( + _migrations0006AddstatuscolumntoversionSql, + "migrations/0006-AddStatusColumnToVersion.sql", + ) +} + +func migrations0006AddstatuscolumntoversionSql() (*asset, error) { + bytes, err := migrations0006AddstatuscolumntoversionSqlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "migrations/0006-AddStatusColumnToVersion.sql", size: 270, mode: os.FileMode(420), modTime: time.Unix(1569276907, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _migrations0007AddrollbackversioncolumntoversionSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x44\x8e\xc1\x4a\xc4\x30\x10\x86\xef\x7d\x8a\x39\xea\x21\x8d\x2e\x2c\x48\x15\xb1\xba\xab\x08\xf5\x22\xbd\x4b\x36\x9d\x4d\x82\x6d\x26\x4c\xa6\x5b\x7c\x24\x5f\xc3\x27\x93\xe0\x16\x8f\xf3\xf3\x7d\xc3\xa7\x14\x4c\x06\xb3\x30\x55\x4a\x81\x17\x49\xb9\xd1\xda\x05\xf1\xf3\xa1\xb6\x34\x69\xa1\x74\x64\x44\x67\x26\xcc\xfa\x1f\x2d\x74\x17\x2c\xc6\x8c\x03\xcc\x71\x40\x06\xf1\x08\x6f\xaf\x3d\x8c\x7f\x73\xb3\x3e\x6c\xb4\x5e\x96\xa5\xa6\x84\x31\xd3\xcc\x16\x6b\x62\xa7\xcf\x54\xd6\x53\x10\x75\x3e\x8a\xf1\x44\xe9\x8b\x83\xf3\x02\x3f\xdf\xb0\xb9\xba\xbe\x81\x9e\x12\x3c\x33\x22\xbc\x94\x06\xb8\x3b\x18\xfb\x89\x71\x78\x90\xa3\xb3\x54\x1a\xef\xab\xaa\xed\xfa\xfd\x3b\xf4\xed\x63\xb7\x87\x6c\x3d\x0e\xf3\x88\xfc\x71\x42\xce\x81\x62\x86\x76\xb7\x03\xa6\x71\x2c\xea\xba\xc2\xc9\xb0\xf5\x86\x2f\x36\xdb\xed\xe5\x6d\xf5\x1b\x00\x00\xff\xff\x65\x4e\x1c\x26\x09\x01\x00\x00") + +func migrations0007AddrollbackversioncolumntoversionSqlBytes() ([]byte, error) { + return bindataRead( + _migrations0007AddrollbackversioncolumntoversionSql, + "migrations/0007-AddRollbackVersionColumnToVersion.sql", + ) +} + +func migrations0007AddrollbackversioncolumntoversionSql() (*asset, error) { + bytes, err := migrations0007AddrollbackversioncolumntoversionSqlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "migrations/0007-AddRollbackVersionColumnToVersion.sql", size: 265, mode: os.FileMode(420), modTime: time.Unix(1569276907, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -229,6 +271,8 @@ var _bindata = map[string]func() (*asset, error){ "migrations/0003-CreateSchedulerVersionsTable.sql": migrations0003CreateschedulerversionstableSql, "migrations/0004-AlterSchedulerTableAddVersion.sql": migrations0004AlterschedulertableaddversionSql, "migrations/0005-AlterVersionType.sql": migrations0005AlterversiontypeSql, + "migrations/0006-AddStatusColumnToVersion.sql": migrations0006AddstatuscolumntoversionSql, + "migrations/0007-AddRollbackVersionColumnToVersion.sql": migrations0007AddrollbackversioncolumntoversionSql, } // AssetDir returns the file names below a certain @@ -277,6 +321,8 @@ var _bintree = &bintree{nil, map[string]*bintree{ "0003-CreateSchedulerVersionsTable.sql": &bintree{migrations0003CreateschedulerversionstableSql, map[string]*bintree{}}, "0004-AlterSchedulerTableAddVersion.sql": &bintree{migrations0004AlterschedulertableaddversionSql, map[string]*bintree{}}, "0005-AlterVersionType.sql": &bintree{migrations0005AlterversiontypeSql, map[string]*bintree{}}, + "0006-AddStatusColumnToVersion.sql": &bintree{migrations0006AddstatuscolumntoversionSql, map[string]*bintree{}}, + "0007-AddRollbackVersionColumnToVersion.sql": &bintree{migrations0007AddrollbackversioncolumntoversionSql, map[string]*bintree{}}, }}, }} diff --git a/mocks/port_chooser.go b/mocks/port_chooser.go index d47631659..de841cef2 100644 --- a/mocks/port_chooser.go +++ b/mocks/port_chooser.go @@ -33,13 +33,13 @@ func (m *MockPortChooser) EXPECT() *MockPortChooserMockRecorder { } // Choose mocks base method -func (m *MockPortChooser) Choose(arg0, arg1, arg2 int) []int { - ret := m.ctrl.Call(m, "Choose", arg0, arg1, arg2) +func (m *MockPortChooser) Choose(start, end, quantity int, excluding []int) []int { + ret := m.ctrl.Call(m, "Choose", start, end, quantity, excluding) ret0, _ := ret[0].([]int) return ret0 } // Choose indicates an expected call of Choose -func (mr *MockPortChooserMockRecorder) Choose(arg0, arg1, arg2 interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Choose", reflect.TypeOf((*MockPortChooser)(nil).Choose), arg0, arg1, arg2) +func (mr *MockPortChooserMockRecorder) Choose(start, end, quantity, excluding interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Choose", reflect.TypeOf((*MockPortChooser)(nil).Choose), start, end, quantity, excluding) } diff --git a/models/constants.go b/models/constants.go index bfecbff77..32e39da37 100644 --- a/models/constants.go +++ b/models/constants.go @@ -120,7 +120,6 @@ var InvalidPodWaitingStates = []string{ "ErrImageNeverPull", "ErrImagePullBackOff", "ImagePullBackOff", - "CrashLoopBackOff", "ErrInvalidImageName", } diff --git a/models/game_room.go b/models/game_room.go index 0b8234da6..3855f2ac3 100644 --- a/models/game_room.go +++ b/models/game_room.go @@ -170,7 +170,7 @@ func createPod( env := append(configYAML.Env, namesEnvVars...) err = mr.WithSegment(SegmentPod, func() error { var err error - pod, err = NewPod(name, env, configYAML, clientset, redisClient) + pod, err = NewPod(name, env, configYAML, clientset, redisClient, mr) return err }) } else if configYAML.Version() == "v2" { @@ -180,7 +180,7 @@ func createPod( containers[i].Env = append(containers[i].Env, namesEnvVars...) } - pod, err = NewPodWithContainers(name, containers, configYAML, clientset, redisClient) + pod, err = NewPodWithContainers(name, containers, configYAML, clientset, redisClient, mr) } if err != nil { return nil, err diff --git a/models/game_room_test.go b/models/game_room_test.go index 7a8c2b379..83fdde538 100644 --- a/models/game_room_test.go +++ b/models/game_room_test.go @@ -144,9 +144,13 @@ var _ = Describe("GameRoomManagement", func() { Describe("Create", func() { It("Should create a pod", func() { + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(scheduler.Name), gomock.Any()). + Return(goredis.NewStringResult("", goredis.Nil)) + mockRedisClient.EXPECT().Get(models.GlobalPortsPoolKey). Return(goredis.NewStringResult(portRange, nil)) - mockPortChooser.EXPECT().Choose(portStart, portEnd, 2).Return([]int{5000, 5001}) + mockPortChooser.EXPECT().Choose(portStart, portEnd, 2, gomock.Any()).Return([]int{5000, 5001}) mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).AnyTimes() mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( func(schedulerName string, statusInfo map[string]interface{}) { @@ -249,9 +253,13 @@ var _ = Describe("GameRoomManagement", func() { Describe("Delete", func() { It("Should delete a pod", func() { + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(scheduler.Name), gomock.Any()). + Return(goredis.NewStringResult("", goredis.Nil)) + mockRedisClient.EXPECT().Get(models.GlobalPortsPoolKey). Return(goredis.NewStringResult(portRange, nil)) - mockPortChooser.EXPECT().Choose(portStart, portEnd, 2).Return([]int{5000, 5001}) + mockPortChooser.EXPECT().Choose(portStart, portEnd, 2, gomock.Any()).Return([]int{5000, 5001}) mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).AnyTimes() mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( func(schedulerName string, statusInfo map[string]interface{}) { @@ -312,9 +320,13 @@ var _ = Describe("GameRoomManagement", func() { Describe("Create", func() { It("Should create a pod", func() { + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(scheduler.Name), gomock.Any()). + Return(goredis.NewStringResult("", goredis.Nil)) + mockRedisClient.EXPECT().Get(models.GlobalPortsPoolKey). Return(goredis.NewStringResult(portRange, nil)) - mockPortChooser.EXPECT().Choose(portStart, portEnd, 2).Return([]int{5000, 5001}) + mockPortChooser.EXPECT().Choose(portStart, portEnd, 2, gomock.Any()).Return([]int{5000, 5001}) mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).AnyTimes() mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( func(schedulerName string, statusInfo map[string]interface{}) { @@ -424,6 +436,11 @@ var _ = Describe("GameRoomManagement", func() { Expect(len(svcs.Items)).To(Equal(0)) }) It("Should return error and remove created pod if service fails to create", func() { + Skip("Maybe it's not relevant now that roommanager.Create doesn't really create the pod in k8s") + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(scheduler.Name), gomock.Any()). + Return(goredis.NewStringResult("", goredis.Nil)) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).AnyTimes() mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( func(schedulerName string, statusInfo map[string]interface{}) { @@ -481,9 +498,13 @@ var _ = Describe("GameRoomManagement", func() { Describe("Delete", func() { It("Should delete a pod", func() { + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(scheduler.Name), gomock.Any()). + Return(goredis.NewStringResult("", goredis.Nil)) + mockRedisClient.EXPECT().Get(models.GlobalPortsPoolKey). Return(goredis.NewStringResult(portRange, nil)) - mockPortChooser.EXPECT().Choose(portStart, portEnd, 2).Return([]int{5000, 5001}) + mockPortChooser.EXPECT().Choose(portStart, portEnd, 2, gomock.Any()).Return([]int{5000, 5001}) mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).AnyTimes() mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( func(schedulerName string, statusInfo map[string]interface{}) { diff --git a/models/interfaces.go b/models/interfaces.go index d8cdbf925..f876b131c 100644 --- a/models/interfaces.go +++ b/models/interfaces.go @@ -40,7 +40,7 @@ type ContainerIface interface { // AddrGetter return IP and ports of a room type AddrGetter interface { - Get(*Room, kubernetes.Interface, redisinterfaces.RedisClient) (*RoomAddresses, error) + Get(*Room, kubernetes.Interface, redisinterfaces.RedisClient, *MixedMetricsReporter) (*RoomAddresses, error) } // RoomManager should create and delete a game room diff --git a/models/namespace.go b/models/namespace.go index 4aab40ed7..2b7aa5425 100644 --- a/models/namespace.go +++ b/models/namespace.go @@ -73,9 +73,13 @@ func (n *Namespace) Delete(clientset kubernetes.Interface) error { } // DeletePods deletes all pods from a kubernetes namespace -func (n *Namespace) DeletePods(clientset kubernetes.Interface, - redisClient redisinterfaces.RedisClient, s *Scheduler) error { - pods, err := clientset.CoreV1().Pods(n.Name).List(metav1.ListOptions{}) +func (n *Namespace) DeletePods( + clientset kubernetes.Interface, + redisClient redisinterfaces.RedisClient, + mr *MixedMetricsReporter, + s *Scheduler, +) error { + podCount, err := GetPodCountFromRedis(redisClient, mr, s.Name) if err != nil { return errors.NewKubernetesError("delete namespace pods error", err) } @@ -84,7 +88,7 @@ func (n *Namespace) DeletePods(clientset kubernetes.Interface, return errors.NewKubernetesError("delete namespace pods error", err) } - for range pods.Items { + for i := 0; i < podCount; i++ { reporters.Report(reportersConstants.EventGruDelete, map[string]interface{}{ reportersConstants.TagGame: s.Game, reportersConstants.TagScheduler: s.Name, diff --git a/models/namespace_int_test.go b/models/namespace_int_test.go index 3062b34b3..e9829dcf7 100644 --- a/models/namespace_int_test.go +++ b/models/namespace_int_test.go @@ -54,7 +54,7 @@ var _ = Describe("Namespace", func() { Cmd: []string{"command"}, } - pod, err := models.NewPod("name", []*models.EnvVar{}, configYaml, clientset, redisClient.Client) + pod, err := models.NewPod("name", []*models.EnvVar{}, configYaml, clientset, redisClient.Client, mmr) Expect(err).NotTo(HaveOccurred()) _, err = pod.Create(clientset) Expect(err).NotTo(HaveOccurred()) @@ -66,7 +66,7 @@ var _ = Describe("Namespace", func() { }).Should(Equal(1)) s := &models.Scheduler{Name: "name", Game: "game"} - err = namespace.DeletePods(clientset, redisClient.Client, s) + err = namespace.DeletePods(clientset, redisClient.Client, mmr, s) Expect(err).NotTo(HaveOccurred()) Eventually(func() int { diff --git a/models/namespace_test.go b/models/namespace_test.go index b70bb9a7b..1d9bd37b2 100644 --- a/models/namespace_test.go +++ b/models/namespace_test.go @@ -10,6 +10,7 @@ package models_test import ( "fmt" + "github.com/go-redis/redis" "github.com/topfreegames/maestro/models" @@ -110,8 +111,13 @@ var _ = Describe("Namespace", func() { It("should fail if namespace does not exist", func() { namespace := models.NewNamespace(name) - err := namespace.DeletePods(clientset, mockRedisClient, s) - Expect(err).NotTo(HaveOccurred()) + + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(s.Name)) + mockPipeline.EXPECT().Exec().Return(nil, redis.Nil) + + err := namespace.DeletePods(clientset, mockRedisClient, mmr, s) + Expect(err).To(HaveOccurred()) }) It("should succeed if namespace exists and has no pods", func() { @@ -119,7 +125,11 @@ var _ = Describe("Namespace", func() { err := namespace.Create(clientset) Expect(err).NotTo(HaveOccurred()) - err = namespace.DeletePods(clientset, mockRedisClient, s) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HLen(models.GetPodMapRedisKey(s.Name)).Return(redis.NewIntResult(0, nil)) + mockPipeline.EXPECT().Exec() + + err = namespace.DeletePods(clientset, mockRedisClient, mmr, s) Expect(err).NotTo(HaveOccurred()) pods, err := clientset.CoreV1().Pods(namespace.Name).List(metav1.ListOptions{}) diff --git a/models/operation_manager.go b/models/operation_manager.go index 6d2e92157..ace7bbacb 100644 --- a/models/operation_manager.go +++ b/models/operation_manager.go @@ -304,20 +304,20 @@ func (o *OperationManager) getOperationRollingProgress( ) (float64, error) { progress := float64(0) - roomsToUpdate, err := GetInvalidRoomsCount(o.redisClient, mr, scheduler.Name) + roomsCount, err := GetPodCountFromRedis(o.redisClient, mr, scheduler.Name) if err != nil { return 0, err } - old, err := GetCurrentInvalidRoomsCount(o.redisClient, mr, scheduler.Name) + invalidCount, err := GetCurrentInvalidRoomsCount(o.redisClient, mr, scheduler.Name) if err != nil { return 0, err } - if roomsToUpdate <= 0 { + if roomsCount <= 0 { progress = 1 } else { - progress = 1 - float64(old)/float64(roomsToUpdate) + progress = 1 - float64(invalidCount)/float64(roomsCount) } // if the percentage of gameservers with the actual version is 100% but the diff --git a/models/pod.go b/models/pod.go index 601a2baf5..e4a40255f 100644 --- a/models/pod.go +++ b/models/pod.go @@ -9,17 +9,19 @@ package models import ( "bytes" + "encoding/json" "fmt" "strings" "text/template" + "github.com/go-redis/redis" redisinterfaces "github.com/topfreegames/extensions/redis/interfaces" reportersConstants "github.com/topfreegames/maestro/reporters/constants" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/topfreegames/maestro/errors" "github.com/topfreegames/maestro/reporters" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/yaml" @@ -132,8 +134,12 @@ type Pod struct { ShutdownTimeout int NodeAffinity string NodeToleration string + IsTerminating bool Containers []*Container Version string + Status v1.PodStatus + Spec v1.PodSpec + NodeName string Environment string } @@ -144,6 +150,7 @@ func NewPod( configYaml *ConfigYAML, clientset kubernetes.Interface, redisClient redisinterfaces.RedisClient, + mr *MixedMetricsReporter, ) (*Pod, error) { pod := &Pod{ Name: name, @@ -174,7 +181,7 @@ func NewPod( } } pod.Containers = []*Container{container} - err := pod.configureHostPorts(configYaml, clientset, redisClient) + err := pod.configureHostPorts(configYaml, clientset, redisClient, mr) if err == nil { reporters.Report(reportersConstants.EventGruNew, map[string]interface{}{ @@ -193,6 +200,7 @@ func NewPodWithContainers( configYaml *ConfigYAML, clientset kubernetes.Interface, redisClient redisinterfaces.RedisClient, + mr *MixedMetricsReporter, ) (*Pod, error) { pod := &Pod{ Game: configYaml.Game, @@ -201,7 +209,7 @@ func NewPodWithContainers( ShutdownTimeout: configYaml.ShutdownTimeout, Containers: containers, } - err := pod.configureHostPorts(configYaml, clientset, redisClient) + err := pod.configureHostPorts(configYaml, clientset, redisClient, mr) if err == nil { reporters.Report(reportersConstants.EventGruNew, map[string]interface{}{ reportersConstants.TagGame: configYaml.Game, @@ -276,7 +284,7 @@ func (p *Pod) Delete(clientset kubernetes.Interface, return nil } -func getContainerWithName(name string, pod *v1.Pod) v1.Container { +func getContainerWithName(name string, pod *Pod) v1.Container { var container v1.Container for _, container = range pod.Spec.Containers { @@ -292,16 +300,16 @@ func (p *Pod) configureHostPorts( configYaml *ConfigYAML, clientset kubernetes.Interface, redisClient redisinterfaces.RedisClient, + mr *MixedMetricsReporter, ) error { - pod, err := clientset.CoreV1().Pods(p.Namespace).Get(p.Name, metav1.GetOptions{}) - if err != nil && !strings.Contains(err.Error(), "not found") { + pod, err := GetPodFromRedis(redisClient, mr, p.Name, p.Namespace) + if err != nil { return errors.NewKubernetesError("could not access kubernetes", err) - } else if err == nil { + } else if err == nil && pod != nil { //pod exists, so just retrieve ports for _, container := range p.Containers { podContainer := getContainerWithName(container.Name, pod) container.Ports = make([]*Port, len(podContainer.Ports)) - for i, port := range podContainer.Ports { container.Ports[i] = &Port{ ContainerPort: int(port.ContainerPort), @@ -334,22 +342,47 @@ func (p *Pod) configureHostPorts( return fmt.Errorf("error reading global port range from redis: %s", err.Error()) } + // save ports already used to avoid duplication + usedPorts := []int{} + for _, container := range p.Containers { - ports := GetRandomPorts(start, end, len(container.Ports)) - containerPorts := make([]*Port, len(container.Ports)) - for i, port := range ports { - containerPorts[i] = &Port{ - ContainerPort: container.Ports[i].ContainerPort, - Name: container.Ports[i].Name, - HostPort: port, - Protocol: container.Ports[i].Protocol, - } + ports := GetRandomPorts(start, end, len(container.Ports), usedPorts) + usedPorts = append(usedPorts, ports...) + containerPorts := []*Port{} + + for j := range container.Ports { + hostPort := ports[0] + ports = ports[1:] + containerPorts = append(containerPorts, &Port{ + ContainerPort: container.Ports[j].ContainerPort, + Name: container.Ports[j].Name, + HostPort: hostPort, + Protocol: container.Ports[j].Protocol, + }) } container.Ports = containerPorts } return nil } +// MarshalToRedis stringfy pod object with the information to save on podMap redis key +func (p *Pod) MarshalToRedis() ([]byte, error) { + + return json.Marshal(map[string]interface{}{ + "name": p.Name, + "status": p.Status, + "version": p.Version, + "nodeName": p.NodeName, + "isTerminating": p.IsTerminating, + "spec": p.Spec, + }) +} + +// UnmarshalFromRedis loads pod string from redis podMap key in a *Pod object +func (p *Pod) UnmarshalFromRedis(pod string) error { + return json.Unmarshal([]byte(pod), p) +} + //PodExists returns true if a pod exists on namespace // returns false if it doesn't // returns false and a error if an error occurs @@ -368,7 +401,7 @@ func PodExists( } // IsPodReady returns true if pod is ready -func IsPodReady(pod *v1.Pod) bool { +func IsPodReady(pod *Pod) bool { status := &pod.Status if status == nil { return false @@ -383,13 +416,16 @@ func IsPodReady(pod *v1.Pod) bool { return false } +func IsPodTerminating(pod *v1.Pod) bool { + return pod.ObjectMeta.DeletionTimestamp != nil +} + // ValidatePodWaitingState returns nil if pod waiting reson is valid and error otherwise // Errors checked: // - ErrImageNeverPull -// - CrashLoopBackOff // - ErrImagePullBackOff // - ErrInvalidImageName -func ValidatePodWaitingState(pod *v1.Pod) error { +func ValidatePodWaitingState(pod *Pod) error { for _, invalidState := range InvalidPodWaitingStates { status := &pod.Status @@ -418,7 +454,7 @@ func checkWaitingReason(status *v1.PodStatus, reason string) bool { // PodPending returns true if pod is with status Pending. // In this case, also returns reason for being pending and message. -func PodPending(pod *v1.Pod) (isPending bool, reason, message string) { +func PodPending(pod *Pod) (isPending bool, reason, message string) { for _, condition := range pod.Status.Conditions { if condition.Status == v1.ConditionFalse { return true, condition.Reason, condition.Message @@ -430,6 +466,120 @@ func PodPending(pod *v1.Pod) (isPending bool, reason, message string) { // IsUnitTest returns true if pod was created using fake client-go // and is not running in a kubernetes cluster. -func IsUnitTest(pod *v1.Pod) bool { +func IsUnitTest(pod *Pod) bool { return len(pod.Status.Phase) == 0 } + +// GetPodMapRedisKey gets the key for string that keeps the pod map from kube on redis +func GetPodMapRedisKey(schedulerName string) string { + return fmt.Sprintf("scheduler:%s:podMap", schedulerName) +} + +// GetPodMapFromRedis loads the pod map from redis +func GetPodMapFromRedis( + redisClient redisinterfaces.RedisClient, + mr *MixedMetricsReporter, + schedulerName string, +) (podMap map[string]*Pod, err error) { + pipe := redisClient.TxPipeline() + cmd := pipe.HGetAll(GetPodMapRedisKey(schedulerName)) + err = mr.WithSegment(SegmentPipeExec, func() error { + var err error + _, err = pipe.Exec() + return err + }) + if err != nil { + return nil, err + } + + podMap = map[string]*Pod{} + + for podName, podStr := range cmd.Val() { + pod := &Pod{} + err = pod.UnmarshalFromRedis(podStr) + if err != nil { + return nil, err + } + + podMap[podName] = pod + } + + return podMap, err +} + +// GetPodCountFromRedis returns the pod count from redis podMap key +func GetPodCountFromRedis( + redisClient redisinterfaces.RedisClient, + mr *MixedMetricsReporter, + schedulerName string, +) (count int, err error) { + pipe := redisClient.TxPipeline() + cmd := pipe.HLen(GetPodMapRedisKey(schedulerName)) + err = mr.WithSegment(SegmentPipeExec, func() error { + var err error + _, err = pipe.Exec() + return err + }) + if err != nil { + return 0, err + } + + return int(cmd.Val()), err +} + +// GetPodFromRedis returns a specific pod from redis +func GetPodFromRedis( + redisClient redisinterfaces.RedisClient, + mr *MixedMetricsReporter, + podName, schedulerName string, +) (pod *Pod, err error) { + podStr, err := redisClient.HGet(GetPodMapRedisKey(schedulerName), podName).Result() + if err == redis.Nil { + return nil, nil + } + if err != nil { + return nil, err + } + + pod = &Pod{} + err = pod.UnmarshalFromRedis(podStr) + + return pod, err +} + +// AddToPodMap adds a pod to redis podMap key +func AddToPodMap( + redisClient redisinterfaces.RedisClient, + mr *MixedMetricsReporter, + pod *Pod, + schedulerName string, +) error { + // convert Pod to []byte + podStr, err := pod.MarshalToRedis() + if err != nil { + return err + } + + // Add pod to redis + err = redisClient.HMSet( + GetPodMapRedisKey(schedulerName), + map[string]interface{}{ + pod.Name: podStr, + }, + ).Err() + + return err +} + +// RemoveFromPodMap removes a pod from redis podMap key +func RemoveFromPodMap( + redisClient redisinterfaces.RedisClient, + mr *MixedMetricsReporter, + podName, schedulerName string, +) error { + // Remove pod from redis + pipe := redisClient.TxPipeline() + pipe.HDel(GetPodMapRedisKey(schedulerName), podName) + _, err := pipe.Exec() + return err +} diff --git a/models/pod_int_test.go b/models/pod_int_test.go index 9b957619d..95c26e9b4 100644 --- a/models/pod_int_test.go +++ b/models/pod_int_test.go @@ -95,7 +95,7 @@ var _ = Describe("Pod", func() { err := ns.Create(clientset) Expect(err).NotTo(HaveOccurred()) - pod, err := models.NewPod(name, env, configYaml, clientset, redisClient.Client) + pod, err := models.NewPod(name, env, configYaml, clientset, redisClient.Client, mmr) Expect(err).NotTo(HaveOccurred()) pod.SetToleration(game) pod.SetVersion("v1.0") diff --git a/models/pod_test.go b/models/pod_test.go index d3ee00cae..f3afbec48 100644 --- a/models/pod_test.go +++ b/models/pod_test.go @@ -9,6 +9,7 @@ package models_test import ( + "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -47,7 +48,11 @@ var _ = Describe("Pod", func() { reportersConstants.TagScheduler: "pong-free-for-all", }) - pod, err := models.NewPod(name, env, configYaml, mockClientset, mockRedisClient) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml.Name), name). + Return(goredis.NewStringResult("", goredis.Nil)) + + pod, err := models.NewPod(name, env, configYaml, mockClientset, mockRedisClient, mmr) Expect(err).NotTo(HaveOccurred()) return pod, err @@ -111,7 +116,7 @@ var _ = Describe("Pod", func() { BeforeEach(func() { mockRedisClient.EXPECT().Get(models.GlobalPortsPoolKey). Return(goredis.NewStringResult(portRange, nil)) - mockPortChooser.EXPECT().Choose(portStart, portEnd, 2).Return([]int{5000, 5001}) + mockPortChooser.EXPECT().Choose(portStart, portEnd, 2, gomock.Any()).Return([]int{5000, 5001}) }) It("should build correct pod struct", func() { @@ -158,17 +163,21 @@ var _ = Describe("Pod", func() { BeforeEach(func() { mockRedisClient.EXPECT().Get(models.GlobalPortsPoolKey). Return(goredis.NewStringResult(portRange, nil)) - mockPortChooser.EXPECT().Choose(portStart, portEnd, 1).Return([]int{5000}) + mockPortChooser.EXPECT().Choose(portStart, portEnd, 1, gomock.Any()).Return([]int{5000}) }) It("should create pod with two containers", func() { - mockPortChooser.EXPECT().Choose(portStart, portEnd, 1).Return([]int{5001}) + mockPortChooser.EXPECT().Choose(portStart, portEnd, 1, gomock.Any()).Return([]int{5001}) mr.EXPECT().Report("gru.new", map[string]interface{}{ reportersConstants.TagGame: "pong", reportersConstants.TagScheduler: "pong-free-for-all", }) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml.Name), name). + Return(goredis.NewStringResult("", goredis.Nil)) + pod, err := models.NewPodWithContainers( name, []*models.Container{ @@ -191,7 +200,7 @@ var _ = Describe("Pod", func() { Command: command, }, }, - configYaml, mockClientset, mockRedisClient, + configYaml, mockClientset, mockRedisClient, mmr, ) Expect(err).NotTo(HaveOccurred()) @@ -257,7 +266,11 @@ var _ = Describe("Pod", func() { reportersConstants.TagScheduler: "pong-free-for-all", }) - mockPortChooser.EXPECT().Choose(portStart, portEnd, 1).Return([]int{5001}) + mockPortChooser.EXPECT().Choose(portStart, portEnd, 1, gomock.Any()).Return([]int{5001}) + + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml.Name), name). + Return(goredis.NewStringResult("", goredis.Nil)) mr.EXPECT().Report("gru.new", map[string]interface{}{ reportersConstants.TagGame: "pong", @@ -286,13 +299,20 @@ var _ = Describe("Pod", func() { Command: command, }, }, - configYaml, mockClientset, mockRedisClient, + configYaml, mockClientset, mockRedisClient, mmr, ) Expect(err).NotTo(HaveOccurred()) firstPod.SetToleration(game) podv1, err := firstPod.Create(mockClientset) Expect(err).NotTo(HaveOccurred()) + firstPod.Spec = podv1.Spec + jsonBytes, err := firstPod.MarshalToRedis() + Expect(err).NotTo(HaveOccurred()) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml.Name), name). + Return(goredis.NewStringResult(string(jsonBytes), nil)) + pod, err := models.NewPodWithContainers( name, []*models.Container{ @@ -315,7 +335,7 @@ var _ = Describe("Pod", func() { Command: command, }, }, - configYaml, mockClientset, mockRedisClient, + configYaml, mockClientset, mockRedisClient, mmr, ) Expect(err).NotTo(HaveOccurred()) pod.SetToleration(game) @@ -378,7 +398,7 @@ var _ = Describe("Pod", func() { BeforeEach(func() { mockRedisClient.EXPECT().Get(models.GlobalPortsPoolKey). Return(goredis.NewStringResult(portRange, nil)) - mockPortChooser.EXPECT().Choose(portStart, portEnd, 2).Return([]int{5000, 5001}) + mockPortChooser.EXPECT().Choose(portStart, portEnd, 2, gomock.Any()).Return([]int{5000, 5001}) }) It("should create a pod in kubernetes", func() { @@ -433,6 +453,10 @@ var _ = Describe("Pod", func() { }) It("should create pod without requests and limits", func() { + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml.Name), name). + Return(goredis.NewStringResult("", goredis.Nil)) + mr.EXPECT().Report("gru.new", map[string]interface{}{ reportersConstants.TagGame: "pong", reportersConstants.TagScheduler: "pong-free-for-all", @@ -449,7 +473,7 @@ var _ = Describe("Pod", func() { Cmd: command, } - pod, err := models.NewPod(name, env, configYaml, mockClientset, mockRedisClient) + pod, err := models.NewPod(name, env, configYaml, mockClientset, mockRedisClient, mmr) Expect(err).NotTo(HaveOccurred()) podv1, err := pod.Create(mockClientset) Expect(err).NotTo(HaveOccurred()) @@ -546,7 +570,7 @@ var _ = Describe("Pod", func() { BeforeEach(func() { mockRedisClient.EXPECT().Get(models.GlobalPortsPoolKey). Return(goredis.NewStringResult(portRange, nil)) - mockPortChooser.EXPECT().Choose(portStart, portEnd, 2).Return([]int{5000, 5001}) + mockPortChooser.EXPECT().Choose(portStart, portEnd, 2, gomock.Any()).Return([]int{5000, 5001}) }) It("should delete a pod from kubernetes", func() { diff --git a/models/port.go b/models/port.go index 2c018a0d7..2715f32cc 100644 --- a/models/port.go +++ b/models/port.go @@ -36,8 +36,8 @@ func GetPortRange(configYaml *ConfigYAML, redis redisinterfaces.RedisClient) (st } // GetRandomPorts returns n random ports within scheduler limits -func GetRandomPorts(start, end, quantity int) []int { - return ThePortChooser.Choose(start, end, quantity) +func GetRandomPorts(start, end, quantity int, excludingPorts []int) []int { + return ThePortChooser.Choose(start, end, quantity, excludingPorts) } // GetGlobalPortRange returns the port range used by default by maestro worker diff --git a/models/port_chooser.go b/models/port_chooser.go index 121e9587a..434850dbc 100644 --- a/models/port_chooser.go +++ b/models/port_chooser.go @@ -9,7 +9,7 @@ import ( // PortChooser has a Choose method that returns // an array of ports type PortChooser interface { - Choose(start, end, quantity int) []int + Choose(start, end, quantity int, excluding []int) []int } var ( @@ -21,15 +21,24 @@ var ( type RandomPortChooser struct{} // Choose initialized an seed and chooses quantity random ports between [start, end] -func (r *RandomPortChooser) Choose(start, end, quantity int) []int { +func (r *RandomPortChooser) Choose(start, end, quantity int, excluding []int) []int { once.Do(func() { source := rand.NewSource(time.Now().Unix()) random = rand.New(source) }) + usedPortMap := map[int]bool{} + for _, port := range excluding { + usedPortMap[port] = true + } + ports := make([]int, quantity) for i := 0; i < quantity; i++ { port := start + random.Intn(end-start) + for usedPortMap[port] { + port = start + random.Intn(end-start) + } + usedPortMap[port] = true ports[i] = port } diff --git a/models/room.go b/models/room.go index 422236dad..61b0c9105 100644 --- a/models/room.go +++ b/models/room.go @@ -14,6 +14,8 @@ import ( "strings" "time" + "github.com/sirupsen/logrus" + "github.com/go-redis/redis" pginterfaces "github.com/topfreegames/extensions/pg/interfaces" "github.com/topfreegames/extensions/redis/interfaces" @@ -63,14 +65,16 @@ func (r RoomAddresses) Clone() *RoomAddresses { // RoomPort struct type RoomPort struct { - Name string `json:"name"` - Port int32 `json:"port"` + Name string `json:"name"` + Protocol string `json:"protocol"` + Port int32 `json:"port"` } func (r RoomPort) Clone() *RoomPort { return &RoomPort{ - Name: r.Name, - Port: r.Port, + Name: r.Name, + Port: r.Port, + Protocol: r.Protocol, } } @@ -347,9 +351,15 @@ func reportStatus(game, scheduler, status, gauge string) error { // ClearAll removes all room keys from redis func (r *Room) ClearAll(redisClient interfaces.RedisClient, mr *MixedMetricsReporter) error { + err := RemoveFromPodMap(redisClient, mr, r.ID, r.SchedulerName) + if err != nil { + return err + } + pipe := redisClient.TxPipeline() + r.clearAllWithPipe(pipe) - err := mr.WithSegment(SegmentPipeExec, func() error { + err = mr.WithSegment(SegmentPipeExec, func() error { _, err := pipe.Exec() return err }) @@ -359,10 +369,16 @@ func (r *Room) ClearAll(redisClient interfaces.RedisClient, mr *MixedMetricsRepo // ClearAllMultipleRooms removes all rooms keys from redis func ClearAllMultipleRooms(redisClient interfaces.RedisClient, mr *MixedMetricsReporter, rooms []*Room) error { pipe := redisClient.TxPipeline() + var err error for _, room := range rooms { + // remove from redis podMap + err = RemoveFromPodMap(redisClient, mr, room.ID, room.SchedulerName) + if err != nil { + return err + } room.clearAllWithPipe(pipe) } - err := mr.WithSegment(SegmentPipeExec, func() error { + err = mr.WithSegment(SegmentPipeExec, func() error { _, err := pipe.Exec() return err }) @@ -409,6 +425,7 @@ func (r *Room) GetRoomInfos( schedulerCache *SchedulerCache, scheduler *Scheduler, addrGetter AddrGetter, + mr *MixedMetricsReporter, ) (map[string]interface{}, error) { if scheduler == nil { cachedScheduler, err := schedulerCache.LoadScheduler(db, r.SchedulerName, true) @@ -417,7 +434,7 @@ func (r *Room) GetRoomInfos( } scheduler = cachedScheduler.Scheduler } - address, err := addrGetter.Get(r, kubernetesClient, redis) + address, err := addrGetter.Get(r, kubernetesClient, redis, mr) if err != nil { return nil, err } @@ -425,17 +442,35 @@ func (r *Room) GetRoomInfos( if len(address.Ports) > 0 { selectedPort = address.Ports[0].Port } - for _, p := range address.Ports { + metadata := map[string]interface{}{ + "ports": make([]map[string]interface{}, len(address.Ports)), + "ipv6Label": address.Ipv6Label, + } + for i, p := range address.Ports { if p.Name == "clientPort" { selectedPort = p.Port } + // save multiple defined ports in metadata to send to forwarders + metadata["ports"].([]map[string]interface{})[i] = map[string]interface{}{ + "port": p.Port, + "name": p.Name, + "protocol": p.Protocol, + } + } + if metadata["ports"] != nil { + metadata["ports"], err = json.Marshal(metadata["ports"]) + if err != nil { + return nil, err + } + metadata["ports"] = string(metadata["ports"].([]byte)) } + return map[string]interface{}{ - "game": scheduler.Game, - "roomId": r.ID, - "host": address.Host, - "ipv6Label": address.Ipv6Label, - "port": selectedPort, + "game": scheduler.Game, + "roomId": r.ID, + "host": address.Host, + "port": selectedPort, + "metadata": metadata, }, nil } @@ -609,19 +644,12 @@ func GetInvalidRoomsKey(schedulerName string) string { return fmt.Sprintf("scheduler:%s:invalidRooms", schedulerName) } -// GetInvalidRoomsCountKey gets the key for the string that keeps count of invalid rooms -func GetInvalidRoomsCountKey(schedulerName string) string { - return fmt.Sprintf("scheduler:%s:invalidRooms:count", schedulerName) -} - // SetInvalidRooms save a room in invalid redis set // A room is considered invalid if its version is not the scheduler current version func SetInvalidRooms(redisClient interfaces.RedisClient, mr *MixedMetricsReporter, schedulerName string, roomIDs []string) error { pipe := redisClient.TxPipeline() pipe.Del(GetInvalidRoomsKey(schedulerName)) - pipe.Del(GetInvalidRoomsCountKey(schedulerName)) pipe.SAdd(GetInvalidRoomsKey(schedulerName), roomIDs) - pipe.Set(GetInvalidRoomsCountKey(schedulerName), len(roomIDs), 2*time.Hour) return mr.WithSegment(SegmentPipeExec, func() error { var err error _, err = pipe.Exec() @@ -648,7 +676,6 @@ func RemoveInvalidRooms(redisClient interfaces.RedisClient, mr *MixedMetricsRepo func RemoveInvalidRoomsKey(redisClient interfaces.RedisClient, mr *MixedMetricsReporter, schedulerName string) error { pipe := redisClient.TxPipeline() pipe.Del(GetInvalidRoomsKey(schedulerName)) - pipe.Del(GetInvalidRoomsCountKey(schedulerName)) err := mr.WithSegment(SegmentPipeExec, func() error { var err error _, err = pipe.Exec() @@ -676,20 +703,25 @@ func GetCurrentInvalidRoomsCount(redisClient interfaces.RedisClient, mr *MixedMe return count, err } -// GetInvalidRoomsCount returns the count of invalid rooms saved on InvalidRoomsCountKey -func GetInvalidRoomsCount(redisClient interfaces.RedisClient, mr *MixedMetricsReporter, schedulerName string) (int, error) { - count := 0 - err := mr.WithSegment(SegmentGet, func() error { - var err error - c, err := redisClient.Get(GetInvalidRoomsCountKey(schedulerName)).Result() - if err == redis.Nil { - return nil - } - if err != nil { - return err - } - count, err = strconv.Atoi(c) - return nil - }) - return count, err +// IsRoomReadyOrOccupied returns true if a room is in ready or occupied status +func IsRoomReadyOrOccupied(logger logrus.FieldLogger, redisClient interfaces.RedisClient, schedulerName, roomName string) bool { + room := NewRoom(roomName, schedulerName) + pipe := redisClient.TxPipeline() + roomIsReady := pipe.SIsMember(GetRoomStatusSetRedisKey(schedulerName, StatusReady), room.GetRoomRedisKey()) + roomIsOccupied := pipe.SIsMember(GetRoomStatusSetRedisKey(schedulerName, StatusOccupied), room.GetRoomRedisKey()) + + _, err := pipe.Exec() + if err != nil { + logger.WithError(err).Error("failed to check room rediness") + return false + } + + isReady, err := roomIsReady.Result() + isOccupied, err := roomIsOccupied.Result() + if err != nil { + logger.WithError(err).Error("failed to check room rediness") + return false + } + + return isReady || isOccupied } diff --git a/models/room_address.go b/models/room_address.go index c76c63ab4..88d1a368b 100644 --- a/models/room_address.go +++ b/models/room_address.go @@ -46,12 +46,15 @@ func NewRoomAddressesFromHostPort( // Get gets room public addresses func (r *RoomAddressesFromHostPort) Get( - room *Room, kubernetesClient kubernetes.Interface, redisClient redisinterfaces.RedisClient, + room *Room, + kubernetesClient kubernetes.Interface, + redisClient redisinterfaces.RedisClient, + mr *MixedMetricsReporter, ) (*RoomAddresses, error) { if addrs := r.fromCache(redisClient, room); addrs != nil { return addrs, nil } - addrs, err := getRoomAddresses(false, r.ipv6KubernetesLabelKey, room, kubernetesClient) + addrs, err := getRoomAddresses(false, r.ipv6KubernetesLabelKey, room, kubernetesClient, redisClient, mr) if err != nil { return nil, err } @@ -136,12 +139,12 @@ func NewRoomAddressesFromNodePort( // Get gets room public addresses func (r *RoomAddressesFromNodePort) Get( - room *Room, kubernetesClient kubernetes.Interface, redisClient redisinterfaces.RedisClient, + room *Room, kubernetesClient kubernetes.Interface, redisClient redisinterfaces.RedisClient, mr *MixedMetricsReporter, ) (*RoomAddresses, error) { if addrs := r.fromCache(redisClient, room); addrs != nil { return addrs, nil } - addrs, err := getRoomAddresses(true, r.ipv6KubernetesLabelKey, room, kubernetesClient) + addrs, err := getRoomAddresses(true, r.ipv6KubernetesLabelKey, room, kubernetesClient, redisClient, mr) if err != nil { return nil, err } @@ -203,13 +206,24 @@ func (r RoomAddressesFromNodePort) buildCacheKey(room *Room) string { return fmt.Sprintf("room-addr-%s-%s", room.SchedulerName, room.ID) } -func getRoomAddresses(IsNodePort bool, ipv6KubernetesLabelKey string, room *Room, kubernetesClient kubernetes.Interface) (*RoomAddresses, error) { +func getRoomAddresses( + IsNodePort bool, + ipv6KubernetesLabelKey string, + room *Room, + kubernetesClient kubernetes.Interface, + redisClient redisinterfaces.RedisClient, + mr *MixedMetricsReporter, +) (*RoomAddresses, error) { rAddresses := &RoomAddresses{} - roomPod, err := kubernetesClient.CoreV1().Pods(room.SchedulerName).Get(room.ID, metav1.GetOptions{}) + roomPod, err := GetPodFromRedis(redisClient, mr, room.ID, room.SchedulerName) if err != nil { return nil, err } + if roomPod == nil { + return nil, fmt.Errorf(`pod "%s" not found on redis podMap`, room.ID) + } + if len(roomPod.Spec.NodeName) == 0 { return rAddresses, nil } @@ -243,8 +257,9 @@ func getRoomAddresses(IsNodePort bool, ipv6KubernetesLabelKey string, room *Room for _, port := range roomSvc.Spec.Ports { if port.NodePort != 0 { rAddresses.Ports = append(rAddresses.Ports, &RoomPort{ - Name: port.Name, - Port: port.NodePort, + Name: port.Name, + Port: port.NodePort, + Protocol: string(port.Protocol), }) } } @@ -262,8 +277,9 @@ func getRoomAddresses(IsNodePort bool, ipv6KubernetesLabelKey string, room *Room for _, port := range container.Ports { if port.HostPort != 0 { rAddresses.Ports = append(rAddresses.Ports, &RoomPort{ - Name: port.Name, - Port: port.HostPort, + Name: port.Name, + Port: port.HostPort, + Protocol: string(port.Protocol), }) } } diff --git a/models/room_address_test.go b/models/room_address_test.go index 3bf638828..de6d68e73 100644 --- a/models/room_address_test.go +++ b/models/room_address_test.go @@ -10,7 +10,7 @@ package models_test import ( "encoding/json" - "fmt" + redismocks "github.com/topfreegames/extensions/redis/mocks" "time" "github.com/btcsuite/btcutil/base58" @@ -23,32 +23,44 @@ import ( . "github.com/onsi/gomega" "github.com/topfreegames/maestro/models" - reportersConstants "github.com/topfreegames/maestro/reporters/constants" v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes/fake" ) +func createPodWithContainers( + clientset *fake.Clientset, + redisClient *redismocks.MockRedisClient, + schedulerName, nodeName, podName string, + containers []v1.Container, +) { + pod := &v1.Pod{} + pod.Spec.NodeName = nodeName + pod.SetName(podName) + pod.Spec.Containers = containers + _, err := clientset.CoreV1().Pods(schedulerName).Create(pod) + Expect(err).NotTo(HaveOccurred()) + + var podModel models.Pod + podModel.Name = podName + podModel.Spec.NodeName = nodeName + podModel.Spec.Containers = containers + + jsonBytes, err := podModel.MarshalToRedis() + Expect(err).NotTo(HaveOccurred()) + + redisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(schedulerName), podModel.Name). + Return(goredis.NewStringResult(string(jsonBytes), nil)) +} + var _ = Describe("AddressGetter", func() { var ( clientset *fake.Clientset - portStart = 5000 - portEnd = 6000 - portRange = fmt.Sprintf("%d-%d", portStart, portEnd) command = []string{ "./room-binary", "-serverType", "6a8e136b-2dc1-417e-bbe8-0f0a2d2df431", } - env = []*models.EnvVar{ - { - Name: "EXAMPLE_ENV_VAR", - Value: "examplevalue", - }, - { - Name: "ANOTHER_ENV_VAR", - Value: "anothervalue", - }, - } game = "pong" image = "pong/pong:v123" name = "pong-free-for-all-0" @@ -106,7 +118,7 @@ var _ = Describe("AddressGetter", func() { node.SetName(nodeName) node.SetLabels(nodeLabels) node.Status.Addresses = []v1.NodeAddress{ - v1.NodeAddress{ + { Type: v1.NodeInternalIP, Address: host, }, @@ -114,16 +126,11 @@ var _ = Describe("AddressGetter", func() { _, err := clientset.CoreV1().Nodes().Create(node) Expect(err).NotTo(HaveOccurred()) - pod := &v1.Pod{} - pod.Spec.NodeName = nodeName - pod.SetName(name) - pod.Spec.Containers = []v1.Container{ + createPodWithContainers(clientset, mockRedisClient, namespace, nodeName, name, []v1.Container{ {Ports: []v1.ContainerPort{ {HostPort: port, Name: "TCP"}, }}, - } - _, err = clientset.CoreV1().Pods(namespace).Create(pod) - Expect(err).NotTo(HaveOccurred()) + }) service := &v1.Service{} service.SetName(name) @@ -151,16 +158,18 @@ var _ = Describe("AddressGetter", func() { Return(goredis.NewStringResult("", goredis.Nil)) mockRedisClient.EXPECT().Set("room-addr-pong-free-for-all-pong-free-for-all-0", gomock.Any(), gomock.Any()). Return(goredis.NewStatusCmd()) - addrs1, err := addrGetter.Get(room, clientset, mockRedisClient) + addrs1, err := addrGetter.Get(room, clientset, mockRedisClient, mmr) Expect(err).NotTo(HaveOccurred()) + step2 := len(clientset.Fake.Actions()) - Expect(step2).To(Equal(step1 + 3)) + Expect(step2).To(Equal(step1 + 2)) b, err := json.Marshal(addrs1) Expect(err).NotTo(HaveOccurred()) mockRedisClient.EXPECT().Get("room-addr-pong-free-for-all-pong-free-for-all-0"). Return(goredis.NewStringResult(string(b), nil)) - addrs2, err := addrGetter.Get(room, clientset, mockRedisClient) + addrs2, err := addrGetter.Get(room, clientset, mockRedisClient, mmr) Expect(err).NotTo(HaveOccurred()) + step3 := len(clientset.Fake.Actions()) Expect(step3).To(Equal(step2)) Expect(addrs2).To(Equal(addrs1)) @@ -173,7 +182,7 @@ var _ = Describe("AddressGetter", func() { node := &v1.Node{} node.SetName(nodeName) node.Status.Addresses = []v1.NodeAddress{ - v1.NodeAddress{ + { Type: v1.NodeExternalDNS, Address: host, }, @@ -181,16 +190,11 @@ var _ = Describe("AddressGetter", func() { _, err := clientset.CoreV1().Nodes().Create(node) Expect(err).NotTo(HaveOccurred()) - pod := &v1.Pod{} - pod.Spec.NodeName = nodeName - pod.SetName(name) - pod.Spec.Containers = []v1.Container{ + createPodWithContainers(clientset, mockRedisClient, namespace, nodeName, name, []v1.Container{ {Ports: []v1.ContainerPort{ {HostPort: port, Name: "TCP"}, }}, - } - _, err = clientset.CoreV1().Pods(namespace).Create(pod) - Expect(err).NotTo(HaveOccurred()) + }) step1 := len(clientset.Fake.Actions()) addrGetter := models.NewRoomAddressesFromHostPort(logger, ipv6KubernetesLabelKey, true, 10*time.Second) @@ -198,16 +202,18 @@ var _ = Describe("AddressGetter", func() { Return(goredis.NewStringResult("", goredis.Nil)) mockRedisClient.EXPECT().Set("room-addr-pong-free-for-all-pong-free-for-all-0", gomock.Any(), gomock.Any()). Return(goredis.NewStatusCmd()) - addrs1, err := addrGetter.Get(room, clientset, mockRedisClient) + addrs1, err := addrGetter.Get(room, clientset, mockRedisClient, mmr) Expect(err).NotTo(HaveOccurred()) + step2 := len(clientset.Fake.Actions()) - Expect(step2).To(Equal(step1 + 2)) + Expect(step2).To(Equal(step1 + 1)) b, err := json.Marshal(addrs1) Expect(err).NotTo(HaveOccurred()) mockRedisClient.EXPECT().Get("room-addr-pong-free-for-all-pong-free-for-all-0"). Return(goredis.NewStringResult(string(b), nil)) - addrs2, err := addrGetter.Get(room, clientset, mockRedisClient) + addrs2, err := addrGetter.Get(room, clientset, mockRedisClient, mmr) Expect(err).NotTo(HaveOccurred()) + step3 := len(clientset.Fake.Actions()) Expect(step3).To(Equal(step2)) Expect(addrs2).To(Equal(addrs1)) @@ -221,29 +227,22 @@ var _ = Describe("AddressGetter", func() { Describe("Get", func() { It("should not crash if pod does not exist", func() { - _, err := addrGetter.Get(room, clientset, mockRedisClient) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(room.SchedulerName), room.ID). + Return(goredis.NewStringResult("", goredis.Nil)) + _, err := addrGetter.Get(room, clientset, mockRedisClient, mmr) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(Equal(`pods "pong-free-for-all-0" not found`)) + Expect(err.Error()).To(Equal(`pod "pong-free-for-all-0" not found on redis podMap`)) }) It("should return no address if no node assigned to the room", func() { - mockRedisClient.EXPECT().Get(models.GlobalPortsPoolKey). - Return(goredis.NewStringResult(portRange, nil)) - mockPortChooser.EXPECT().Choose(portStart, portEnd, 2).Return([]int{5000, 5001}) + createPodWithContainers(clientset, mockRedisClient, namespace, "", name, []v1.Container{}) - mr.EXPECT().Report("gru.new", map[string]interface{}{ - reportersConstants.TagGame: "pong", - reportersConstants.TagScheduler: "pong-free-for-all", - }) - - pod, err := models.NewPod(name, env, configYaml, mockClientset, mockRedisClient) - Expect(err).NotTo(HaveOccurred()) - _, err = pod.Create(clientset) - Expect(err).NotTo(HaveOccurred()) - svc := models.NewService(pod.Name, configYaml) + svc := models.NewService(name, configYaml) svc.Create(clientset) room := models.NewRoom(name, namespace) - addresses, err := addrGetter.Get(room, clientset, mockRedisClient) + + addresses, err := addrGetter.Get(room, clientset, mockRedisClient, mmr) Expect(err).NotTo(HaveOccurred()) Expect(len(addresses.Ports)).To(Equal(0)) }) @@ -253,7 +252,7 @@ var _ = Describe("AddressGetter", func() { node.SetName(nodeName) node.SetLabels(nodeLabels) node.Status.Addresses = []v1.NodeAddress{ - v1.NodeAddress{ + { Type: v1.NodeInternalIP, Address: host, }, @@ -261,16 +260,11 @@ var _ = Describe("AddressGetter", func() { _, err := clientset.CoreV1().Nodes().Create(node) Expect(err).NotTo(HaveOccurred()) - pod := &v1.Pod{} - pod.Spec.NodeName = nodeName - pod.SetName(name) - pod.Spec.Containers = []v1.Container{ + createPodWithContainers(clientset, mockRedisClient, namespace, nodeName, name, []v1.Container{ {Ports: []v1.ContainerPort{ {HostPort: port, Name: "TCP"}, }}, - } - _, err = clientset.CoreV1().Pods(namespace).Create(pod) - Expect(err).NotTo(HaveOccurred()) + }) service := &v1.Service{} service.SetName(name) @@ -292,7 +286,7 @@ var _ = Describe("AddressGetter", func() { _, err = clientset.CoreV1().Services(namespace).Create(service) Expect(err).NotTo(HaveOccurred()) - roomAddresses, err := addrGetter.Get(room, clientset, mockRedisClient) + roomAddresses, err := addrGetter.Get(room, clientset, mockRedisClient, mmr) Expect(err).NotTo(HaveOccurred()) Expect(roomAddresses.Host).To(Equal(host)) @@ -304,6 +298,7 @@ var _ = Describe("AddressGetter", func() { &models.RoomPort{ Name: "TCP", Port: port, + Protocol: "TCP", })) }) @@ -319,16 +314,11 @@ var _ = Describe("AddressGetter", func() { _, err := clientset.CoreV1().Nodes().Create(node) Expect(err).NotTo(HaveOccurred()) - pod := &v1.Pod{} - pod.Spec.NodeName = nodeName - pod.SetName(name) - pod.Spec.Containers = []v1.Container{ + createPodWithContainers(clientset, mockRedisClient, namespace, nodeName, name, []v1.Container{ {Ports: []v1.ContainerPort{ {HostPort: port, Name: "TCP"}, }}, - } - _, err = clientset.CoreV1().Pods(namespace).Create(pod) - Expect(err).NotTo(HaveOccurred()) + }) service := &v1.Service{} service.SetName(name) @@ -350,7 +340,7 @@ var _ = Describe("AddressGetter", func() { _, err = clientset.CoreV1().Services(namespace).Create(service) Expect(err).NotTo(HaveOccurred()) - roomAddresses, err := addrGetter.Get(room, clientset, mockRedisClient) + roomAddresses, err := addrGetter.Get(room, clientset, mockRedisClient, mmr) Expect(err).NotTo(HaveOccurred()) Expect(roomAddresses.Host).To(Equal(host)) Expect(roomAddresses.Ipv6Label).To(Equal("")) @@ -359,17 +349,14 @@ var _ = Describe("AddressGetter", func() { &models.RoomPort{ Name: "TCP", Port: port, + Protocol: "TCP", })) }) It("should return error if there is no node", func() { - pod := &v1.Pod{} - pod.SetName(name) - pod.Spec.NodeName = nodeName - _, err := clientset.CoreV1().Pods(namespace).Create(pod) - Expect(err).NotTo(HaveOccurred()) + createPodWithContainers(clientset, mockRedisClient, namespace, nodeName, name, []v1.Container{}) - _, err = addrGetter.Get(room, clientset, mockRedisClient) + _, err := addrGetter.Get(room, clientset, mockRedisClient, mmr) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("nodes \"node-name\" not found")) }) @@ -381,29 +368,21 @@ var _ = Describe("AddressGetter", func() { Describe("Get", func() { It("should not crash if pod does not exist", func() { - _, err := addrGetter.Get(room, clientset, mockRedisClient) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(namespace), name). + Return(goredis.NewStringResult("", goredis.Nil)) + _, err := addrGetter.Get(room, clientset, mockRedisClient, mmr) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(Equal(`pods "pong-free-for-all-0" not found`)) + Expect(err.Error()).To(Equal(`pod "pong-free-for-all-0" not found on redis podMap`)) }) It("should return no address if no node assigned to the room", func() { - mockRedisClient.EXPECT().Get(models.GlobalPortsPoolKey). - Return(goredis.NewStringResult(portRange, nil)) - mockPortChooser.EXPECT().Choose(portStart, portEnd, 2).Return([]int{5000, 5001}) + createPodWithContainers(clientset, mockRedisClient, namespace, "", name, []v1.Container{}) - mr.EXPECT().Report("gru.new", map[string]interface{}{ - reportersConstants.TagGame: "pong", - reportersConstants.TagScheduler: "pong-free-for-all", - }) - - pod, err := models.NewPod(name, env, configYaml, mockClientset, mockRedisClient) - Expect(err).NotTo(HaveOccurred()) - _, err = pod.Create(clientset) - Expect(err).NotTo(HaveOccurred()) - svc := models.NewService(pod.Name, configYaml) + svc := models.NewService(name, configYaml) svc.Create(clientset) room := models.NewRoom(name, namespace) - addresses, err := addrGetter.Get(room, clientset, mockRedisClient) + addresses, err := addrGetter.Get(room, clientset, mockRedisClient, mmr) Expect(err).NotTo(HaveOccurred()) Expect(len(addresses.Ports)).To(Equal(0)) }) @@ -420,18 +399,13 @@ var _ = Describe("AddressGetter", func() { _, err := clientset.CoreV1().Nodes().Create(node) Expect(err).NotTo(HaveOccurred()) - pod := &v1.Pod{} - pod.Spec.NodeName = nodeName - pod.SetName(name) - pod.Spec.Containers = []v1.Container{ + createPodWithContainers(clientset, mockRedisClient, namespace, nodeName, name, []v1.Container{ {Ports: []v1.ContainerPort{ {HostPort: port, Name: "TCP"}, }}, - } - _, err = clientset.CoreV1().Pods(namespace).Create(pod) - Expect(err).NotTo(HaveOccurred()) + }) - roomAddresses, err := addrGetter.Get(room, clientset, mockRedisClient) + roomAddresses, err := addrGetter.Get(room, clientset, mockRedisClient, mmr) Expect(err).NotTo(HaveOccurred()) Expect(roomAddresses.Host).To(Equal(host)) Expect(roomAddresses.Ports).To(HaveLen(1)) @@ -443,13 +417,9 @@ var _ = Describe("AddressGetter", func() { }) It("should return error if there is no node", func() { - pod := &v1.Pod{} - pod.SetName(name) - pod.Spec.NodeName = nodeName - _, err := clientset.CoreV1().Pods(namespace).Create(pod) - Expect(err).NotTo(HaveOccurred()) + createPodWithContainers(clientset, mockRedisClient, namespace, nodeName, name, []v1.Container{}) - _, err = addrGetter.Get(room, clientset, mockRedisClient) + _, err := addrGetter.Get(room, clientset, mockRedisClient, mmr) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("nodes \"node-name\" not found")) }) diff --git a/models/room_test.go b/models/room_test.go index d2a671389..89ab06134 100644 --- a/models/room_test.go +++ b/models/room_test.go @@ -253,6 +253,10 @@ var _ = Describe("Room", func() { pKey := models.GetRoomPingRedisKey(room.SchedulerName) payload := &models.RoomStatusPayload{Status: status} + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HDel(models.GetPodMapRedisKey(schedulerName), room.ID) + mockPipeline.EXPECT().Exec() + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) mockPipeline.EXPECT().SRem(models.GetRoomStatusSetRedisKey(schedulerName, models.StatusReady), rKey) mockPipeline.EXPECT().ZRem(models.GetLastStatusRedisKey(schedulerName, models.StatusReady), name) @@ -378,6 +382,10 @@ var _ = Describe("Room", func() { rKey := room.GetRoomRedisKey() pKey := models.GetRoomPingRedisKey(room.SchedulerName) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HDel(models.GetPodMapRedisKey(scheduler), name) + mockPipeline.EXPECT().Exec() + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) mockPipeline.EXPECT().Del(rKey) mockPipeline.EXPECT().ZRem(pKey, room.ID) @@ -398,22 +406,12 @@ var _ = Describe("Room", func() { name := "pong-free-for-all-0" scheduler := "pong-free-for-all" room := models.NewRoom(name, scheduler) - rKey := room.GetRoomRedisKey() - pKey := models.GetRoomPingRedisKey(room.SchedulerName) mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - mockPipeline.EXPECT().Del(rKey) - for _, mt := range allMetrics { - mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(room.SchedulerName, mt), room.ID) - } - mockPipeline.EXPECT().ZRem(pKey, room.ID) - for _, st := range allStatus { - mockPipeline.EXPECT().SRem(models.GetRoomStatusSetRedisKey(room.SchedulerName, st), rKey) - mockPipeline.EXPECT().ZRem(models.GetLastStatusRedisKey(room.SchedulerName, st), room.ID) - } - mockPipeline.EXPECT().Exec().Return([]redis.Cmder{}, errors.New("some error in redis")) - + mockPipeline.EXPECT().HDel(models.GetPodMapRedisKey(scheduler), name) + mockPipeline.EXPECT().Exec().Return(nil, errors.New("some error in redis")) err := room.ClearAll(mockRedisClient, mmr) + Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("some error in redis")) }) diff --git a/testing/common.go b/testing/common.go index a05a42b88..5609ad46c 100644 --- a/testing/common.go +++ b/testing/common.go @@ -510,6 +510,20 @@ func MockCreateScheduler( calls.Append( MockInsertScheduler(mockDb, nil)) + calls.Add( + mockRedisClient.EXPECT(). + TxPipeline(). + Return(mockPipeline)) + + calls.Add( + mockPipeline.EXPECT(). + HGetAll(models.GetPodMapRedisKey(configYaml.Name)). + Return(goredis.NewStringStringMapResult(map[string]string{}, nil))) + + calls.Add(mockPipeline.EXPECT().Exec()) + + calls.Add(MockAnyRunningPod(mockRedisClient, configYaml.Name, configYaml.AutoScaling.Min * 2)) + calls.Add( mockRedisClient.EXPECT(). TxPipeline(). @@ -538,9 +552,6 @@ func MockCreateScheduler( Exec(). Times(configYaml.AutoScaling.Min)) - calls.Append( - MockGetPortsFromPool(&configYaml, mockRedisClient, mockPortChooser, workerPortRange, portStart, portEnd, 0)) - calls.Append( MockUpdateScheduler(mockDb, nil, nil)) @@ -587,7 +598,7 @@ func MockGetPortsFromPool( ports[i] = portStart + i } mockPortChooser.EXPECT(). - Choose(portStart, portEnd, nPorts). + Choose(portStart, portEnd, nPorts, gomock.Any()). Return(ports). Times(callTimes) } @@ -633,7 +644,7 @@ func MockGetPortsFromPoolAnyTimes( ports[i] = portStart + i } mockPortChooser.EXPECT(). - Choose(portStart, portEnd, nPorts). + Choose(portStart, portEnd, nPorts, gomock.Any()). Return(ports). AnyTimes() } @@ -1068,6 +1079,71 @@ func MockGetRegisteredRoomsPerStatus( mockPipeline.EXPECT().Exec().Return(nil, err) } +func MockListPods( + mockPipeline *redismocks.MockPipeliner, + mockRedisClient *redismocks.MockRedisClient, + schedulerName string, + rooms []string, + err error, +) *gomock.Call { + result := make(map[string]string, len(rooms)) + for _, room := range rooms { + result[room] = fmt.Sprintf(`{"name": "%s", "version": "v1.0"}`, room) + } + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HGetAll( + models.GetPodMapRedisKey(schedulerName)). + Return(goredis.NewStringStringMapResult(result, nil)) + execCall := mockPipeline.EXPECT().Exec() + if err != nil { + execCall.Return([]goredis.Cmder{}, err) + } + return execCall +} + +func MockAnyRunningPod( + mockRedisClient *redismocks.MockRedisClient, + schedulerName string, + times int, +) *gomock.Call { + runningPod := `{"status":{"phase":"Running", "conditions": [{"type":"Ready","status":"True"}]}}` + return mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(schedulerName), gomock.Any()). + Return(goredis.NewStringResult(runningPod, nil)). + Times(times) +} + +func MockRunningPod( + mockRedisClient *redismocks.MockRedisClient, + schedulerName string, + podName string, +) *gomock.Call { + runningPod := fmt.Sprintf(`{"name":"%s", "status":{"phase":"Running", "conditions": [{"type":"Ready","status":"True"}]}}`, podName) + return mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(schedulerName), podName). + Return(goredis.NewStringResult(runningPod, nil)) +} + +func MockPodNotFound( + mockRedisClient *redismocks.MockRedisClient, + schedulerName string, + podName string, +) *gomock.Call { + return mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(schedulerName), podName). + Return(goredis.NewStringResult("", goredis.Nil)) +} + +func MockPodsNotFound( + mockRedisClient *redismocks.MockRedisClient, + schedulerName string, + amount int, +) { + for i := 0; i < amount; i++ { + MockPodNotFound(mockRedisClient, schedulerName, fmt.Sprintf("room-%d", i)) + } +} + // MockSavingRoomsMetricses mocks the call to redis to save a sorted set with pods metricses func MockSavingRoomsMetricses( mockRedisClient *redismocks.MockRedisClient, @@ -1120,6 +1196,26 @@ func MockSetScallingAmount( return nil } +func MockSetScallingAmountAndReturnExec( + mockRedis *redismocks.MockRedisClient, + mockPipeline *redismocks.MockPipeliner, + configYaml *models.ConfigYAML, + currrentRooms int, +) *gomock.Call { + mockRedis.EXPECT().TxPipeline().Return(mockPipeline) + + creating := models.GetRoomStatusSetRedisKey(configYaml.Name, "creating") + ready := models.GetRoomStatusSetRedisKey(configYaml.Name, "ready") + occupied := models.GetRoomStatusSetRedisKey(configYaml.Name, "occupied") + terminating := models.GetRoomStatusSetRedisKey(configYaml.Name, "terminating") + + mockPipeline.EXPECT().SCard(creating).Return(goredis.NewIntResult(int64(0), nil)) + mockPipeline.EXPECT().SCard(ready).Return(goredis.NewIntResult(int64(currrentRooms), nil)) + mockPipeline.EXPECT().SCard(occupied).Return(goredis.NewIntResult(int64(0), nil)) + mockPipeline.EXPECT().SCard(terminating).Return(goredis.NewIntResult(int64(0), nil)) + return mockPipeline.EXPECT().Exec() +} + // MockSetScallingAmountWithRoomStatusCount mocks the call to adjust the scaling amount based on min and max limits func MockSetScallingAmountWithRoomStatusCount( mockRedis *redismocks.MockRedisClient, @@ -1232,11 +1328,11 @@ func MockRedisReadyPop( amount int, ) { readyKey := models.GetRoomStatusSetRedisKey(schedulerName, models.StatusReady) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) for i := 0; i < amount; i++ { - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) mockPipeline.EXPECT().SPop(readyKey).Return(goredis.NewStringResult(fmt.Sprintf("room-%d", i), nil)) - mockPipeline.EXPECT().Exec() } + mockPipeline.EXPECT().Exec() } // MockClearAll mocks models.Room.ClearAll method @@ -1275,6 +1371,7 @@ func MockClearAll( mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(schedulerName, mt), room.ID) } mockPipeline.EXPECT().Del(room.GetRoomRedisKey()) + mockPipeline.EXPECT().HDel(models.GetPodMapRedisKey(schedulerName), room.ID) } } @@ -1285,6 +1382,8 @@ func MockScaleUp( schedulerName string, times int, ) { + MockListPods(mockPipeline, mockRedisClient, schedulerName, []string{}, nil) + MockAnyRunningPod(mockRedisClient, schedulerName, 2 * times) mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(times) mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( func(schedulerName string, statusInfo map[string]interface{}) { @@ -1331,7 +1430,7 @@ func TransformLegacyInMetricsTrigger(autoScalingInfo *models.AutoScaling) { autoScalingInfo.Up.MetricsTrigger = append( autoScalingInfo.Up.MetricsTrigger, &models.ScalingPolicyMetricsTrigger{ - Type: models.LegacyAutoScalingPolicyType, + Type: models.RoomAutoScalingPolicyType, Usage: autoScalingInfo.Up.Trigger.Usage, Limit: autoScalingInfo.Up.Trigger.Limit, Threshold: autoScalingInfo.Up.Trigger.Threshold, @@ -1345,7 +1444,7 @@ func TransformLegacyInMetricsTrigger(autoScalingInfo *models.AutoScaling) { autoScalingInfo.Down.MetricsTrigger = append( autoScalingInfo.Down.MetricsTrigger, &models.ScalingPolicyMetricsTrigger{ - Type: models.LegacyAutoScalingPolicyType, + Type: models.RoomAutoScalingPolicyType, Usage: autoScalingInfo.Down.Trigger.Usage, Limit: autoScalingInfo.Down.Trigger.Limit, Threshold: autoScalingInfo.Down.Trigger.Threshold, @@ -1410,7 +1509,7 @@ func CreatePod(clientset *fake.Clientset, cpuRequests, memRequests, schedulerNam } // CreateTestRooms returns a map of string slices with names of test rooms with the 4 possible status -func CreateTestRooms(clientset *fake.Clientset, schedulerName string, expC *models.RoomsStatusCount) map[string][]string { +func CreateTestRooms(clientset *fake.Clientset, schedulerName string, expC *models.RoomsStatusCount) ([]string, map[string][]string) { rooms := make(map[string][]string, 4) statusIdx := []string{models.StatusReady, models.StatusOccupied, models.StatusTerminating, models.StatusCreating} statusCount := []int{expC.Ready, expC.Occupied, expC.Terminating, expC.Creating} @@ -1418,14 +1517,17 @@ func CreateTestRooms(clientset *fake.Clientset, schedulerName string, expC *mode rooms[models.StatusOccupied] = make([]string, expC.Occupied) rooms[models.StatusCreating] = make([]string, expC.Creating) rooms[models.StateTerminating] = make([]string, expC.Terminating) + + roomNames := make([]string, 0, expC.Total()) for idx, numPods := range statusCount { for i := 0; i < numPods; i++ { roomName := fmt.Sprintf("test-%s-%d", statusIdx[idx], i) + roomNames = append(roomNames, roomName) rooms[statusIdx[idx]][i] = fmt.Sprintf("scheduler:%s:rooms:%s", schedulerName, roomName) CreatePod(clientset, "1.0", "1Ki", schedulerName, roomName, roomName) } } - return rooms + return roomNames, rooms } // CreatePodMetricsList returns a fakeMetricsClientset with reactor to PodMetricses Get call @@ -1639,9 +1741,6 @@ func MockGetInvalidRooms( mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) mockPipeline.EXPECT().Exec() mockPipeline.EXPECT().SCard(models.GetInvalidRoomsKey(schedulerName)).Return(goredis.NewIntResult(int64(currentInvalidCount), nil)) - - retGet := goredis.NewStringResult(strconv.Itoa(invalidCount), err) - mockRedisClient.EXPECT().Get(models.GetInvalidRoomsCountKey(schedulerName)).Return(retGet) } // MockRemoveInvalidRoomsKey mocks removing invalid rooms keys from redis @@ -1653,9 +1752,6 @@ func MockRemoveInvalidRoomsKey( mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) mockPipeline.EXPECT().Exec() - mockPipeline.EXPECT(). - Del(models.GetInvalidRoomsCountKey(schedulerName)) - mockPipeline.EXPECT(). Del(models.GetInvalidRoomsKey(schedulerName)) } diff --git a/watcher/watcher.go b/watcher/watcher.go index 18648aecc..a09d8a4be 100644 --- a/watcher/watcher.go +++ b/watcher/watcher.go @@ -9,9 +9,11 @@ package watcher import ( "context" + "errors" e "errors" "fmt" "math" + "math/rand" "os" "os/signal" "strconv" @@ -23,11 +25,12 @@ import ( uuid "github.com/satori/go.uuid" "github.com/topfreegames/extensions/clock" pginterfaces "github.com/topfreegames/extensions/pg/interfaces" - redis "github.com/topfreegames/extensions/redis" - kubernetesExtensions "github.com/topfreegames/go-extensions-k8s-client-go/kubernetes" + "github.com/topfreegames/extensions/redis" "github.com/topfreegames/maestro/constants" reportersConstants "github.com/topfreegames/maestro/reporters/constants" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/watch" "github.com/sirupsen/logrus" "github.com/spf13/viper" @@ -44,12 +47,14 @@ import ( metricsClient "k8s.io/metrics/pkg/client/clientset/versioned" ) -func createRoomUsages(pods *v1.PodList) ([]*models.RoomUsage, map[string]int) { - roomUsages := make([]*models.RoomUsage, len(pods.Items)) - roomUsagesIdxMap := make(map[string]int, len(pods.Items)) - for i, pod := range pods.Items { - roomUsages[i] = &models.RoomUsage{Name: pod.Name, Usage: float64(math.MaxInt64)} +func createRoomUsages(pods map[string]*models.Pod) ([]*models.RoomUsage, map[string]int) { + roomUsages := make([]*models.RoomUsage, len(pods)) + roomUsagesIdxMap := make(map[string]int, len(pods)) + i := 0 + for podName, pod := range pods { + roomUsages[i] = &models.RoomUsage{Name: podName, Usage: float64(math.MaxInt64)} roomUsagesIdxMap[pod.Name] = i + i++ } return roomUsages, roomUsagesIdxMap @@ -66,6 +71,7 @@ type Watcher struct { RoomsStatusesReportPeriod int EnsureCorrectRoomsPeriod time.Duration PodStatesCountPeriod time.Duration + KubeCacheTTL time.Duration Config *viper.Viper DB pginterfaces.DB KubernetesClient kubernetes.Interface @@ -118,6 +124,7 @@ func NewWatcher( occupiedTimeout int64, eventForwarders []*eventforwarder.Info, ) *Watcher { + logger.Infof("%s", "Starting NewWatcher") w := &Watcher{ Config: config, Logger: logger, @@ -144,6 +151,7 @@ func (w *Watcher) loadConfigurationDefaults() { w.Config.SetDefault("watcher.podStatesCountPeriod", 1*time.Minute) w.Config.SetDefault("watcher.lockKey", "maestro-lock-key") w.Config.SetDefault("watcher.lockTimeoutMs", 180000) + w.Config.SetDefault("watcher.maxScaleUpAmount", 300) w.Config.SetDefault("watcher.gracefulShutdownTimeout", 300) w.Config.SetDefault("pingTimeout", 30) w.Config.SetDefault("occupiedTimeout", 60*60) @@ -228,12 +236,12 @@ func (w *Watcher) Start() { ticker := time.NewTicker(time.Duration(w.AutoScalingPeriod) * time.Second) defer ticker.Stop() - tickerEnsure := time.NewTicker(w.EnsureCorrectRoomsPeriod) - defer tickerEnsure.Stop() tickerStateCount := time.NewTicker(w.PodStatesCountPeriod) defer tickerStateCount.Stop() go w.reportRoomsStatusesRoutine() + stopKubeWatch := make(chan struct{}) + go w.configureKubeWatch(stopKubeWatch) for w.Run == true { l = w.Logger.WithFields(logrus.Fields{ @@ -246,6 +254,7 @@ func (w *Watcher) Start() { w.PodStatesCount() case sig := <-sigchan: l.Warnf("caught signal %v: terminating\n", sig) + close(stopKubeWatch) w.Run = false } } @@ -328,18 +337,11 @@ func (w *Watcher) AddUtilizationMetricsToRedis() error { } // Load pods and set their usage to MaxInt64 for all resources - var pods *v1.PodList - err = w.MetricsReporter.WithSegment(models.SegmentPod, func() error { - var err error - pods, err = w.KubernetesClient.CoreV1().Pods(w.SchedulerName).List(metav1.ListOptions{}) - return err - }) + var pods map[string]*models.Pod + pods, err = w.listPods() if err != nil { logger.WithError(err).Error("failed to list pods on namespace") return err - } else if len(pods.Items) == 0 { - logger.Warn("empty list of pods on namespace") - return err } // Load pods metricses @@ -571,9 +573,19 @@ func (w *Watcher) forwardRemovalRoomEvent(logger *logrus.Entry, rooms []string) _, err := eventforwarder.ForwardRoomEvent( context.Background(), - w.EventForwarders, w.RedisClient.Client, w.DB, w.KubernetesClient, - room, models.RoomTerminated, eventforwarder.PingTimeoutEvent, - metadatas[roomName], nil, w.Logger, w.RoomAddrGetter) + w.EventForwarders, + w.RedisClient.Client, + w.DB, + w.KubernetesClient, + w.MetricsReporter, + room, + models.RoomTerminated, + eventforwarder.PingTimeoutEvent, + metadatas[roomName], + nil, + w.Logger, + w.RoomAddrGetter, + ) if err != nil { logger.WithError(err).Error("pingTimeout event forward failed") } @@ -581,26 +593,29 @@ func (w *Watcher) forwardRemovalRoomEvent(logger *logrus.Entry, rooms []string) return nil } -func (w *Watcher) listPods() (pods []v1.Pod, err error) { - k := w.KubernetesClient - podList, err := k.CoreV1().Pods(w.SchedulerName).List( - metav1.ListOptions{}) +func (w *Watcher) listPods() (podMap map[string]*models.Pod, err error) { + logger := w.Logger.WithFields(logrus.Fields{ + "operation": "watcher.listPods", + }) + + podMap, err = models.GetPodMapFromRedis(w.RedisClient.Client, w.MetricsReporter, w.SchedulerName) if err != nil { + logger.WithError(err).Error("failed to list pods on redis") return nil, err } - - return podList.Items, nil + logger.Debug("got pod map from redis") + return podMap, nil } // filterPodsByName returns a []v1.Pod with pods which names are in podNames -func (w *Watcher) filterPodsByName(logger *logrus.Entry, pods []v1.Pod, podNames []string) (filteredPods []v1.Pod) { +func (w *Watcher) filterPodsByName(logger *logrus.Entry, pods map[string]*models.Pod, podNames []string) (filteredPods []*models.Pod) { podNameMap := map[string]bool{} for _, name := range podNames { podNameMap[name] = true } - for _, pod := range pods { - if podNameMap[pod.GetName()] { + for podName, pod := range pods { + if podNameMap[podName] { filteredPods = append(filteredPods, pod) } } @@ -609,13 +624,13 @@ func (w *Watcher) filterPodsByName(logger *logrus.Entry, pods []v1.Pod, podNames } // zombie rooms are the ones that are in terminating state but the pods doesn't exist -func (w *Watcher) removeZombies(pods []v1.Pod, rooms []string) ([]string, error) { +func (w *Watcher) removeZombies(pods map[string]*models.Pod, rooms []string) ([]string, error) { zombieRooms := []*models.Room{} zombieRoomsNames := []string{} liveKubePods := map[string]bool{} - for _, pod := range pods { - liveKubePods[pod.GetName()] = true + for podName := range pods { + liveKubePods[podName] = true } for _, room := range rooms { if _, ok := liveKubePods[room]; !ok { @@ -637,7 +652,7 @@ func (w *Watcher) RemoveDeadRooms() error { "operation": "watcher.RemoveDeadRooms", }) - pods := []v1.Pod{} + pods := map[string]*models.Pod{} // get rooms with no ping roomsNoPingSince, err := w.roomsWithNoPing(logger) @@ -678,10 +693,12 @@ func (w *Watcher) RemoveDeadRooms() error { return err } - l := logger.WithFields(logrus.Fields{ - "rooms": fmt.Sprintf("%v", roomsRemoved), - }) - l.Info("successfully deleted zombie rooms") + if len(roomsRemoved) > 0 { + l := logger.WithFields(logrus.Fields{ + "rooms": fmt.Sprintf("%v", roomsRemoved), + }) + l.Info("successfully deleted zombie rooms") + } } if len(roomsNoPingSince) > 0 || len(roomsOnOccupiedTimeout) > 0 { @@ -692,7 +709,7 @@ func (w *Watcher) RemoveDeadRooms() error { return err } } - podsToReplace := w.filterPodsByName(logger, pods, append(roomsNoPingSince, roomsOnOccupiedTimeout...)) + podsToDelete := w.filterPodsByName(logger, pods, append(roomsNoPingSince, roomsOnOccupiedTimeout...)) // load scheduler from database scheduler := models.NewScheduler(w.SchedulerName, "", "") @@ -712,24 +729,22 @@ func (w *Watcher) RemoveDeadRooms() error { return err } - timeoutErr, _, err := controller.SegmentAndReplacePods( - context.Background(), + var timeoutErr bool + timeoutErr, _, err = controller.DeletePodsAndWait( logger, w.RoomManager, w.MetricsReporter, w.KubernetesClient, - w.DB, w.RedisClient.Client, willTimeoutAt, configYAML, - podsToReplace, scheduler, nil, - w.Config.GetInt("watcher.maxSurge"), + podsToDelete, &clock.Clock{}, ) - if timeoutErr != nil { + if timeoutErr { logger.WithError(err).Error("timeout replacing pods on RemoveDeadRooms") } @@ -737,7 +752,7 @@ func (w *Watcher) RemoveDeadRooms() error { logger.WithError(err).Error("replacing pods returned error on RemoveDeadRooms") } - if timeoutErr == nil && err == nil { + if timeoutErr == false && err == nil { l := logger.WithFields(logrus.Fields{ "rooms": fmt.Sprintf("%v", roomsNoPingSince), }) @@ -844,6 +859,7 @@ func (w *Watcher) AutoScale() error { scaling.Delta, timeoutSec, false, + w.Config, ) scheduler.State = models.StateInSync scheduler.StateLastChangedAt = nowTimestamp @@ -857,11 +873,12 @@ func (w *Watcher) AutoScale() error { scaleDown := func() error { return controller.ScaleDown( + context.Background(), logger, w.RoomManager, w.MetricsReporter, w.DB, - w.RedisClient.Client, + w.RedisClient, w.KubernetesClient, scheduler, -scaling.Delta, @@ -953,7 +970,7 @@ func (w *Watcher) transformLegacyInMetricsTrigger(autoScalingInfo *models.AutoSc autoScalingInfo.Up.MetricsTrigger = append( autoScalingInfo.Up.MetricsTrigger, &models.ScalingPolicyMetricsTrigger{ - Type: models.LegacyAutoScalingPolicyType, + Type: models.RoomAutoScalingPolicyType, Usage: autoScalingInfo.Up.Trigger.Usage, Limit: autoScalingInfo.Up.Trigger.Limit, Threshold: autoScalingInfo.Up.Trigger.Threshold, @@ -967,7 +984,7 @@ func (w *Watcher) transformLegacyInMetricsTrigger(autoScalingInfo *models.AutoSc autoScalingInfo.Down.MetricsTrigger = append( autoScalingInfo.Down.MetricsTrigger, &models.ScalingPolicyMetricsTrigger{ - Type: models.LegacyAutoScalingPolicyType, + Type: models.RoomAutoScalingPolicyType, Usage: autoScalingInfo.Down.Trigger.Usage, Limit: autoScalingInfo.Down.Trigger.Limit, Threshold: autoScalingInfo.Down.Trigger.Threshold, @@ -1118,35 +1135,29 @@ func (w *Watcher) checkMetricsTrigger( return scaling, nil } -func (w *Watcher) getInvalidPodsAndPodNames( - podList *v1.PodList, +func (w *Watcher) getIncorrectAndUnregisteredPods( + logger logrus.FieldLogger, + podMap map[string]*models.Pod, scheduler *models.Scheduler, -) (invalidPods []v1.Pod, invalidPodNames []string, err error) { - concat := func(pods []v1.Pod, err error) error { - if err != nil { - return err - } - invalidPods = append(invalidPods, pods...) - return nil - } - - err = concat(w.podsOfIncorrectVersion(podList, scheduler)) +) (invalidPods, unregisteredPods []*models.Pod, err error) { + incorrectPods, err := w.podsOfIncorrectVersion(podMap, scheduler) if err != nil { return nil, nil, err } - err = concat(w.podsNotRegistered(podList)) + unregisteredPods, err = w.podsNotRegistered(podMap) if err != nil { return nil, nil, err } if len(invalidPods) > 0 { - for _, pod := range invalidPods { - invalidPodNames = append(invalidPodNames, pod.GetName()) - } + logger.WithFields(logrus.Fields{ + "incorrectVersion": len(incorrectPods), + "unregistered": len(unregisteredPods), + }).Info("replacing invalid pods") } - return invalidPods, invalidPodNames, err + return incorrectPods, unregisteredPods, err } func (w *Watcher) getOperation(ctx context.Context, logger logrus.FieldLogger) (operationManager *models.OperationManager, err error) { @@ -1154,7 +1165,7 @@ func (w *Watcher) getOperation(ctx context.Context, logger logrus.FieldLogger) ( w.SchedulerName, w.RedisClient.Trace(ctx), logger, ) - currentOpKey, _ := operationManager.CurrentOperation() + currentOpKey, err := operationManager.CurrentOperation() if err != nil { return nil, err } @@ -1219,8 +1230,7 @@ func (w *Watcher) EnsureCorrectRooms() error { } // list current pods - k := kubernetesExtensions.TryWithContext(w.KubernetesClient, ctx) - pods, err := controller.ListCurrentPods(w.MetricsReporter, k, w.SchedulerName) + pods, err := w.listPods() if err != nil { logger.WithError(err).Error("failed to list pods on namespace") return err @@ -1228,19 +1238,37 @@ func (w *Watcher) EnsureCorrectRooms() error { // get invalid pods (wrong versions and pods not registered in redis) logger.Info("searching for invalid pods") - invalidPods, invalidPodNames, err := w.getInvalidPodsAndPodNames(pods, scheduler) + incorrectPods, unregisteredPods, err := w.getIncorrectAndUnregisteredPods(logger, pods, scheduler) if err != nil { logger.WithError(err).Error("failed to get invalid pods") return err } + // get incorrect pod names + var incorrectPodNames []string + if len(incorrectPods) > 0 { + for _, pod := range incorrectPods { + incorrectPodNames = append(incorrectPodNames, pod.Name) + } + } + + // get unregistered pod names + var unregisteredPodNames []string + if len(unregisteredPods) > 0 { + for _, pod := range unregisteredPods { + unregisteredPodNames = append(unregisteredPodNames, pod.Name) + } + } + + invalidPods := append(incorrectPods, unregisteredPods...) + if len(invalidPods) <= 0 { + // delete invalidRooms key for safety + models.RemoveInvalidRoomsKey(w.RedisClient.Trace(ctx), w.MetricsReporter, w.SchedulerName) logger.Debug("no invalid pods to replace") return nil } - logger.WithField("invalidPods", invalidPodNames).Info("replacing invalid pods") - // get operation manager if it exists. // It won't exist if not in a UpdateSchedulerConfig operation operationManager, err := w.getOperation(ctx, logger) @@ -1251,15 +1279,26 @@ func (w *Watcher) EnsureCorrectRooms() error { // save invalid pods in redis to track rolling update progress if operationManager != nil { - err = models.SetInvalidRooms(w.RedisClient.Trace(ctx), w.MetricsReporter, w.SchedulerName, invalidPodNames) + status, err := operationManager.Get(operationManager.GetOperationKey()) if err != nil { logger.WithError(err).Error("error trying to save invalid rooms to track progress") return err } - err = operationManager.SetDescription(models.OpManagerRollingUpdate) - if err != nil { - logger.WithError(err).Error("error trying to set opmanager to rolling update status") - return err + + // don't remove unregistered rooms if in a rolling update + invalidPods = incorrectPods + + if status["description"] != models.OpManagerRollingUpdate { + err = models.SetInvalidRooms(w.RedisClient.Trace(ctx), w.MetricsReporter, w.SchedulerName, incorrectPodNames) + if err != nil { + logger.WithError(err).Error("error trying to save invalid rooms to track progress") + return err + } + err = operationManager.SetDescription(models.OpManagerRollingUpdate) + if err != nil { + logger.WithError(err).Error("error trying to set opmanager to rolling update status") + return err + } } } @@ -1267,6 +1306,8 @@ func (w *Watcher) EnsureCorrectRooms() error { timeoutSec := w.Config.GetInt("updateTimeoutSeconds") timeoutDur := time.Duration(timeoutSec) * time.Second willTimeoutAt := time.Now().Add(timeoutDur) + + logger.Infof("replacing pods with %d seconds of timeout", timeoutSec) timeoutErr, _, err := controller.SegmentAndReplacePods( ctx, logger, @@ -1281,6 +1322,7 @@ func (w *Watcher) EnsureCorrectRooms() error { scheduler, operationManager, w.Config.GetInt("watcher.maxSurge"), + w.Config.GetInt("watcher.goroutinePoolSize"), &clock.Clock{}, ) @@ -1306,16 +1348,15 @@ func (w *Watcher) EnsureCorrectRooms() error { } func (w *Watcher) podsNotRegistered( - pods *v1.PodList, -) ([]v1.Pod, error) { - registered, err := models.GetAllRegisteredRooms(w.RedisClient.Client, - w.SchedulerName) + pods map[string]*models.Pod, +) ([]*models.Pod, error) { + registered, err := models.GetAllRegisteredRooms(w.RedisClient.Client, w.SchedulerName) if err != nil { return nil, err } - notRegistered := []v1.Pod{} - for _, pod := range pods.Items { + notRegistered := []*models.Pod{} + for _, pod := range pods { if _, ok := registered[pod.Name]; !ok { notRegistered = append(notRegistered, pod) } @@ -1343,10 +1384,10 @@ func (w *Watcher) splitedVersion(version string) (majorInt, minorInt int, err er } func (w *Watcher) podsOfIncorrectVersion( - pods *v1.PodList, + pods map[string]*models.Pod, scheduler *models.Scheduler, -) ([]v1.Pod, error) { - incorrectPods := []v1.Pod{} +) ([]*models.Pod, error) { + incorrectPods := []*models.Pod{} schedulerMajorVersion, _, err := w.splitedVersion(scheduler.Version) if err != nil { @@ -1361,8 +1402,8 @@ func (w *Watcher) podsOfIncorrectVersion( } } - for _, pod := range pods.Items { - podMajorVersion, _, err := w.splitedVersion(pod.Labels["version"]) + for _, pod := range pods { + podMajorVersion, _, err := w.splitedVersion(pod.Version) if err != nil { return nil, err } @@ -1389,9 +1430,7 @@ func (w *Watcher) PodStatesCount() { logger := w.Logger.WithField("method", "PodStatesCount") - logger.Info("listing pods on namespace") - k := kubernetesExtensions.TryWithContext(w.KubernetesClient, context.Background()) - pods, err := k.CoreV1().Pods(w.SchedulerName).List(metav1.ListOptions{}) + pods, err := w.listPods() if err != nil { logger.WithError(err).Error("failed to list pods") return @@ -1406,7 +1445,7 @@ func (w *Watcher) PodStatesCount() { v1.PodUnknown: 0, } - for _, pod := range pods.Items { + for _, pod := range pods { stateCount[pod.Status.Phase]++ for _, status := range pod.Status.ContainerStatuses { logger.Debugf("termination state: %+v", status) @@ -1467,3 +1506,95 @@ func (w *Watcher) checkIfUsageIsAboveLimit( } return false } + +func (w *Watcher) configureWatcher() (watch.Interface, error) { + timeout := int64(5.0 * 60 * (rand.Float64() + 1.0)) + return w.KubernetesClient.CoreV1().Pods(w.SchedulerName). + Watch(metav1.ListOptions{ + Watch: true, + TimeoutSeconds: &timeout, + FieldSelector: fields.Everything().String(), + }) +} + +func (w *Watcher) watchPods(watcher watch.Interface, stopCh <-chan struct{}) error { + defer watcher.Stop() + +loop: + for { + select { + case <-stopCh: + return errors.New("stop channel") + + case event, ok := <-watcher.ResultChan(): + if !ok { + break loop + } + + if event.Type == watch.Error { + return nil + } + + switch event.Type { + case watch.Added, watch.Modified: + logger := w.Logger.WithFields(logrus.Fields{ + "operation": "watcher.kubeWatch.CreateOrUpdatePod", + }) + + // logger.Debug("new pod detected: ", event.Type) + if kubePod, ok := event.Object.(*v1.Pod); ok { + // create Pod from v1.Pod + pod := &models.Pod{ + Name: kubePod.GetName(), + Version: kubePod.GetLabels()["version"], + NodeName: kubePod.Spec.NodeName, + Status: kubePod.Status, + Spec: kubePod.Spec, + IsTerminating: models.IsPodTerminating(kubePod), + } + + err := models.AddToPodMap(w.RedisClient.Client, w.MetricsReporter, pod, w.SchedulerName) + if err != nil { + logger.WithError(err).Error("failed to add pod to redis podMap key") + } + } else { + logger.Error("obj received is not of type *v1.Pod") + } + case watch.Deleted: + logger := w.Logger.WithFields(logrus.Fields{ + "operation": "watcher.kubeWatch.DeletePod", + }) + + // logger.Debug("new pod removed") + if kubePod, ok := event.Object.(*v1.Pod); ok { + // Remove pod from redis + err := models.RemoveFromPodMap(w.RedisClient.Client, w.MetricsReporter, kubePod.GetName(), w.SchedulerName) + if err != nil { + logger.WithError(err).Errorf("failed to remove pod %s from redis", kubePod.GetName()) + } + room := models.NewRoom(kubePod.GetName(), w.SchedulerName) + err = room.ClearAll(w.RedisClient.Client, w.MetricsReporter) + if err != nil { + logger.WithError(err).Errorf("failed to clearAll %s from redis", kubePod.GetName()) + } + } else { + logger.Error("obj received is not of type *v1.Pod or cache.DeletedFinalStateUnknown") + } + } + } + } + return nil +} + +func (w *Watcher) configureKubeWatch(stopCh <-chan struct{}) error { + for { + watcher, err := w.configureWatcher() + if err != nil { + return err + } + + if err := w.watchPods(watcher, stopCh); err != nil { + return err + } + } +} diff --git a/watcher/watcher_test.go b/watcher/watcher_test.go index 34e9be37d..a6962aee7 100644 --- a/watcher/watcher_test.go +++ b/watcher/watcher_test.go @@ -64,6 +64,12 @@ autoscaling: usage: 60 time: 100 threshold: 80 + metricsTrigger: + - type: legacy + usage: 60 + time: 100 + threshold: 80 + delta: 2 cooldown: 200 down: delta: 1 @@ -71,6 +77,12 @@ autoscaling: usage: 30 time: 500 threshold: 80 + metricsTrigger: + - type: legacy + usage: 30 + time: 500 + threshold: 80 + delta: -1 cooldown: 500 env: - name: MY_ENV_VAR @@ -103,41 +115,13 @@ autoscaling: time: 100 threshold: 80 limit: 70 - cooldown: 200 - down: - delta: 1 - trigger: - usage: 30 - time: 500 - threshold: 80 - cooldown: 500 -env: - - name: MY_ENV_VAR - value: myvalue -cmd: - - "./room" -` - yamlWithMinZero = ` -name: controller-name -game: controller -image: controller/controller:v123 -occupiedTimeout: 300 -limits: - memory: "66Mi" - cpu: "2" -limits: - memory: "66Mi" - cpu: "2" -shutdownTimeout: 20 -autoscaling: - min: 0 - up: - delta: 2 - trigger: + metricsTrigger: + - type: legacy usage: 60 time: 100 threshold: 80 limit: 70 + delta: 2 cooldown: 200 down: delta: 1 @@ -145,50 +129,18 @@ autoscaling: usage: 30 time: 500 threshold: 80 - cooldown: 500 -env: - - name: MY_ENV_VAR - value: myvalue -cmd: - - "./room" -` - yamlWithDownDelta5 = ` -name: controller-name -game: controller -image: controller/controller:v123 -occupiedTimeout: 300 -limits: - memory: "66Mi" - cpu: "2" -limits: - memory: "66Mi" - cpu: "2" -shutdownTimeout: 20 -autoscaling: - min: 3 - up: - delta: 2 - trigger: - usage: 60 - time: 100 - threshold: 80 - cooldown: 200 - down: - delta: 5 - trigger: + metricsTrigger: + - type: legacy usage: 30 time: 500 threshold: 80 + delta: -1 cooldown: 500 env: - name: MY_ENV_VAR value: myvalue cmd: - "./room" -forwarders: - plugin: - name: - enabled: true ` yamlWithLegacyDownAndMetricsUpTrigger = ` name: controller-name @@ -216,10 +168,6 @@ autoscaling: up: delta: 1 trigger: - usage: 70 - time: 10 - metricsTrigger: - - type: room usage: 50 time: 200 limit: 85 @@ -231,6 +179,12 @@ autoscaling: usage: 30 time: 100 threshold: 80 + metricsTrigger: + - type: legacy + usage: 30 + time: 100 + threshold: 80 + delta: -2 cooldown: 60 ` yamlWithLegacyUpAndMetricsDownTrigger = ` @@ -257,16 +211,16 @@ autoscaling: min: 2 max: 10 up: - delta: 2 - trigger: + metricsTrigger: + - type: legacy + delta: 2 usage: 50 time: 200 limit: 85 threshold: 80 cooldown: 30 down: - metricsTrigger: - - type: room + trigger: usage: 30 time: 100 threshold: 80 @@ -533,26 +487,37 @@ var _ = Describe("Watcher", func() { } // ScaleUp - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal("creating")) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml.Name), gomock.Any()).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml.Name, "creating"), gomock.Any()).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().Exec().Times(configYaml.AutoScaling.Up.Delta) + testing.MockScaleUp(mockPipeline, mockRedisClient, configYaml.Name, configYaml.AutoScaling.Up.Delta) // UpdateScheduler testing.MockUpdateSchedulerStatusAndDo(func(_ *models.Scheduler, _ string, scheduler *models.Scheduler) { w.Run = false }, mockDb, nil, nil) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().Del(models.GetInvalidRoomsKey(configYaml.Name)) + mockPipeline.EXPECT().Exec() + + names := make([]string, 0, configYaml.AutoScaling.Up.Delta) + for i := 0; i < configYaml.AutoScaling.Up.Delta; i++ { + names = append(names, fmt.Sprintf("room-%d", i)) + } + + testing.MockListPods(mockPipeline, mockRedisClient, configYaml.Name, names, nil) + mockRedisClient.EXPECT().HMSet(models.GetPodMapRedisKey(configYaml.Name), gomock.Any()).Return(redis.NewStatusResult("", nil)).AnyTimes() + // LeaveCriticalSection (unlock done by redis-lock) mockRedisClient.EXPECT().Eval(gomock.Any(), []string{terminationLockKey}, gomock.Any()).Return(redis.NewCmdResult(nil, nil)).AnyTimes() mockRedisClient.EXPECT().Eval(gomock.Any(), []string{downscalingLockKey}, gomock.Any()).Return(redis.NewCmdResult(nil, nil)).AnyTimes() - w.Start() + + Expect(func() { + go func() { + defer GinkgoRecover() + w.Start() + }() + }).ShouldNot(Panic()) + Eventually(func() bool { return w.Run }).Should(BeTrue()) + time.Sleep(10 * time.Second) }) It("should not panic if error acquiring lock", func() { @@ -920,8 +885,7 @@ var _ = Describe("Watcher", func() { // Mock removal from redis ready set testing.MockRedisReadyPop(mockPipeline, mockRedisClient, configYaml.Name, simSpec.deltaExpected) - // Mock ClearAll - testing.MockClearAll(mockPipeline, mockRedisClient, configYaml.Name, simSpec.deltaExpected) + testing.MockPodsNotFound(mockRedisClient, configYaml.Name, simSpec.deltaExpected) // Mock UpdateScheduler testing.MockUpdateSchedulerStatusAndDo(func(_ *models.Scheduler, _ string, scheduler *models.Scheduler) { @@ -992,16 +956,7 @@ var _ = Describe("Watcher", func() { } // ScaleUp - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal("creating")) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml.Name), gomock.Any()).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml.Name, "creating"), gomock.Any()).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().Exec().Times(configYaml.AutoScaling.Up.Delta) + testing.MockScaleUp(mockPipeline, mockRedisClient, configYaml.Name, configYaml.AutoScaling.Up.Delta) // UpdateScheduler testing.MockUpdateSchedulerStatusAndDo(func(base *models.Scheduler, query string, scheduler *models.Scheduler) { @@ -1038,16 +993,7 @@ var _ = Describe("Watcher", func() { Expect(err).NotTo(HaveOccurred()) // ScaleUp - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal("creating")) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml.Name), gomock.Any()) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml.Name, "creating"), gomock.Any()) - mockPipeline.EXPECT().Exec() + testing.MockScaleUp(mockPipeline, mockRedisClient, configYaml.Name, 1) // UpdateScheduler testing.MockUpdateSchedulerStatusAndDo(func(_ *models.Scheduler, _ string, scheduler *models.Scheduler) { @@ -1093,16 +1039,7 @@ var _ = Describe("Watcher", func() { Expect(err).NotTo(HaveOccurred()) // ScaleUp - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal("creating")) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml.Name), gomock.Any()).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml.Name, "creating"), gomock.Any()).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().Exec().Times(configYaml.AutoScaling.Up.Delta) + testing.MockScaleUp(mockPipeline, mockRedisClient, configYaml.Name, configYaml.AutoScaling.Up.Delta) // UpdateScheduler testing.MockUpdateSchedulerStatusAndDo(func(_ *models.Scheduler, _ string, scheduler *models.Scheduler) { @@ -1159,37 +1096,8 @@ var _ = Describe("Watcher", func() { ) Expect(err).NotTo(HaveOccurred()) - for i := 0; i < 5; i++ { - pod := &v1.Pod{} - pod.Name = fmt.Sprintf("room-%d", i) - pod.Spec.Containers = []v1.Container{ - {Ports: []v1.ContainerPort{ - {HostPort: int32(5000 + i), Name: "TCP"}, - }}, - } - pod.Status.Phase = v1.PodPending - _, err := clientset.CoreV1().Pods(configYaml.Name).Create(pod) - Expect(err).NotTo(HaveOccurred()) - } - - readyKey := models.GetRoomStatusSetRedisKey(configYaml.Name, models.StatusReady) - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - mockPipeline.EXPECT().SPop(readyKey).Return(redis.NewStringResult("room-0", nil)) - mockPipeline.EXPECT().Exec() - - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - for range allStatus { - mockPipeline.EXPECT(). - SRem(gomock.Any(), gomock.Any()) - mockPipeline.EXPECT(). - ZRem(gomock.Any(), gomock.Any()) - } - mockPipeline.EXPECT().ZRem(models.GetRoomPingRedisKey(configYaml.Name), gomock.Any()) - for _, mt := range allMetrics { - mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(configYaml.Name, mt), gomock.Any()) - } - mockPipeline.EXPECT().Del(gomock.Any()) - mockPipeline.EXPECT().Exec() + testing.MockRedisReadyPop(mockPipeline, mockRedisClient, configYaml.Name, 1) + testing.MockPodsNotFound(mockRedisClient, configYaml.Name, 1) testing.MockUpdateSchedulerStatusAndDo(func(_ *models.Scheduler, _ string, scheduler *models.Scheduler) { Expect(scheduler.State).To(Equal("in-sync")) @@ -1328,18 +1236,12 @@ var _ = Describe("Watcher", func() { var configYaml models.ConfigYAML err := yaml.Unmarshal([]byte(yaml1), &configYaml) Expect(err).NotTo(HaveOccurred()) + scheduler := models.NewScheduler(configYaml.Name, configYaml.Game, yaml1) + scaleUpAmount := 5 - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(scaleUpAmount) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal(models.StatusCreating)) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(scaleUpAmount) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml.Name), gomock.Any()).Times(scaleUpAmount) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml.Name, "creating"), gomock.Any()).Times(scaleUpAmount) - mockPipeline.EXPECT().Exec().Times(scaleUpAmount) + + testing.MockScaleUp(mockPipeline, mockRedisClient, configYaml.Name, 5) err = testing.MockSetScallingAmount( mockRedisClient, @@ -1352,7 +1254,7 @@ var _ = Describe("Watcher", func() { ) Expect(err).NotTo(HaveOccurred()) - err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, scaleUpAmount, timeoutSec, true) + err = controller.ScaleUp(logger, roomManager, mr, mockDb, mockRedisClient, clientset, scheduler, scaleUpAmount, timeoutSec, true, config) Expect(err).NotTo(HaveOccurred()) // Mock MetricsTrigger Up get usage percentages @@ -1386,20 +1288,7 @@ var _ = Describe("Watcher", func() { mockPipeline.EXPECT().Exec() for _, name := range names { - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - room := models.NewRoom(name, scheduler.Name) - for _, status := range allStatus { - mockPipeline.EXPECT(). - SRem(models.GetRoomStatusSetRedisKey(room.SchedulerName, status), room.GetRoomRedisKey()) - mockPipeline.EXPECT(). - ZRem(models.GetLastStatusRedisKey(room.SchedulerName, status), room.ID) - } - mockPipeline.EXPECT().ZRem(models.GetRoomPingRedisKey(scheduler.Name), room.ID) - for _, mt := range allMetrics { - mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(room.SchedulerName, mt), gomock.Any()) - } - mockPipeline.EXPECT().Del(room.GetRoomRedisKey()) - mockPipeline.EXPECT().Exec() + testing.MockPodNotFound(mockRedisClient, configYaml.Name, name) } // UpdateScheduler @@ -1520,17 +1409,8 @@ var _ = Describe("Watcher", func() { ) Expect(err).NotTo(HaveOccurred()) - // ScaleUp - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal("creating")) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml.Name), gomock.Any()).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml.Name, "creating"), gomock.Any()).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().Exec().Times(configYaml.AutoScaling.Up.Delta) + + testing.MockScaleUp(mockPipeline, mockRedisClient, configYaml.Name, configYaml.AutoScaling.Up.Delta) // UpdateScheduler testing.MockUpdateSchedulerStatusAndDo(func(_ *models.Scheduler, _ string, scheduler *models.Scheduler) { @@ -1612,17 +1492,8 @@ var _ = Describe("Watcher", func() { ) } - // ScaleUp - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal("creating")) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml.Name), gomock.Any()).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml.Name, "creating"), gomock.Any()).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().Exec().Times(configYaml.AutoScaling.Up.Delta) + + testing.MockScaleUp(mockPipeline, mockRedisClient, configYaml.Name, configYaml.AutoScaling.Up.Delta) // UpdateScheduler testing.MockUpdateSchedulerStatusAndDo(func(_ *models.Scheduler, _ string, scheduler *models.Scheduler) { @@ -1717,37 +1588,8 @@ var _ = Describe("Watcher", func() { ) } - for i := 0; i < 5; i++ { - pod := &v1.Pod{} - pod.Name = fmt.Sprintf("room-%d", i) - pod.Spec.Containers = []v1.Container{ - {Ports: []v1.ContainerPort{ - {HostPort: int32(5000 + i), Name: "TCP"}, - }}, - } - pod.Status.Phase = v1.PodPending - _, err := clientset.CoreV1().Pods(configYaml.Name).Create(pod) - Expect(err).NotTo(HaveOccurred()) - } - - readyKey := models.GetRoomStatusSetRedisKey(configYaml.Name, models.StatusReady) - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - mockPipeline.EXPECT().SPop(readyKey).Return(redis.NewStringResult("room-0", nil)) - mockPipeline.EXPECT().Exec() - - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - for range allStatus { - mockPipeline.EXPECT(). - SRem(gomock.Any(), gomock.Any()) - mockPipeline.EXPECT(). - ZRem(gomock.Any(), gomock.Any()) - } - mockPipeline.EXPECT().ZRem(models.GetRoomPingRedisKey(configYaml.Name), gomock.Any()) - for _, mt := range allMetrics { - mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(configYaml.Name, mt), gomock.Any()) - } - mockPipeline.EXPECT().Del(gomock.Any()) - mockPipeline.EXPECT().Exec() + testing.MockRedisReadyPop(mockPipeline, mockRedisClient, configYaml.Name, 1) + testing.MockPodsNotFound(mockRedisClient, configYaml.Name, 1) testing.MockUpdateSchedulerStatusAndDo(func(_ *models.Scheduler, _ string, scheduler *models.Scheduler) { Expect(scheduler.State).To(Equal("in-sync")) @@ -1779,17 +1621,7 @@ var _ = Describe("Watcher", func() { ) Expect(err).NotTo(HaveOccurred()) - // ScaleUp - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal("creating")) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml.Name), gomock.Any()).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml.Name, "creating"), gomock.Any()).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().Exec().Times(configYaml.AutoScaling.Up.Delta) + testing.MockScaleUp(mockPipeline, mockRedisClient, configYaml.Name, configYaml.AutoScaling.Up.Delta) // UpdateScheduler testing.MockUpdateSchedulerStatusAndDo(func(_ *models.Scheduler, _ string, scheduler *models.Scheduler) { @@ -1824,17 +1656,7 @@ var _ = Describe("Watcher", func() { ) Expect(err).NotTo(HaveOccurred()) - // ScaleUp - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().HMSet(gomock.Any(), gomock.Any()).Do( - func(schedulerName string, statusInfo map[string]interface{}) { - Expect(statusInfo["status"]).To(Equal("creating")) - Expect(statusInfo["lastPing"]).To(BeNumerically("~", time.Now().Unix(), 1)) - }, - ).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().ZAdd(models.GetRoomPingRedisKey(configYaml.Name), gomock.Any()).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().SAdd(models.GetRoomStatusSetRedisKey(configYaml.Name, "creating"), gomock.Any()).Times(configYaml.AutoScaling.Up.Delta) - mockPipeline.EXPECT().Exec().Times(configYaml.AutoScaling.Up.Delta) + testing.MockScaleUp(mockPipeline, mockRedisClient, configYaml.Name, configYaml.AutoScaling.Up.Delta) // UpdateScheduler testing.MockUpdateSchedulerStatusAndDo(func(_ *models.Scheduler, _ string, scheduler *models.Scheduler) { @@ -1953,9 +1775,7 @@ var _ = Describe("Watcher", func() { // Mock removal from redis ready set testing.MockRedisReadyPop(mockPipeline, mockRedisClient, configYaml.Name, configYaml.AutoScaling.Down.Delta) - - // Mock ClearAll - testing.MockClearAll(mockPipeline, mockRedisClient, configYaml.Name, configYaml.AutoScaling.Down.Delta) + testing.MockPodsNotFound(mockRedisClient, configYaml.Name, configYaml.AutoScaling.Down.Delta) // Mock UpdateScheduler testing.MockUpdateSchedulerStatusAndDo(func(_ *models.Scheduler, _ string, scheduler *models.Scheduler) { @@ -2072,9 +1892,7 @@ var _ = Describe("Watcher", func() { // Mock removal from redis ready set testing.MockRedisReadyPop(mockPipeline, mockRedisClient, configYaml.Name, expC.Total()-configYaml.AutoScaling.Min) - - // Mock ClearAll - testing.MockClearAll(mockPipeline, mockRedisClient, configYaml.Name, expC.Total()-configYaml.AutoScaling.Min) + testing.MockPodsNotFound(mockRedisClient, configYaml.Name, expC.Total()-configYaml.AutoScaling.Min) // Mock UpdateScheduler testing.MockUpdateSchedulerStatusAndDo(func(_ *models.Scheduler, _ string, scheduler *models.Scheduler) { @@ -2149,9 +1967,7 @@ var _ = Describe("Watcher", func() { // Mock removal from redis ready set testing.MockRedisReadyPop(mockPipeline, mockRedisClient, configYaml.Name, expC.Total()-configYaml.AutoScaling.Max) - - // Mock ClearAll - testing.MockClearAll(mockPipeline, mockRedisClient, configYaml.Name, expC.Total()-configYaml.AutoScaling.Max) + testing.MockPodsNotFound(mockRedisClient, configYaml.Name, expC.Total()-configYaml.AutoScaling.Max) // Mock UpdateScheduler testing.MockUpdateSchedulerStatusAndDo(func(_ *models.Scheduler, _ string, scheduler *models.Scheduler) { @@ -2194,7 +2010,7 @@ var _ = Describe("Watcher", func() { ) } - metricTrigger := configYaml.AutoScaling.Up.MetricsTrigger[0] + metricTrigger := mockAutoScaling.Up.MetricsTrigger[0] // [Occupied / (Total + Delta)] = Usage/100 occupied := float64(expC.Occupied) @@ -2492,7 +2308,7 @@ var _ = Describe("Watcher", func() { ) } - metricTrigger := configYaml.AutoScaling.Down.MetricsTrigger[0] + metricTrigger := mockAutoScaling.Down.MetricsTrigger[0] // [Occupied / (Total + Delta)] = Usage/100 occupied := float64(expC.Occupied) @@ -2505,8 +2321,7 @@ var _ = Describe("Watcher", func() { // Mock removal from redis ready set testing.MockRedisReadyPop(mockPipeline, mockRedisClient, configYaml.Name, deltaInt) - // Mock ClearAll - testing.MockClearAll(mockPipeline, mockRedisClient, configYaml.Name, deltaInt) + testing.MockPodsNotFound(mockRedisClient, configYaml.Name, deltaInt) // Mock UpdateScheduler testing.MockUpdateSchedulerStatusAndDo(func(_ *models.Scheduler, _ string, scheduler *models.Scheduler) { @@ -2626,8 +2441,7 @@ var _ = Describe("Watcher", func() { // Mock removal from redis ready set testing.MockRedisReadyPop(mockPipeline, mockRedisClient, configYaml.Name, expC.Total()-configYaml.AutoScaling.Min) - // Mock ClearAll - testing.MockClearAll(mockPipeline, mockRedisClient, configYaml.Name, expC.Total()-configYaml.AutoScaling.Min) + testing.MockPodsNotFound(mockRedisClient, configYaml.Name, expC.Total()-configYaml.AutoScaling.Min) // Mock UpdateScheduler testing.MockUpdateSchedulerStatusAndDo(func(_ *models.Scheduler, _ string, scheduler *models.Scheduler) { @@ -2703,8 +2517,7 @@ var _ = Describe("Watcher", func() { // Mock removal from redis ready set testing.MockRedisReadyPop(mockPipeline, mockRedisClient, configYaml.Name, expC.Total()-configYaml.AutoScaling.Max) - // Mock ClearAll - testing.MockClearAll(mockPipeline, mockRedisClient, configYaml.Name, expC.Total()-configYaml.AutoScaling.Max) + testing.MockPodsNotFound(mockRedisClient, configYaml.Name, expC.Total()-configYaml.AutoScaling.Max) // Mock UpdateScheduler testing.MockUpdateSchedulerStatusAndDo(func(_ *models.Scheduler, _ string, scheduler *models.Scheduler) { @@ -2751,7 +2564,7 @@ var _ = Describe("Watcher", func() { testing.MockScaleUp( mockPipeline, mockRedisClient, configYaml.Name, - configYaml.AutoScaling.Up.Delta, + mockAutoScaling.Up.MetricsTrigger[0].Delta, ) // Mock UpdateScheduler @@ -2921,7 +2734,7 @@ var _ = Describe("Watcher", func() { testing.MockScaleUp( mockPipeline, mockRedisClient, configYaml.Name, - configYaml.AutoScaling.Up.Delta, + mockAutoScaling.Up.MetricsTrigger[0].Delta, ) // Mock UpdateScheduler @@ -3046,8 +2859,7 @@ var _ = Describe("Watcher", func() { // Mock removal from redis ready set testing.MockRedisReadyPop(mockPipeline, mockRedisClient, configYaml.Name, deltaInt) - // Mock ClearAll - testing.MockClearAll(mockPipeline, mockRedisClient, configYaml.Name, deltaInt) + testing.MockPodsNotFound(mockRedisClient, configYaml.Name, deltaInt) // Mock UpdateScheduler testing.MockUpdateSchedulerStatusAndDo(func(_ *models.Scheduler, _ string, scheduler *models.Scheduler) { @@ -3167,8 +2979,7 @@ var _ = Describe("Watcher", func() { // Mock removal from redis ready set testing.MockRedisReadyPop(mockPipeline, mockRedisClient, configYaml.Name, expC.Total()-configYaml.AutoScaling.Min) - // Mock ClearAll - testing.MockClearAll(mockPipeline, mockRedisClient, configYaml.Name, expC.Total()-configYaml.AutoScaling.Min) + testing.MockPodsNotFound(mockRedisClient, configYaml.Name, expC.Total()-configYaml.AutoScaling.Min) // Mock UpdateScheduler testing.MockUpdateSchedulerStatusAndDo(func(_ *models.Scheduler, _ string, scheduler *models.Scheduler) { @@ -3244,8 +3055,7 @@ var _ = Describe("Watcher", func() { // Mock removal from redis ready set testing.MockRedisReadyPop(mockPipeline, mockRedisClient, configYaml.Name, expC.Total()-configYaml.AutoScaling.Max) - // Mock ClearAll - testing.MockClearAll(mockPipeline, mockRedisClient, configYaml.Name, expC.Total()-configYaml.AutoScaling.Max) + testing.MockPodsNotFound(mockRedisClient, configYaml.Name, expC.Total()-configYaml.AutoScaling.Max) // Mock UpdateScheduler testing.MockUpdateSchedulerStatusAndDo(func(_ *models.Scheduler, _ string, scheduler *models.Scheduler) { @@ -4044,7 +3854,10 @@ var _ = Describe("Watcher", func() { Game: "game", Image: "img", } - pod, err := models.NewPod(name, nil, configYaml, clientset, mockRedisClient) + mockRedisClient.EXPECT(). + HGet(models.GetPodMapRedisKey(configYaml.Name), name). + Return(redis.NewStringResult("", redis.Nil)) + pod, err := models.NewPod(name, nil, configYaml, clientset, mockRedisClient, mr) if err != nil { return err } @@ -4074,13 +3887,14 @@ var _ = Describe("Watcher", func() { mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient).AnyTimes() }) - It("should call controller DeleteRoomsNoPingSince and DeleteRoomsOccupiedTimeout", func() { + It("should call controller roomsWithNoPing and roomsWithOccupationTimeout", func() { schedulerName := configYaml.Name pKey := models.GetRoomPingRedisKey(schedulerName) lKey := models.GetLastStatusRedisKey(schedulerName, models.StatusOccupied) ts := time.Now().Unix() - w.Config.GetInt64("pingTimeout") createNamespace(schedulerName, clientset) - // DeleteRoomsNoPingSince + + // roomsWithNoPing expectedRooms := []string{"room1", "room2", "room3"} mockRedisClient.EXPECT().ZRangeByScore( pKey, @@ -4090,9 +3904,9 @@ var _ = Describe("Watcher", func() { max, err := strconv.Atoi(zrangeby.Max) Expect(err).NotTo(HaveOccurred()) Expect(max).To(BeNumerically("~", ts, 1*time.Second)) - }).Return(redis.NewStringSliceResult(expectedRooms, nil)).AnyTimes() + }).Return(redis.NewStringSliceResult([]string{"room1"}, nil)).AnyTimes() - // DeleteRoomsOccupiedTimeout + // roomsWithOccupationTimeout ts = time.Now().Unix() - w.OccupiedTimeout mockRedisClient.EXPECT().ZRangeByScore( lKey, @@ -4102,7 +3916,7 @@ var _ = Describe("Watcher", func() { max, err := strconv.Atoi(zrangeby.Max) Expect(err).NotTo(HaveOccurred()) Expect(max).To(BeNumerically("~", ts, 1*time.Second)) - }).Return(redis.NewStringSliceResult(expectedRooms, nil)).AnyTimes() + }).Return(redis.NewStringSliceResult([]string{"room2","room3"}, nil)).AnyTimes() for _, roomName := range expectedRooms { err := createPod(roomName, schedulerName, clientset) @@ -4123,9 +3937,18 @@ var _ = Describe("Watcher", func() { testing.MockCreateRoomsAnyTimes(mockRedisClient, mockPipeline, &configYaml, 0) // Mock get terminating rooms + testing.MockListPods(mockPipeline, mockRedisClient, schedulerName, []string{"room1","room2","room3"}, nil) testing.MockRemoveZombieRooms(mockPipeline, mockRedisClient, []string{"scheduler:controller-name:rooms:room-0"}, schedulerName) for _, roomName := range expectedRooms { + runningCall := testing.MockRunningPod(mockRedisClient, configYaml.Name, roomName) + + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HDel(models.GetPodMapRedisKey(configYaml.Name), roomName) + mockPipeline.EXPECT().Exec() + + testing.MockPodNotFound(mockRedisClient, configYaml.Name, roomName).After(runningCall) + room := models.NewRoom(roomName, schedulerName) mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) for _, status := range allStatus { @@ -4155,32 +3978,11 @@ var _ = Describe("Watcher", func() { }, map[string]interface{}(nil)) } - mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) - mockPipeline.EXPECT().HGet("scheduler:controller-name:rooms:room1", "metadata") - mockPipeline.EXPECT().HGet("scheduler:controller-name:rooms:room2", "metadata") - mockPipeline.EXPECT().HGet("scheduler:controller-name:rooms:room3", "metadata") - mockPipeline.EXPECT().Exec().Return([]redis.Cmder{ - redis.NewStringResult(`{"region": "us"}`, nil), - redis.NewStringResult(`{"region": "us"}`, nil), - redis.NewStringResult(`{"region": "us"}`, nil), - }, nil) - - for _, roomName := range expectedRooms { - mockEventForwarder.EXPECT().Forward(gomock.Any(), models.RoomTerminated, - map[string]interface{}{ - "game": schedulerName, - "host": "", - "port": int32(0), - "roomId": roomName, - "metadata": map[string]interface{}{"region": "us"}, - }, map[string]interface{}(nil)) - } - testing.MockLoadScheduler(configYaml.Name, mockDb). Do(func(scheduler *models.Scheduler, query string, modifier string) { scheduler.YAML = yaml1 scheduler.Game = schedulerName - }).Times(7) + }).Times(4) Expect(func() { w.RemoveDeadRooms() }).ShouldNot(Panic()) }) @@ -4245,9 +4047,14 @@ var _ = Describe("Watcher", func() { }) } + testing.MockCreateRoomsAnyTimes(mockRedisClient, mockPipeline, &configYaml, 0) + // Mock get terminating rooms + testing.MockListPods(mockPipeline, mockRedisClient, schedulerName, []string{}, nil) testing.MockRemoveZombieRooms(mockPipeline, mockRedisClient, expectedRooms, schedulerName) + testing.MockListPods(mockPipeline, mockRedisClient, schedulerName, []string{}, nil) + testing.MockLoadScheduler(configYaml.Name, mockDb). Do(func(scheduler *models.Scheduler, query string, modifier string) { scheduler.YAML = yaml1 @@ -4339,7 +4146,8 @@ var _ = Describe("Watcher", func() { ) // Create test rooms names for each status - rooms := testing.CreateTestRooms(clientset, configYaml.Name, expC) + roomNames, rooms := testing.CreateTestRooms(clientset, configYaml.Name, expC) + testing.MockListPods(mockPipeline, mockRedisClient, configYaml.Name, roomNames, nil) // Mock saving CPU for all ready and occupied rooms testing.MockSavingRoomsMetricses( @@ -4415,7 +4223,8 @@ var _ = Describe("Watcher", func() { ) // Create test rooms names for each status - rooms := testing.CreateTestRooms(clientset, configYaml.Name, expC) + roomNames, rooms := testing.CreateTestRooms(clientset, configYaml.Name, expC) + testing.MockListPods(mockPipeline, mockRedisClient, configYaml.Name, roomNames, nil) // Mock saving CPU for all ready and occupied rooms testing.MockSavingRoomsMetricses( @@ -4485,6 +4294,7 @@ var _ = Describe("Watcher", func() { ) fakeMetricsClient := testing.CreatePodsMetricsList(containerMetrics, []string{}, configYaml.Name, nil) + testing.MockListPods(mockPipeline, mockRedisClient, configYaml.Name, []string{}, nil) w = watcher.NewWatcher(config, logger, @@ -4513,6 +4323,9 @@ var _ = Describe("Watcher", func() { testing.MockLoadScheduler(configYaml.Name, mockDb).Do(func(scheduler *models.Scheduler, query string, modifier string) { scheduler.YAML = yaml1 }) + + config.Set("watcher.goroutinePoolSize", 1) + w = watcher.NewWatcher( config, logger, mr, mockDb, redisClient, clientset, metricsClientset, configYaml.Name, configYaml.Game, occupiedTimeout, @@ -4543,8 +4356,7 @@ var _ = Describe("Watcher", func() { It("should return error if fail to read rooms from redis", func() { testing.MockSelectScheduler(yaml1, mockDb, nil) - testing.MockGetRegisteredRooms(mockRedisClient, mockPipeline, - w.SchedulerName, [][]string{}, errDB) + testing.MockListPods(mockPipeline, mockRedisClient, configYaml.Name, []string{}, errDB) err := w.EnsureCorrectRooms() @@ -4557,17 +4369,26 @@ var _ = Describe("Watcher", func() { room := models.NewRoom(podNames[0], w.SchedulerName) testing.MockSelectScheduler(yaml1, mockDb, nil) - testing.MockGetRegisteredRooms(mockRedisClient, mockPipeline, - w.SchedulerName, [][]string{{room.GetRoomRedisKey()}}, nil) + testing.MockListPods(mockPipeline, mockRedisClient, w.SchedulerName, podNames, nil) + testing.MockGetRegisteredRooms(mockRedisClient, mockPipeline, w.SchedulerName, [][]string{ {}, {room.GetRoomRedisKey()} }, nil) opManager := models.NewOperationManager(configYaml.Name, mockRedisClient, logger) testing.MockGetCurrentOperationKey(opManager, mockRedisClient, nil) // Create room - testing.MockCreateRoomsAnyTimes(mockRedisClient, mockPipeline, &configYaml, 0) + testing.MockCreateRoomsAnyTimes(mockRedisClient, mockPipeline, &configYaml, 1) testing.MockGetPortsFromPoolAnyTimes(&configYaml, mockRedisClient, mockPortChooser, models.NewPortRange(5000, 6000).String(), 5000, 6000) + testing.MockAnyRunningPod(mockRedisClient, w.SchedulerName, 2) + + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().SIsMember(models.GetRoomStatusSetRedisKey(configYaml.Name, "ready"), gomock.Any()).Return(redis.NewBoolResult(true, nil)) + mockPipeline.EXPECT().SIsMember(models.GetRoomStatusSetRedisKey(configYaml.Name, "occupied"), gomock.Any()).Return(redis.NewBoolResult(false, nil)) + exec1 := mockPipeline.EXPECT().Exec().Return(nil,nil) + + testing.MockRunningPod(mockRedisClient, w.SchedulerName, "room-2") + for _, podName := range podNames { pod := &v1.Pod{} pod.SetName(podName) @@ -4578,6 +4399,11 @@ var _ = Describe("Watcher", func() { } room = models.NewRoom(podNames[1], w.SchedulerName) + + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HDel(models.GetPodMapRedisKey(w.SchedulerName), room.ID) + exec2 := mockPipeline.EXPECT().Exec().Return(nil, nil).After(exec1) + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) for _, status := range allStatus { mockPipeline.EXPECT(). @@ -4593,13 +4419,12 @@ var _ = Describe("Watcher", func() { mockPipeline.EXPECT().ZRem(models.GetRoomMetricsRedisKey(w.SchedulerName, mt), gomock.Any()).AnyTimes() } mockPipeline.EXPECT().Del(room.GetRoomRedisKey()).AnyTimes() - mockPipeline.EXPECT().Exec().Return(nil, errDB) - mockPipeline.EXPECT().Exec().Return(nil, nil) + mockPipeline.EXPECT().Exec().Return(nil, errDB).After(exec2) err := w.EnsureCorrectRooms() Expect(err).ToNot(HaveOccurred()) - Expect(hook.LastEntry().Message).To(Equal(fmt.Sprintf("error deleting pod %s", podNames[1]))) + Expect(hook.Entries).To(testing.ContainLogMessage(fmt.Sprintf("error deleting pod %s", podNames[1]))) }) It("should delete invalid pods", func() { @@ -4607,12 +4432,27 @@ var _ = Describe("Watcher", func() { room := models.NewRoom(podNames[0], w.SchedulerName) testing.MockSelectScheduler(yaml1, mockDb, nil) + testing.MockListPods(mockPipeline, mockRedisClient, w.SchedulerName, podNames, nil) testing.MockGetRegisteredRooms(mockRedisClient, mockPipeline, w.SchedulerName, [][]string{{room.GetRoomRedisKey()}}, nil) opManager := models.NewOperationManager(configYaml.Name, mockRedisClient, logger) testing.MockGetCurrentOperationKey(opManager, mockRedisClient, nil) + // Create room + testing.MockCreateRoomsAnyTimes(mockRedisClient, mockPipeline, &configYaml, 1) + testing.MockGetPortsFromPoolAnyTimes(&configYaml, mockRedisClient, mockPortChooser, + models.NewPortRange(5000, 6000).String(), 5000, 6000) + + testing.MockAnyRunningPod(mockRedisClient, w.SchedulerName, 2) + + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().SIsMember(models.GetRoomStatusSetRedisKey(configYaml.Name, "ready"), gomock.Any()).Return(redis.NewBoolResult(true, nil)) + mockPipeline.EXPECT().SIsMember(models.GetRoomStatusSetRedisKey(configYaml.Name, "occupied"), gomock.Any()).Return(redis.NewBoolResult(false, nil)) + mockPipeline.EXPECT().Exec().Return(nil,nil) + + runningPod := testing.MockRunningPod(mockRedisClient, w.SchedulerName, "room-2") + for _, podName := range podNames { pod := &v1.Pod{} pod.SetName(podName) @@ -4622,13 +4462,13 @@ var _ = Describe("Watcher", func() { Expect(err).ToNot(HaveOccurred()) } - // Create room - testing.MockCreateRoomsAnyTimes(mockRedisClient, mockPipeline, &configYaml, 1) - testing.MockGetPortsFromPoolAnyTimes(&configYaml, mockRedisClient, mockPortChooser, - models.NewPortRange(5000, 6000).String(), 5000, 6000) - // Delete old rooms + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT().HDel(models.GetPodMapRedisKey(w.SchedulerName), "room-2") + mockPipeline.EXPECT().Exec() + testing.MockRemoveAnyRoomsFromRedisAnyTimes(mockRedisClient, mockPipeline, &configYaml, nil, 1) + testing.MockPodNotFound(mockRedisClient, w.SchedulerName, "room-2").After(runningPod) err := w.EnsureCorrectRooms() @@ -4668,10 +4508,11 @@ var _ = Describe("Watcher", func() { nPods := 3 reason := "bug" + pods := make(map[string]string) for idx := 1; idx <= nPods; idx++ { - pod := &v1.Pod{} - pod.SetName(fmt.Sprintf("pod-%d", idx)) - pod.SetNamespace(w.SchedulerName) + var pod models.Pod + pod.Name = fmt.Sprintf("pod-%d", idx) + pod.Namespace = w.SchedulerName pod.Status = v1.PodStatus{ Phase: v1.PodPending, ContainerStatuses: []v1.ContainerStatus{{ @@ -4682,11 +4523,19 @@ var _ = Describe("Watcher", func() { }, }}, } - - _, err := clientset.CoreV1().Pods(w.SchedulerName).Create(pod) + jsonBytes, err := pod.MarshalToRedis() Expect(err).ToNot(HaveOccurred()) + pods[pod.Name] = string(jsonBytes) } + mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline) + mockPipeline.EXPECT(). + HGetAll(models.GetPodMapRedisKey(w.SchedulerName)). + Return(redis.NewStringStringMapResult(pods, nil)) + mockPipeline.EXPECT().Exec() + + mockReporter.EXPECT().Report(reportersConstants.EventResponseTime, gomock.Any()) + stateCount := map[v1.PodPhase]int{ v1.PodPending: nPods, v1.PodRunning: 0, diff --git a/worker/worker.go b/worker/worker.go index eb6ab9d61..8c940f684 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -28,7 +28,7 @@ import ( metricsClient "k8s.io/metrics/pkg/client/clientset/versioned" pginterfaces "github.com/topfreegames/extensions/pg/interfaces" - redis "github.com/topfreegames/extensions/redis" + "github.com/topfreegames/extensions/redis" redisinterfaces "github.com/topfreegames/extensions/redis/interfaces" ) @@ -44,6 +44,7 @@ type Worker struct { InCluster bool KubeconfigPath string KubernetesClient kubernetes.Interface + KubernetesClientWatcher kubernetes.Interface KubernetesMetricsClient metricsClient.Interface Logger logrus.FieldLogger MetricsReporter *models.MixedMetricsReporter @@ -134,6 +135,7 @@ func (w *Worker) configureForwarders() { func (w *Worker) configureKubernetesClient(kubernetesClientOrNil kubernetes.Interface, kubernetesMetricsClientOrNil metricsClient.Interface) error { w.KubernetesClient = kubernetesClientOrNil + w.KubernetesClientWatcher = kubernetesClientOrNil w.KubernetesMetricsClient = kubernetesMetricsClientOrNil if w.KubernetesClient != nil && w.KubernetesMetricsClient != nil { @@ -145,12 +147,23 @@ func (w *Worker) configureKubernetesClient(kubernetesClientOrNil kubernetes.Inte return err } + timeout := w.Config.Get("extensions.kubernetesClient.timeout") + w.Config.Set("extensions.kubernetesClient.timeout", 0) + clientsetWatcher, _, err := extensions.GetKubernetesClient(w.Logger, w.Config, w.InCluster, w.KubeconfigPath) + if err != nil { + return err + } + w.Config.Set("extensions.kubernetesClient.timeout", timeout) + if w.KubernetesClient == nil { w.KubernetesClient = clientset } if w.KubernetesMetricsClient == nil { w.KubernetesMetricsClient = metricsClientset } + if w.KubernetesClientWatcher == nil { + w.KubernetesClientWatcher = clientsetWatcher + } return nil } @@ -301,7 +314,7 @@ func (w *Worker) EnsureRunningWatchers(schedulerNames []string) { w.MetricsReporter, w.DB, w.RedisClient, - w.KubernetesClient, + w.KubernetesClientWatcher, w.KubernetesMetricsClient, schedulerName, gameName,