Skip to content

Commit

Permalink
Merge pull request #92 from topfreegames/refactor/rolling-update-prog…
Browse files Browse the repository at this point in the history 
…ress

Refactor/rolling update progress
  • Loading branch information
@victor-carvalho
victor-carvalho authored Jun 29, 2020
2 parents 9a817a4 + b701a38 commit 82080ea
Show file tree
Hide file tree
Showing 43 changed files with 2,103 additions and 1,607 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ _testmain.go
# IDE files
.vscode
.history
.idea

# Python
__pycache__
Expand Down
2 changes: 2 additions & 0 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ RUN cd /go/src/github.com/topfreegames/maestro && \

FROM debian:buster-slim

RUN apt update && apt install -y ca-certificates openssl

WORKDIR /app

COPY --from=build-env /app/maestro /app/maestro
Expand Down
7 changes: 6 additions & 1 deletion api/room_handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ func (g *RoomPingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
g.App.RedisClient.Trace(ctx),
g.App.DBClient.WithContext(ctx),
kubernetesClient,
mr,
room, fmt.Sprintf("ping%s", strings.Title(payload.Status)),
"",
payload.Metadata,
Expand Down Expand Up @@ -184,6 +185,7 @@ func NewRoomEventHandler(a *App) *RoomEventHandler {
// ServeHTTP method
func (g *RoomEventHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
mr := metricsReporterFromCtx(ctx)
l := middleware.GetLogger(ctx)
params := roomParamsFromContext(ctx)
payload := roomEventPayloadFromCtx(ctx)
Expand Down Expand Up @@ -216,6 +218,7 @@ func (g *RoomEventHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
g.App.RedisClient.Trace(ctx),
g.App.DBClient.WithContext(ctx),
kubernetesClient,
mr,
room,
"roomEvent",
"",
Expand Down Expand Up @@ -298,6 +301,7 @@ func (g *RoomStatusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
g.App.RedisClient.Trace(ctx),
g.App.DBClient.WithContext(ctx),
kubernetesClient,
mr,
room, payload.Status, "",
payload.Metadata,
g.App.SchedulerCache,
Expand Down Expand Up @@ -328,6 +332,7 @@ func NewRoomAddressHandler(a *App) *RoomAddressHandler {
// ServerHTTP method
func (h *RoomAddressHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
mr := metricsReporterFromCtx(ctx)
l := middleware.GetLogger(ctx)
params := roomParamsFromContext(ctx)

Expand All @@ -340,7 +345,7 @@ func (h *RoomAddressHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {

room := models.NewRoom(params.Name, params.Scheduler)
kubernetesClient := kubernetes.TryWithContext(h.App.KubernetesClient, ctx)
roomAddresses, err := h.App.RoomAddrGetter.Get(room, kubernetesClient, h.App.RedisClient.Trace(ctx))
roomAddresses, err := h.App.RoomAddrGetter.Get(room, kubernetesClient, h.App.RedisClient.Trace(ctx), mr)

if err != nil {
status := http.StatusInternalServerError
Expand Down
106 changes: 92 additions & 14 deletions api/room_handler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,19 +67,24 @@ forwarders:
createNamespace := func(name string, clientset kubernetes.Interface) error {
return models.NewNamespace(name).Create(clientset)
}
createPod := func(name, namespace string, clientset kubernetes.Interface) error {
createPod := func(name, namespace string, clientset kubernetes.Interface) (*models.Pod, error) {
configYaml := &models.ConfigYAML{
Name: namespace,
Game: "game",
Image: "img",
}

pod, err := models.NewPod(name, nil, configYaml, mockClientset, mockRedisClient)
MockPodNotFound(mockRedisClient, namespace, name)
pod, err := models.NewPod(name, nil, configYaml, mockClientset, mockRedisClient, mmr)
if err != nil {
return err
return nil, err
}
_, err = pod.Create(clientset)
return err
podv1, err := pod.Create(clientset)
if err != nil {
return nil, err
}
pod.Spec = podv1.Spec
return pod, nil
}

BeforeEach(func() { // Record HTTP responses.
Expand Down Expand Up @@ -257,10 +262,12 @@ forwarders:

Context("with eventforwarders", func() {
var app *api.App
var pod *models.Pod
game := "somegame"
BeforeEach(func() {
var err error
createNamespace(namespace, clientset)
err := createPod(roomName, namespace, clientset)
pod, err = createPod(roomName, namespace, clientset)
Expect(err).NotTo(HaveOccurred())
app, err = api.NewApp("0.0.0.0", 9998, config, logger, false, "", mockDb, mockCtxWrapper, mockRedisClient, mockRedisTraceWrapper, clientset, metricsClientset)
Expect(err).NotTo(HaveOccurred())
Expand All @@ -287,6 +294,12 @@ forwarders:
scheduler.Game = game
})

jsonBytes, err := pod.MarshalToRedis()
Expect(err).NotTo(HaveOccurred())
mockRedisClient.EXPECT().
HGet(models.GetPodMapRedisKey(namespace), roomName).
Return(redis.NewStringResult(string(jsonBytes), nil))

mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient)
mockRedisClient.EXPECT().
HGet("scheduler:schedulerName:rooms:roomName", "metadata").
Expand Down Expand Up @@ -315,6 +328,7 @@ forwarders:
"metadata": map[string]interface{}{
"ipv6Label": "",
"region": "us",
"ports": "[]",
},
}, gomock.Any())

Expand All @@ -333,6 +347,13 @@ forwarders:
})
request, _ = http.NewRequest("PUT", url, reader)

jsonBytes, err := pod.MarshalToRedis()
Expect(err).NotTo(HaveOccurred())
mockRedisClient.EXPECT().
HGet(models.GetPodMapRedisKey(namespace), roomName).
Return(redis.NewStringResult(string(jsonBytes), nil))


mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient)
MockLoadScheduler(namespace, mockDb).
Do(func(scheduler *models.Scheduler, query string, modifier string) {
Expand Down Expand Up @@ -363,6 +384,7 @@ forwarders:
"metadata": map[string]interface{}{
"ipv6Label": "",
"region": "us",
"ports": "[]",
},
}, gomock.Any())

Expand Down Expand Up @@ -541,10 +563,12 @@ forwarders:
Context("with eventforwarders", func() {
// TODO map status from api to something standard
var app *api.App
var pod *models.Pod
game := "somegame"
BeforeEach(func() {
var err error
createNamespace(namespace, clientset)
err := createPod(roomName, namespace, clientset)
pod, err = createPod(roomName, namespace, clientset)
Expect(err).NotTo(HaveOccurred())
app, err = api.NewApp("0.0.0.0", 9998, config, logger, false, "", mockDb, mockCtxWrapper, mockRedisClient, mockRedisTraceWrapper, clientset, metricsClientset)
Expect(err).NotTo(HaveOccurred())
Expand Down Expand Up @@ -584,6 +608,13 @@ forwarders:
})
request, _ = http.NewRequest("PUT", url, reader)

jsonBytes, err := pod.MarshalToRedis()
Expect(err).NotTo(HaveOccurred())
mockRedisClient.EXPECT().
HGet(models.GetPodMapRedisKey(namespace), roomName).
Return(redis.NewStringResult(string(jsonBytes), nil))


mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient)
mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline)
mockPipeline.EXPECT().HMSet(rKey, map[string]interface{}{
Expand Down Expand Up @@ -622,6 +653,12 @@ forwarders:
})
request, _ = http.NewRequest("PUT", url, reader)

jsonBytes, err := pod.MarshalToRedis()
Expect(err).NotTo(HaveOccurred())
mockRedisClient.EXPECT().
HGet(models.GetPodMapRedisKey(namespace), roomName).
Return(redis.NewStringResult(string(jsonBytes), nil))

mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient)
mockRedisClient.EXPECT().TxPipeline().Return(mockPipeline)
mockPipeline.EXPECT().HMSet(rKey, map[string]interface{}{
Expand Down Expand Up @@ -649,6 +686,7 @@ forwarders:
Expect(infos["metadata"]).To(BeEquivalentTo(map[string]interface{}{
"type": "sometype",
"ipv6Label": "",
"ports": "[]",
}))
})
mockEventForwarder2.EXPECT().Forward(gomock.Any(), status, gomock.Any(), gomock.Any()).Do(
Expand All @@ -658,6 +696,7 @@ forwarders:
Expect(infos["metadata"]).To(BeEquivalentTo(map[string]interface{}{
"type": "sometype",
"ipv6Label": "",
"ports": "[]",
}))
})

Expand Down Expand Up @@ -848,10 +887,12 @@ forwarders:
Describe("POST /scheduler/{schedulerName}/rooms/{roomName}/roomevent", func() {
url := "/scheduler/schedulerName/rooms/roomName/roomevent"
var app *api.App
var pod *models.Pod

BeforeEach(func() {
var err error
createNamespace(namespace, clientset)
err := createPod("roomName", namespace, clientset)
pod, err = createPod("roomName", namespace, clientset)
Expect(err).NotTo(HaveOccurred())
app, err = api.NewApp("0.0.0.0", 9998, config, logger, false, "", mockDb, mockCtxWrapper, mockRedisClient, mockRedisTraceWrapper, clientset, metricsClientset)
Expect(err).NotTo(HaveOccurred())
Expand Down Expand Up @@ -898,6 +939,12 @@ forwarders:
scheduler.Game = game
})

jsonBytes, err := pod.MarshalToRedis()
Expect(err).NotTo(HaveOccurred())
mockRedisClient.EXPECT().
HGet(models.GetPodMapRedisKey(namespace), "roomName").
Return(redis.NewStringResult(string(jsonBytes), nil))

mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient)
mockEventForwarder1.EXPECT().Forward(gomock.Any(), "roomEvent", gomock.Any(), gomock.Any()).Return(
int32(500), "", errors.New("some error occurred"),
Expand All @@ -906,7 +953,7 @@ forwarders:
app.Router.ServeHTTP(recorder, request)
Expect(recorder.Code).To(Equal(500))
var obj map[string]interface{}
err := json.Unmarshal([]byte(recorder.Body.String()), &obj)
err = json.Unmarshal([]byte(recorder.Body.String()), &obj)
Expect(err).NotTo(HaveOccurred())
Expect(obj["code"]).To(Equal("MAE-000"))
Expect(obj["error"]).To(Equal("Room event forward failed"))
Expand All @@ -922,6 +969,13 @@ forwarders:
"metadata": make(map[string]interface{}),
})
request, _ = http.NewRequest("POST", url, reader)

jsonBytes, err := pod.MarshalToRedis()
Expect(err).NotTo(HaveOccurred())
mockRedisClient.EXPECT().
HGet(models.GetPodMapRedisKey(namespace), "roomName").
Return(redis.NewStringResult(string(jsonBytes), nil))

mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient)
MockLoadScheduler("schedulerName", mockDb).Do(func(scheduler *models.Scheduler, query string, modifier string) {
scheduler.YAML = yamlStr
Expand All @@ -935,7 +989,7 @@ forwarders:
app.Router.ServeHTTP(recorder, request)
Expect(recorder.Code).To(Equal(500))
var obj map[string]interface{}
err := json.Unmarshal([]byte(recorder.Body.String()), &obj)
err = json.Unmarshal([]byte(recorder.Body.String()), &obj)
Expect(err).NotTo(HaveOccurred())
Expect(obj["code"]).To(Equal("MAE-000"))
Expect(obj["error"]).To(Equal("room event forward failed"))
Expand All @@ -957,14 +1011,20 @@ forwarders:
scheduler.Game = game
})

jsonBytes, err := pod.MarshalToRedis()
Expect(err).NotTo(HaveOccurred())
mockRedisClient.EXPECT().
HGet(models.GetPodMapRedisKey(namespace), "roomName").
Return(redis.NewStringResult(string(jsonBytes), nil))

mockEventForwarder1.EXPECT().Forward(gomock.Any(), "roomEvent", gomock.Any(), gomock.Any()).Return(
int32(200), "all went well", nil,
)

app.Router.ServeHTTP(recorder, request)
Expect(recorder.Code).To(Equal(200))
var obj map[string]interface{}
err := json.Unmarshal([]byte(recorder.Body.String()), &obj)
err = json.Unmarshal([]byte(recorder.Body.String()), &obj)
Expect(err).NotTo(HaveOccurred())
Expect(obj["success"]).To(Equal(true))
Expect(obj["message"]).To(Equal("all went well"))
Expand Down Expand Up @@ -1007,11 +1067,21 @@ forwarders:
err := ns.Create(clientset)
Expect(err).NotTo(HaveOccurred())

pod, err := models.NewPod(name, nil, configYaml, mockClientset, mockRedisClient)
mockRedisClient.EXPECT().
HGet(models.GetPodMapRedisKey(namespace), name).
Return(redis.NewStringResult("", redis.Nil))

pod, err := models.NewPod(name, nil, configYaml, mockClientset, mockRedisClient, mmr)
Expect(err).NotTo(HaveOccurred())
_, err = pod.Create(clientset)
Expect(err).NotTo(HaveOccurred())

jsonBytes, err := pod.MarshalToRedis()
Expect(err).NotTo(HaveOccurred())
mockRedisClient.EXPECT().
HGet(models.GetPodMapRedisKey(namespace), "roomName").
Return(redis.NewStringResult(string(jsonBytes), nil))

mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient)
url := fmt.Sprintf(
"/scheduler/%s/rooms/%s/address",
Expand All @@ -1036,13 +1106,21 @@ forwarders:
err := ns.Create(clientset)
Expect(err).NotTo(HaveOccurred())

pod, err := models.NewPod(name, nil, configYaml, mockClientset, mockRedisClient)
mockRedisClient.EXPECT().
HGet(models.GetPodMapRedisKey(namespace), name).
Return(redis.NewStringResult("", redis.Nil))
pod, err := models.NewPod(name, nil, configYaml, mockClientset, mockRedisClient, mmr)
Expect(err).NotTo(HaveOccurred())
_, err = pod.Create(clientset)
Expect(err).NotTo(HaveOccurred())

mockRedisTraceWrapper.EXPECT().WithContext(gomock.Any(), mockRedisClient).Return(mockRedisClient)
namespace := "unexisting-name"

mockRedisClient.EXPECT().
HGet(models.GetPodMapRedisKey("unexisting-name"), name).
Return(redis.NewStringResult("", redis.Nil))

url := fmt.Sprintf(
"/scheduler/%s/rooms/%s/address",
namespace,
Expand All @@ -1058,7 +1136,7 @@ forwarders:
err = json.Unmarshal(recorder.Body.Bytes(), &obj)
Expect(err).NotTo(HaveOccurred())
Expect(obj).To(HaveKeyWithValue("code", "MAE-000"))
Expect(obj).To(HaveKeyWithValue("description", "pods \"roomName\" not found"))
Expect(obj).To(HaveKeyWithValue("description", "pod \"roomName\" not found on redis podMap"))
Expect(obj).To(HaveKeyWithValue("error", "Address handler error"))
Expect(obj).To(HaveKeyWithValue("success", false))
})
Expand Down
4 changes: 3 additions & 1 deletion api/scheduler_handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -583,12 +583,14 @@ func (g *SchedulerScaleHandler) ServeHTTP(w http.ResponseWriter, r *http.Request

db := g.App.DBClient.WithContext(r.Context())
err := controller.ScaleScheduler(
r.Context(),
logger,
g.App.RoomManager,
mr,
db,
g.App.RedisClient.Trace(r.Context()),
g.App.RedisClient,
g.App.KubernetesClient,
g.App.Config,
g.App.Config.GetInt("scaleUpTimeoutSeconds"), g.App.Config.GetInt("scaleDownTimeoutSeconds"),
scaleParams.ScaleUp, scaleParams.ScaleDown, scaleParams.Replicas,
params.SchedulerName,
Expand Down
Loading

0 comments on commit 82080ea

Please sign in to comment.