Skip to content

Commit

Permalink
feat: add labels to the maestro scheduler (#603)
Browse files Browse the repository at this point in the history 
* feat: add labels to the maestro scheduler
  • Loading branch information
@reinaldooli
reinaldooli authored Dec 1, 2023
1 parent 5164101 commit 46b7421
Show file tree
Hide file tree
Showing 25 changed files with 899 additions and 745 deletions.
3 changes: 2 additions & 1 deletion e2e/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ require (
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect
github.com/imdario/mergo v0.3.11 // indirect
github.com/json-iterator/go v1.1.12 // indirect
Expand Down Expand Up @@ -86,7 +85,9 @@ require (
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/klog/v2 v2.30.0 // indirect
k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
sigs.k8s.io/yaml v1.2.0 // indirect
)
Expand Down
9 changes: 4 additions & 5 deletions e2e/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,7 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7
github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
Expand Down Expand Up @@ -556,6 +557,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
Expand Down Expand Up @@ -722,8 +724,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
Expand Down Expand Up @@ -1293,20 +1295,17 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo=
k8s.io/api v0.22.1 h1:ISu3tD/jRhYfSW8jI/Q1e+lRxkR7w9UwQEZ7FgslrwY=
k8s.io/api v0.22.1/go.mod h1:bh13rkTp3F1XEaLGykbyRD2QaTTzPm0e/BMd8ptFONY=
k8s.io/api v0.23.0 h1:WrL1gb73VSC8obi8cuYETJGXEoFNEh3LU0Pt+Sokgro=
k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg=
k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
k8s.io/apimachinery v0.22.1 h1:DTARnyzmdHMz7bFWFDDm22AM4pLWTQECMpRTFu2d2OM=
k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0=
k8s.io/apimachinery v0.23.0 h1:mIfWRMjBuMdolAWJ3Fd+aPTMv3X9z+waiARMpvvb0HQ=
k8s.io/apimachinery v0.23.0/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc=
k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU=
k8s.io/apiserver v0.22.1 h1:Ul9Iv8OMB2s45h2tl5XWPpAZo1VPIJ/6N+MESeed7L8=
k8s.io/apiserver v0.22.1/go.mod h1:2mcM6dzSt+XndzVQJX21Gx0/Klo7Aen7i0Ai6tIa400=
k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y=
k8s.io/client-go v0.22.1 h1:jW0ZSHi8wW260FvcXHkIa0NLxFBQszTlhiAVsU5mopw=
k8s.io/client-go v0.22.1/go.mod h1:BquC5A4UOo4qVDUtoc04/+Nxp1MeHcVc1HJm1KmG8kk=
k8s.io/client-go v0.23.0 h1:vcsOqyPq7XV3QmQRCBH/t9BICJM9Q1M18qahjv+rebY=
k8s.io/client-go v0.23.0/go.mod h1:hrDnpnK1mSr65lHHcUuIZIXDgEbzc7/683c6hyG4jTA=
Expand All @@ -1319,7 +1318,6 @@ k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAE
k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM=
k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw=
k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
Expand All @@ -1331,6 +1329,7 @@ k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs=
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
Expand Down
6 changes: 5 additions & 1 deletion e2e/suites/create_new_scheduler_version_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -529,6 +529,8 @@ func createMinorVersionAndAssertNoPodsReplace(t *testing.T, kubeClient kubernete
},
},
},
Annotations: scheduler.Annotations,
Labels: scheduler.Labels,
}
updateResponse := &maestroApiV1.NewSchedulerVersionResponse{}

Expand Down Expand Up @@ -557,8 +559,10 @@ func createMinorVersionAndAssertNoPodsReplace(t *testing.T, kubeClient kubernete
for _, pod := range podsAfterUpdate.Items {
podsNameAfterUpdate = append(podsNameAfterUpdate, pod.Name)
}

require.ElementsMatch(t, podsNameAfterUpdate, podsNameBeforeUpdate)

require.Contains(t, getSchedulerResponse.Scheduler.Annotations, updateRequest.Annotations)
require.Contains(t, getSchedulerResponse.Scheduler.Labels, updateRequest.Labels)

require.Equal(t, expectedNewVersion, getSchedulerResponse.Scheduler.Spec.Version)
}
6 changes: 6 additions & 0 deletions e2e/suites/create_scheduler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,12 @@ func TestCreateScheduler(t *testing.T) {
Start: 40000,
End: 60000,
},
Annotations: map[string]string{
"annotation1": "value1",
},
Labels: map[string]string{
"label1": "value1",
},
}

createResponse := &maestrov1.CreateSchedulerResponse{}
Expand Down
2 changes: 0 additions & 2 deletions e2e/suites/switch_active_version_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,6 @@ import (
"testing"
"time"

"github.com/golang/protobuf/ptypes/duration"

_struct "github.com/golang/protobuf/ptypes/struct"
"google.golang.org/protobuf/types/known/durationpb"
"google.golang.org/protobuf/types/known/structpb"
Expand Down
19 changes: 10 additions & 9 deletions internal/adapters/runtime/kubernetes/game_room_convert.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,17 +69,18 @@ var invalidPodWaitingStates = []string{

func convertGameRoomSpec(scheduler entities.Scheduler, gameRoomName string, gameRoomSpec game_room.Spec) (*v1.Pod, error) {
defaultAnnotations := map[string]string{safeToEvictAnnotation: safeToEvictValue}
defaultLabels := map[string]string{
maestroLabelKey: maestroLabelValue,
schedulerLabelKey: scheduler.Name,
versionLabelKey: gameRoomSpec.Version,
}

pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: gameRoomName,
Namespace: scheduler.Name,
Labels: map[string]string{
maestroLabelKey: maestroLabelValue,
schedulerLabelKey: scheduler.Name,
versionLabelKey: gameRoomSpec.Version,
},
Annotations: mergeAnnotations(defaultAnnotations, scheduler.Annotations),
Name: gameRoomName,
Namespace: scheduler.Name,
Labels: mergeMaps(defaultLabels, scheduler.Labels),
Annotations: mergeMaps(defaultAnnotations, scheduler.Annotations),
},
Spec: v1.PodSpec{
TerminationGracePeriodSeconds: convertTerminationGracePeriod(gameRoomSpec),
Expand Down Expand Up @@ -380,7 +381,7 @@ func convertPod(pod *v1.Pod, nodeAddress string) (*game_room.Instance, error) {
}, nil
}

func mergeAnnotations(target map[string]string, source map[string]string) map[string]string {
func mergeMaps(target map[string]string, source map[string]string) map[string]string {
if target == nil {
return source
}
Expand Down
83 changes: 65 additions & 18 deletions internal/adapters/runtime/kubernetes/game_room_convert_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -372,16 +372,18 @@ func TestConvertContainer(t *testing.T) {

func TestConvertGameSpec(t *testing.T) {
cases := map[string]struct {
schedulerID string
scheduler entities.Scheduler
roomName string
annotations map[string]string
gameSpec game_room.Spec
expectedPod v1.Pod
withError bool
}{
"without containers": {
schedulerID: "sample",
roomName: "roomName",
scheduler: entities.Scheduler{
Name: "sample",
},
roomName: "roomName",
gameSpec: game_room.Spec{
Version: "version",
},
Expand All @@ -401,8 +403,10 @@ func TestConvertGameSpec(t *testing.T) {
},
},
"with containers": {
schedulerID: "sample",
roomName: "roomName",
scheduler: entities.Scheduler{
Name: "sample",
},
roomName: "roomName",
gameSpec: game_room.Spec{
Version: "version",
Containers: []game_room.Container{
Expand Down Expand Up @@ -432,8 +436,10 @@ func TestConvertGameSpec(t *testing.T) {
},
},
"with toleration": {
schedulerID: "sample",
roomName: "roomName",
scheduler: entities.Scheduler{
Name: "sample",
},
roomName: "roomName",
gameSpec: game_room.Spec{
Version: "version",
Toleration: "some-toleration",
Expand All @@ -459,8 +465,10 @@ func TestConvertGameSpec(t *testing.T) {
},
},
"with affinity": {
schedulerID: "sample",
roomName: "roomName",
scheduler: entities.Scheduler{
Name: "sample",
},
roomName: "roomName",
gameSpec: game_room.Spec{
Version: "version",
Affinity: "sample-affinity",
Expand All @@ -484,8 +492,13 @@ func TestConvertGameSpec(t *testing.T) {
},
},
"with annotations": {
schedulerID: "sample",
roomName: "roomName",
scheduler: entities.Scheduler{
Name: "sample",
Annotations: map[string]string{
"imageregistry": "https://hub.docker.com/",
},
},
roomName: "roomName",
gameSpec: game_room.Spec{
Version: "version",
Affinity: "sample-affinity",
Expand All @@ -509,9 +522,42 @@ func TestConvertGameSpec(t *testing.T) {
},
},
},
"with labels": {
scheduler: entities.Scheduler{
Name: "sample",
Labels: map[string]string{
"my-label": "my-label-value",
},
},
roomName: "roomName",
gameSpec: game_room.Spec{
Version: "version",
Affinity: "sample-affinity",
},
expectedPod: v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "roomName",
Namespace: "sample",
Labels: map[string]string{
maestroLabelKey: maestroLabelValue,
schedulerLabelKey: "sample",
versionLabelKey: "version",
"my-label": "my-label-value",
},
Annotations: map[string]string{
safeToEvictAnnotation: safeToEvictValue,
},
},
Spec: v1.PodSpec{
Affinity: &v1.Affinity{},
},
},
},
"with termination grace period": {
schedulerID: "sample",
roomName: "roomName",
scheduler: entities.Scheduler{
Name: "sample",
},
roomName: "roomName",
gameSpec: game_room.Spec{
Version: "version",
TerminationGracePeriod: 10 * time.Second,
Expand Down Expand Up @@ -539,11 +585,11 @@ func TestConvertGameSpec(t *testing.T) {
for name, test := range cases {
t.Run(name, func(t *testing.T) {

scheduler := entities.Scheduler{
Name: test.schedulerID,
Annotations: test.expectedPod.ObjectMeta.Annotations,
}
res, err := convertGameRoomSpec(scheduler, test.roomName, test.gameSpec)
//scheduler := entities.Scheduler{
// Name: test.scheduler,
// Annotations: test.expectedPod.ObjectMeta.Annotations,
//}
res, err := convertGameRoomSpec(test.scheduler, test.roomName, test.gameSpec)
if test.withError {
require.Error(t, err)
return
Expand All @@ -554,6 +600,7 @@ func TestConvertGameSpec(t *testing.T) {
require.Equal(t, test.expectedPod.ObjectMeta.Name, res.ObjectMeta.Name)
require.Equal(t, test.expectedPod.ObjectMeta.Namespace, res.ObjectMeta.Namespace)
require.Equal(t, test.expectedPod.ObjectMeta.Annotations, res.ObjectMeta.Annotations)
require.Equal(t, test.expectedPod.ObjectMeta.Labels, res.ObjectMeta.Labels)
require.Equal(t, len(test.expectedPod.Spec.Containers), len(res.Spec.Containers))
require.Equal(t, len(test.expectedPod.Spec.Tolerations), len(res.Spec.Tolerations))

Expand Down
3 changes: 3 additions & 0 deletions internal/adapters/storage/postgres/scheduler/db_scheduler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ type schedulerInfo struct {
Forwarders []*forwarder.Forwarder
Autoscaling *autoscaling.Autoscaling
Annotations map[string]string
Labels map[string]string
LastDownscaleAt time.Time
}

Expand All @@ -78,6 +79,7 @@ func NewDBScheduler(scheduler *entities.Scheduler) *Scheduler {
Forwarders: scheduler.Forwarders,
Autoscaling: scheduler.Autoscaling,
Annotations: scheduler.Annotations,
Labels: scheduler.Labels,
LastDownscaleAt: scheduler.LastDownscaleAt,
}
yamlBytes, _ := yaml.Marshal(info)
Expand All @@ -102,6 +104,7 @@ func (s *Scheduler) ToScheduler() (*entities.Scheduler, error) {
Game: s.Game,
State: s.State,
Annotations: info.Annotations,
Labels: info.Labels,
Spec: game_room.Spec{
Version: s.Version,
TerminationGracePeriod: info.TerminationGracePeriod,
Expand Down
1 change: 1 addition & 0 deletions internal/adapters/storage/postgres/scheduler/db_scheduler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@ func TestScheduler_ToScheduler(t *testing.T) {
},
},
Annotations: map[string]string{"imageregistry": "https://hub.docker.com/"},
Labels: map[string]string{"scheduler": "scheduler-name"},
},
{
Name: "scheduler-6",
Expand Down
7 changes: 7 additions & 0 deletions internal/api/handlers/requestadapters/schedulers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,10 @@ func FromApiPatchSchedulerRequestToChangeMap(request *api.PatchSchedulerRequest)
patchMap[patch.LabelAnnotations] = request.GetAnnotations()
}

if request.Labels != nil {
patchMap[patch.LabelLabels] = request.GetLabels()
}

return patchMap
}

Expand Down Expand Up @@ -125,6 +129,7 @@ func FromApiCreateSchedulerRequestToEntity(request *api.CreateSchedulerRequest)
schedulerAutoscaling,
fromApiForwarders(request.GetForwarders()),
request.GetAnnotations(),
request.GetLabels(),
)
}

Expand Down Expand Up @@ -161,6 +166,7 @@ func FromApiNewSchedulerVersionRequestToEntity(request *api.NewSchedulerVersionR
schedulerAutoscaling,
fromApiForwarders(request.GetForwarders()),
request.GetAnnotations(),
request.GetLabels(),
)
}

Expand All @@ -183,6 +189,7 @@ func FromEntitySchedulerToResponse(entity *entities.Scheduler) (*api.Scheduler,
Autoscaling: getAutoscaling(entity.Autoscaling),
Forwarders: forwarders,
Annotations: entity.Annotations,
Labels: entity.Labels,
}, nil
}

Expand Down
Loading

0 comments on commit 46b7421

Please sign in to comment.