From df49f7b2c85c14ee36eb75d8e58ec60216dc2d53 Mon Sep 17 00:00:00 2001 From: Arsalan Khan Date: Fri, 13 Jan 2023 13:37:55 +0100 Subject: [PATCH 1/4] fix #1 for too many opens files --- ci/autoscaler/scripts/vars.source.sh | 2 +- src/acceptance/helpers/spaces.go | 2 +- .../run_performance/run_performance_test.go | 14 +++---- .../setup_performance_suite_test.go | 42 ++++++++++++++++++- 4 files changed, 48 insertions(+), 12 deletions(-) diff --git a/ci/autoscaler/scripts/vars.source.sh b/ci/autoscaler/scripts/vars.source.sh index d4360ba98e..d7a36e67ed 100644 --- a/ci/autoscaler/scripts/vars.source.sh +++ b/ci/autoscaler/scripts/vars.source.sh @@ -104,7 +104,7 @@ debug "SERVICE_OFFERING_ENABLED: ${SERVICE_OFFERING_ENABLED}" export GINKGO_OPTS=${GINKGO_OPTS:-"--fail-fast"} -export PERFORMANCE_APP_COUNT="${PERFORMANCE_APP_COUNT:-20}" +export PERFORMANCE_APP_COUNT="${PERFORMANCE_APP_COUNT:-3000}" debug "PERFORMANCE_APP_COUNT: ${PERFORMANCE_APP_COUNT}" export PERFORMANCE_APP_PERCENTAGE_TO_SCALE="${PERFORMANCE_APP_PERCENTAGE_TO_SCALE:-30}" diff --git a/src/acceptance/helpers/spaces.go b/src/acceptance/helpers/spaces.go index 9245203574..34f114d77a 100644 --- a/src/acceptance/helpers/spaces.go +++ b/src/acceptance/helpers/spaces.go @@ -38,7 +38,7 @@ func GetTestSpaces(orgGuid string, cfg *config.Config) []string { spaceNames = append(spaceNames, space.Name) } } - ginkgo.GinkgoWriter.Printf("\nGot orgs: %s\n", spaceNames) + ginkgo.GinkgoWriter.Printf("\nGot spaces: %s\n", spaceNames) return spaceNames } diff --git a/src/acceptance/run_performance/run_performance_test.go b/src/acceptance/run_performance/run_performance_test.go index 849d5290bf..e3fa392e58 100644 --- a/src/acceptance/run_performance/run_performance_test.go +++ b/src/acceptance/run_performance/run_performance_test.go @@ -13,6 +13,9 @@ import ( "github.com/onsi/gomega/gmeasure" ) +const pollTime = 10 * time.Second +const desiredScalingTime = 20 * time.Minute + var _ = Describe("Scale in and out (eg: 30%) percentage of apps", func() { var ( appsToScaleCount int @@ -66,11 +69,7 @@ var _ = Describe("Scale in and out (eg: 30%) percentage of apps", func() { experiment.Sample(func(i int) { defer GinkgoRecover() appName := startedApps[i].Name - appGUID, err := helpers.GetAppGuid(cfg, appName) - if err != nil { - errors.Store(appName, err) - } - pollTime := 10 * time.Second + appGUID := startedApps[i].Guid wg := sync.WaitGroup{} wg.Add(1) @@ -102,13 +101,12 @@ var _ = Describe("Scale in and out (eg: 30%) percentage of apps", func() { wg.Done() }) wg.Wait() - atomic.AddInt32(&doneAppsCount, 1) fmt.Printf("Scaled-in apps: %d/%d\n", atomic.LoadInt32(&doneAppsCount), actualAppsToScaleCount) }, samplingConfig) - - Eventually(func() int32 { return atomic.LoadInt32(&doneAppsCount) }, 10*time.Minute, 10*time.Second).Should(BeEquivalentTo(actualAppsToScaleCount)) + fmt.Printf("\nWaiting %s minutes to finish scaling...", desiredScalingTime) + Eventually(func() int32 { return atomic.LoadInt32(&doneAppsCount) }, desiredScalingTime, 10*time.Second).Should(BeEquivalentTo(actualAppsToScaleCount)) checkMedianDurationFor(experiment, "scale-out") checkMedianDurationFor(experiment, "scale-in") }) diff --git a/src/acceptance/setup_performance/setup_performance_suite_test.go b/src/acceptance/setup_performance/setup_performance_suite_test.go index 1a8d9e81e4..b43254ac23 100644 --- a/src/acceptance/setup_performance/setup_performance_suite_test.go +++ b/src/acceptance/setup_performance/setup_performance_suite_test.go @@ -4,6 +4,8 @@ import ( "acceptance/config" . "acceptance/helpers" "fmt" + "github.com/KevinJCross/cf-test-helpers/v2/cf" + "github.com/onsi/gomega/gexec" "os" "strconv" "sync" @@ -111,11 +113,47 @@ func deleteExistingServiceInstances(workerId int, servicesChan chan string, setu func cleanup() { setup = workflowhelpers.NewTestSuiteSetup(cfg) workflowhelpers.AsUser(setup.AdminUserContext(), cfg.DefaultTimeoutDuration(), func() { - DeleteOrgs(GetTestOrgs(cfg), time.Duration(120)*time.Second) if cfg.UseExistingOrganization { + // cf target to org + targetOrg(setup) + orgGuid := GetOrgGuid(cfg, cfg.ExistingOrganization) - DeleteSpaces(cfg.ExistingOrganization, GetTestSpaces(orgGuid, cfg), 0*time.Second) + spaceNames := GetTestSpaces(orgGuid, cfg) + if len(spaceNames) == 0 { + return + } + //TODO - Do it with multiple go processes1 + deleteAllServices(orgGuid) + // delete all apps in a test space - only from first space - what if two spaces are present + deleteAllApps(spaceNames[0]) + DeleteSpaces(cfg.ExistingOrganization, GetTestSpaces(orgGuid, cfg), cfg.DefaultTimeoutDuration()) + } else { + DeleteOrgs(GetTestOrgs(cfg), time.Duration(120)*time.Second) } + }) } + +func deleteAllApps(spaceName string) { + apps := GetApps(cfg, setup.GetOrganizationName(), spaceName, "node-custom-metric-benchmark-") + fmt.Printf("\nExisting apps found %d", len(apps)) + for _, appName := range apps { + fmt.Printf(" - deleting app %s\n", appName) + DeleteTestApp(appName, cfg.DefaultTimeoutDuration()) + } +} + +func deleteAllServices(orgGuid string) { + services := GetServices(cfg, orgGuid, GetSpaceGuid(cfg, orgGuid)) + fmt.Printf("\nExisting services found %d", len(services)) + for _, service := range services { + fmt.Printf(" - deleting service instance %s\n", service) + DeleteServiceInstance(cfg, setup, service) + } +} + +func targetOrg(setup *workflowhelpers.ReproducibleTestSuiteSetup) { + cmd := cf.Cf("target", "-o", setup.GetOrganizationName()).Wait(cfg.DefaultTimeoutDuration()) + Expect(cmd).To(gexec.Exit(0), fmt.Sprintf("failed cf target org %s : %s", setup.GetOrganizationName(), string(cmd.Err.Contents()))) +} From 053368108e449dd3bcace6072fef262650425da9 Mon Sep 17 00:00:00 2001 From: Arsalan Khan Date: Mon, 16 Jan 2023 10:29:31 +0100 Subject: [PATCH 2/4] add clean up (app,services) in before suite --- ci/autoscaler/scripts/vars.source.sh | 4 +- src/acceptance/helpers/org.go | 1 - src/acceptance/helpers/spaces.go | 1 - .../run_performance/run_performance_test.go | 8 +- .../setup_performance_suite_test.go | 127 ++++++++++-------- 5 files changed, 80 insertions(+), 61 deletions(-) diff --git a/ci/autoscaler/scripts/vars.source.sh b/ci/autoscaler/scripts/vars.source.sh index d7a36e67ed..ea62ce8baa 100644 --- a/ci/autoscaler/scripts/vars.source.sh +++ b/ci/autoscaler/scripts/vars.source.sh @@ -104,13 +104,13 @@ debug "SERVICE_OFFERING_ENABLED: ${SERVICE_OFFERING_ENABLED}" export GINKGO_OPTS=${GINKGO_OPTS:-"--fail-fast"} -export PERFORMANCE_APP_COUNT="${PERFORMANCE_APP_COUNT:-3000}" +export PERFORMANCE_APP_COUNT="${PERFORMANCE_APP_COUNT:-50}" debug "PERFORMANCE_APP_COUNT: ${PERFORMANCE_APP_COUNT}" export PERFORMANCE_APP_PERCENTAGE_TO_SCALE="${PERFORMANCE_APP_PERCENTAGE_TO_SCALE:-30}" debug "PERFORMANCE_APP_PERCENTAGE_TO_SCALE: ${PERFORMANCE_APP_PERCENTAGE_TO_SCALE}" -export PERFORMANCE_SETUP_WORKERS="${PERFORMANCE_SETUP_WORKERS:-50}" +export PERFORMANCE_SETUP_WORKERS="${PERFORMANCE_SETUP_WORKERS:-20}" debug "PERFORMANCE_SETUP_WORKERS: ${PERFORMANCE_SETUP_WORKERS}" export PERFORMANCE_TEARDOWN=${PERFORMANCE_TEARDOWN:-true} diff --git a/src/acceptance/helpers/org.go b/src/acceptance/helpers/org.go index 22eec4da81..a923513523 100644 --- a/src/acceptance/helpers/org.go +++ b/src/acceptance/helpers/org.go @@ -43,7 +43,6 @@ func DeleteOrgs(orgs []string, timeout time.Duration) { } fmt.Printf("\nDeleting orgs: %s ", strings.Join(orgs, ", ")) - for _, org := range orgs { deleteOrg := cf.Cf("delete-org", org, "-f").Wait(timeout) Expect(deleteOrg).To(Exit(0), fmt.Sprintf("unable to delete org %s", org)) diff --git a/src/acceptance/helpers/spaces.go b/src/acceptance/helpers/spaces.go index 34f114d77a..5a946ea40f 100644 --- a/src/acceptance/helpers/spaces.go +++ b/src/acceptance/helpers/spaces.go @@ -58,7 +58,6 @@ func DeleteSpaces(orgName string, spaces []string, timeout time.Duration) { } fmt.Printf("\nDeleting spaces: %s ", strings.Join(spaces, ", ")) - for _, spaceName := range spaces { if timeout.Seconds() != 0 { deleteSpace := cf.Cf("delete-space", "-f", "-o", orgName, spaceName).Wait(timeout) diff --git a/src/acceptance/run_performance/run_performance_test.go b/src/acceptance/run_performance/run_performance_test.go index e3fa392e58..eb0be8635b 100644 --- a/src/acceptance/run_performance/run_performance_test.go +++ b/src/acceptance/run_performance/run_performance_test.go @@ -14,7 +14,7 @@ import ( ) const pollTime = 10 * time.Second -const desiredScalingTime = 20 * time.Minute +const desiredScalingTime = 300 * time.Minute var _ = Describe("Scale in and out (eg: 30%) percentage of apps", func() { var ( @@ -52,12 +52,12 @@ var _ = Describe("Scale in and out (eg: 30%) percentage of apps", func() { actualAppsToScaleCount = int(math.RoundToEven(float64(len(startedApps) * percentageToScale / 100))) fmt.Printf("\nDesired Scaling %d apps: \n", appsToScaleCount) - fmt.Printf("Actual Scaling %d apps (based on sucessuful apps pushed): \n\n", actualAppsToScaleCount) + fmt.Printf("Actual Scaling %d apps (based on sucessuful apps pushed) \n\n", actualAppsToScaleCount) samplingConfig = gmeasure.SamplingConfig{ N: actualAppsToScaleCount, - NumParallel: actualAppsToScaleCount, - Duration: 60 * time.Minute, + NumParallel: 100, + Duration: 300 * time.Minute, } experiment = gmeasure.NewExperiment("Scaling Benchmark") }) diff --git a/src/acceptance/setup_performance/setup_performance_suite_test.go b/src/acceptance/setup_performance/setup_performance_suite_test.go index b43254ac23..733db26ae7 100644 --- a/src/acceptance/setup_performance/setup_performance_suite_test.go +++ b/src/acceptance/setup_performance/setup_performance_suite_test.go @@ -34,7 +34,6 @@ func TestSetup(t *testing.T) { } var _ = BeforeSuite(func() { - var spaceGuid, orgGuid string if os.Getenv("SKIP_TEARDOWN") == "true" { fmt.Println("Skipping Teardown...") @@ -47,12 +46,11 @@ var _ = BeforeSuite(func() { EnableServiceAccess(setup, cfg, setup.GetOrganizationName()) workflowhelpers.AsUser(setup.AdminUserContext(), cfg.DefaultTimeoutDuration(), func() { - _, orgGuid, _, spaceGuid = GetOrgSpaceNamesAndGuids(cfg, setup.GetOrganizationName()) + _, orgGuid, _, spaceGuid := GetOrgSpaceNamesAndGuids(cfg, setup.GetOrganizationName()) + Expect(spaceGuid).NotTo(BeEmpty()) updateOrgQuotaForPerformanceTest(orgGuid) }) - cleanUpServiceInstanceInParallel(setup, orgGuid, spaceGuid) - if cfg.IsServiceOfferingEnabled() { CheckServiceExists(cfg, setup.TestSpace.SpaceName(), cfg.ServiceName) } @@ -79,78 +77,101 @@ func updateOrgQuotaForPerformanceTest(orgGuid string) { } } -func cleanUpServiceInstanceInParallel(setup *workflowhelpers.ReproducibleTestSuiteSetup, orgGuid string, spaceGuid string) { - waitGroup := sync.WaitGroup{} - servicesChan := make(chan string) - - serviceInstances := GetServices(cfg, orgGuid, spaceGuid) - if len(serviceInstances) != 0 { - fmt.Printf("\ndeleting existing service instances: %d\n", len(serviceInstances)) - for i := 0; i < len(serviceInstances); i++ { - waitGroup.Add(1) - i := i - go deleteExistingServiceInstances(i, servicesChan, setup, orgGuid, spaceGuid, &waitGroup) - } - for _, serviceInstanceName := range serviceInstances { - servicesChan <- serviceInstanceName - } - close(servicesChan) - waitGroup.Wait() - } -} - -func deleteExistingServiceInstances(workerId int, servicesChan chan string, setup *workflowhelpers.ReproducibleTestSuiteSetup, orgGuid string, spaceGuid string, wg *sync.WaitGroup) { - fmt.Printf("Worker %d - Delete Service Instance starting...\n", workerId) - defer wg.Done() - defer GinkgoRecover() - for instanceName := range servicesChan { - fmt.Printf("worker %d - deleting service instance - %s\n", workerId, instanceName) - DeleteServiceInstance(cfg, setup, instanceName) - } - fmt.Printf("worker %d - Delete Service Instance finished...\n", workerId) -} - func cleanup() { setup = workflowhelpers.NewTestSuiteSetup(cfg) workflowhelpers.AsUser(setup.AdminUserContext(), cfg.DefaultTimeoutDuration(), func() { - + fmt.Println("\nCleaning up test leftovers...") if cfg.UseExistingOrganization { // cf target to org targetOrg(setup) - orgGuid := GetOrgGuid(cfg, cfg.ExistingOrganization) spaceNames := GetTestSpaces(orgGuid, cfg) if len(spaceNames) == 0 { return } - //TODO - Do it with multiple go processes1 - deleteAllServices(orgGuid) + //TODO - Do it with multiple go processes + waitGroup := sync.WaitGroup{} + waitGroup.Add(2) + + deleteAllServices(cfg.Performance.SetupWorkers, orgGuid, GetSpaceGuid(cfg, orgGuid), &waitGroup) // delete all apps in a test space - only from first space - what if two spaces are present - deleteAllApps(spaceNames[0]) + deleteAllApps(cfg.Performance.SetupWorkers, orgGuid, GetSpaceGuid(cfg, orgGuid), &waitGroup) + fmt.Println("\nWaiting for services and apps to be deleted...") + waitGroup.Wait() DeleteSpaces(cfg.ExistingOrganization, GetTestSpaces(orgGuid, cfg), cfg.DefaultTimeoutDuration()) } else { DeleteOrgs(GetTestOrgs(cfg), time.Duration(120)*time.Second) } - }) } -func deleteAllApps(spaceName string) { - apps := GetApps(cfg, setup.GetOrganizationName(), spaceName, "node-custom-metric-benchmark-") - fmt.Printf("\nExisting apps found %d", len(apps)) - for _, appName := range apps { - fmt.Printf(" - deleting app %s\n", appName) - DeleteTestApp(appName, cfg.DefaultTimeoutDuration()) +func deleteAllServices(workersCount int, orgGuid string, spaceGuid string, parentWaitGroup *sync.WaitGroup) { + defer parentWaitGroup.Done() + defer GinkgoRecover() + waitGroup := sync.WaitGroup{} + servicesChan := make(chan string) + + services := GetServices(cfg, orgGuid, spaceGuid) + if len(services) == 0 { + fmt.Printf("- deleting existing service instances: %d\n", len(services)) + return + } + fmt.Printf("- deleting existing service instances: %d\n", len(services)) + for i := 1; i <= workersCount; i++ { + waitGroup.Add(1) + go deleteExistingServiceInstances(i, servicesChan, setup, &waitGroup) + } + for _, serviceInstanceName := range services { + servicesChan <- serviceInstanceName + } + close(servicesChan) + waitGroup.Wait() +} + +func deleteAllApps(workersCount int, orgGuid string, spaceName string, parentWaitGroup *sync.WaitGroup) { + defer parentWaitGroup.Done() + defer GinkgoRecover() + waitGroup := sync.WaitGroup{} + appsChan := make(chan string) + + apps := GetApps(cfg, orgGuid, spaceName, "node-custom-metric-benchmark-") + fmt.Printf("\ndeleting existing app instances: %d", len(apps)) + if len(apps) == 0 { + return + } + for i := 1; i <= workersCount; i++ { + waitGroup.Add(1) + go deleteExistingApps(i, appsChan, &waitGroup) + } + for _, app := range apps { + appsChan <- app + } + close(appsChan) + waitGroup.Wait() +} + +func deleteExistingServiceInstances(workerId int, servicesChan chan string, setup *workflowhelpers.ReproducibleTestSuiteSetup, wg *sync.WaitGroup) { + defer wg.Done() + defer GinkgoRecover() + + //fmt.Printf("Worker %d - Delete Service Instance starting...\n", workerId) + for instanceName := range servicesChan { + fmt.Printf("worker %d - deleting service instance - %s\n", workerId, instanceName) + DeleteServiceInstance(cfg, setup, instanceName) } + //fmt.Printf("worker %d - Delete Service Instance finished...\n", workerId) } -func deleteAllServices(orgGuid string) { - services := GetServices(cfg, orgGuid, GetSpaceGuid(cfg, orgGuid)) - fmt.Printf("\nExisting services found %d", len(services)) - for _, service := range services { - fmt.Printf(" - deleting service instance %s\n", service) - DeleteServiceInstance(cfg, setup, service) +func deleteExistingApps(workerId int, appsChan chan string, wg *sync.WaitGroup) { + defer wg.Done() + defer GinkgoRecover() + + fmt.Printf("Worker %d - Delete app starting...\n", workerId) + for appName := range appsChan { + fmt.Printf("worker %d - deleting app instance - %s\n", workerId, appName) + DeleteTestApp(appName, cfg.DefaultTimeoutDuration()) } + fmt.Printf("worker %d - Delete app finished...\n", workerId) } func targetOrg(setup *workflowhelpers.ReproducibleTestSuiteSetup) { From 26776a182db97ab50e34bfa665ebe00c2a77cf9a Mon Sep 17 00:00:00 2001 From: Arsalan Khan Date: Mon, 16 Jan 2023 11:59:24 +0100 Subject: [PATCH 3/4] refactored code to better readibility --- src/acceptance/helpers/spaces.go | 3 +- .../run_performance/run_performance_test.go | 56 ++++++----- .../setup_performance_suite_test.go | 7 +- .../setup_performance_test.go | 93 ++++--------------- 4 files changed, 50 insertions(+), 109 deletions(-) diff --git a/src/acceptance/helpers/spaces.go b/src/acceptance/helpers/spaces.go index 5a946ea40f..c2f35be1ba 100644 --- a/src/acceptance/helpers/spaces.go +++ b/src/acceptance/helpers/spaces.go @@ -56,8 +56,7 @@ func DeleteSpaces(orgName string, spaces []string, timeout time.Duration) { if len(spaces) == 0 { return } - - fmt.Printf("\nDeleting spaces: %s ", strings.Join(spaces, ", ")) + fmt.Printf("\nDeleting spaces: %s \n", strings.Join(spaces, ", ")) for _, spaceName := range spaces { if timeout.Seconds() != 0 { deleteSpace := cf.Cf("delete-space", "-f", "-o", orgName, spaceName).Wait(timeout) diff --git a/src/acceptance/run_performance/run_performance_test.go b/src/acceptance/run_performance/run_performance_test.go index eb0be8635b..5cbe38291e 100644 --- a/src/acceptance/run_performance/run_performance_test.go +++ b/src/acceptance/run_performance/run_performance_test.go @@ -23,7 +23,7 @@ var _ = Describe("Scale in and out (eg: 30%) percentage of apps", func() { appCount int samplingConfig gmeasure.SamplingConfig experiment *gmeasure.Experiment - doneAppsCount int32 + scaledInAppsCount int32 scaledOutAppsCount int32 errors sync.Map startedApps []helpers.AppInfo @@ -73,16 +73,7 @@ var _ = Describe("Scale in and out (eg: 30%) percentage of apps", func() { wg := sync.WaitGroup{} wg.Add(1) - experiment.MeasureDuration("scale-out", func() { - scaleOut := func() (int, error) { - helpers.SendMetric(cfg, appName, 550) - return helpers.RunningInstances(appGUID, 20*time.Second) - } - Eventually(scaleOut).WithPolling(pollTime).WithTimeout(5*time.Minute).Should(Equal(2), - fmt.Sprintf("Failed to scale out app: %s", appName)) - fmt.Printf("\nfinished scaling-out app: %s at index %d\n", appName, i) - wg.Done() - }) + experiment.MeasureDuration("scale-out", scaleOutApp(appName, appGUID, &wg)) wg.Wait() atomic.AddInt32(&scaledOutAppsCount, 1) @@ -90,29 +81,44 @@ var _ = Describe("Scale in and out (eg: 30%) percentage of apps", func() { wg = sync.WaitGroup{} wg.Add(1) - experiment.MeasureDuration("scale-in", func() { - scaleIn := func() (int, error) { - helpers.SendMetric(cfg, appName, 100) - return helpers.RunningInstances(appGUID, 20*time.Second) - } - Eventually(scaleIn).WithPolling(pollTime).WithTimeout(5*time.Minute).Should(Equal(1), - fmt.Sprintf("Failed to scale in app: %s", appName)) - fmt.Printf("\nfinished scaling-in app: %s at index %d\n", appName, i) - wg.Done() - }) + experiment.MeasureDuration("scale-in", scaleInApp(appName, appGUID, &wg)) wg.Wait() - atomic.AddInt32(&doneAppsCount, 1) - fmt.Printf("Scaled-in apps: %d/%d\n", atomic.LoadInt32(&doneAppsCount), actualAppsToScaleCount) + atomic.AddInt32(&scaledInAppsCount, 1) + fmt.Printf("Scaled-in apps: %d/%d\n", atomic.LoadInt32(&scaledInAppsCount), actualAppsToScaleCount) }, samplingConfig) - fmt.Printf("\nWaiting %s minutes to finish scaling...", desiredScalingTime) - Eventually(func() int32 { return atomic.LoadInt32(&doneAppsCount) }, desiredScalingTime, 10*time.Second).Should(BeEquivalentTo(actualAppsToScaleCount)) + fmt.Printf("Waiting %s minutes to finish scaling...\n\n", desiredScalingTime) + Eventually(func() int32 { return atomic.LoadInt32(&scaledInAppsCount) }, desiredScalingTime, 10*time.Second).Should(BeEquivalentTo(actualAppsToScaleCount)) checkMedianDurationFor(experiment, "scale-out") checkMedianDurationFor(experiment, "scale-in") }) }) }) +func scaleInApp(appName string, appGUID string, wg *sync.WaitGroup) func() { + return func() { + scaleIn := func() (int, error) { + helpers.SendMetric(cfg, appName, 100) + return helpers.RunningInstances(appGUID, 20*time.Second) + } + Eventually(scaleIn).WithPolling(pollTime).WithTimeout(5*time.Minute).Should(Equal(1), + fmt.Sprintf("Failed to scale in app: %s", appName)) + wg.Done() + } +} + +func scaleOutApp(appName string, appGUID string, wg *sync.WaitGroup) func() { + return func() { + scaleOut := func() (int, error) { + helpers.SendMetric(cfg, appName, 550) + return helpers.RunningInstances(appGUID, 20*time.Second) + } + Eventually(scaleOut).WithPolling(pollTime).WithTimeout(5*time.Minute).Should(Equal(2), + fmt.Sprintf("Failed to scale out app: %s", appName)) + wg.Done() + } +} + func checkMedianDurationFor(experiment *gmeasure.Experiment, statName string) { scaleOutStats := experiment.GetStats(statName) medianScaleOutDuration := scaleOutStats.DurationFor(gmeasure.StatMedian) diff --git a/src/acceptance/setup_performance/setup_performance_suite_test.go b/src/acceptance/setup_performance/setup_performance_suite_test.go index 733db26ae7..f76ceb3329 100644 --- a/src/acceptance/setup_performance/setup_performance_suite_test.go +++ b/src/acceptance/setup_performance/setup_performance_suite_test.go @@ -82,7 +82,6 @@ func cleanup() { workflowhelpers.AsUser(setup.AdminUserContext(), cfg.DefaultTimeoutDuration(), func() { fmt.Println("\nCleaning up test leftovers...") if cfg.UseExistingOrganization { - // cf target to org targetOrg(setup) orgGuid := GetOrgGuid(cfg, cfg.ExistingOrganization) spaceNames := GetTestSpaces(orgGuid, cfg) @@ -135,7 +134,7 @@ func deleteAllApps(workersCount int, orgGuid string, spaceName string, parentWai appsChan := make(chan string) apps := GetApps(cfg, orgGuid, spaceName, "node-custom-metric-benchmark-") - fmt.Printf("\ndeleting existing app instances: %d", len(apps)) + fmt.Printf("\n- deleting existing app instances: %d\n", len(apps)) if len(apps) == 0 { return } @@ -154,24 +153,20 @@ func deleteExistingServiceInstances(workerId int, servicesChan chan string, setu defer wg.Done() defer GinkgoRecover() - //fmt.Printf("Worker %d - Delete Service Instance starting...\n", workerId) for instanceName := range servicesChan { fmt.Printf("worker %d - deleting service instance - %s\n", workerId, instanceName) DeleteServiceInstance(cfg, setup, instanceName) } - //fmt.Printf("worker %d - Delete Service Instance finished...\n", workerId) } func deleteExistingApps(workerId int, appsChan chan string, wg *sync.WaitGroup) { defer wg.Done() defer GinkgoRecover() - fmt.Printf("Worker %d - Delete app starting...\n", workerId) for appName := range appsChan { fmt.Printf("worker %d - deleting app instance - %s\n", workerId, appName) DeleteTestApp(appName, cfg.DefaultTimeoutDuration()) } - fmt.Printf("worker %d - Delete app finished...\n", workerId) } func targetOrg(setup *workflowhelpers.ReproducibleTestSuiteSetup) { diff --git a/src/acceptance/setup_performance/setup_performance_test.go b/src/acceptance/setup_performance/setup_performance_test.go index 7dc01a484e..9f9689013d 100644 --- a/src/acceptance/setup_performance/setup_performance_test.go +++ b/src/acceptance/setup_performance/setup_performance_test.go @@ -8,14 +8,16 @@ import ( "time" . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) -var _ = Describe("Prepare test apps based on benchmark inputs", func() { +var _ = Describe("Prepare test apps based on performance inputs", func() { var ( appName string runningAppsCount int32 pendingApps sync.Map errors sync.Map + itSpecText string ) AfterEach(func() { @@ -30,7 +32,6 @@ var _ = Describe("Prepare test apps based on benchmark inputs", func() { }) }) BeforeEach(func() { - // Refactored version today starts wg := sync.WaitGroup{} queue := make(chan string) @@ -39,9 +40,8 @@ var _ = Describe("Prepare test apps based on benchmark inputs", func() { for i := 1; i <= workerCount; i++ { wg.Add(1) - go appHandler(queue, &runningAppsCount, &errors, &wg) + go appHandler(queue, &runningAppsCount, &pendingApps, &errors, &wg) } - for i := 1; i <= cfg.Performance.AppCount; i++ { appName = fmt.Sprintf("node-custom-metric-benchmark-%d", i) desiredApps = append(desiredApps, appName) @@ -49,67 +49,42 @@ var _ = Describe("Prepare test apps based on benchmark inputs", func() { } fmt.Printf("desired app count: %d\n", len(desiredApps)) appNameGenerator(queue, desiredApps) - + itSpecText = fmt.Sprintf(" should be equal to %d", cfg.Performance.AppCount) close(queue) - fmt.Println("\nWaiting for goroutines to finish...") + fmt.Println("\nWaiting for app to finish...") wg.Wait() fmt.Printf("\nTotal Running apps: %d/%dn", atomic.LoadInt32(&runningAppsCount), cfg.Performance.AppCount) - - // Refactored version today ends - - /*workerCount := cfg.Performance.SetupWorkers - appsChan := make(chan string) - - wg := sync.WaitGroup{} - // ToDo: Why starting workers first than putting the appname to appChannel. - // it should be .. first appchanel then start workers - fmt.Println(fmt.Sprintf("\nStarting %d workers...", cfg.Performance.SetupWorkers)) - for i := 0; i < workerCount; i++ { - wg.Add(1) - time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond) - go worker(appsChan, &runningAppsCount, &pendingApps, &errors, &wg) - } - - fmt.Println(fmt.Sprintf("Deploying %d apps", cfg.Performance.AppCount)) - for i := 0; i < cfg.Performance.AppCount; i++ { - appName = fmt.Sprintf("node-custom-metric-benchmark-%d", i) - pendingApps.Store(appName, 1) - appsChan <- appName - } - - close(appsChan) - wg.Wait()*/ }) - Context("when scaling by custom metrics", func() { - It("should scale out and scale in", func() { - //Eventually(func() int32 { return atomic.LoadInt32(&runningAppsCount) }, 3*time.Minute, 5*time.Second).Should(BeEquivalentTo(cfg.Performance.AppCount)) + Context("App count", func() { + It(itSpecText, func() { + Eventually(func() int32 { + return atomic.LoadInt32(&runningAppsCount) + }, + 300*time.Minute, 5*time.Second). + Should(BeEquivalentTo(cfg.Performance.AppCount)) }) }) }) func appNameGenerator(ch chan<- string, desiredApps []string) { for _, app := range desiredApps { - msg := fmt.Sprintf("Start [ %s ] ", app) - fmt.Println(msg) ch <- app } } -func appHandler(ch <-chan string, runningAppsCount *int32, errors *sync.Map, wg *sync.WaitGroup) { +func appHandler(ch <-chan string, runningAppsCount *int32, pendingApps *sync.Map, errors *sync.Map, wg *sync.WaitGroup) { defer wg.Done() defer GinkgoRecover() for appName := range ch { fmt.Printf("- pushing app [ %s ] \n", appName) - pushAppAndBindService(appName, runningAppsCount, errors) - fmt.Printf("Done [ %s ] \n", appName) + pushAppAndBindService(appName, runningAppsCount, pendingApps, errors) time.Sleep(time.Millisecond) } } -func pushAppAndBindService(appName string, runningApps *int32, errors *sync.Map) { - // TODO Refactoring required....May be separate service creation and app pushing e.g parellel +func pushAppAndBindService(appName string, runningApps *int32, pendingApps *sync.Map, errors *sync.Map) { err := helpers.CreateTestAppFromDropletByName(cfg, nodeAppDropletPath, appName, 1) if err != nil { errors.Store(appName, err) @@ -133,40 +108,6 @@ func pushAppAndBindService(appName string, runningApps *int32, errors *sync.Map) return } atomic.AddInt32(runningApps, 1) - //pendingApps.Delete(appName) + pendingApps.Delete(appName) fmt.Printf("- Running apps: %d/%d - %s\n", atomic.LoadInt32(runningApps), cfg.Performance.AppCount, appName) } - -/*func worker(appsChan chan string, runningApps *int32, pendingApps *sync.Map, errors *sync.Map, wg *sync.WaitGroup) { - // Decreasing internal counter for wait-group as soon as goroutine finishes - defer wg.Done() - defer GinkgoRecover() - for appName := range appsChan { - err := helpers.CreateTestAppFromDropletByName(cfg, nodeAppDropletPath, appName, 1) - if err != nil { - errors.Store(appName, err) - continue - } - policy := helpers.GenerateDynamicScaleOutAndInPolicy(1, 2, "test_metric", 500, 500) - appGUID, err := helpers.GetAppGuid(cfg, appName) - if err != nil { - errors.Store(appName, err) - continue - } - _, err = helpers.CreatePolicyWithErr(cfg, appName, appGUID, policy) - if err != nil { - errors.Store(appName, err) - continue - } - helpers.CreateCustomMetricCred(cfg, appName, appGUID) - err = helpers.StartAppWithErr(appName, cfg.CfPushTimeoutDuration()) - if err != nil { - errors.Store(appName, err) - continue - } - atomic.AddInt32(runningApps, 1) - pendingApps.Delete(appName) - fmt.Printf("Running apps: %d/%d - %s\n", atomic.LoadInt32(runningApps), cfg.Performance.AppCount, appName) - } -} -*/ From 746b44e0960cc8cda3fda49f74da96f8c81ea52e Mon Sep 17 00:00:00 2001 From: Arsalan Khan Date: Mon, 16 Jan 2023 16:02:59 +0100 Subject: [PATCH 4/4] fix golangci-lint --fix and Remove --path-prefix flag Already [opened Issue](https://github.com/golangci/golangci-lint/pull/2293) --- Makefile | 3 ++- src/acceptance/run_performance/run_performance_test.go | 2 +- .../setup_performance/setup_performance_suite_test.go | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 7d49529da5..98a1b12b6e 100644 --- a/Makefile +++ b/Makefile @@ -253,7 +253,8 @@ lint-actions: $(addprefix lint_,$(go_modules)): lint_%: @echo " - linting: $(patsubst lint_%,%,$@)" - @pushd src/$(patsubst lint_%,%,$@) >/dev/null && golangci-lint run --path-prefix=src/$(patsubst lint_%,%,$@) --config ${lint_config} ${OPTS} + @echo " receiving OPTS ${OPTS}" + @pushd src/$(patsubst lint_%,%,$@) >/dev/null && golangci-lint run --config ${lint_config} ${OPTS} .PHONY: spec-test spec-test: diff --git a/src/acceptance/run_performance/run_performance_test.go b/src/acceptance/run_performance/run_performance_test.go index 5cbe38291e..d349f27360 100644 --- a/src/acceptance/run_performance/run_performance_test.go +++ b/src/acceptance/run_performance/run_performance_test.go @@ -56,7 +56,7 @@ var _ = Describe("Scale in and out (eg: 30%) percentage of apps", func() { samplingConfig = gmeasure.SamplingConfig{ N: actualAppsToScaleCount, - NumParallel: 100, + NumParallel: 100, // number of sample to execute at a time Duration: 300 * time.Minute, } experiment = gmeasure.NewExperiment("Scaling Benchmark") diff --git a/src/acceptance/setup_performance/setup_performance_suite_test.go b/src/acceptance/setup_performance/setup_performance_suite_test.go index f76ceb3329..0d20efd017 100644 --- a/src/acceptance/setup_performance/setup_performance_suite_test.go +++ b/src/acceptance/setup_performance/setup_performance_suite_test.go @@ -4,14 +4,15 @@ import ( "acceptance/config" . "acceptance/helpers" "fmt" - "github.com/KevinJCross/cf-test-helpers/v2/cf" - "github.com/onsi/gomega/gexec" "os" "strconv" "sync" "testing" "time" + "github.com/KevinJCross/cf-test-helpers/v2/cf" + "github.com/onsi/gomega/gexec" + "github.com/KevinJCross/cf-test-helpers/v2/workflowhelpers" . "github.com/onsi/ginkgo/v2" @@ -88,7 +89,6 @@ func cleanup() { if len(spaceNames) == 0 { return } - //TODO - Do it with multiple go processes waitGroup := sync.WaitGroup{} waitGroup.Add(2)