From d2e959c9d967aed05d2e8f628c9acb2b0a36626d Mon Sep 17 00:00:00 2001 From: Jack He Date: Mon, 25 Nov 2024 19:52:26 -0800 Subject: [PATCH] Revert "Standard SecNetPerf Scenarios (#4607)" This reverts commit 3eb3fe018be2f15e2d4a7bc298d17b77241a835c. --- scripts/secnetperf-helpers.psm1 | 41 +++++++++++++++-------------- scripts/secnetperf.ps1 | 27 ++++++++++++------- src/inc/msquichelper.h | 2 +- src/perf/lib/PerfClient.cpp | 46 --------------------------------- src/perf/lib/SecNetPerfMain.cpp | 24 +---------------- 5 files changed, 40 insertions(+), 100 deletions(-) diff --git a/scripts/secnetperf-helpers.psm1 b/scripts/secnetperf-helpers.psm1 index 0a768477ee..0ecd7c94b0 100644 --- a/scripts/secnetperf-helpers.psm1 +++ b/scripts/secnetperf-helpers.psm1 @@ -524,7 +524,7 @@ function Get-LatencyOutput { # Invokes secnetperf with the given arguments for both TCP and QUIC. function Invoke-Secnetperf { - param ($Session, $RemoteName, $RemoteDir, $UserName, $SecNetPerfPath, $LogProfile, $Scenario, $io, $Filter, $Environment, $RunId, $SyncerSecret) + param ($Session, $RemoteName, $RemoteDir, $UserName, $SecNetPerfPath, $LogProfile, $TestId, $ExeArgs, $io, $Filter, $Environment, $RunId, $SyncerSecret) $values = @(@(), @()) $latency = $null @@ -536,25 +536,26 @@ function Invoke-Secnetperf { $tcpSupported = 0 } $metric = "throughput" - if ($Scenario.Contains("rps")) { + if ($exeArgs.Contains("conns:16cpu")) { # TODO: figure out a better way to detect max RPS tests $metric = "rps" - } elseif ($Scenario.Contains("latency")) { + } elseif ($exeArgs.Contains("plat:1")) { $metric = "latency" $latency = @(@(), @()) $extraOutput = Repo-Path "latency.txt" if (!$isWindows) { chmod +rw "$extraOutput" } - } elseif ($Scenario.Contains("hps")) { + } elseif ($exeArgs.Contains("prate:1")) { $metric = "hps" } for ($tcp = 0; $tcp -le $tcpSupported; $tcp++) { # Set up all the parameters and paths for running the test. + $execMode = $ExeArgs.Substring(0, $ExeArgs.IndexOf(" ")) # First arg is the exec mode $clientPath = Repo-Path $SecNetPerfPath - $serverArgs = "-scenario:$Scenario -io:$io" - $clientArgs = "-target:$RemoteName -scenario:$Scenario -io:$io -tcp:$tcp -trimout -watchdog:25000" + $serverArgs = "$execMode -io:$io" + $clientArgs = "-target:$RemoteName $ExeArgs -tcp:$tcp -trimout -watchdog:25000" if ($io -eq "xdp" -or $io -eq "qtip") { $serverArgs += " -pollidle:10000" $clientArgs += " -pollidle:10000" @@ -583,9 +584,9 @@ function Invoke-Secnetperf { $useSudo = (!$isWindows -and $io -eq "xdp") if ($tcp -eq 0) { - $artifactName = "$Scenario-quic" + $artifactName = "$TestId-quic" } else { - $artifactName = "$Scenario-tcp" + $artifactName = "$TestId-tcp" } New-Item -ItemType Directory "artifacts/logs/$artifactName" -ErrorAction Ignore | Out-Null $artifactDir = Repo-Path "artifacts/logs/$artifactName" @@ -714,14 +715,14 @@ function Invoke-Secnetperf { } } -function CheckRegressionResult($values, $scenario, $transport, $regressionJson, $envStr) { +function CheckRegressionResult($values, $testid, $transport, $regressionJson, $envStr) { $sum = 0 foreach ($item in $values) { $sum += $item } $avg = $sum / $values.Length - $Scenario = "$scenario-$transport" + $Testid = "$testid-$transport" $res = @{ Baseline = "N/A" @@ -733,14 +734,14 @@ function CheckRegressionResult($values, $scenario, $transport, $regressionJson, } try { - $res.Baseline = $regressionJson.$Scenario.$envStr.baseline - $res.BestResult = $regressionJson.$Scenario.$envStr.BestResult - $res.BestResultCommit = $regressionJson.$Scenario.$envStr.BestResultCommit + $res.Baseline = $regressionJson.$Testid.$envStr.baseline + $res.BestResult = $regressionJson.$Testid.$envStr.BestResult + $res.BestResultCommit = $regressionJson.$Testid.$envStr.BestResultCommit $res.CumulativeResult = $avg $res.AggregateFunction = "AVG" if ($avg -lt $res.Baseline) { - Write-GHError "Regression detected in $Scenario for $envStr. See summary table for details." + Write-GHError "Regression detected in $Testid for $envStr. See summary table for details." $res.HasRegression = $true } } catch { @@ -750,7 +751,7 @@ function CheckRegressionResult($values, $scenario, $transport, $regressionJson, return $res } -function CheckRegressionLat($values, $regressionJson, $scenario, $transport, $envStr) { +function CheckRegressionLat($values, $regressionJson, $testid, $transport, $envStr) { # TODO: Right now, we are not using a watermark based method for regression detection of latency percentile values because we don't know how to determine a "Best Ever" distribution. # (we are just looking at P0, P50, P99 columns, and computing the baseline for each percentile as the mean - 2 * std of the last 20 runs. ) @@ -763,7 +764,7 @@ function CheckRegressionLat($values, $regressionJson, $scenario, $transport, $en } $RpsAvg /= $NumRuns - $Scenario = "$scenario-$transport" + $Testid = "$testid-$transport" $res = @{ Baseline = "N/A" @@ -775,14 +776,14 @@ function CheckRegressionLat($values, $regressionJson, $scenario, $transport, $en } try { - $res.Baseline = $regressionJson.$Scenario.$envStr.baseline - $res.BestResult = $regressionJson.$Scenario.$envStr.BestResult - $res.BestResultCommit = $regressionJson.$Scenario.$envStr.BestResultCommit + $res.Baseline = $regressionJson.$Testid.$envStr.baseline + $res.BestResult = $regressionJson.$Testid.$envStr.BestResult + $res.BestResultCommit = $regressionJson.$Testid.$envStr.BestResultCommit $res.CumulativeResult = $RpsAvg $res.AggregateFunction = "AVG" if ($RpsAvg -lt $res.Baseline) { - Write-GHError "RPS Regression detected in $Scenario for $envStr. See summary table for details." + Write-GHError "RPS Regression detected in $Testid for $envStr. See summary table for details." $res.HasRegression = $true } } catch { diff --git a/scripts/secnetperf.ps1 b/scripts/secnetperf.ps1 index 5abff3b504..332e2fe3c4 100644 --- a/scripts/secnetperf.ps1 +++ b/scripts/secnetperf.ps1 @@ -251,11 +251,17 @@ if ($isWindows) { $kernelVersion = bash -c "uname -r" $json["os_version"] = "$osName $kernelVersion" } +$allTests = [System.Collections.Specialized.OrderedDictionary]::new() -# Test all supported scenarios. -$allScenarios = @("upload", "download", "hps", "rps", "rps-multi", "latency") +# > All tests: +$allTests["tput-up"] = "-exec:maxtput -up:12s -ptput:1" +$allTests["tput-down"] = "-exec:maxtput -down:12s -ptput:1" +$allTests["hps-conns-100"] = "-exec:maxtput -rconn:1 -share:1 -conns:100 -run:12s -prate:1" +$allTests["rps-up-512-down-4000"] = "-exec:lowlat -rstream:1 -up:512 -down:4000 -run:20s -plat:1" +$allTests["max-rps-up-512-down-4000"] = "-exec:lowlat -conns:16cpu -streams:10 -rstream:1 -up:512 -down:4000 -run:20s -plat:1" $hasFailures = $false +$json["run_args"] = $allTests try { @@ -330,8 +336,9 @@ $regressionJson = Get-Content -Raw -Path "watermark_regression.json" | ConvertFr # Run all the test cases. Write-Host "Setup complete! Running all tests" -foreach ($scenario in $allScenarios) { - $Output = Invoke-Secnetperf $Session $RemoteName $RemoteDir $UserName $SecNetPerfPath $LogProfile $scenario $io $filter $environment $RunId $SyncerSecret +foreach ($testId in $allTests.Keys) { + $ExeArgs = $allTests[$testId] + " -io:$io" + $Output = Invoke-Secnetperf $Session $RemoteName $RemoteDir $UserName $SecNetPerfPath $LogProfile $testId $ExeArgs $io $filter $environment $RunId $SyncerSecret $Test = $Output[-1] if ($Test.HasFailures) { $hasFailures = $true } @@ -342,15 +349,15 @@ foreach ($scenario in $allScenarios) { } else { $transport = "quic" } - $json["$scenario-$transport"] = $Test.Values[$tcp] + $json["$testId-$transport"] = $Test.Values[$tcp] if ($Test.Metric -eq "latency") { - $json["$scenario-$transport-lat"] = $Test.Latency[$tcp] - $LatencyRegression = CheckRegressionLat $Test.Values[$tcp] $regressionJson $scenario $transport "$os-$arch-$environment-$io-$tls" - $json["$scenario-$transport-regression"] = $LatencyRegression + $json["$testId-$transport-lat"] = $Test.Latency[$tcp] + $LatencyRegression = CheckRegressionLat $Test.Values[$tcp] $regressionJson $testId $transport "$os-$arch-$environment-$io-$tls" + $json["$testId-$transport-regression"] = $LatencyRegression } else { - $ResultRegression = CheckRegressionResult $Test.Values[$tcp] $scenario $transport $regressionJson "$os-$arch-$environment-$io-$tls" - $json["$scenario-$transport-regression"] = $ResultRegression + $ResultRegression = CheckRegressionResult $Test.Values[$tcp] $testId $transport $regressionJson "$os-$arch-$environment-$io-$tls" + $json["$testId-$transport-regression"] = $ResultRegression } } } diff --git a/src/inc/msquichelper.h b/src/inc/msquichelper.h index 529c68413d..4572ddbd3e 100644 --- a/src/inc/msquichelper.h +++ b/src/inc/msquichelper.h @@ -304,7 +304,7 @@ IsValue( _In_z_ const char* toTestAgainst ) { - return _strnicmp(name, toTestAgainst, strlen(toTestAgainst)) == 0; + return _strnicmp(name, toTestAgainst, CXPLAT_MIN(strlen(name), strlen(toTestAgainst))) == 0; } inline diff --git a/src/perf/lib/PerfClient.cpp b/src/perf/lib/PerfClient.cpp index 1b6542be87..5ae2d86897 100644 --- a/src/perf/lib/PerfClient.cpp +++ b/src/perf/lib/PerfClient.cpp @@ -108,52 +108,6 @@ PerfClient::Init( CountMult[0] = CxPlatProcCount(); - // - // Scenario profile sets new defauls for values below, that may then be - // further overridden by command line arguments. - // - const char* ScenarioStr = GetValue(argc, argv, "scenario"); - if (ScenarioStr != nullptr) { - if (IsValue(ScenarioStr, "upload")) { - Upload = S_TO_US(12); // 12 seconds - Timed = TRUE; - PrintThroughput = TRUE; - } else if (IsValue(ScenarioStr, "download")) { - Download = S_TO_US(12); // 12 seconds - Timed = TRUE; - PrintThroughput = TRUE; - } else if (IsValue(ScenarioStr, "hps")) { - ConnectionCount = 16 * CxPlatProcCount(); - RunTime = S_TO_US(12); // 12 seconds - RepeatConnections = TRUE; - PrintIoRate = TRUE; - } else if (IsValue(ScenarioStr, "rps-multi")) { - Upload = 512; - Download = 4000; - ConnectionCount = 16 * CxPlatProcCount(); - StreamCount = 100; - RunTime = S_TO_US(20); // 20 seconds - RepeatStreams = TRUE; - PrintLatency = TRUE; - } else if (IsValue(ScenarioStr, "rps")) { - Upload = 512; - Download = 4000; - StreamCount = 100; - RunTime = S_TO_US(20); // 20 seconds - RepeatStreams = TRUE; - PrintLatency = TRUE; - } else if (IsValue(ScenarioStr, "latency")) { - Upload = 512; - Download = 4000; - RunTime = S_TO_US(20); // 20 seconds - RepeatStreams = TRUE; - PrintLatency = TRUE; - } else { - WriteOutput("Failed to parse scenario profile[%s]!\n", ScenarioStr); - return QUIC_STATUS_INVALID_PARAMETER; - } - } - // // Remote target/server options // diff --git a/src/perf/lib/SecNetPerfMain.cpp b/src/perf/lib/SecNetPerfMain.cpp index dc7ef17942..7e6f3ba3a8 100644 --- a/src/perf/lib/SecNetPerfMain.cpp +++ b/src/perf/lib/SecNetPerfMain.cpp @@ -78,8 +78,6 @@ PrintHelp( " -platency<0/1> Print latency statistics. (def:0)\n" "\n" " Scenario options:\n" - " -scenario: Scenario profile to use.\n" - " - {upload, download, hps, rps, rps-multi, latency}.\n" " -conns:<####> The number of connections to use. (def:1)\n" " -streams:<####> The number of streams to send on at a time. (def:0)\n" " -upload:<####>[unit] The length of bytes to send on each stream, with an optional (time or length) unit. (def:0)\n" @@ -208,25 +206,6 @@ QuicMainStart( return Status; } - const char* ScenarioStr = GetValue(argc, argv, "scenario"); - if (ScenarioStr != nullptr) { - if (IsValue(ScenarioStr, "upload") || - IsValue(ScenarioStr, "download") || - IsValue(ScenarioStr, "hps")) { - PerfDefaultExecutionProfile = QUIC_EXECUTION_PROFILE_TYPE_MAX_THROUGHPUT; - TcpDefaultExecutionProfile = TCP_EXECUTION_PROFILE_MAX_THROUGHPUT; - } else if ( - IsValue(ScenarioStr, "rps") || - IsValue(ScenarioStr, "rps-multi") || - IsValue(ScenarioStr, "latency")) { - PerfDefaultExecutionProfile = QUIC_EXECUTION_PROFILE_LOW_LATENCY; - TcpDefaultExecutionProfile = TCP_EXECUTION_PROFILE_LOW_LATENCY; - } else { - WriteOutput("Failed to parse scenario profile[%s]!\n", ScenarioStr); - return QUIC_STATUS_INVALID_PARAMETER; - } - } - const char* ExecStr = GetValue(argc, argv, "exec"); if (ExecStr != nullptr) { if (IsValue(ExecStr, "lowlat")) { @@ -240,8 +219,7 @@ QuicMainStart( } else if (IsValue(ExecStr, "realtime")) { PerfDefaultExecutionProfile = QUIC_EXECUTION_PROFILE_TYPE_REAL_TIME; } else { - WriteOutput("Failed to parse execution profile[%s]!\n", ExecStr); - return QUIC_STATUS_INVALID_PARAMETER; + WriteOutput("Failed to parse execution profile[%s], use lowlat as default for QUIC, lowlat as default for TCP.\n", ExecStr); } }