Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Standard SecNetPerf Scenarios #4607

Merged
merged 6 commits into from
Nov 26, 2024
Merged

Standard SecNetPerf Scenarios #4607

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e51dc04
Standard SecNetPerf Scenarios
nibanks Oct 10, 2024
fadaad2
Fix compiler errors
nibanks Oct 10, 2024
10a156e
nits
nibanks Oct 11, 2024
3548145
reorder IsValue checks
nibanks Oct 11, 2024
3afdfbf
Fix string compare helper
nibanks Oct 11, 2024
410e492
More streams
nibanks Oct 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 20 additions & 21 deletions scripts/secnetperf-helpers.psm1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -524,7 +524,7 @@ function Get-LatencyOutput {

# Invokes secnetperf with the given arguments for both TCP and QUIC.
function Invoke-Secnetperf {
param ($Session, $RemoteName, $RemoteDir, $UserName, $SecNetPerfPath, $LogProfile, $TestId, $ExeArgs, $io, $Filter, $Environment, $RunId, $SyncerSecret)
param ($Session, $RemoteName, $RemoteDir, $UserName, $SecNetPerfPath, $LogProfile, $Scenario, $io, $Filter, $Environment, $RunId, $SyncerSecret)
nibanks marked this conversation as resolved.
Show resolved Hide resolved

$values = @(@(), @())
$latency = $null
Expand All @@ -536,26 +536,25 @@ function Invoke-Secnetperf {
$tcpSupported = 0
}
$metric = "throughput"
if ($exeArgs.Contains("conns:16cpu")) { # TODO: figure out a better way to detect max RPS tests
if ($Scenario.Contains("rps")) {
$metric = "rps"
} elseif ($exeArgs.Contains("plat:1")) {
} elseif ($Scenario.Contains("latency")) {
$metric = "latency"
$latency = @(@(), @())
$extraOutput = Repo-Path "latency.txt"
if (!$isWindows) {
chmod +rw "$extraOutput"
}
} elseif ($exeArgs.Contains("prate:1")) {
} elseif ($Scenario.Contains("hps")) {
$metric = "hps"
}

for ($tcp = 0; $tcp -le $tcpSupported; $tcp++) {

# Set up all the parameters and paths for running the test.
$execMode = $ExeArgs.Substring(0, $ExeArgs.IndexOf(" ")) # First arg is the exec mode
$clientPath = Repo-Path $SecNetPerfPath
$serverArgs = "$execMode -io:$io"
$clientArgs = "-target:$RemoteName $ExeArgs -tcp:$tcp -trimout -watchdog:25000"
$serverArgs = "-scenario:$Scenario -io:$io"
$clientArgs = "-target:$RemoteName -scenario:$Scenario -io:$io -tcp:$tcp -trimout -watchdog:25000"
if ($io -eq "xdp" -or $io -eq "qtip") {
$serverArgs += " -pollidle:10000"
$clientArgs += " -pollidle:10000"
Expand Down Expand Up @@ -584,9 +583,9 @@ function Invoke-Secnetperf {
$useSudo = (!$isWindows -and $io -eq "xdp")

if ($tcp -eq 0) {
$artifactName = "$TestId-quic"
$artifactName = "$Scenario-quic"
} else {
$artifactName = "$TestId-tcp"
$artifactName = "$Scenario-tcp"
}
New-Item -ItemType Directory "artifacts/logs/$artifactName" -ErrorAction Ignore | Out-Null
$artifactDir = Repo-Path "artifacts/logs/$artifactName"
Expand Down Expand Up @@ -715,14 +714,14 @@ function Invoke-Secnetperf {
}
}

function CheckRegressionResult($values, $testid, $transport, $regressionJson, $envStr) {
function CheckRegressionResult($values, $scenario, $transport, $regressionJson, $envStr) {

$sum = 0
foreach ($item in $values) {
$sum += $item
}
$avg = $sum / $values.Length
$Testid = "$testid-$transport"
$Scenario = "$scenario-$transport"

$res = @{
Baseline = "N/A"
Expand All @@ -734,14 +733,14 @@ function CheckRegressionResult($values, $testid, $transport, $regressionJson, $e
}

try {
$res.Baseline = $regressionJson.$Testid.$envStr.baseline
$res.BestResult = $regressionJson.$Testid.$envStr.BestResult
$res.BestResultCommit = $regressionJson.$Testid.$envStr.BestResultCommit
$res.Baseline = $regressionJson.$Scenario.$envStr.baseline
$res.BestResult = $regressionJson.$Scenario.$envStr.BestResult
$res.BestResultCommit = $regressionJson.$Scenario.$envStr.BestResultCommit
$res.CumulativeResult = $avg
$res.AggregateFunction = "AVG"

if ($avg -lt $res.Baseline) {
Write-GHError "Regression detected in $Testid for $envStr. See summary table for details."
Write-GHError "Regression detected in $Scenario for $envStr. See summary table for details."
$res.HasRegression = $true
}
} catch {
Expand All @@ -751,7 +750,7 @@ function CheckRegressionResult($values, $testid, $transport, $regressionJson, $e
return $res
}

function CheckRegressionLat($values, $regressionJson, $testid, $transport, $envStr) {
function CheckRegressionLat($values, $regressionJson, $scenario, $transport, $envStr) {

# TODO: Right now, we are not using a watermark based method for regression detection of latency percentile values because we don't know how to determine a "Best Ever" distribution.
# (we are just looking at P0, P50, P99 columns, and computing the baseline for each percentile as the mean - 2 * std of the last 20 runs. )
Expand All @@ -764,7 +763,7 @@ function CheckRegressionLat($values, $regressionJson, $testid, $transport, $envS
}

$RpsAvg /= $NumRuns
$Testid = "$testid-$transport"
$Scenario = "$scenario-$transport"

$res = @{
Baseline = "N/A"
Expand All @@ -776,14 +775,14 @@ function CheckRegressionLat($values, $regressionJson, $testid, $transport, $envS
}

try {
$res.Baseline = $regressionJson.$Testid.$envStr.baseline
$res.BestResult = $regressionJson.$Testid.$envStr.BestResult
$res.BestResultCommit = $regressionJson.$Testid.$envStr.BestResultCommit
$res.Baseline = $regressionJson.$Scenario.$envStr.baseline
$res.BestResult = $regressionJson.$Scenario.$envStr.BestResult
$res.BestResultCommit = $regressionJson.$Scenario.$envStr.BestResultCommit
$res.CumulativeResult = $RpsAvg
$res.AggregateFunction = "AVG"

if ($RpsAvg -lt $res.Baseline) {
Write-GHError "RPS Regression detected in $Testid for $envStr. See summary table for details."
Write-GHError "RPS Regression detected in $Scenario for $envStr. See summary table for details."
$res.HasRegression = $true
}
} catch {
Expand Down
23 changes: 9 additions & 14 deletions scripts/secnetperf.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,11 +254,7 @@ if ($isWindows) {
$allTests = [System.Collections.Specialized.OrderedDictionary]::new()

# > All tests:
$allTests["tput-up"] = "-exec:maxtput -up:12s -ptput:1"
$allTests["tput-down"] = "-exec:maxtput -down:12s -ptput:1"
$allTests["hps-conns-100"] = "-exec:maxtput -rconn:1 -share:1 -conns:100 -run:12s -prate:1"
$allTests["rps-up-512-down-4000"] = "-exec:lowlat -rstream:1 -up:512 -down:4000 -run:20s -plat:1"
$allTests["max-rps-up-512-down-4000"] = "-exec:lowlat -conns:16cpu -streams:10 -rstream:1 -up:512 -down:4000 -run:20s -plat:1"
$allTests = @("upload", "download", "hps", "rps-single", "rps-multi", "latency")
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this is effectively a breaking change, from an identifier tput-up to upload (and so on), and I might need to keep the old ID and map it to the scenario. @ProjectsByJackHe please let me know what you think.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ProjectsByJackHe as we discussed in our meeting yesterday, we need to ensure we're Ok with this change, and then update the dashboard and other netperf files as necessary. Please take a look. Thanks!

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Tracking: microsoft/netperf#401


$hasFailures = $false
$json["run_args"] = $allTests
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, if netperf will always use the default options (stream count, connection count...) for each scenario, then this is fine. But otherwise, we might still need this.

Expand Down Expand Up @@ -336,9 +332,8 @@ $regressionJson = Get-Content -Raw -Path "watermark_regression.json" | ConvertFr

# Run all the test cases.
Write-Host "Setup complete! Running all tests"
foreach ($testId in $allTests.Keys) {
$ExeArgs = $allTests[$testId] + " -io:$io"
$Output = Invoke-Secnetperf $Session $RemoteName $RemoteDir $UserName $SecNetPerfPath $LogProfile $testId $ExeArgs $io $filter $environment $RunId $SyncerSecret
foreach ($scenario in $allTests) {
$Output = Invoke-Secnetperf $Session $RemoteName $RemoteDir $UserName $SecNetPerfPath $LogProfile $scenario $io $filter $environment $RunId $SyncerSecret
$Test = $Output[-1]
if ($Test.HasFailures) { $hasFailures = $true }

Expand All @@ -349,15 +344,15 @@ foreach ($testId in $allTests.Keys) {
} else {
$transport = "quic"
}
$json["$testId-$transport"] = $Test.Values[$tcp]
$json["$scenario-$transport"] = $Test.Values[$tcp]

if ($Test.Metric -eq "latency") {
$json["$testId-$transport-lat"] = $Test.Latency[$tcp]
$LatencyRegression = CheckRegressionLat $Test.Values[$tcp] $regressionJson $testId $transport "$os-$arch-$environment-$io-$tls"
$json["$testId-$transport-regression"] = $LatencyRegression
$json["$scenario-$transport-lat"] = $Test.Latency[$tcp]
$LatencyRegression = CheckRegressionLat $Test.Values[$tcp] $regressionJson $scenario $transport "$os-$arch-$environment-$io-$tls"
$json["$scenario-$transport-regression"] = $LatencyRegression
} else {
$ResultRegression = CheckRegressionResult $Test.Values[$tcp] $testId $transport $regressionJson "$os-$arch-$environment-$io-$tls"
$json["$testId-$transport-regression"] = $ResultRegression
$ResultRegression = CheckRegressionResult $Test.Values[$tcp] $scenario $transport $regressionJson "$os-$arch-$environment-$io-$tls"
$json["$scenario-$transport-regression"] = $ResultRegression
}
}
}
Expand Down
46 changes: 46 additions & 0 deletions src/perf/lib/PerfClient.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,52 @@ PerfClient::Init(

CountMult[0] = CxPlatProcCount();

//
// Scenario profile sets new defauls for values below, that may then be
// further overridden by command line arguments.
//
const char* ScenarioStr = GetValue(argc, argv, "scenario");
if (ScenarioStr != nullptr) {
if (IsValue(ScenarioStr, "upload")) {
Upload = S_TO_US(12); // 12 seconds
Timed = TRUE;
PrintThroughput = TRUE;
} else if (IsValue(ScenarioStr, "download")) {
Download = S_TO_US(12); // 12 seconds
Timed = TRUE;
PrintThroughput = TRUE;
} else if (IsValue(ScenarioStr, "hps")) {
ConnectionCount = 16 * CxPlatProcCount();
RunTime = S_TO_US(12); // 12 seconds
RepeatConnections = TRUE;
PrintIoRate = TRUE;
} else if (IsValue(ScenarioStr, "rps-single")) {
Upload = 512;
Download = 4000;
StreamCount = 100;
RunTime = S_TO_US(20); // 20 seconds
RepeatStreams = TRUE;
PrintLatency = TRUE;
} else if (IsValue(ScenarioStr, "rps-multi")) {
Upload = 512;
Download = 4000;
ConnectionCount = 16 * CxPlatProcCount();
StreamCount = 10;
RunTime = S_TO_US(20); // 20 seconds
RepeatStreams = TRUE;
PrintLatency = TRUE;
} else if (IsValue(ScenarioStr, "latency")) {
Upload = 512;
Download = 4000;
RunTime = S_TO_US(20); // 20 seconds
RepeatStreams = TRUE;
PrintLatency = TRUE;
} else {
WriteOutput("Failed to parse scenario profile[%s]!\n");
return QUIC_STATUS_INVALID_PARAMETER;
}
}

//
// Remote target/server options
//
Expand Down
24 changes: 23 additions & 1 deletion src/perf/lib/SecNetPerfMain.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ PrintHelp(
" -platency<0/1> Print latency statistics. (def:0)\n"
"\n"
" Scenario options:\n"
" -scenario:<profile> Scenario profile to use.\n"
" - {upload, download, hps, rps-single, rps-multi, latency}.\n"
nibanks marked this conversation as resolved.
Show resolved Hide resolved
" -conns:<####> The number of connections to use. (def:1)\n"
" -streams:<####> The number of streams to send on at a time. (def:0)\n"
" -upload:<####>[unit] The length of bytes to send on each stream, with an optional (time or length) unit. (def:0)\n"
Expand Down Expand Up @@ -199,6 +201,25 @@ QuicMainStart(
return Status;
}

const char* ScenarioStr = GetValue(argc, argv, "scenario");
if (ScenarioStr != nullptr) {
if (IsValue(ScenarioStr, "upload") ||
IsValue(ScenarioStr, "download") ||
IsValue(ScenarioStr, "hps")) {
PerfDefaultExecutionProfile = QUIC_EXECUTION_PROFILE_TYPE_MAX_THROUGHPUT;
TcpDefaultExecutionProfile = TCP_EXECUTION_PROFILE_MAX_THROUGHPUT;
} else if (
IsValue(ScenarioStr, "rps-single") ||
IsValue(ScenarioStr, "rps-multi") ||
IsValue(ScenarioStr, "latency")) {
PerfDefaultExecutionProfile = QUIC_EXECUTION_PROFILE_LOW_LATENCY;
TcpDefaultExecutionProfile = TCP_EXECUTION_PROFILE_LOW_LATENCY;
} else {
WriteOutput("Failed to parse scenario profile[%s]!\n");
return QUIC_STATUS_INVALID_PARAMETER;
}
}

const char* ExecStr = GetValue(argc, argv, "exec");
if (ExecStr != nullptr) {
if (IsValue(ExecStr, "lowlat")) {
Expand All @@ -212,7 +233,8 @@ QuicMainStart(
} else if (IsValue(ExecStr, "realtime")) {
PerfDefaultExecutionProfile = QUIC_EXECUTION_PROFILE_TYPE_REAL_TIME;
} else {
WriteOutput("Failed to parse execution profile[%s], use lowlat as default for QUIC, lowlat as default for TCP.\n", ExecStr);
WriteOutput("Failed to parse execution profile[%s]!\n");
return QUIC_STATUS_INVALID_PARAMETER;
}
}

Expand Down
Loading