Refactor QUIC to be compatible with Scenario-Based Secnet runs. (#414)

* still exercise the save pipeline on PR, just don't commit anything. * fixing copilot bugs... * use name to filter out bad files, run ls to print dir structure * these changes are good * all dashboard changes to ensure forwards and backwards compat with secnet data versions * get generate-summary to work
microsoft · Nov 26, 2024 · 0306907 · 0306907
1 parent 34ad1ee
commit 0306907
Show file tree

Hide file tree

Showing 7 changed files with 322 additions and 174 deletions.
diff --git a/.github/workflows/generate-summary.ps1 b/.github/workflows/generate-summary.ps1
@@ -51,10 +51,11 @@ function Write-ThroughputRow {
         else { $row += " $(($Results[$i] / 1000000).ToString('F2')) |" }
     }
 
-    $row += " " + $Regression.CumulativeResult + " |"
-    $row += " " + $Regression.Baseline + " |"
-    if ($Regression.BestResultCommit -eq "N/A") { $row += "N/A |" }
-    else { $row += "[" + $Regression.BestResult + "](https://github.com/microsoft/msquic/commit/" + $Regression.BestResultCommit + ") |" }
+    # TODO: Regression detection is heinously broken. Let's reduce the noise.
+    # $row += " " + $Regression.CumulativeResult + " |"
+    # $row += " " + $Regression.Baseline + " |"
+    # if ($Regression.BestResultCommit -eq "N/A") { $row += "N/A |" }
+    # else { $row += "[" + $Regression.BestResult + "](https://github.com/microsoft/msquic/commit/" + $Regression.BestResultCommit + ") |" }
 
     $Script:markdown += $row
 }
@@ -78,10 +79,11 @@ function Write-HpsRow {
         else { $row += " $($Results[$i]) |" }
     }
 
-    $row += " " + $Regression.CumulativeResult + " |"
-    $row += " " + $Regression.Baseline + " |"
-    if ($Regression.BestResultCommit -eq "N/A") { $row += "N/A |" }
-    else { $row += "[" + $Regression.BestResult + "](https://github.com/microsoft/msquic/commit/" + $Regression.BestResultCommit + ") |" }
+    # TODO: Regression detection is heinously broken. Let's reduce the noise.
+    # $row += " " + $Regression.CumulativeResult + " |"
+    # $row += " " + $Regression.Baseline + " |"
+    # if ($Regression.BestResultCommit -eq "N/A") { $row += "N/A |" }
+    # else { $row += "[" + $Regression.BestResult + "](https://github.com/microsoft/msquic/commit/" + $Regression.BestResultCommit + ") |" }
 
     $Script:markdown += $row
 }
@@ -106,10 +108,11 @@ function Write-RpsRow {
             $row += " $($Results[$i+$j]) |"
         }
 
-        $row += " " + $Regression.CumulativeResult + " |"
-        $row += " " + $Regression.Baseline + " |"
-        if ($Regression.BestResultCommit -eq "N/A") { $row += "N/A |" }
-        else { $row += "[" + $Regression.BestResult + "](https://github.com/microsoft/msquic/commit/" + $Regression.BestResultCommit + ") |" }
+        # TODO: Regression detection is heinously broken. Let's reduce the noise.
+        # $row += " " + $Regression.CumulativeResult + " |"
+        # $row += " " + $Regression.Baseline + " |"
+        # if ($Regression.BestResultCommit -eq "N/A") { $row += "N/A |" }
+        # else { $row += "[" + $Regression.BestResult + "](https://github.com/microsoft/msquic/commit/" + $Regression.BestResultCommit + ") |" }
 
         $Script:markdown += $row
     }
@@ -128,8 +131,8 @@ $hasRegression = $false
 # Write the Upload table.
 $markdown = @"
 # Upload Throughput (Gbps)
-| Pass/Fail | Env | OS | Version | Arch | TLS | IO | Transport | Result 1 | Result 2 | Result 3 | Average | Noise | BestEver |
-| --------- | --- | -- | ------- | ---- | --- | -- | --------- | -------- | -------- | -------- | ------- | ----- | -------- |
+| Pass/Fail | Env | OS | Version | Arch | TLS | IO | Transport | Result 1 | Result 2 | Result 3 |
+| --------- | --- | -- | ------- | ---- | --- | -- | --------- | -------- | -------- | -------- |
 "@
 foreach ($file in $files) {
     Write-Host "Upload Tput: Processing $file..."
@@ -154,31 +157,31 @@ foreach ($file in $files) {
     }
 
     try {
-        $RegressionQuic = $json.'tput-up-quic-regression'
+        $RegressionQuic = $json.'upload-quic-regression'
         $RegressionQuic.Baseline = CleanResult ($RegressionQuic.Baseline / $RegressionQuic.BestResult) # transform baseline to "noise"
         $RegressionQuic.CumulativeResult = CleanResult ($RegressionQuic.CumulativeResult / 1000000)
         $RegressionQuic.BestResult = CleanResult ($RegressionQuic.BestResult / 1000000)
         $hasRegression = $hasRegression -or $RegressionQuic.HasRegression
     } catch { Write-Host $_ }
 
     try {
-        $RegressionTcp = $json.'tput-up-tcp-regression'
+        $RegressionTcp = $json.'upload-tcp-regression'
          $RegressionTcp.Baseline = CleanResult ($RegressionTcp.Baseline / $RegressionTcp.BestResult)
         $RegressionTcp.CumulativeResult = CleanResult ($RegressionTcp.CumulativeResult / 1000000)
         $RegressionTcp.BestResult = CleanResult ($RegressionTcp.BestResult / 1000000)
         $hasRegression = $hasRegression -or $RegressionTcp.HasRegression
     } catch { Write-Host $_ }
 
-    try { Write-ThroughputRow $file.Name "quic" $json.'tput-up-quic' $RegressionQuic } catch { Write-Host $_ }
-    try { Write-ThroughputRow $file.Name "tcp" $json.'tput-up-tcp' $RegressionTcp } catch { Write-Host $_ }
+    try { Write-ThroughputRow $file.Name "quic" $json.'upload-quic' $RegressionQuic } catch { Write-Host $_ }
+    try { Write-ThroughputRow $file.Name "tcp" $json.'upload-tcp' $RegressionTcp } catch { Write-Host $_ }
 }
 
 # Write the Download table.
 $markdown += @"
 `n
 # Download Throughput (Gbps)
-| Pass/Fail | Env | OS | Version | Arch | TLS | IO | Transport | Result 1 | Result 2 | Result 3 | Average | Noise | BestEver |
-| --------- | --- | -- | ------- | ---- | --- | -- | --------- | -------- | -------- | -------- | ------- | ----- | -------- |
+| Pass/Fail | Env | OS | Version | Arch | TLS | IO | Transport | Result 1 | Result 2 | Result 3 |
+| --------- | --- | -- | ------- | ---- | --- | -- | --------- | -------- | -------- | -------- |
 "@
 foreach ($file in $files) {
     Write-Host "Download Tput: Processing $file..."
@@ -202,31 +205,31 @@ foreach ($file in $files) {
         HasRegression = $false
     }
     try {
-        $RegressionQuic = $json.'tput-down-quic-regression'
+        $RegressionQuic = $json.'download-quic-regression'
         $RegressionQuic.Baseline = CleanResult ($RegressionQuic.Baseline / $RegressionQuic.BestResult)
         $RegressionQuic.CumulativeResult = CleanResult ($RegressionQuic.CumulativeResult / 1000000)
         $RegressionQuic.BestResult = CleanResult ($RegressionQuic.BestResult / 1000000)
         $hasRegression = $hasRegression -or $RegressionQuic.HasRegression
     } catch { Write-Host $_ }
 
     try {
-        $RegressionTcp = $json.'tput-down-tcp-regression'
+        $RegressionTcp = $json.'download-tcp-regression'
         $RegressionTcp.Baseline = CleanResult ($RegressionTcp.Baseline / $RegressionTcp.BestResult)
         $RegressionTcp.CumulativeResult = CleanResult ($RegressionTcp.CumulativeResult / 1000000)
         $RegressionTcp.BestResult = CleanResult ($RegressionTcp.BestResult / 1000000)
         $hasRegression = $hasRegression -or $RegressionTcp.HasRegression
     } catch { Write-Host $_ }
 
-    try { Write-ThroughputRow $file.Name "quic" $json.'tput-down-quic' $RegressionQuic } catch { Write-Host $_ }
-    try { Write-ThroughputRow $file.Name "tcp" $json.'tput-down-tcp' $RegressionTcp } catch { Write-Host $_ }
+    try { Write-ThroughputRow $file.Name "quic" $json.'download-quic' $RegressionQuic } catch { Write-Host $_ }
+    try { Write-ThroughputRow $file.Name "tcp" $json.'download-tcp' $RegressionTcp } catch { Write-Host $_ }
 }
 
 # Write the HPS table.
 $markdown += @"
 `n
 # Handshakes Per Second (HPS)
-| Pass/Fail | Env | OS | Version | Arch | TLS | IO | Transport | Result 1 | Result 2 | Result 3 | Average | Noise | BestEver |
-| --------- | --- | -- | ------- | ---- | --- | -- | --------- | -------- | -------- | -------- | ------- | ----- | -------- |
+| Pass/Fail | Env | OS | Version | Arch | TLS | IO | Transport | Result 1 | Result 2 | Result 3 |
+| --------- | --- | -- | ------- | ---- | --- | -- | --------- | -------- | -------- | -------- |
 "@
 foreach ($file in $files) {
     Write-Host "HPS: Processing $file..."
@@ -249,31 +252,31 @@ foreach ($file in $files) {
     }
 
     try {
-        $RegressionQuic = $json.'hps-conns-100-quic-regression'
+        $RegressionQuic = $json.'hps-quic-regression'
         $RegressionQuic.CumulativeResult = CleanResult $RegressionQuic.CumulativeResult
         $RegressionQuic.Baseline = CleanResult ($RegressionQuic.Baseline / $RegressionQuic.BestResult)
         $RegressionQuic.BestResult = CleanResult $RegressionQuic.BestResult
         $hasRegression = $hasRegression -or $RegressionQuic.HasRegression
     } catch { Write-Host $_ }
 
     try {
-        $RegressionTcp = $json.'hps-conns-100-tcp-regression'
+        $RegressionTcp = $json.'hps-tcp-regression'
         $RegressionTcp.CumulativeResult = CleanResult $RegressionTcp.CumulativeResult
         $RegressionTcp.Baseline = CleanResult ($RegressionTcp.Baseline / $RegressionTcp.BestResult)
         $RegressionTcp.BestResult = CleanResult $RegressionTcp.BestResult
         $hasRegression = $hasRegression -or $RegressionTcp.HasRegression
     } catch { Write-Host $_ }
 
-    try { Write-HpsRow $file.Name "quic" $json.'hps-conns-100-quic' $RegressionQuic } catch { Write-Host $_ }
-    try { Write-HpsRow $file.Name "tcp" $json.'hps-conns-100-tcp' $RegressionTcp } catch { Write-Host $_ }
+    try { Write-HpsRow $file.Name "quic" $json.'hps-quic' $RegressionQuic } catch { Write-Host $_ }
+    try { Write-HpsRow $file.Name "tcp" $json.'hps-tcp' $RegressionTcp } catch { Write-Host $_ }
 }
 
 # Write the RPS table.
 $markdown += @"
 `n
 # Request Per Second (RPS) and Latency (µs)
-| Pass/Fail | Env | OS | Version | Arch | TLS | IO | Transport | Min | P50 | P90 | P99 | P99.9 | P99.99 | P99.999 | P99.9999 | RPS | Average | Noise | BestRPS |
-| --------- | --- | -- | ------- | ---- | --- | -- | --------- | --- | --- | --- | --- | ----- | ------ | ------- | -------- | --- | ------- | ----- | ------- |
+| Pass/Fail | Env | OS | Version | Arch | TLS | IO | Transport | Min | P50 | P90 | P99 | P99.9 | P99.99 | P99.999 | P99.9999 | RPS |
+| --------- | --- | -- | ------- | ---- | --- | -- | --------- | --- | --- | --- | --- | ----- | ------ | ------- | -------- | --- |
 "@
 foreach ($file in $files) {
     # TODO: Right now, we are not using a watermark based method for regression detection of latency percentile values because we don't know how to determine a "Best Ever" distribution.
@@ -299,23 +302,23 @@ foreach ($file in $files) {
     }
 
     try {
-        $RegressionQuic = $json.'rps-up-512-down-4000-quic-regression'
+        $RegressionQuic = $json.'rps-quic-regression'
         $RegressionQuic.CumulativeResult = CleanResult ($RegressionQuic.CumulativeResult)
         $RegressionQuic.Baseline = CleanResult ($RegressionQuic.Baseline / $RegressionQuic.BestResult)
         $RegressionQuic.BestResult = CleanResult ($RegressionQuic.BestResult)
         $hasRegression = $hasRegression -or $RegressionQuic.HasRegression
     } catch { Write-Host $_ }
 
     try {
-        $RegressionTcp = $json.'rps-up-512-down-4000-tcp-regression'
+        $RegressionTcp = $json.'rps-tcp-regression'
         $RegressionTcp.CumulativeResult = CleanResult ($RegressionTcp.CumulativeResult)
         $RegressionTcp.Baseline = CleanResult ($RegressionTcp.Baseline / $RegressionTcp.BestResult)
         $RegressionTcp.BestResult = CleanResult ($RegressionTcp.BestResult)
         $hasRegression = $hasRegression -or $RegressionTcp.HasRegression
     } catch { Write-Host $_ }
 
-    try { Write-RpsRow $file.Name "quic" $json.'rps-up-512-down-4000-quic' $RegressionQuic } catch { Write-Host $_ }
-    try { Write-RpsRow $file.Name "tcp" $json.'rps-up-512-down-4000-tcp' $RegressionTcp } catch { Write-Host $_ }
+    try { Write-RpsRow $file.Name "quic" $json.'rps-quic' $RegressionQuic } catch { Write-Host $_ }
+    try { Write-RpsRow $file.Name "tcp" $json.'rps-tcp' $RegressionTcp } catch { Write-Host $_ }
 }
 
 # Write the markdown to the console and to the summary file.

diff --git a/.github/workflows/quic.yml b/.github/workflows/quic.yml
@@ -413,7 +413,6 @@ jobs:
     permissions: write-all
     name: Persist Results in DB
     needs: [run-secnetperf, run-secnetperf-1es] # TODO: Add 'observe-lab' once we fully transition to a stateless lab.
-    if: ${{ (github.event_name == 'repository_dispatch' && github.event.client_payload.pr == '') || inputs.commit }}
     strategy:
       fail-fast: false
     runs-on: 'ubuntu-20.04'
@@ -426,9 +425,27 @@ jobs:
     - uses: actions/download-artifact@8caf195ad4b1dee92908e23f56eeb0696f1dd42d
       with:
         pattern: json-test-results-*
-    - run: python sql.py --featureint 2
+    - name: Remove deprecated python scripts
+      run: rm -rf *.py
+    - name: Fetch sql.py and generate_historical_data.py
+      run: |
+        wget https://raw.githubusercontent.com/microsoft/netperf/main/pipeline/sql.py
+        wget https://raw.githubusercontent.com/microsoft/netperf/main/pipeline/generate_historical_data.py
+    - run: python sql.py
+    - run: python generate_historical_data.py
+    - name: Move all generated historical data json files to a new folder
+      run: |
+        mkdir history_pages
+        mv historical*.json history_pages
+    - run: ls
+    - name: Upload history pages
+      uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882
+      with:
+        name: history_pages
+        path: history_pages
     - run: ls
     - name: Git commit
+      if: ${{ (github.event_name == 'repository_dispatch' && github.event.client_payload.pr == '') || inputs.commit }}
       run: 'git config user.name "QUIC Dev[bot]" && git config user.email "[email protected]" && git add netperf.sqlite && git add full_latencies && git commit -m "Update DB" && git push'
 
   update-intermediary-dashboard-json:
@@ -445,11 +462,21 @@ jobs:
         repository: microsoft/netperf
         ref: deploy
     - run: 'rm -rf *.json'
+    - uses: actions/download-artifact@8caf195ad4b1dee92908e23f56eeb0696f1dd42d
+      with:
+        name: history_pages
+        path: history_pages
+    - run: ls
+    - name: Flatten directory structure
+      run: |
+        mv history_pages/*.json .
+        rm -rf history_pages
     - uses: actions/download-artifact@8caf195ad4b1dee92908e23f56eeb0696f1dd42d
       with:
         pattern: "json-test-results-*"
-    - run: python generate_historical_data.py
+    - run: ls
     - name: Git commit
+      if: ${{ (github.event_name == 'repository_dispatch' && github.event.client_payload.pr == '') || inputs.commit }}
       run: 'git config user.name "QUIC Dev[bot]" && git config user.email "[email protected]" && git pull && git add *.json && git commit -m "Update intermediary dashboard files" && git push'
 
   regression-detection:
@@ -468,5 +495,6 @@ jobs:
         ref: sqlite
     - run: ls
     - run: python regression.py
-    - name: Git commit # TODO: Squash history for this branch to keep size small
+    - name: Git commit
+      if: ${{ (github.event_name == 'repository_dispatch' && github.event.client_payload.pr == '') || inputs.commit }}
       run: 'git config user.name "QUIC Dev[bot]" && git config user.email "[email protected]" && git add *.json && git commit -m "Update regression metrics" && git push'
diff --git a/dashboard/src/pages/hps.jsx b/dashboard/src/pages/hps.jsx
@@ -25,7 +25,30 @@ document.addEventListener('mouseup', function() {
     isMouseDown = false;
 });
 
-// ----------------------------------------------------------------------
+function accessData(envStr, data, newKey, oldKey) {
+  const HISTORY_SIZE = 20;
+  if (!(envStr in data)) {
+    alert(`Could not find ${envStr} in data`);
+    console.error(`Could not find ${envStr} in data`);
+    return [];
+  }
+  const envData = data[envStr];
+  let outputData = [];
+  if (oldKey in envData) {
+    outputData = envData[oldKey]['data'].slice().reverse();
+  } else {
+    console.log("OLD KEY DOES NOT EXIST", oldKey);
+  }
+  if (newKey in envData) {
+    outputData = outputData.concat(envData[newKey]['data'].slice().reverse());
+  } else {
+    console.log("NEW KEY DOES NOT EXIST", newKey);
+  }
+  while (outputData.length > HISTORY_SIZE) {
+    outputData.shift();
+  }
+  return outputData;
+}
 
 export default function HpsPage() {
 
@@ -43,15 +66,15 @@ export default function HpsPage() {
 
   if (data) {
     // TODO: Should we find the max of windows / linux run and use that as our baseline?
-    let rep = data[`${windowsOs}-${env}-iocp-schannel`][`${testType}-tcp`]['data'].slice().reverse();
-    let linuxRep = data[`${linuxOs}-${env}-epoll-openssl`][`${testType}-tcp`]['data'].slice().reverse();
+    let rep = accessData(`${windowsOs}-${env}-iocp-schannel`, data, `hps-tcp`, `${testType}-tcp`);
+    let linuxRep = accessData(`${linuxOs}-${env}-epoll-openssl`, data, `hps-tcp`, `${testType}-tcp`);
     let indices = Array.from({length: Math.max(rep.length, linuxRep.length)}, (_, i) => i);
     indices.reverse();
-    const tcpiocp = data[`${windowsOs}-${env}-iocp-schannel`][`${testType}-tcp`]['data'].slice().reverse();
-    const quiciocp = data[`${windowsOs}-${env}-iocp-schannel`][`${testType}-quic`]['data'].slice().reverse();
-    const tcpepoll = data[`${linuxOs}-${env}-epoll-openssl`][`${testType}-tcp`]['data'].slice().reverse();
-    const quicepoll = data[`${linuxOs}-${env}-epoll-openssl`][`${testType}-quic`]['data'].slice().reverse();
-    const quicxdp = data[`${windowsOs}-${env}-xdp-schannel`][`${testType}-quic`]['data'].slice().reverse();
+    const tcpiocp = accessData(`${windowsOs}-${env}-iocp-schannel`, data, `hps-tcp`, `${testType}-tcp`);
+    const quiciocp = accessData(`${windowsOs}-${env}-iocp-schannel`, data, `hps-quic`, `${testType}-quic`);
+    const tcpepoll = accessData(`${linuxOs}-${env}-epoll-openssl`, data, `hps-tcp`, `${testType}-tcp`);
+    const quicepoll = accessData(`${linuxOs}-${env}-epoll-openssl`, data, `hps-quic`, `${testType}-quic`);
+    const quicxdp = accessData(`${windowsOs}-${env}-xdp-schannel`, data, `hps-quic`, `${testType}-quic`);
     // const quicwsk = data[`${windowsOs}-${env}-wsk-schannel`][`${testType}-quic`]['data'].slice().reverse();
 
     const TCPIOCP = tcpiocp.map(x => x[0]);