Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HW test failure slacking #112

Merged
merged 1 commit into from
Dec 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions ghaf-main-pipeline.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ properties([

// Record failed target(s)
def failedTargets = []

def failedHWTests = []
def target_jobs = [:]

////////////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -136,10 +136,16 @@ pipeline {
servername = sh(script: 'uname -n', returnStdout: true).trim()
echo "Server name:$servername"
def formattedFailedMessage = ""
def formattedHWFailedMessage = ""
def line5=""
def line6=""
if (failedTargets) {
formattedFailedMessage = failedTargets.collect { "- ${it.trim()}" }.join("\n")
} else {
formattedFailedMessage = "No failed build targets"
formattedHWFailedMessage = failedHWTests.collect { "- ${it.trim()}" }.join("\n")
line5="\n*Failed HW test targets:*".stripIndent()
line6="\n${formattedHWFailedMessage}".stripIndent()
}
if (servername=="ghaf-jenkins-controller-prod") {
serverchannel="ghaf-build" // prod main build failures channel
Expand All @@ -152,7 +158,9 @@ pipeline {
${line1}
${line2}
${line3}
${line4}""".stripIndent()
${line4}
${line5}
${line6}""".stripIndent()
slackSend (
channel: "$serverchannel",
color: "danger",
Expand Down
18 changes: 13 additions & 5 deletions utils.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,7 @@ def ghaf_hw_test(String flakeref, String device_config, String testset='_boot_')
// Add a link to failed test job(s) on the calling pipeline
def test_href = "<a href=\"${job.absoluteUrl}\">⛔ ${flakeref_trimmed}</a>"
currentBuild.description = "${currentBuild.description}<br>${test_href}"
return
return flakeref
mnokka-unikie marked this conversation as resolved.
Show resolved Hide resolved
}
// Copy test results from agent to controller to 'test-results' directory
copyArtifacts(
Expand All @@ -243,6 +243,7 @@ def ghaf_hw_test(String flakeref, String device_config, String testset='_boot_')
)
// Archive the test results
archive_artifacts("ghaf-hw-test", flakeref_trimmed)
return null
}

def nix_eval_jobs(List<Map> targets) {
Expand Down Expand Up @@ -310,7 +311,6 @@ def create_parallel_stages(List<Map> targets, String testset='_boot_bat_perf_',
def displayName = "${it.target} (${it.system})"
def targetAttr = "${it.system}.${it.target}"
def scsdir = "scs/${targetAttr}/scs"
def target = "${it.target}"
target_jobs[displayName] = {
stage("Build ${displayName}") {
def opts = ""
Expand All @@ -322,7 +322,7 @@ def create_parallel_stages(List<Map> targets, String testset='_boot_bat_perf_',
try {
if (it.error) {
error("Error in evaluation! ${it.error}")
}
}
timestampBegin = sh(script: "date +%s", returnStdout: true).trim()
sh "nix build -L ${it.drvPath}\\^* ${opts}"
timestampEnd = sh(script: "date +%s", returnStdout: true).trim()
Expand All @@ -339,7 +339,7 @@ def create_parallel_stages(List<Map> targets, String testset='_boot_bat_perf_',
unstable("FAILED: ${displayName}")
currentBuild.result = "FAILURE"
if (failedTargets != null) {
failedTargets.add(target)
failedTargets.add(targetAttr)
}
println "Error: ${e.toString()}"
}
Expand Down Expand Up @@ -421,7 +421,15 @@ def create_parallel_stages(List<Map> targets, String testset='_boot_bat_perf_',
if (testset != null && it.hwtest_device != null) {
stage("Test ${displayName}") {
script {
ghaf_hw_test(targetAttr, it.hwtest_device, testset)
flakeref=ghaf_hw_test(targetAttr, it.hwtest_device, '_boot_bat_perf_')
println ("Test creation and execution done")
if (flakeref==null) {
println("Test was OK")
}
else {
failedHWTests.add(flakeref)
println ("ERROR FOUND, target: ${flakeref}")
}
}
}
}
Expand Down