Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Build the main pipeline in parallel #77

Merged
merged 2 commits into from
Oct 17, 2024
Merged

Build the main pipeline in parallel #77

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
baaffc6
Build the main pipeline in parallel
joinemm Oct 9, 2024
0fcebfe
Remove testagent_ prefix and use device names as labels
joinemm Oct 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 80 additions & 31 deletions ghaf-main-pipeline.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,39 @@ properties([
githubProjectProperty(displayName: '', projectUrlStr: REPO_URL),
])

////////////////////////////////////////////////////////////////////////////////
// Which attribute of the flake to evaluate for building
def flakeAttr = ".#hydraJobs"

// Target names must be direct children of the above
def targets = [
[ target: "docs.aarch64-linux",
hwtest_device: null ],
[ target: "docs.x86_64-linux",
hwtest_device: null ],
[ target: "generic-x86_64-debug.x86_64-linux",
hwtest_device: "nuc" ],
[ target: "lenovo-x1-carbon-gen11-debug.x86_64-linux",
hwtest_device: "lenovo-x1" ],
[ target: "microchip-icicle-kit-debug-from-x86_64.x86_64-linux",
hwtest_device: null ],
[ target: "nvidia-jetson-orin-agx-debug.aarch64-linux",
hwtest_device: "orin-agx" ],
[ target: "nvidia-jetson-orin-agx-debug-from-x86_64.x86_64-linux",
hwtest_device: "orin-agx" ],
[ target: "nvidia-jetson-orin-nx-debug.aarch64-linux",
hwtest_device: "orin-nx" ],
[ target: "nvidia-jetson-orin-nx-debug-from-x86_64.x86_64-linux",
hwtest_device: "orin-nx" ],
]

target_jobs = [:]

pipeline {
agent { label 'built-in' }
triggers {
pollSCM('* * * * *')
}
options {
disableConcurrentBuilds()
timestamps ()
buildDiscarder(logRotator(numToKeepStr: '100'))
}
Expand All @@ -44,43 +68,68 @@ pipeline {
}
}
}
stage('Build x86_64') {
steps {
dir(WORKDIR) {
script {
utils.nix_build('.#packages.x86_64-linux.nvidia-jetson-orin-agx-debug-from-x86_64', 'archive')
utils.nix_build('.#packages.x86_64-linux.nvidia-jetson-orin-nx-debug-from-x86_64', 'archive')
utils.nix_build('.#packages.x86_64-linux.lenovo-x1-carbon-gen11-debug', 'archive')
utils.nix_build('.#packages.x86_64-linux.microchip-icicle-kit-debug-from-x86_64', 'archive')
utils.nix_build('.#packages.x86_64-linux.generic-x86_64-debug', 'archive')
utils.nix_build('.#packages.x86_64-linux.doc')
}
}
}
}
stage('Build aarch64') {

stage('Evaluate') {
steps {
dir(WORKDIR) {
script {
utils.nix_build('.#packages.aarch64-linux.nvidia-jetson-orin-agx-debug', 'archive')
utils.nix_build('.#packages.aarch64-linux.nvidia-jetson-orin-nx-debug', 'archive')
utils.nix_build('.#packages.aarch64-linux.doc')
// nix-eval-jobs is used to evaluate the given flake attribute, and output target information into jobs.json
sh "nix-eval-jobs --gc-roots-dir gcroots --flake ${flakeAttr} --force-recurse > jobs.json"

// jobs.json is parsed using jq. target's name and derivation path are appended as space separated row into jobs.txt
sh "jq -r '.attr + \" \" + .drvPath' < jobs.json > jobs.txt"

targets.each {
def target = it['target']

// row that matches this target is grepped from jobs.txt, extracting the pre-evaluated derivation path
def drvPath = sh (script: "cat jobs.txt | grep ${target} | cut -d ' ' -f 2", returnStdout: true).trim()

target_jobs[target] = {
stage("Build ${target}") {
def opts = ""
if (it['hwtest_device'] != null) {
opts = "--out-link archive/${target}"
} else {
opts = "--no-link"
}
try {
if (drvPath) {
sh "nix build -L ${drvPath}\\^* ${opts}"
} else {
error("Target \"${target}\" was not found in ${flakeAttr}")
}
} catch (InterruptedException e) {
throw e
} catch (Exception e) {
unstable("FAILED: ${target}")
currentBuild.result = "FAILURE"
println "Error: ${e.toString()}"
}
}

if (it['hwtest_device'] != null) {
stage("Archive ${target}") {
script {
utils.archive_artifacts("archive", target)
}
}

stage("Test ${target}") {
vjuntunen marked this conversation as resolved.
Show resolved Hide resolved
utils.ghaf_parallel_hw_test(target, it['hwtest_device'])
}
}
}
}
}
}
}
}
stage('HW test') {

stage('Build targets') {
steps {
dir(WORKDIR) {
script {
utils.ghaf_hw_test('.#packages.x86_64-linux.nvidia-jetson-orin-agx-debug-from-x86_64', 'orin-agx')
utils.ghaf_hw_test('.#packages.aarch64-linux.nvidia-jetson-orin-agx-debug', 'orin-agx')
utils.ghaf_hw_test('.#packages.x86_64-linux.nvidia-jetson-orin-nx-debug-from-x86_64', 'orin-nx')
utils.ghaf_hw_test('.#packages.aarch64-linux.nvidia-jetson-orin-nx-debug', 'orin-nx')
utils.ghaf_hw_test('.#packages.x86_64-linux.lenovo-x1-carbon-gen11-debug', 'lenovo-x1')
utils.ghaf_hw_test('.#packages.x86_64-linux.generic-x86_64-debug', 'nuc')
utils.ghaf_hw_test('.#packages.x86_64-linux.microchip-icicle-kit-debug-from-x86_64', 'riscv')
}
script {
parallel target_jobs
}
}
}
Expand Down
8 changes: 4 additions & 4 deletions utils.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ def sign_file(String path, String sigfile, String cert="INT-Ghaf-Devenv-Common")
}

def ghaf_hw_test(String flakeref, String device_config, String testset='_boot_') {
testagent_nodes = nodesByLabel(label: 'testagent', offline: false)
testagent_nodes = nodesByLabel(label: "$device_config", offline: false)
if (!testagent_nodes) {
println "Warning: Skipping HW test '$flakeref', no test agents online"
unstable("No test agents online")
Expand Down Expand Up @@ -217,7 +217,7 @@ def ghaf_hw_test(String flakeref, String device_config, String testset='_boot_')
job: "ghaf-hw-test",
propagate: false,
parameters: [
string(name: "LABEL", value: "testagent"),
string(name: "LABEL", value: "$device_config"),
string(name: "DEVICE_CONFIG_NAME", value: "$device_config"),
string(name: "IMG_URL", value: "$img_url"),
string(name: "DESC", value: "$description"),
Expand Down Expand Up @@ -247,7 +247,7 @@ def ghaf_hw_test(String flakeref, String device_config, String testset='_boot_')
}

def ghaf_parallel_hw_test(String flakeref, String device_config, String testset='_boot_') {
testagent_nodes = nodesByLabel(label: "testagent_$device_config", offline: false)
testagent_nodes = nodesByLabel(label: "$device_config", offline: false)
if (!testagent_nodes) {
println "Warning: Skipping HW test '$flakeref', no test agents online"
unstable("No test agents online")
Expand Down Expand Up @@ -275,7 +275,7 @@ def ghaf_parallel_hw_test(String flakeref, String device_config, String testset=
job: "ghaf-parallel-hw-test",
propagate: false,
parameters: [
string(name: "LABEL", value: "testagent_$device_config"),
string(name: "LABEL", value: "$device_config"),
string(name: "DEVICE_CONFIG_NAME", value: "$device_config"),
string(name: "IMG_URL", value: "$img_url"),
string(name: "DESC", value: "$description"),
Expand Down
Loading