From e596b41c9caeaa3cd40d3116ffbdea660415cc86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 26 Nov 2024 14:02:35 +0200 Subject: [PATCH] Refactor utils.py 1. Introduce typehints 2. Include what you use. --- common_factories.py | 30 ++- master-bintars/master.cfg | 2 +- master-docker-nonstandard-2/master.cfg | 14 +- master-docker-nonstandard/master.cfg | 9 +- master-galera/master.cfg | 8 +- master-libvirt/master.cfg | 1 + master-nonlatent/master.cfg | 6 +- master-protected-branches/master.cfg | 3 - master.cfg | 10 +- utils.py | 277 ++++++++++++------------- 10 files changed, 184 insertions(+), 176 deletions(-) diff --git a/common_factories.py b/common_factories.py index f31a7f31..47407024 100644 --- a/common_factories.py +++ b/common_factories.py @@ -142,7 +142,7 @@ def addPostTests(factory): steps.SetPropertyFromCommand( command="basename mariadb-*-linux-*.tar.gz", property="mariadb_binary", - doStepIf=savePackage, + doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), ) ) factory.addStep( @@ -158,7 +158,7 @@ def addPostTests(factory): && sync /packages/%(prop:tarbuildnum)s """ ), - doStepIf=savePackage, + doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), ) ) factory.addStep( @@ -174,7 +174,10 @@ def addPostTests(factory): "master_branch": Property("master_branch"), "parentbuildername": Property("buildername"), }, - doStepIf=lambda step: savePackage(step) and hasEco(step), + doStepIf=( + lambda step: savePackageIfBranchMatch(step, savedPackageBranches) + and hasEco(step) + ), ) ) factory.addStep( @@ -747,7 +750,10 @@ def getRpmAutobakeFactory(mtrDbPool): sync /packages/%(prop:tarbuildnum)s """ ), - doStepIf=lambda step: hasFiles(step) and savePackage(step), + doStepIf=( + lambda step: hasFiles(step) + and savePackageIfBranchMatch(step, savedPackageBranches) + ), descriptionDone=util.Interpolate( """ Repository available with: curl %(kw:url)s/%(prop:tarbuildnum)s/%(prop:buildername)s/MariaDB.repo -o /etc/yum.repos.d/MariaDB.repo""", @@ -784,9 +790,11 @@ def getRpmAutobakeFactory(mtrDbPool): "master_branch": Property("master_branch"), "parentbuildername": Property("buildername"), }, - doStepIf=lambda step: hasInstall(step) - and savePackage(step) - and hasFiles(step), + doStepIf=( + lambda step: hasInstall(step) + and savePackageIfBranchMatch(step, savedPackageBranches) + and hasFiles(step) + ), ) ) f_rpm_autobake.addStep( @@ -801,9 +809,11 @@ def getRpmAutobakeFactory(mtrDbPool): "master_branch": Property("master_branch"), "parentbuildername": Property("buildername"), }, - doStepIf=lambda step: hasUpgrade(step) - and savePackage(step) - and hasFiles(step), + doStepIf=( + lambda step: hasUpgrade(step) + and savePackageIfBranchMatch(step, savedPackageBranches) + and hasFiles(step) + ), ) ) f_rpm_autobake.addStep( diff --git a/master-bintars/master.cfg b/master-bintars/master.cfg index 22dd4327..e608ef60 100644 --- a/master-bintars/master.cfg +++ b/master-bintars/master.cfg @@ -265,7 +265,7 @@ def getBintarFactory( + " && sync /packages/" + "%(prop:tarbuildnum)s" ), - doStepIf=savePackage, + doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), ) ) diff --git a/master-docker-nonstandard-2/master.cfg b/master-docker-nonstandard-2/master.cfg index fb8b6305..1012f71f 100644 --- a/master-docker-nonstandard-2/master.cfg +++ b/master-docker-nonstandard-2/master.cfg @@ -502,7 +502,6 @@ f_big_test.addStep( command="basename mariadb-*-linux-*.tar.gz", property="mariadb_binary" ) ) -# f_big_test.addStep(steps.ShellCommand(name='save_packages', timeout=7200, haltOnFailure=True, command=util.Interpolate('mkdir -p ' + '/packages/' + '%(prop:tarbuildnum)s' + '/' + '%(prop:buildername)s'+ ' && sha256sum %(prop:mariadb_binary)s >> sha256sums.txt && cp ' + '%(prop:mariadb_binary)s sha256sums.txt' + ' /packages/' + '%(prop:tarbuildnum)s' + '/' + '%(prop:buildername)s' + '/' + ' && sync /packages/' + '%(prop:tarbuildnum)s'), doStepIf=savePackage)) f_big_test.addStep( steps.ShellCommand( name="cleanup", command="rm -r * .* 2> /dev/null || true", alwaysRun=True @@ -757,7 +756,8 @@ f_without_server.addStep( + " && sync /packages/" + "%(prop:tarbuildnum)s" ), - doStepIf=savePackage, + doStepIf=lambda step: savePackageIfBranchMatch(step, + savedPackageBranches), ) ) f_without_server.addStep( @@ -1044,7 +1044,8 @@ f_bintar.addStep( steps.SetPropertyFromCommand( command="basename mariadb-*-linux-*.tar.gz", property="mariadb_binary", - doStepIf=savePackage, + doStepIf=lambda step: savePackageIfBranchMatch(step, + savedPackageBranches), ) ) f_bintar.addStep( @@ -1068,7 +1069,8 @@ f_bintar.addStep( + " && sync /packages/" + "%(prop:tarbuildnum)s" ), - doStepIf=savePackage, + doStepIf=lambda step: savePackageIfBranchMatch(step, + savedPackageBranches), ) ) f_bintar.addStep( @@ -1085,7 +1087,9 @@ f_bintar.addStep( "master_branch": Property("master_branch"), "parentbuildername": Property("buildername"), }, - doStepIf=lambda step: savePackage(step) and hasEco(step), + doStepIf=(lambda step: savePackageIfBranchMatch(step, + savedPackageBranches) + and hasEco(step)), ) ) f_bintar.addStep( diff --git a/master-docker-nonstandard/master.cfg b/master-docker-nonstandard/master.cfg index 992739a7..c229674f 100644 --- a/master-docker-nonstandard/master.cfg +++ b/master-docker-nonstandard/master.cfg @@ -997,7 +997,6 @@ f_big_test.addStep( command="basename mariadb-*-linux-*.tar.gz", property="mariadb_binary" ) ) -# f_big_test.addStep(steps.ShellCommand(name='save_packages', timeout=7200, haltOnFailure=True, command=util.Interpolate('mkdir -p ' + '/packages/' + '%(prop:tarbuildnum)s' + '/' + '%(prop:buildername)s'+ ' && sha256sum %(prop:mariadb_binary)s >> sha256sums.txt && cp ' + '%(prop:mariadb_binary)s sha256sums.txt' + ' /packages/' + '%(prop:tarbuildnum)s' + '/' + '%(prop:buildername)s' + '/' + ' && sync /packages/' + '%(prop:tarbuildnum)s'), doStepIf=savePackage)) f_big_test.addStep( steps.ShellCommand( name="cleanup", command="rm -r * .* 2> /dev/null || true", alwaysRun=True @@ -1134,7 +1133,7 @@ f_without_server.addStep( + " && sync /packages/" + "%(prop:tarbuildnum)s" ), - doStepIf=savePackage, + doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), ) ) f_without_server.addStep( @@ -1422,7 +1421,7 @@ f_bintar.addStep( steps.SetPropertyFromCommand( command="basename mariadb-*-linux-*.tar.gz", property="mariadb_binary", - doStepIf=savePackage, + doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), ) ) f_bintar.addStep( @@ -1446,7 +1445,7 @@ f_bintar.addStep( + " && sync /packages/" + "%(prop:tarbuildnum)s" ), - doStepIf=savePackage, + doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), ) ) f_bintar.addStep( @@ -1463,7 +1462,7 @@ f_bintar.addStep( "master_branch": Property("master_branch"), "parentbuildername": Property("buildername"), }, - doStepIf=lambda step: savePackage(step) and hasEco(step), + doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches) and hasEco(step), ) ) f_bintar.addStep( diff --git a/master-galera/master.cfg b/master-galera/master.cfg index 00a47898..c14ea5c1 100644 --- a/master-galera/master.cfg +++ b/master-galera/master.cfg @@ -265,7 +265,7 @@ def dpkgDeb(): """ ), ], - doStepIf=lambda step: savePackage(step, savedPackageBranches), + doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), ) @@ -284,7 +284,7 @@ def rpmSave(): """ ), ], - doStepIf=lambda step: savePackage(step, savedPackageBranches), + doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), ) @@ -366,7 +366,7 @@ EOF """, url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"), ), - doStepIf=lambda step: savePackage(step, savedPackageBranches), + doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), ) ) f_deb_build.addStep( @@ -423,7 +423,7 @@ EOF """, url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"), ), - doStepIf=lambda step: savePackage(step, savedPackageBranches), + doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), ) ) f_rpm_build.addStep( diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg index 2838d542..e35b064a 100644 --- a/master-libvirt/master.cfg +++ b/master-libvirt/master.cfg @@ -19,6 +19,7 @@ sys.setrecursionlimit(10000) sys.path.insert(0, "/srv/buildbot/master") from utils import * +from constants import os_info # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. diff --git a/master-nonlatent/master.cfg b/master-nonlatent/master.cfg index beb77370..7213b84f 100644 --- a/master-nonlatent/master.cfg +++ b/master-nonlatent/master.cfg @@ -535,7 +535,8 @@ f_windows_msi.addStep( '/' ), - doStepIf=savePackage, + doStepIf=lambda step: savePackageIfBranchMatch(step, + savedPackageBranches), ) ) f_windows_msi.addStep( @@ -557,7 +558,8 @@ f_windows_msi.addStep( '%(prop:buildername)s' '/' ), - doStepIf=savePackage, + doStepIf=lambda step: savePackageIfBranchMatch(step, + savedPackageBranches), ) ) f_windows_msi.addStep( diff --git a/master-protected-branches/master.cfg b/master-protected-branches/master.cfg index 91b7aa5b..714c0c80 100644 --- a/master-protected-branches/master.cfg +++ b/master-protected-branches/master.cfg @@ -422,9 +422,6 @@ f_tarball.addStep( doStepIf=lambda step: isStagingBranch(step), ) ) -# f_tarball.addStep(steps.ShellSequence( commands=[ -# util.ShellArg(command=util.Interpolate("git checkout " + "%(prop:staging_branch)s"), logfile="rebase"), -# util.ShellArg(command=util.Interpolate("git merge %(prop:branch)s"), logfile="rebase")], workdir="build/server", haltOnFailure="true", doStepIf=ifStagingSucceeding)) f_tarball.addStep( steps.ShellCommand( name="cleanup", command="rm -r * .* 2> /dev/null || true", alwaysRun=True diff --git a/master.cfg b/master.cfg index 1ec15fcc..17cdea0c 100644 --- a/master.cfg +++ b/master.cfg @@ -235,7 +235,7 @@ f_deb_autobake.addStep( "master_branch": Property("master_branch"), "parentbuildername": Property("buildername"), }, - doStepIf=lambda step: savePackage(step), + doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches), ) ) f_deb_autobake.addStep( @@ -250,7 +250,9 @@ f_deb_autobake.addStep( "master_branch": Property("master_branch"), "parentbuildername": Property("buildername"), }, - doStepIf=lambda step: hasInstall(step) and savePackage(step) and hasFiles(step), + doStepIf=(lambda step: hasInstall(step) + and savePackageIfBranchMatch(step, savedPackageBranches) + and hasFiles(step)), ) ) f_deb_autobake.addStep( @@ -265,7 +267,9 @@ f_deb_autobake.addStep( "master_branch": Property("master_branch"), "parentbuildername": Property("buildername"), }, - doStepIf=lambda step: hasUpgrade(step) and savePackage(step) and hasFiles(step), + doStepIf=(lambda step: hasUpgrade(step) + and savePackageIfBranchMatch(step, savedPackageBranches) + and hasFiles(step)), ) ) f_deb_autobake.addStep( diff --git a/utils.py b/utils.py index 1027d8ce..6332961c 100644 --- a/utils.py +++ b/utils.py @@ -1,20 +1,21 @@ import fnmatch import os import re -import sys -from datetime import datetime, timedelta +from datetime import datetime +from typing import Any, Tuple import docker from pyzabbix import ZabbixAPI -from twisted.internet import defer +from buildbot.buildrequest import BuildRequest +from buildbot.interfaces import IProperties +from buildbot.master import BuildMaster from buildbot.plugins import steps, util, worker -from buildbot.process.properties import Properties, Property -from buildbot.process.remotecommand import RemoteCommand +from buildbot.process.builder import Builder +from buildbot.process.buildstep import BuildStep from buildbot.process.results import FAILURE -from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver -from buildbot.steps.shell import Compile, SetPropertyFromCommand, ShellCommand, Test -from buildbot.steps.source.github import GitHub +from buildbot.process.workerforbuilder import AbstractWorkerForBuilder +from buildbot.worker import AbstractWorker from constants import ( DEVELOPMENT_BRANCH, MTR_ENV, @@ -25,7 +26,6 @@ builders_install, builders_s3_mtr, builders_upgrade, - os_info, releaseBranches, savedPackageBranches, ) @@ -34,8 +34,8 @@ exec(open("/srv/buildbot/master/master-private.cfg").read(), private_config, {}) -def envFromProperties(envlist): - d = dict() +def envFromProperties(envlist: list[str]) -> dict[str, str]: + d = {} for e in envlist: d[e] = util.Interpolate(f"%(prop:{e})s") d["tarbuildnum"] = util.Interpolate("%(prop:tarbuildnum)s") @@ -43,7 +43,7 @@ def envFromProperties(envlist): return d -def getScript(scriptname): +def getScript(scriptname: str) -> steps.ShellCommand: branch = os.getenv("BRANCH", default="main") return steps.ShellCommand( name=f"fetch_{scriptname}", @@ -68,25 +68,22 @@ def getScript(scriptname): # Helper function that creates a worker instance. def createWorker( - worker_name_prefix, - worker_id, - worker_type, - dockerfile, - jobs=5, - save_packages=False, - shm_size="15G", - worker_name_suffix="", - volumes=[ + worker_name_prefix: str, + worker_id: str, + worker_type: str, + dockerfile: str, + jobs: int = 5, + save_packages: bool = False, + shm_size: str = "15G", + worker_name_suffix: str = "", + volumes: list[str] = [ "/srv/buildbot/ccache:/mnt/ccache", "/srv/buildbot/packages:/mnt/packages", MASTER_PACKAGES + "/:/packages", ], -): - worker_name = worker_name_prefix + str(worker_id) + "-docker" - name = worker_name + worker_type - - i = worker_id - tls = None +) -> Tuple[str, worker.DockerLatentWorker]: + worker_name = f"{worker_name_prefix}{worker_id}-docker" + name = f"{worker_name}{worker_type}" if worker_name_prefix.startswith("hz"): b_name = "x64-bbw" @@ -123,7 +120,7 @@ def createWorker( docker_host=private_config["private"]["docker_workers"][worker_name], image=image_str, dockerfile=dockerfile_str, - tls=tls, + tls=None, autopull=True, alwaysPull=need_pull, followStartupLogs=False, @@ -143,8 +140,8 @@ def createWorker( return ((base_name, name + worker_name_suffix), worker_instance) -def printEnv(): - return ShellCommand( +def printEnv() -> steps.ShellCommand: + return steps.ShellCommand( name="Environment details", command=[ "bash", @@ -162,8 +159,8 @@ def printEnv(): ) -def getSourceTarball(): - return ShellCommand( +def getSourceTarball() -> steps.ShellCommand: + return steps.ShellCommand( name="get_tarball", description="get source tarball", descriptionDone="get source tarball...done", @@ -181,8 +178,8 @@ def getSourceTarball(): ) -def saveLogs(): - return ShellCommand( +def saveLogs() -> steps.ShellCommand: + return steps.ShellCommand( name="Save logs", description="saving logs", descriptionDone="save logs...done", @@ -201,8 +198,8 @@ def saveLogs(): ) -def createDebRepo(): - return ShellCommand( +def createDebRepo() -> steps.ShellCommand: + return steps.ShellCommand( name="Create deb repository", haltOnFailure=True, command=[ @@ -219,12 +216,15 @@ def createDebRepo(): """ ), ], - doStepIf=lambda step: hasFiles(step) and savePackage(step), + doStepIf=( + lambda step: hasFiles(step) + and savePackageIfBranchMatch(step, savedPackageBranches) + ), ) -def uploadDebArtifacts(): - return ShellCommand( +def uploadDebArtifacts() -> steps.ShellCommand: + return steps.ShellCommand( name="Upload artifacts", timeout=7200, haltOnFailure=True, @@ -258,7 +258,10 @@ def uploadDebArtifacts(): """ ), ], - doStepIf=lambda step: hasFiles(step) and savePackage(step), + doStepIf=( + lambda step: hasFiles(step) + and savePackageIfBranchMatch(step, savedPackageBranches) + ), descriptionDone=util.Interpolate( """ Use """ @@ -269,16 +272,17 @@ def uploadDebArtifacts(): ) -def staging_branch_fn(branch): +def staging_branch_fn(branch: str) -> bool: return fnmatch.fnmatch(branch, "prot-st-*") -def fnmatch_any(s, list_of_patterns): - return any(fnmatch.fnmatch(s, p) for p in list_of_patterns) +def fnmatch_any(branch: str, patterns: list[str]) -> bool: + return any(fnmatch.fnmatch(branch, pattern) for pattern in patterns) # Priority filter based on saved package branches -def nextBuild(bldr, requests): +def nextBuild(builder: Builder, + requests: list[BuildRequest]) -> str: for r in requests: if fnmatch_any(r.sources[""].branch, releaseBranches): return r @@ -288,13 +292,17 @@ def nextBuild(bldr, requests): return requests[0] -def canStartBuild(builder, wfb, request): - worker = wfb.worker - if not "s390x" in worker.name: +def canStartBuild( + builder: Builder, wfb: AbstractWorkerForBuilder, request: BuildRequest +) -> bool: + worker: AbstractWorker = wfb.worker + if "s390x" not in worker.name: return True - worker_prefix = "-".join((worker.name).split("-")[0:2]) + worker_prefix = "-".join(worker.name.split("-")[0:2]) worker_name = private_config["private"]["worker_name_mapping"][worker_prefix] + # TODO(cvicentiu) this could be done with a yield to not have the master + # stuck until the network operation is completed. load = getMetric(worker_name, "BB_accept_new_build") if float(load) > 60: @@ -309,42 +317,45 @@ def canStartBuild(builder, wfb, request): @util.renderer -def mtrJobsMultiplier(props): +def mtrJobsMultiplier(props: IProperties) -> int: jobs = props.getProperty("jobs", default=20) return jobs * 2 -# ls2string gets the output of ls and returns a space delimited string with the files and directories -def ls2string(rc, stdout, stderr): - lsFilenames = [] +# ls2string gets the output of ls and returns a space delimited string with +# the files and directories +def ls2string(rc: int, stdout: str, stderr: str) -> dict[str, str]: + ls_filenames = [] + for line in stdout.strip().split("\n"): + line = line.strip() + if not line: + ls_filenames.append(line) - for l in stdout.strip().split("\n"): - if l != "": - lsFilenames.append(l.strip()) + return {"packages": " ".join(ls_filenames)} - return {"packages": " ".join(lsFilenames)} +# ls2list gets the output of ls and returns a list with the files and +# directories +def ls2list(rc: int, stdout: str, stderr: str) -> dict[str, list[str]]: + ls_filenames = [] -# ls2list gets the output of ls and returns a list with the files and directories -def ls2list(rc, stdout, stderr): - lsFilenames = [] + for line in stdout.strip().split("\n"): + line = line.strip() + if not line: + ls_filenames.append(line) - for l in stdout.strip().split("\n"): - if l != "": - lsFilenames.append(l.strip()) - - return {"packages": lsFilenames} + return {"packages": ls_filenames} # Save packages for current branch? -def savePackage(step, savedBranches=savedPackageBranches): +def savePackageIfBranchMatch(step: BuildStep, branch_match: str) -> bool: return step.getProperty("save_packages") and fnmatch_any( - step.getProperty("branch"), savedBranches + step.getProperty("branch"), branch_match ) # Return a HTML file that contains links to MTR logs -def getHTMLLogString(base_path="./buildbot"): +def getHTMLLogString(base_path: str = "./buildbot") -> str: return f""" mkdir -p {base_path} echo ' @@ -357,11 +368,11 @@ def getHTMLLogString(base_path="./buildbot"): ' >> {base_path}/mysql_logs.html""" -def hasFailed(step): +def hasFailed(step: BuildStep) -> bool: return step.build.results == FAILURE -def createVar(base_path="./buildbot", output_dir=""): +def createVar(base_path: str = "./buildbot", output_dir: str = "") -> str: return f""" if [[ -d ./mysql-test/var ]]; then typeset -r var_tarball_list="var_tarball_list.txt" @@ -393,7 +404,7 @@ def createVar(base_path="./buildbot", output_dir=""): # Function to move the MTR logs to a known location so that they can be saved -def moveMTRLogs(base_path="./buildbot", output_dir=""): +def moveMTRLogs(base_path: str = "./buildbot", output_dir: str = "") -> str: return f""" mkdir -p {base_path}/logs/{output_dir} @@ -429,7 +440,7 @@ def moveMTRLogs(base_path="./buildbot", output_dir=""): @util.renderer -def dockerfile(props): +def dockerfile(props: IProperties) -> str: worker = props.getProperty("workername") return ( "https://github.com/MariaDB/buildbot/tree/main/dockerfiles/" @@ -439,40 +450,35 @@ def dockerfile(props): # checks if the list of files is empty -def hasFiles(step): - if len(step.getProperty("packages")) < 1: - return False - return True - +def hasFiles(step: BuildStep) -> bool: + return len(step.getProperty("packages")) >= 1 -def hasInstall(props): - builderName = str(props.getProperty("buildername")) +def hasInstall(step: BuildStep) -> bool: + builder_name = step.getProperty("buildername") for b in builders_install: - if builderName in b: + if builder_name in b: return True return False -def hasUpgrade(props): - builderName = str(props.getProperty("buildername")) - +def hasUpgrade(step: BuildStep) -> bool: + builder_name = step.getProperty("buildername") for b in builders_upgrade: - if builderName in b: + if builder_name in b: return True return False -def hasEco(props): - builderName = str(props.getProperty("buildername")) - +def hasEco(step: BuildStep) -> bool: + builder_name = step.getProperty("buildername") for b in builders_eco: - if builderName in b: + if builder_name in b: return True return False -def hasCompat(step): +def hasCompat(step: BuildStep) -> bool: builderName = str(step.getProperty("buildername")) # For s390x and the listed distros there are no compat files @@ -494,13 +500,13 @@ def hasCompat(step): return True -def hasDockerLibrary(step): +def hasDockerLibrary(step: BuildStep) -> bool: # Can only build with a saved package - if not savePackage(step): + if not savePackageIfBranchMatch(step, savedPackageBranches): return False - branch = str(step.getProperty("master_branch")) - builderName = str(step.getProperty("buildername")) + branch = step.getProperty("master_branch") + builder_name = step.getProperty("buildername") # from https://github.com/MariaDB/mariadb-docker/blob/next/update.sh#L7-L15 if fnmatch.fnmatch(branch, "10.[56]"): @@ -511,14 +517,14 @@ def hasDockerLibrary(step): dockerbase = "ubuntu-2404-deb-autobake" # UBI images - if branch != "10.5" and builderName.endswith("amd64-rhel-9-rpm-autobake"): + if branch != "10.5" and builder_name.endswith("amd64-rhel-9-rpm-autobake"): return True # We only build on the above autobakes for all architectures - return builderName.endswith(dockerbase) + return builder_name.endswith(dockerbase) -def filterBranch(step): +def filterBranch(step: BuildStep) -> str: if "10.5" in step.getProperty("branch"): return False if "10.6" in step.getProperty("branch"): @@ -527,84 +533,72 @@ def filterBranch(step): # check if branch is a staging branch -def isStagingBranch(step): - if staging_branch_fn(step.getProperty("branch")): - return True - else: - return False - - -# returns true if build is succeeding -def ifStagingSucceeding(step): - if isStagingBranch(step): - step.setProperty("build_results", step.build.results) - return step.build.results in ("SUCCESS", "WARNINGS") - else: - return False +def isStagingBranch(step: BuildStep) -> bool: + return staging_branch_fn(step.getProperty("branch")) # set step's waitForFinish to True if staging branch -def waitIfStaging(step): +def waitIfStaging(step: BuildStep) -> bool: if isStagingBranch(step): step.waitForFinish = True return True -def hasAutobake(props): - builderName = props.getProperty("buildername") +def hasAutobake(step: BuildStep) -> bool: + builder_name = step.getProperty("buildername") for b in builders_autobake: - if builderName in b: + if builder_name in b: return True return False -def hasGalera(props): - builderName = str(props.getProperty("buildername")) - +def hasGalera(step: BuildStep) -> bool: + builder_name = step.getProperty("buildername") for b in builders_galera_mtr: - if builderName in b: + if builder_name in b: return True return False def hasS3(props): - builderName = str(props.getProperty("buildername")) - + builder_name = props.getProperty("buildername") for b in builders_s3_mtr: - if builderName == b: + if builder_name == b: return True return False -def hasBigtest(props): - builderName = str(props.getProperty("buildername")) - +def hasBigtest(step: BuildStep) -> bool: + builder_name = step.getProperty("buildername") for b in builders_big: - if builderName in b: + if builder_name in b: return True return False -def hasRpmLint(step): - builderName = str(step.getProperty("buildername")) +def hasRpmLint(step: BuildStep) -> str: + builder_name = step.getProperty("buildername") # The step fails on s390x SLES 12 due to permissions issues - if "s390x-sles-12" in builderName: + if "s390x-sles-12" in builder_name: return False return True @util.renderer -def getArch(props): +def getArch(props: IProperties) -> str: buildername = props.getProperty("buildername") return buildername.split("-")[0] -##### Builder priority -# Prioritize builders. At this point, only the Windows builders need a higher priority -# since the others run on dedicated machines. -def prioritizeBuilders(buildmaster, builders): - """amd64-windows-* builders should have the highest priority since they are used for - protected branches""" +# Builder priority +# +# Prioritize builders. At this point, only the Windows builders need a higher +# priority since the others run on dedicated machines. +def prioritizeBuilders( + buildmaster: BuildMaster, builders: list[Builder] +) -> list[Builder]: + """amd64-windows-* builders should have the highest priority since they are + used for protected branches""" builderPriorities = { "amd64-windows": 0, "amd64-windows-packages": 1, @@ -613,13 +607,11 @@ def prioritizeBuilders(buildmaster, builders): return builders -##### Zabbix helper -def getMetric(hostname, metric): +# Zabbix helper +def getMetric(hostname: str, metric: str) -> Any: # set API zapi = ZabbixAPI(private_config["private"]["zabbix_server"]) - zapi.session.verify = True - zapi.timeout = 10 zapi.login(api_token=private_config["private"]["zabbix_token"]) @@ -648,17 +640,17 @@ def getMetric(hostname, metric): return last_value -def read_template(template_name): +def read_template(template_name: str) -> str: with open(f"/srv/buildbot/master/script_templates/{template_name}.sh") as f: return f.read() -def isJepsenBranch(step): +def isJepsenBranch(step: BuildStep) -> bool: return step.getProperty("branch").startswith("jpsn") @util.renderer -def mtrEnv(props) -> dict: +def mtrEnv(props: IProperties) -> dict: """ Renders the MTR environment variables. @@ -678,5 +670,4 @@ def mtrEnv(props) -> dict: if key not in mtr_add_env: mtr_add_env[key] = value return mtr_add_env - else: - return MTR_ENV + return MTR_ENV