Refactor utils.py

1. Introduce typehints 2. Include what you use.
MariaDB · Jan 7, 2025 · e596b41 · e596b41
1 parent 9aa4446
commit e596b41
Show file tree

Hide file tree

Showing 10 changed files with 184 additions and 176 deletions.
diff --git a/common_factories.py b/common_factories.py
@@ -142,7 +142,7 @@ def addPostTests(factory):
         steps.SetPropertyFromCommand(
             command="basename mariadb-*-linux-*.tar.gz",
             property="mariadb_binary",
-            doStepIf=savePackage,
+            doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches),
         )
     )
     factory.addStep(
@@ -158,7 +158,7 @@ def addPostTests(factory):
         && sync /packages/%(prop:tarbuildnum)s
         """
             ),
-            doStepIf=savePackage,
+            doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches),
         )
     )
     factory.addStep(
@@ -174,7 +174,10 @@ def addPostTests(factory):
                 "master_branch": Property("master_branch"),
                 "parentbuildername": Property("buildername"),
             },
-            doStepIf=lambda step: savePackage(step) and hasEco(step),
+            doStepIf=(
+                lambda step: savePackageIfBranchMatch(step, savedPackageBranches)
+                and hasEco(step)
+            ),
         )
     )
     factory.addStep(
@@ -747,7 +750,10 @@ def getRpmAutobakeFactory(mtrDbPool):
                 sync /packages/%(prop:tarbuildnum)s
 """
             ),
-            doStepIf=lambda step: hasFiles(step) and savePackage(step),
+            doStepIf=(
+                lambda step: hasFiles(step)
+                and savePackageIfBranchMatch(step, savedPackageBranches)
+            ),
             descriptionDone=util.Interpolate(
                 """
 Repository available with: curl %(kw:url)s/%(prop:tarbuildnum)s/%(prop:buildername)s/MariaDB.repo -o /etc/yum.repos.d/MariaDB.repo""",
@@ -784,9 +790,11 @@ def getRpmAutobakeFactory(mtrDbPool):
                 "master_branch": Property("master_branch"),
                 "parentbuildername": Property("buildername"),
             },
-            doStepIf=lambda step: hasInstall(step)
-            and savePackage(step)
-            and hasFiles(step),
+            doStepIf=(
+                lambda step: hasInstall(step)
+                and savePackageIfBranchMatch(step, savedPackageBranches)
+                and hasFiles(step)
+            ),
         )
     )
     f_rpm_autobake.addStep(
@@ -801,9 +809,11 @@ def getRpmAutobakeFactory(mtrDbPool):
                 "master_branch": Property("master_branch"),
                 "parentbuildername": Property("buildername"),
             },
-            doStepIf=lambda step: hasUpgrade(step)
-            and savePackage(step)
-            and hasFiles(step),
+            doStepIf=(
+                lambda step: hasUpgrade(step)
+                and savePackageIfBranchMatch(step, savedPackageBranches)
+                and hasFiles(step)
+            ),
         )
     )
     f_rpm_autobake.addStep(

diff --git a/master-bintars/master.cfg b/master-bintars/master.cfg
@@ -265,7 +265,7 @@ def getBintarFactory(
                 + " && sync /packages/"
                 + "%(prop:tarbuildnum)s"
             ),
-            doStepIf=savePackage,
+            doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches),
         )
     )
 

diff --git a/master-docker-nonstandard-2/master.cfg b/master-docker-nonstandard-2/master.cfg
@@ -502,7 +502,6 @@ f_big_test.addStep(
         command="basename mariadb-*-linux-*.tar.gz", property="mariadb_binary"
     )
 )
-# f_big_test.addStep(steps.ShellCommand(name='save_packages', timeout=7200, haltOnFailure=True, command=util.Interpolate('mkdir -p ' + '/packages/' + '%(prop:tarbuildnum)s' + '/' + '%(prop:buildername)s'+ ' && sha256sum %(prop:mariadb_binary)s >> sha256sums.txt  && cp ' + '%(prop:mariadb_binary)s sha256sums.txt' + ' /packages/' + '%(prop:tarbuildnum)s' + '/' + '%(prop:buildername)s' + '/' +  ' && sync /packages/' + '%(prop:tarbuildnum)s'), doStepIf=savePackage))
 f_big_test.addStep(
     steps.ShellCommand(
         name="cleanup", command="rm -r * .* 2> /dev/null || true", alwaysRun=True
@@ -757,7 +756,8 @@ f_without_server.addStep(
             + " && sync /packages/"
             + "%(prop:tarbuildnum)s"
         ),
-        doStepIf=savePackage,
+        doStepIf=lambda step: savePackageIfBranchMatch(step,
+                                                       savedPackageBranches),
     )
 )
 f_without_server.addStep(
@@ -1044,7 +1044,8 @@ f_bintar.addStep(
     steps.SetPropertyFromCommand(
         command="basename mariadb-*-linux-*.tar.gz",
         property="mariadb_binary",
-        doStepIf=savePackage,
+        doStepIf=lambda step: savePackageIfBranchMatch(step,
+                                                       savedPackageBranches),
     )
 )
 f_bintar.addStep(
@@ -1068,7 +1069,8 @@ f_bintar.addStep(
             + " && sync /packages/"
             + "%(prop:tarbuildnum)s"
         ),
-        doStepIf=savePackage,
+        doStepIf=lambda step: savePackageIfBranchMatch(step,
+                                                       savedPackageBranches),
     )
 )
 f_bintar.addStep(
@@ -1085,7 +1087,9 @@ f_bintar.addStep(
             "master_branch": Property("master_branch"),
             "parentbuildername": Property("buildername"),
         },
-        doStepIf=lambda step: savePackage(step) and hasEco(step),
+        doStepIf=(lambda step: savePackageIfBranchMatch(step,
+                                                        savedPackageBranches)
+                  and hasEco(step)),
     )
 )
 f_bintar.addStep(

diff --git a/master-docker-nonstandard/master.cfg b/master-docker-nonstandard/master.cfg
@@ -997,7 +997,6 @@ f_big_test.addStep(
         command="basename mariadb-*-linux-*.tar.gz", property="mariadb_binary"
     )
 )
-# f_big_test.addStep(steps.ShellCommand(name='save_packages', timeout=7200, haltOnFailure=True, command=util.Interpolate('mkdir -p ' + '/packages/' + '%(prop:tarbuildnum)s' + '/' + '%(prop:buildername)s'+ ' && sha256sum %(prop:mariadb_binary)s >> sha256sums.txt  && cp ' + '%(prop:mariadb_binary)s sha256sums.txt' + ' /packages/' + '%(prop:tarbuildnum)s' + '/' + '%(prop:buildername)s' + '/' +  ' && sync /packages/' + '%(prop:tarbuildnum)s'), doStepIf=savePackage))
 f_big_test.addStep(
     steps.ShellCommand(
         name="cleanup", command="rm -r * .* 2> /dev/null || true", alwaysRun=True
@@ -1134,7 +1133,7 @@ f_without_server.addStep(
             + " && sync /packages/"
             + "%(prop:tarbuildnum)s"
         ),
-        doStepIf=savePackage,
+        doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches),
     )
 )
 f_without_server.addStep(
@@ -1422,7 +1421,7 @@ f_bintar.addStep(
     steps.SetPropertyFromCommand(
         command="basename mariadb-*-linux-*.tar.gz",
         property="mariadb_binary",
-        doStepIf=savePackage,
+        doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches),
     )
 )
 f_bintar.addStep(
@@ -1446,7 +1445,7 @@ f_bintar.addStep(
             + " && sync /packages/"
             + "%(prop:tarbuildnum)s"
         ),
-        doStepIf=savePackage,
+        doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches),
     )
 )
 f_bintar.addStep(
@@ -1463,7 +1462,7 @@ f_bintar.addStep(
             "master_branch": Property("master_branch"),
             "parentbuildername": Property("buildername"),
         },
-        doStepIf=lambda step: savePackage(step) and hasEco(step),
+        doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches) and hasEco(step),
     )
 )
 f_bintar.addStep(

diff --git a/master-galera/master.cfg b/master-galera/master.cfg
@@ -265,7 +265,7 @@ def dpkgDeb():
 """
             ),
         ],
-        doStepIf=lambda step: savePackage(step, savedPackageBranches),
+        doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches),
     )
 
 
@@ -284,7 +284,7 @@ def rpmSave():
 """
             ),
         ],
-        doStepIf=lambda step: savePackage(step, savedPackageBranches),
+        doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches),
     )
 
 
@@ -366,7 +366,7 @@ EOF
 """,
             url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"),
         ),
-        doStepIf=lambda step: savePackage(step, savedPackageBranches),
+        doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches),
     )
 )
 f_deb_build.addStep(
@@ -423,7 +423,7 @@ EOF
 """,
             url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"),
         ),
-        doStepIf=lambda step: savePackage(step, savedPackageBranches),
+        doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches),
     )
 )
 f_rpm_build.addStep(

diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg
@@ -19,6 +19,7 @@ sys.setrecursionlimit(10000)
 sys.path.insert(0, "/srv/buildbot/master")
 
 from utils import *
+from constants import os_info
 
 # This is the dictionary that the buildmaster pays attention to. We also use
 # a shorter alias to save typing.

diff --git a/master-nonlatent/master.cfg b/master-nonlatent/master.cfg
@@ -535,7 +535,8 @@ f_windows_msi.addStep(
             '/'
 
         ),
-        doStepIf=savePackage,
+        doStepIf=lambda step: savePackageIfBranchMatch(step,
+                                                       savedPackageBranches),
     )
 )
 f_windows_msi.addStep(
@@ -557,7 +558,8 @@ f_windows_msi.addStep(
             '%(prop:buildername)s'
             '/'
         ),
-        doStepIf=savePackage,
+        doStepIf=lambda step: savePackageIfBranchMatch(step,
+                                                       savedPackageBranches),
     )
 )
 f_windows_msi.addStep(

diff --git a/master-protected-branches/master.cfg b/master-protected-branches/master.cfg
@@ -422,9 +422,6 @@ f_tarball.addStep(
         doStepIf=lambda step: isStagingBranch(step),
     )
 )
-# f_tarball.addStep(steps.ShellSequence( commands=[
-#    util.ShellArg(command=util.Interpolate("git checkout " + "%(prop:staging_branch)s"), logfile="rebase"),
-#    util.ShellArg(command=util.Interpolate("git merge %(prop:branch)s"), logfile="rebase")], workdir="build/server", haltOnFailure="true", doStepIf=ifStagingSucceeding))
 f_tarball.addStep(
     steps.ShellCommand(
         name="cleanup", command="rm -r * .* 2> /dev/null || true", alwaysRun=True

diff --git a/master.cfg b/master.cfg
@@ -235,7 +235,7 @@ f_deb_autobake.addStep(
             "master_branch": Property("master_branch"),
             "parentbuildername": Property("buildername"),
         },
-        doStepIf=lambda step: savePackage(step),
+        doStepIf=lambda step: savePackageIfBranchMatch(step, savedPackageBranches),
     )
 )
 f_deb_autobake.addStep(
@@ -250,7 +250,9 @@ f_deb_autobake.addStep(
             "master_branch": Property("master_branch"),
             "parentbuildername": Property("buildername"),
         },
-        doStepIf=lambda step: hasInstall(step) and savePackage(step) and hasFiles(step),
+        doStepIf=(lambda step: hasInstall(step)
+                  and savePackageIfBranchMatch(step, savedPackageBranches)
+                  and hasFiles(step)),
     )
 )
 f_deb_autobake.addStep(
@@ -265,7 +267,9 @@ f_deb_autobake.addStep(
             "master_branch": Property("master_branch"),
             "parentbuildername": Property("buildername"),
         },
-        doStepIf=lambda step: hasUpgrade(step) and savePackage(step) and hasFiles(step),
+        doStepIf=(lambda step: hasUpgrade(step)
+                  and savePackageIfBranchMatch(step, savedPackageBranches)
+                  and hasFiles(step)),
     )
 )
 f_deb_autobake.addStep(