Refactor locks.py

locks.py

@@ -1,47 +1,25 @@
-from twisted.internet import defer
+import os
 
-from buildbot.plugins import *
-from buildbot.process.properties import Properties, Property
-from buildbot.process.remotecommand import RemoteCommand
-from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver
-from buildbot.steps.shell import Compile, SetPropertyFromCommand, ShellCommand, Test
-from buildbot.steps.source.github import GitHub
-from constants import *
+import yaml
 
-####### LOCKS
-# main_master_lock = util.MasterLock('main_master_lock', maxCount=30)
+from buildbot.plugins import util
 
-hz_bbw1_lock = util.MasterLock("hz_bbw1_lock", maxCount=9)
-hz_bbw2_lock = util.MasterLock("hz_bbw2_lock", maxCount=1)
-hz_bbw4_lock = util.MasterLock("hz_bbw4_lock", maxCount=9)
-hz_bbw5_lock = util.MasterLock("hz_bbw5_lock", maxCount=9)
-amd_bbw1_lock = util.MasterLock("amd_bbw1_lock", maxCount=4)
-amd_bbw2_lock = util.MasterLock("amd_bbw2_lock", maxCount=6)
-intel_bbw1_lock = util.MasterLock("intel_bbw1_lock", maxCount=5)
-p9_rhel8_bbw1_lock = util.MasterLock("p9_rhel8_bbw1_lock", maxCount=6)
-p9_db_bbw1_lock = util.MasterLock("p9_db_bbw1_lock", maxCount=8)
-p9_raptor_bbw1_lock = util.MasterLock("p9_raptor_bbw1_lock", maxCount=6)
-aarch_bbw1_lock = util.MasterLock("aarch64_bbw1_lock", maxCount=2)
-aarch_bbw2_lock = util.MasterLock("aarch64_bbw2_lock", maxCount=2)
-aarch_bbw3_lock = util.MasterLock("aarch64_bbw3_lock", maxCount=2)
-aarch_bbw4_lock = util.MasterLock("aarch64_bbw4_lock", maxCount=2)
-aarch_bbw5_lock = util.MasterLock("aarch64_bbw5_lock", maxCount=15)
-aarch_bbw6_lock = util.MasterLock("aarch64_bbw6_lock", maxCount=15)
-aarch_bbw7_lock = util.MasterLock("aarch64_bbw7_lock", maxCount=15)
-apexis_bbw1_lock = util.MasterLock("apexis_bbw1_lock", maxCount=1)
-apexis_bbw2_lock = util.MasterLock("apexis_bbw2_lock", maxCount=1)
-apexis_bbw3_lock = util.MasterLock("apexis_bbw3_lock", maxCount=6)
-bg_bbw1_lock = util.MasterLock("bg_bbw1_lock", maxCount=3)
-bg_bbw2_lock = util.MasterLock("bg_bbw2_lock", maxCount=2)
-bg_bbw3_lock = util.MasterLock("bg_bbw3_lock", maxCount=2)
-bg_bbw4_lock = util.MasterLock("bg_bbw4_lock", maxCount=2)
-win_bbw1_lock = util.MasterLock("win_bbw1_lock", maxCount=1)
-win_bbw2_lock = util.MasterLock("win_bbw2_lock", maxCount=4)
-s390x_bbw1_lock = util.MasterLock("s390x_bbw1_lock", maxCount=3)
-s390x_bbw2_lock = util.MasterLock("s390x_bbw2_lock", maxCount=3)
-s390x_bbw3_lock = util.MasterLock("s390x_bbw3_lock", maxCount=3)
-s390x_bbw4_lock = util.MasterLock("s390x_bbw4_lock", maxCount=3)
-s390x_bbw5_lock = util.MasterLock("s390x_bbw5_lock", maxCount=3)
+# Local
+from constants import builders_install, builders_upgrade, github_status_builders
+
+LOCKS: dict[str, util.MasterLock] = {}
+# worker_locks.yaml currently is in the same folder as locks.py.
+# TODO: re-evaluate if this is the right place after multi-master
+# is refactored to use a single base master.cfg.
+with open(
+    os.path.join(os.path.dirname(os.path.realpath(__file__)), "worker_locks.yaml"),
+    encoding="utf-8",
+) as file:
+    locks = yaml.safe_load(file)
+    for worker_name in locks:
+        LOCKS[worker_name] = util.MasterLock(
+            f"{worker_name}_lock", maxCount=locks[worker_name]
+        )
 
 
 @util.renderer
@@ -57,71 +35,7 @@ def getLocks(props):
         or builder_name in builders_upgrade
     ):
         return []
-    locks = []
-    # else:
-    #    locks = [main_master_lock.access('counting')]
-
-    if "hz-bbw1-docker" in worker_name:
-        locks = locks + [hz_bbw1_lock.access("counting")]
-    if "hz-bbw2-docker" in worker_name:
-        locks = locks + [hz_bbw2_lock.access("counting")]
-    if "hz-bbw4-docker" in worker_name:
-        locks = locks + [hz_bbw4_lock.access("counting")]
-    if "hz-bbw5-docker" in worker_name:
-        locks = locks + [hz_bbw5_lock.access("counting")]
-    if "intel-bbw1-docker" in worker_name:
-        locks = locks + [intel_bbw1_lock.access("counting")]
-    if "ppc64le-rhel8-bbw1-docker" in worker_name:
-        locks = locks + [p9_rhel8_bbw1_lock.access("counting")]
-    if "ppc64le-db-bbw1-docker" in worker_name:
-        locks = locks + [p9_db_bbw1_lock.access("counting")]
-    if "ppc64le-raptor-bbw1-docker" in worker_name:
-        locks = locks + [p9_raptor_bbw1_lock.access("counting")]
-    if "aarch64-bbw1-docker" in worker_name:
-        locks = locks + [aarch_bbw1_lock.access("counting")]
-    if "aarch64-bbw2-docker" in worker_name:
-        locks = locks + [aarch_bbw2_lock.access("counting")]
-    if "aarch64-bbw3-docker" in worker_name:
-        locks = locks + [aarch_bbw3_lock.access("counting")]
-    if "aarch64-bbw4-docker" in worker_name:
-        locks = locks + [aarch_bbw4_lock.access("counting")]
-    if "aarch64-bbw5-docker" in worker_name:
-        locks = locks + [aarch_bbw5_lock.access("counting")]
-    if "aarch64-bbw6-docker" in worker_name:
-        locks = locks + [aarch_bbw6_lock.access("counting")]
-    if "aarch64-bbw7-docker" in worker_name:
-        locks = locks + [aarch_bbw7_lock.access("counting")]
-    if "fjord1-docker" in worker_name:
-        locks = locks + [apexis_bbw1_lock.access("counting")]
-    if "fjord2-docker" in worker_name:
-        locks = locks + [apexis_bbw2_lock.access("counting")]
-    if "ns-x64-bbw1-docker" in worker_name:
-        locks = locks + [bg_bbw1_lock.access("counting")]
-    if "ns-x64-bbw2-docker" in worker_name:
-        locks = locks + [bg_bbw2_lock.access("counting")]
-    if "ns-x64-bbw3-docker" in worker_name:
-        locks = locks + [bg_bbw3_lock.access("counting")]
-    if "ns-x64-bbw4-docker" in worker_name:
-        locks = locks + [bg_bbw4_lock.access("counting")]
-    if "bbw1-docker-windows" in worker_name:
-        locks = locks + [win_bbw1_lock.access("counting")]
-    if "bbw2-docker-windows" in worker_name:
-        locks = locks + [win_bbw2_lock.access("counting")]
-    if "s390x-bbw1-docker" in worker_name:
-        locks = locks + [s390x_bbw1_lock.access("counting")]
-    if "s390x-bbw2-docker" in worker_name:
-        locks = locks + [s390x_bbw2_lock.access("counting")]
-    if "s390x-bbw3-docker" in worker_name:
-        locks = locks + [s390x_bbw3_lock.access("counting")]
-    if "s390x-bbw4-docker" in worker_name:
-        locks = locks + [s390x_bbw4_lock.access("counting")]
-    if "s390x-bbw5-docker" in worker_name:
-        locks = locks + [s390x_bbw5_lock.access("counting")]
-    if "amd-bbw1-docker" in worker_name:
-        locks = locks + [amd_bbw1_lock.access("counting")]
-    if "amd-bbw2-docker" in worker_name:
-        locks = locks + [amd_bbw2_lock.access("counting")]
-    if "apexis-bbw3-docker" in worker_name:
-        locks = locks + [apexis_bbw3_lock.access("counting")]
 
-    return locks
+    if worker_name not in LOCKS:
+        return []
+    return [LOCKS[worker_name].access("counting")]

worker_locks.yaml

@@ -0,0 +1,33 @@
+# This file keeps a maximum count of concurrent builds per worker.
+# Later, this should be refactored to have this information as part of
+# other worker metadata, such as IP, connection string, etc.
+# The keys are worker names, as used in locks.py.
+# TODO: There should be a single place where these keys are defined, currently
+# there are more places.
+hz-bbw1-docker: 9
+hz-bbw2-docker: 1
+hz-bbw4-docker: 9
+hz-bbw5-docker: 9
+amd-bbw1-docker: 4
+amd-bbw2-docker: 6
+intel-bbw1-docker: 5
+ppc64le-rhel8-docker: 6
+ppc64le-db-docker: 8
+ppc64le-raptor-docker: 6
+aarch64-bbw1-docker: 2
+aarch64-bbw2-docker: 2
+aarch64-bbw3-docker: 2
+aarch64-bbw4-docker: 2
+aarch64-bbw5-docker: 15
+aarch64-bbw6-docker: 15
+aarch64-bbw7-docker: 15
+apexis-bbw3-docker: 6
+ns-x64-bbw1-docker: 3
+ns-x64-bbw2-docker: 2
+ns-x64-bbw3-docker: 2
+ns-x64-bbw4-docker: 2
+s390x-bbw1-docker: 3
+s390x-bbw2-docker: 3
+s390x-bbw3-docker: 3
+s390x-bbw4-docker: 3
+s390x-bbw5-docker: 3