Merge 9938e47 into sapling-pr-archive-ktf

ktf · Jul 18, 2024 · 9a241b4 · 9a241b4
2 parents 024e21f + 9938e47
commit 9a241b4
Show file tree

Hide file tree

Showing 11 changed files with 80 additions and 32 deletions.
diff --git a/.github/workflows/o2-full-deps.yml b/.github/workflows/o2-full-deps.yml
@@ -27,30 +27,37 @@ jobs:
         include:
           - el_version: el7
             container: centos:7
-            alibuild_tag: v1.16.1
+            alibuild_tag: v1.17.7
           - el_version: el8
             container: almalinux:8
-            alibuild_tag: v1.16.1
+            alibuild_tag: v1.17.7
           - el_version: el9
             container: almalinux:9
-            alibuild_tag: v1.16.1
+            alibuild_tag: v1.17.7
           - el_version: fedora
-            container: fedora:33
-            alibuild_tag: v1.16.1
+            container: fedora:40
+            alibuild_tag: v1.17.7
 
     name: RPM (${{ matrix.el_version }})
     container: ${{ matrix.container }}
     env:
       ALIBUILD_TAG: ${{ matrix.alibuild_tag }}
       DISTRO: ${{ matrix.el_version }}
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: 'true' # Only required for CentOS 7
 
     steps:
       # For rpms/*.spec
       - uses: actions/checkout@v3
-
       - name: Install prerequisites
         run: |
-          set -x
+          set -ex
+          if [[ "$DISTRO" == "el7" ]]; then
+            sed -i.bak -e 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
+            sed -i.bak -r -e 's|# ?baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
+            find /etc/yum.repos.d -name "*.bak" -delete
+            yum clean all
+            yum update -y
+          fi
           yum clean all
           yum update -y
           yum install -y rpm-build scl-utils-build createrepo unzip git python3 python3-pip python3-setuptools
@@ -61,7 +68,7 @@ jobs:
 
       - name: Build the ${{ matrix.el_version }} RPM and create a yum repo
         run: |
-          set -x
+          set -ex
           rpmbuild -ba "rpms/o2-prereq-$DISTRO.spec"
           git clone -b "$ALIBUILD_TAG" https://github.com/alisw/alibuild
           cd alibuild
@@ -70,8 +77,8 @@ jobs:
             *)   deps=python3,git,python3-pyyaml,python3-requests,python3-distro ;;
           esac
           python3 setup.py bdist_rpm --requires "$deps"
-          cp dist/alibuild-*.noarch.rpm ~/rpmbuild/RPMS/$(uname -m)/
-          cd  ~/rpmbuild/RPMS/$(uname -m)/
+          cp dist/alibuild-*.noarch.rpm ~/rpmbuild/RPMS/"$(uname -m)"/
+          cd  ~/rpmbuild/RPMS/"$(uname -m)"/
           createrepo .
           ls -la
           case "$DISTRO" in
@@ -92,11 +99,12 @@ jobs:
           - focal   # 20.04
           - jammy   # 22.04
           - mantic  # 23.10
+          - noble   # 24.04
 
     name: DEB (${{ matrix.ubuntu_codename }})
     container: ubuntu:${{ matrix.ubuntu_codename }}
     env:
-      ALIBUILD_TAG: v1.16.1
+      ALIBUILD_TAG: v1.17.7
       ALIBUILD_DISTRO: ${{ matrix.ubuntu_codename }}
       DEBIAN_FRONTEND: noninteractive
 

diff --git a/ci/repo-config/mesosci/slc7-aliphysics/aliphysics-root6.env b/ci/repo-config/mesosci/slc7-aliphysics/aliphysics-root6.env
@@ -1,6 +1,6 @@
 CI_NAME=build_AliPhysics_root6
 PACKAGE=AliPhysics
-ALIBUILD_DEFAULTS=o2
+ALIBUILD_DEFAULTS=ali
 PR_REPO=alisw/AliPhysics
 PR_BRANCH=master
 TRUSTED_USERS=

diff --git a/ci/repo-config/mesosci/slc7/aliroot-alidist.env b/ci/repo-config/mesosci/slc7/aliroot-alidist.env
@@ -1,6 +1,6 @@
 CI_NAME=build_AliRoot_alidist
 PACKAGE=AliRoot
-ALIBUILD_DEFAULTS=o2
+ALIBUILD_DEFAULTS=ali
 PR_REPO=alisw/alidist
 PR_BRANCH=master
 NO_ASSUME_CONSISTENT_EXTERNALS=

diff --git a/ci/repo-config/mesosci/slc7/aliroot.env b/ci/repo-config/mesosci/slc7/aliroot.env
@@ -2,7 +2,7 @@ CI_NAME=build_AliRoot_o2
 PACKAGE=AliRoot
 PR_REPO=alisw/AliRoot
 PR_BRANCH=master
-ALIBUILD_DEFAULTS=o2
+ALIBUILD_DEFAULTS=ali
 TRUSTED_USERS=
 TASK_PRIORITY=10
 NO_ASSUME_CONSISTENT_EXTERNALS=

diff --git a/ci/run-http.sh b/ci/run-http.sh
@@ -16,11 +16,12 @@ ALI_BOT_DIR="$(dirname "$0")"
 ALI_BOT_DIR="$(cd "$ALI_BOT_DIR/.."; pwd)"
 
 # Move into the scratch directory
-cd "$MESOS_SANDBOX" || :
+cd "$NOMAD_TASK_DIR" || :
 
 # Install with pip
 export PYTHONUSERBASE=$PWD/python_local
 export PATH="$PYTHONUSERBASE/bin:$PATH"
+pip3 install --user --upgrade pip
 pip3 install --ignore-installed --upgrade --user -e "${ALI_BOT_DIR}[services]"
 
 # Setup GitLab credentials (to push new data)

diff --git a/publish/aliPublish.conf b/publish/aliPublish.conf
@@ -344,24 +344,43 @@ architectures:
     AliEn: el9-x86_64
     RPM: false
     include:
-      grid-base-packages: true
-      googlebenchmark: true
-      json-c: true
-      MPFR: true
-      ITSResponse: true
-      Python-modules-list: true
-      double-conversion: true
-      FairCMakeModules: true
-      Alice-GRID-Utils: true
-      libwebsockets: true
       AliEn-CAs: true
+      Alice-GRID-Utils: true
       ApMon-CPP: true
-      GMP: true
-      capstone: true
       CMake: true
+      CodingGuidelines: true
+      FairCMakeModules: true
+      GMP: true
+      HEPscore-CCDB: true
+      ITSResponse: true
+      IgProf: true
+      MCSingContainer: true
+      MPFR: true
+      O2-customization: true
+      Python-modules-list: true
+      QualityControl: true
+      RapidJSON: true
+      UUID: true
       abseil: true
+      alibuild-recipe-tools: true
+      autotools: true
       bz2: true
-      IgProf: true
+      capstone: true
+      cub: true
+      defaults-release: true
+      double-conversion: true
+      flatbuffers: true
+      fmt: true
+      googlebenchmark: true
+      googletest: true
+      grid-base-packages: true
+      hijing: true
+      json-c: true
+      libtirpc: true
+      libwebsockets: true
+      ninja: true
+      oniguruma: true
+      re2: true
       O2PDPSuite:
         - ^async-.*$
         # Tags for testing compatibility with Hyperloop.

diff --git a/report-pr-errors b/report-pr-errors
@@ -160,10 +160,13 @@ class Logs(object):
         self.build_successful = args.success
         self.pr_built = parse_pr(args.pr)
         self.main_packages = args.main_packages
+        self.alibuild_version = None
+        self.alidist_version = None
 
     def parse(self):
         self.find()
         self.grep()
+        self.get_versions()
         self.cat(self.full_log)
         self.generate_pretty_log()
         if self.is_branch:
@@ -314,6 +317,17 @@ class Logs(object):
                                    .format(self.all_logs[-1], err)]
         self.preview_error_log = '\n'.join(error_log_lines).strip()
 
+    def get_versions(self):
+        # The versions are defined differently for both job runner setups
+        # Jenkins (daily builds):
+        #   - aliBuild: ALIBUILD_SLUG
+        #   - alidist: ALIDIST_SLUG (if not set it defaults to alidist@master)
+        # Nomad (CI checks):
+        #   - aliBuild: INSTALL_ALIBUILD
+        #   - alidist: (defaults to alidist@master)
+        self.alibuild_version = os.getenv('ALIBUILD_SLUG') or os.getenv('INSTALL_ALIBUILD') or 'Not pinned (most likely alibuild@master)'
+        self.alidist_version = os.getenv('ALIDIST_SLUG') or 'Not pinned (most likely alidist@master)'
+
     def generate_pretty_log(self):
         '''Extract error messages from logs.
 
@@ -368,6 +382,8 @@ class Logs(object):
                 'nomad_alloc_id': os.getenv('NOMAD_ALLOC_ID', ''),
                 'nomad_short_alloc_id': os.getenv('NOMAD_SHORT_ALLOC_ID',
                                                   os.getenv('NOMAD_ALLOC_ID', '(no Nomad ID)')),
+                'alibuild_version': self.alibuild_version,
+                'alidist_version': self.alidist_version,
             })
 
     def cat(self, target_file, no_delete=False):
@@ -586,6 +602,10 @@ PRETTY_LOG_TEMPLATE = '''\
         <td><code>nomad alloc exec %(nomad_short_alloc_id)s bash</code></td></tr>
     <tr><th>Stream logs</th>
         <td><code>nomad alloc logs -stderr -tail -f %(nomad_short_alloc_id)s</code></td></tr>
+    <tr><th>Alibuild version</th>
+        <td>%(alibuild_version)s</td></tr>
+    <tr><th>Alidist version</th>
+        <td>%(alidist_version)s</td></tr>
   </table>
   <p>The code that finds and extracts error messages may have missed some.
      Check the <a href="%(log_url)s">full build log</a> if you suspect the build failed for reasons not listed below.</p>

diff --git a/rpms/o2-prereq-el7.spec b/rpms/o2-prereq-el7.spec
@@ -14,7 +14,7 @@ Name: %scl_name
 Version: 1
 Release: 1%{?dist}
 License: GPLv2+
-Requires: make rsync glew-devel epel-release python-requests python-yaml pigz which git mysql-devel curl curl-devel bzip2 bzip2-devel unzip autoconf automake texinfo gettext gettext-devel libtool freetype freetype-devel libpng libpng-devel sqlite sqlite-devel ncurses-devel mesa-libGLU-devel libX11-devel libXpm-devel libXext-devel libXft-devel libXi-devel libxml2 libxml2-devel motif motif-devel kernel-devel pciutils-devel kmod-devel bison flex perl-ExtUtils-Embed environment-modules tk-devel libXinerama-devel libXcursor-devel libXrandr-devel
+Requires: make rsync glew-devel epel-release python-requests python-yaml pigz which git mysql-devel curl curl-devel bzip2 bzip2-devel unzip autoconf automake texinfo gettext gettext-devel libtool freetype freetype-devel libpng libpng-devel sqlite sqlite-devel ncurses-devel mesa-libGLU-devel libX11-devel libXpm-devel libXext-devel libXft-devel libXi-devel libxml2 libxml2-devel motif motif-devel kernel-devel pciutils-devel kmod-devel bison flex perl-ExtUtils-Embed environment-modules tk-devel libXinerama-devel libXcursor-devel libXrandr-devel readline-devel
 BuildRequires: scl-utils-build
 
 %description

diff --git a/rpms/o2-prereq-el8.spec b/rpms/o2-prereq-el8.spec
@@ -14,7 +14,7 @@ Name: %scl_name
 Version: 1
 Release: 1%{?dist}
 License: GPLv2+
-Requires: make rsync glew-devel python3-requests python3-yaml pigz which git mysql-devel curl curl-devel bzip2 bzip2-devel unzip autoconf automake texinfo gettext gettext-devel libtool freetype freetype-devel libpng libpng-devel sqlite sqlite-devel ncurses-devel mesa-libGLU-devel libX11-devel libXpm-devel libXext-devel libXft-devel libXi-devel libxml2 libxml2-devel motif motif-devel kernel-devel pciutils-devel kmod-devel bison flex perl-ExtUtils-Embed environment-modules tk-devel libXinerama-devel libXcursor-devel libXrandr-devel subversion-devel apr-util-devel apr-devel
+Requires: make rsync glew-devel python3-requests python3-yaml pigz which git mysql-devel curl curl-devel bzip2 bzip2-devel unzip autoconf automake texinfo gettext gettext-devel libtool freetype freetype-devel libpng libpng-devel sqlite sqlite-devel ncurses-devel mesa-libGLU-devel libX11-devel libXpm-devel libXext-devel libXft-devel libXi-devel libxml2 libxml2-devel motif motif-devel kernel-devel pciutils-devel kmod-devel bison flex perl-ExtUtils-Embed environment-modules tk-devel libXinerama-devel libXcursor-devel libXrandr-devel subversion-devel apr-util-devel apr-devel readline-devel
 BuildRequires: scl-utils-build
 
 %description

diff --git a/rpms/o2-prereq-el9.spec b/rpms/o2-prereq-el9.spec
@@ -14,7 +14,7 @@ Name: %scl_name
 Version: 1
 Release: 1%{?dist}
 License: GPLv2+
-Requires: make rsync glew-devel pigz which git mariadb-devel curl curl-devel bzip2 bzip2-devel unzip autoconf automake texinfo gettext gettext-devel libtool freetype freetype-devel libpng libpng-devel sqlite sqlite-devel ncurses-devel mesa-libGLU-devel libX11-devel libXpm-devel libXext-devel libXft-devel libXi-devel libxml2 libxml2-devel motif motif-devel kernel-devel pciutils-devel kmod-devel bison flex perl-ExtUtils-Embed perl-FindBin environment-modules tk-devel libXinerama-devel libXcursor-devel libXrandr-devel subversion-devel apr-util-devel apr-devel
+Requires: make rsync glew-devel pigz which git mariadb-devel curl curl-devel bzip2 bzip2-devel unzip autoconf automake texinfo gettext gettext-devel libtool freetype freetype-devel libpng libpng-devel sqlite sqlite-devel ncurses-devel mesa-libGLU-devel libX11-devel libXpm-devel libXext-devel libXft-devel libXi-devel libxml2 libxml2-devel motif motif-devel kernel-devel pciutils-devel kmod-devel bison flex perl-ExtUtils-Embed perl-FindBin environment-modules tk-devel libXinerama-devel libXcursor-devel libXrandr-devel subversion-devel apr-util-devel apr-devel readline-devel
 BuildRequires: scl-utils-build
 
 %description

diff --git a/rpms/o2-prereq-fedora.spec b/rpms/o2-prereq-fedora.spec
@@ -15,7 +15,7 @@ Name: %scl_name
 Version: 1
 Release: 1%{?dist}
 License: GPLv2+
-Requires: make python3-requests python3-yaml pigz which git mariadb-devel curl curl-devel bzip2 bzip2-devel unzip autoconf automake texinfo gettext gettext-devel libtool freetype freetype-devel libpng libpng-devel sqlite sqlite-devel ncurses-devel mesa-libGLU-devel libX11-devel libXpm-devel libXext-devel libXft-devel libXi-devel libxml2 libxml2-devel motif motif-devel kernel-devel pciutils-devel kmod-devel bison flex perl-ExtUtils-Embed environment-modules tk-devel libXinerama-devel libXcursor-devel libXrandr-devel
+Requires: make python3-requests python3-yaml pigz which git mariadb-devel curl curl-devel bzip2 bzip2-devel unzip autoconf automake texinfo gettext gettext-devel libtool freetype freetype-devel libpng libpng-devel sqlite sqlite-devel ncurses-devel mesa-libGLU-devel libX11-devel libXpm-devel libXext-devel libXft-devel libXi-devel libxml2 libxml2-devel motif motif-devel kernel-devel pciutils-devel kmod-devel bison flex perl-ExtUtils-Embed environment-modules tk-devel libXinerama-devel libXcursor-devel libXrandr-devel readline-devel
 BuildRequires: scl-utils-build
 
 %description