Merge e71da4f into sapling-pr-archive-ktf

.github/workflows/c++-code-formatting.yml

@@ -32,7 +32,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           persist-credentials: false
@@ -148,7 +148,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           persist-credentials: false
@@ -194,6 +194,7 @@ jobs:
           incorrect_files=()
           for file in "${files[@]}"; do
             case $file in
+              */3rdparty/*) continue ;;  # ignore vendored files
               *.cxx|*.h|*.C) first=$cpp_first rest=$cpp_rest ;;
               *.cmake|*CMakeLists.txt) first=$hash_first rest=$hash_rest ;;
               *) echo "error: unknown file type for $file" >&2; exit 1 ;;
@@ -245,7 +246,7 @@ jobs:
           sudo -n apt install dos2unix
 
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           persist-credentials: false
@@ -301,7 +302,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           persist-credentials: false
@@ -383,7 +384,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           persist-credentials: false

.github/workflows/cacheo2package.yml

@@ -0,0 +1,53 @@
+---
+# Laungh the CacheO2Package job in Jenkins. Useful for caching the build
+# results for CI, so there are minimal delays when merging critical packages
+name: Cache O2 Package
+
+'on':
+  workflow_dispatch:
+    inputs:
+      package_name:
+        type: string
+        description: Name of the package to cache
+        default: 'O2'
+      alidist_slug:
+        type: string
+        description: Alidist version to use for the package (group/repo[@branch])
+        default: 'alisw/alidist@master'
+      alibuild_slug:
+        type: string
+        description: Alibuild version to use for the package. If empty, the latest version from pypi is used.
+
+permissions: {}
+
+jobs:
+  cache-o2-package:
+    runs-on: ubuntu-latest
+
+    env:
+      ALIDIST_SLUG: ${{ inputs.alidist_slug }}
+      ALIBUILD_SLUG: ${{ inputs.alibuild_slug }}
+      PACKAGE_NAME: ${{ inputs.package_name }}
+      JENKINS_URL: ${{ secrets.JENKINS_URL }}
+      SSO_AUTH_URL: ${{ secrets.SSO_AUTH_URL }}
+      CLIENT_ID: ${{ secrets.SSO_JENKINS_API_CLIENT_ID }}
+      CLIENT_SECRET: ${{ secrets.SSO_JENKINS_API_CLIENT_SECRET }}
+      TARGET_APP: ${{ secrets.SSO_JENKINS_API_TARGET_APP }}
+      JOB_NAME: 'CacheO2Package'
+
+    steps:
+      - name: Launch the CacheO2Package job in Jenkins
+        run: |
+          # Login against SSO
+          TOKEN="$(curl --location -X POST "$SSO_AUTH_URL" \
+          --header 'Content-Type: application/x-www-form-urlencoded' \
+          --data-urlencode 'grant_type=client_credentials' \
+          --data-urlencode "client_id=$CLIENT_ID" \
+          --data-urlencode "client_secret=$CLIENT_SECRET" \
+          --data-urlencode "audience=$TARGET_APP" | jq -r '.access_token')"
+
+          # Trigger the Jenkins job
+          curl "$JENKINS_URL/job/$JOB_NAME/buildWithParameters" \
+              -H "Authorization: Bearer $TOKEN"                 \
+              --data "PACKAGE_NAME=$PACKAGE_NAME"               \
+              --data "ALIDIST_SLUG=$ALIDIST_SLUG"

.github/workflows/clean-pr-checks.yml

@@ -76,9 +76,10 @@ jobs:
                 }
               }
             }
-          owner: ${{ inputs.owner }}
-          name: ${{ inputs.repo }}
-          pr: ${{ inputs.pr }}
+          variables: |
+            owner: ${{ inputs.owner }}
+            name: ${{ inputs.repo }}
+            pr: ${{ inputs.pr }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

.github/workflows/o2-full-deps.yml

@@ -1,7 +1,7 @@
 ---
 # Create repositories with all O2 prerequisites plus aliBuild and push them
 # upstream (e.g. to S3).
-name: Build o2-full-deps repositories
+name: "Build o2-full-deps repositories"
 
 # Controls when the action will run. This workflow is triggered manually.
 "on":
@@ -27,18 +27,18 @@ jobs:
         include:
           - el_version: el7
             container: centos:7
-            alibuild_tag: v1.17.10
+            alibuild_tag: v1.17.12
           - el_version: el8
             container: almalinux:8
-            alibuild_tag: v1.17.10
+            alibuild_tag: v1.17.12
           - el_version: el9
             container: almalinux:9
-            alibuild_tag: v1.17.7
+            alibuild_tag: v1.17.12
           - el_version: fedora
             container: fedora:40
-            alibuild_tag: v1.17.7
+            alibuild_tag: v1.17.12
 
-    name: RPM (${{ matrix.el_version }})
+    name: "RPM (${{ matrix.el_version }})"
     container: ${{ matrix.container }}
     env:
       ALIBUILD_TAG: ${{ matrix.alibuild_tag }}
@@ -47,8 +47,9 @@ jobs:
 
     steps:
       # For rpms/*.spec
-      - uses: actions/checkout@v3
-      - name: Install prerequisites
+      - uses: actions/checkout@v4
+
+      - name: "Install prerequisites"
         run: |
           set -ex
           if [[ "$DISTRO" == "el7" ]]; then
@@ -66,7 +67,7 @@ jobs:
           ${{ secrets.S3_RCLONE_CONFIG }}
           EOF
 
-      - name: Build the ${{ matrix.el_version }} RPM and create a yum repo
+      - name: "Build the ${{ matrix.el_version }} RPM and create a yum repo"
         run: |
           set -ex
           rpmbuild -ba "rpms/o2-prereq-$DISTRO.spec"
@@ -95,7 +96,6 @@ jobs:
       fail-fast: false
       matrix:
         ubuntu_codename:
-          - bionic  # 18.04
           - focal   # 20.04
           - jammy   # 22.04
           - mantic  # 23.10
@@ -104,30 +104,31 @@ jobs:
     name: DEB (${{ matrix.ubuntu_codename }})
     container: ubuntu:${{ matrix.ubuntu_codename }}
     env:
-      ALIBUILD_TAG: v1.17.7
+      ALIBUILD_TAG: v1.17.12
       ALIBUILD_DISTRO: ${{ matrix.ubuntu_codename }}
       DEBIAN_FRONTEND: noninteractive
-
     steps:
-      - name: Install prerequisites
+      - name: "Install prerequisites"
         run: |
           apt update
           apt install -y vim git devscripts build-essential lintian debhelper dh-python python3-all python3-setuptools python3-setuptools-scm
-          gpg --import - <<\EOF
-          ${{ secrets.LAUNCHPAD_PPA_GPG_KEY }}
-          EOF
+          mkdir -p ~/.gnupg
+          chmod 700 ~/.gnupg
+          echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
+          echo -n "${{ secrets.LAUNCHPAD_PPA_GPG_KEY }}" | gpg --no-tty --batch --import
 
-      - name: Build a DEB for Ubuntu and push it to the PPA
+      - name: "Build a DEB for Ubuntu and push it to the PPA"
         run: |
           set -x
-          git clone -b "$ALIBUILD_TAG" https://github.com/alisw/alibuild
+          git clone  https://github.com/alisw/alibuild
           cd alibuild
+          rm -rf .git
           cat <<EOF > debian/changelog
           python3-alibuild (${ALIBUILD_TAG#v}+$ALIBUILD_DISTRO) $ALIBUILD_DISTRO; urgency=medium
 
             * Bump to $ALIBUILD_TAG
 
            -- Giulio Eulisse <[email protected]>  $(date -R -u)
           EOF
-          debuild --no-lintian -k48F330BAFFA564EF2383E2B472E9262B5C0D9DE5 -S
+          debuild --no-lintian -kA3A177D7C6BAD5C044ACAC10F56D273E83B84A47 -S -p'gpg --no-tty --passphrase ${{ secrets.LAUNCHPAD_PPA_GPG_PASS }}'
           dput ppa:alisw/ppa "../python3-alibuild_${ALIBUILD_TAG#v}+${ALIBUILD_DISTRO}_source.changes"

.github/workflows/pr-security-approval.yml

@@ -50,9 +50,10 @@ jobs:
                 }
               }
             }
-          owner: ${{ github.event.repository.owner.login }}
-          name: ${{ github.event.repository.name }}
-          pr: ${{ github.event.pull_request.number }}
+          variables: |
+            owner: ${{ github.event.repository.owner.login }}
+            name: ${{ github.event.repository.name }}
+            pr: ${{ github.event.pull_request.number }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

alibot_helpers/github_utilities.py

@@ -251,6 +251,8 @@ def post(self, url, data, stable_api=True, **kwds):
         data = json.dumps(data) if type(data) == dict else data
         response = requests.post(url=url, data=data, headers=headers)
         sc = response.status_code
+        if sc == 422:
+            print(f"GitHub error: Unprocessable Entity", file=sys.stderr)
         return sc
 
     @trace

analytics/report-analytics

@@ -4,12 +4,8 @@ import argparse
 import os
 import subprocess
 import sys
-try:
-    from commands import getstatusoutput
-    from urllib import urlopen
-except ImportError:
-    from subprocess import getstatusoutput
-    from urllib.request import urlopen
+from subprocess import getstatusoutput
+from urllib.request import urlopen
 
 
 def python_version():

ci/build-loop.sh

@@ -9,7 +9,10 @@
 . build-helpers.sh
 get_config
 
-ensure_vars CI_NAME CHECK_NAME PR_REPO PR_BRANCH PACKAGE ALIBUILD_DEFAULTS
+PR_START_TIME=$(TZ=Europe/Zurich date +'%a %H:%M CET')
+echo "$PR_START_TIME: Started building check $CHECK_NAME for $PR_REPO@$PR_HASH on $host_id"
+
+ensure_vars CI_NAME CHECK_NAME PR_REPO PR_BRANCH PACKAGE ALIBUILD_DEFAULTS PR_START_TIME
 
 : "${WORKERS_POOL_SIZE:=1}" "${WORKER_INDEX:=0}" "${PR_REPO_CHECKOUT:=$(basename "$PR_REPO")}"
 
@@ -47,12 +50,12 @@ case "$BUILD_TYPE" in
   # Create a status on GitHub showing the build start time, but only if this is
   # the first build of this check!
   # If we are running in Nomad, add a link to the this allocation.
-  untested) report_pr_errors --pending -m "Started $(TZ=Europe/Zurich date +'%a %H:%M CET') on $host_id" \
+  untested) report_pr_errors --pending -m "Started $PR_START_TIME on $host_id" \
                              ${NOMAD_ALLOC_ID:+--log-url "https://alinomad.cern.ch/ui/allocations/$NOMAD_ALLOC_ID"} ;;
   # Rebuilds only change the existing status's message, keeping the red status
   # and URL intact.
   failed) set-github-status -k -c "$PR_REPO@$PR_HASH" -s "$CHECK_NAME/$(build_type_to_status "$BUILD_TYPE")" \
-                            -m "Rechecking since $(TZ=Europe/Zurich date +'%a %H:%M CET') on $host_id" ;;
+                            -m "Rechecking since $PR_START_TIME on $host_id" ;;
   # See above for why we don't update the status for green checks.
   succeeded) ;;
 esac

ci/repo-config/DEFAULTS.env

@@ -6,6 +6,6 @@ BUILD_SUFFIX=master
 MAX_DIFF_SIZE=20000000
 TIMEOUT=120
 LONG_TIMEOUT=36000
-DOCKER_EXTRA_ARGS='--tmpfs=/dev/shm:rw,size=8g,mode=1777'
-INSTALL_ALIBUILD='alisw/[email protected].10#egg=alibuild'
+DOCKER_EXTRA_ARGS='--tmpfs=/dev/shm:rw,size=10g,mode=1777'
+INSTALL_ALIBUILD='alisw/[email protected].12#egg=alibuild'
 INSTALL_ALIBOT='alisw/ali-bot@master#egg=ali-bot'

ci/repo-config/mesosci/slc7-aliphysics/aliphysics-root6.env

@@ -1,6 +1,6 @@
 CI_NAME=build_AliPhysics_root6
 PACKAGE=AliPhysics
-ALIBUILD_DEFAULTS=ali
+ALIBUILD_DEFAULTS=o2
 PR_REPO=alisw/AliPhysics
 PR_BRANCH=master
 TRUSTED_USERS=

ci/repo-config/mesosci/slc7/aliroot-alidist.env

@@ -1,6 +1,6 @@
 CI_NAME=build_AliRoot_alidist
 PACKAGE=AliRoot
-ALIBUILD_DEFAULTS=ali
+ALIBUILD_DEFAULTS=o2
 PR_REPO=alisw/alidist
 PR_BRANCH=master
 NO_ASSUME_CONSISTENT_EXTERNALS=

ci/repo-config/mesosci/slc7/aliroot.env

@@ -2,7 +2,7 @@ CI_NAME=build_AliRoot_o2
 PACKAGE=AliRoot
 PR_REPO=alisw/AliRoot
 PR_BRANCH=master
-ALIBUILD_DEFAULTS=ali
+ALIBUILD_DEFAULTS=o2
 TRUSTED_USERS=
 TASK_PRIORITY=10
 NO_ASSUME_CONSISTENT_EXTERNALS=

ci/repo-config/mesosci/slc7-o2physics/o2-o2dpg.env → ci/repo-config/mesosci/slc9-o2physics/o2-o2dpg.env

@@ -11,5 +11,4 @@ AliceO2Group/AliceO2 dev O2
 AliceO2Group/O2Physics master
 AliceO2Group/QualityControl master
 alisw/alidist master"
-DOCKER_EXTRA_ARGS='--tmpfs=/dev/shm:rw,size=8g,mode=1777'
 ONLY_RUN_WHEN_CHANGED='DATA/'

ci/repo-config/mesosdaq/slc7/qualitycontrol-fullsystest.env → ci/repo-config/mesosdaq/cs8/qualitycontrol-fullsystest.env

@@ -12,5 +12,4 @@ AliceO2Group/AliceO2 dev O2
 AliceO2Group/O2Physics master
 AliceO2Group/O2DPG master
 alisw/alidist master"
-DOCKER_EXTRA_ARGS='--tmpfs=/dev/shm:rw,size=8g,mode=1777'
-DONT_USE_COMMENTS=1
+DONT_USE_COMMENTS=1