Cleanups to rpm/deb install upgrade scripts

master-libvirt/master.cfg

@@ -198,12 +198,27 @@ def getPAMTestStep():
         command=["./pam-test.sh"],
     )
 
+def getSaveArtifactsStep():
+    return steps.DirectoryUpload(
+        name="save mariadb log files",
+        doStepIf=hasFailed,
+        workersrc="/home/buildbot/logs/",
+        masterdest=util.Interpolate(
+            "/srv/buildbot/packages/"
+            + "%(prop:tarbuildnum)s"
+            + "/logs/"
+            + "%(prop:buildername)s"
+        ),
+    )
+
+
 # FACTORY
 
 ## f_deb_install
 f_deb_install = util.BuildFactory()
 f_deb_install.addStep(getScript("deb-install.sh"))
 f_deb_install.addStep(getDebInstallStep())
+f_deb_install.addStep(getSaveArtifactsStep())
 f_deb_install.addStep(getScript("pam-test.sh"))
 f_deb_install.addStep(getPAMTestStep())
 
@@ -212,11 +227,13 @@ f_deb_upgrade = util.BuildFactory()
 f_deb_upgrade.addStep(getMajorVersionStep())
 f_deb_upgrade.addStep(getScript("deb-upgrade.sh"))
 f_deb_upgrade.addStep(getDebUpgradeStep())
+f_deb_upgrade.addStep(getSaveArtifactsStep())
 
 ## f_rpm_install
 f_rpm_install = util.BuildFactory()
 f_rpm_install.addStep(getScript("rpm-install.sh"))
 f_rpm_install.addStep(getRpmInstallStep())
+f_rpm_install.addStep(getSaveArtifactsStep())
 f_rpm_install.addStep(getScript("pam-test.sh"))
 f_rpm_install.addStep(getPAMTestStep())
 
@@ -225,6 +242,7 @@ f_rpm_upgrade = util.BuildFactory()
 f_rpm_upgrade.addStep(getMajorVersionStep())
 f_rpm_upgrade.addStep(getScript("rpm-upgrade.sh"))
 f_rpm_upgrade.addStep(getRpmUpgradeStep())
+f_rpm_upgrade.addStep(getSaveArtifactsStep())
 
 ####### WORKERS and BUILDERS
 

scripts/bash_lib.sh

@@ -437,27 +437,37 @@ upgrade_test_type() {
   esac
 }
 
-get_columnstore_logs() {
+save_failure_logs() {
+  local logdir=../logs/
+  local logfile=$logdir/$test_mode.log
+  mkdir -p $logdir
+  bb_log_err "Previous step failed, saving mariadb logs"
   if [[ $test_mode == "columnstore" ]]; then
     bb_log_info "storing Columnstore logs in columnstore_logs"
     set +ex
-    # It is done in such a weird way, because Columnstore currently makes its logs hard to read
-    # //TEMP this is fragile and weird (test that /var/log/mariadb/columnstore exist)
-    for f in $(sudo ls /var/log/mariadb/columnstore | xargs); do
-      f=/var/log/mariadb/columnstore/$f
-      echo "----------- $f -----------" >>/home/buildbot/columnstore_logs
-      sudo cat "$f" 1>>/home/buildbot/columnstore_logs 2>&1
+    for f in $(sudo find /tmp/columnstore_tmp_files /var/log/mariadb/columnstore -type f); do
+      echo "----------- $f -----------" >>"$logfile"
+      sudo cat "$f" 1>>"$logfile" 2>&1
     done
-    for f in /tmp/columnstore_tmp_files/*; do
-      echo "----------- $f -----------" >>/home/buildbot/columnstore_logs
-      sudo cat "$f" | sudo tee -a /home/buildbot/columnstore_logs 2>&1
+    for s in mcs-writeengineserver mcs-controllernode.service mcs-ddlproc.service mcs-dmlproc.service \
+            mcs-loadbrm.service mcs-primproc.service [email protected]; do
+        echo "----------- $s -----------" >>"$logfile"
+        sudo journalctl -u "$s" | tee -a "$logfile" 2>&1
     done
+    if [ -d /var/lib/columnstore ]; then
+      tar -Jcvf $logdir/columnstore.tar.bz2 /var/lib/columnstore
+    fi
+  fi
+  echo "----------- mariadb.service -----------" >>"$logfile"
+  sudo journalctl -u mariadb.service | tee -a "$logfile" 2>&1
+  if [ -f "$logfile" ]; then
+    bzip2 "$logfile"
   fi
+  sudo find /var/lib/systemd/coredump/ -type -f -exec mv {} $logdir \;
 }
 
 check_mariadb_server_and_create_structures() {
   # All the commands below should succeed
-  set -e
   sudo mariadb -e "CREATE DATABASE db"
   sudo mariadb -e "CREATE TABLE db.t_innodb(a1 SERIAL, c1 CHAR(8)) ENGINE=InnoDB; INSERT INTO db.t_innodb VALUES (1,'foo'),(2,'bar')"
   sudo mariadb -e "CREATE TABLE db.t_myisam(a2 SERIAL, c2 CHAR(8)) ENGINE=MyISAM; INSERT INTO db.t_myisam VALUES (1,'foo'),(2,'bar')"
@@ -468,19 +478,14 @@ check_mariadb_server_and_create_structures() {
   sudo mariadb -e "CREATE PROCEDURE db.p() SELECT * FROM db.v_merge"
   sudo mariadb -e "CREATE FUNCTION db.f() RETURNS INT DETERMINISTIC RETURN 1"
   if [[ $test_mode == "columnstore" ]]; then
-    if ! sudo mariadb -e "CREATE TABLE db.t_columnstore(a INT, c VARCHAR(8)) ENGINE=ColumnStore; SHOW CREATE TABLE db.t_columnstore; INSERT INTO db.t_columnstore VALUES (1,'foo'),(2,'bar')"; then
-      get_columnstore_logs
-      exit 1
-    fi
+    sudo mariadb -e "CREATE TABLE db.t_columnstore(a INT, c VARCHAR(8)) ENGINE=ColumnStore; SHOW CREATE TABLE db.t_columnstore; INSERT INTO db.t_columnstore VALUES (1,'foo'),(2,'bar')"
   fi
-  set +e
 }
 
 check_mariadb_server_and_verify_structures() {
   # Print "have_xx" capabilitites for the new server
   sudo mariadb -e "select 'Stat' t, variable_name name, variable_value val from information_schema.global_status where variable_name like '%have%' union select 'Vars' t, variable_name name, variable_value val from information_schema.global_variables where variable_name like '%have%' order by t, name"
   # All the commands below should succeed
-  set -e
   sudo mariadb -e "select @@version, @@version_comment"
   sudo mariadb -e "SHOW TABLES IN db"
   sudo mariadb -e "SELECT * FROM db.t_innodb; INSERT INTO db.t_innodb VALUES (3,'foo'),(4,'bar')"
@@ -495,12 +500,8 @@ check_mariadb_server_and_verify_structures() {
   sudo mariadb -e "SELECT db.f()"
 
   if [[ $test_mode == "columnstore" ]]; then
-    if ! sudo mariadb -e "SELECT * FROM db.t_columnstore; INSERT INTO db.t_columnstore VALUES (3,'foo'),(4,'bar')"; then
-      get_columnstore_logs
-      exit 1
-    fi
+    sudo mariadb -e "SELECT * FROM db.t_columnstore; INSERT INTO db.t_columnstore VALUES (3,'foo'),(4,'bar')"
   fi
-  set +e
 }
 
 control_mariadb_server() {
@@ -512,6 +513,11 @@ control_mariadb_server() {
     no)
       sudo /etc/init.d/mysql "$1"
       ;;
+    *)
+      bb_log_warn "should never happen, check your configuration:"
+      bb_log_warn "(systemdCapability property is not set or is set to a wrong value ($systemdCapability))"
+    ;;
+
   esac
 }
 

scripts/deb-install.sh

@@ -69,6 +69,9 @@ fi
 # apt get update may be running in the background (Ubuntu start).
 apt_get_update
 
+# set -e already set at start of script
+trap save_failure_logs ERR
+
 sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
   apt-get install -y $package_list $columnstore_package_list"
 
@@ -78,28 +81,8 @@ wait_for_mariadb_upgrade
 # To avoid confusing errors in further logic, do an explicit check whether the
 # service is up and running
 if [[ $systemdCapability == "yes" ]]; then
-  if ! sudo systemctl status mariadb --no-pager; then
-    sudo journalctl -xe --no-pager
-    bb_log_warn "mariadb service isn't running properly after installation"
-    if echo "$package_list" | grep -q columnstore; then
-      bb_log_info "It is likely to be caused by ColumnStore"
-      bb_log_info "problems upon installation, getting the logs"
-      set +e
-      # It is done in such a weird way, because Columnstore currently makes its
-      # logs hard to read
-      for f in $(sudo ls /var/log/mariadb/columnstore | xargs); do
-        f=/var/log/mariadb/columnstore/$f
-        echo "----------- $f -----------"
-        sudo cat "$f"
-      done
-      for f in /tmp/columnstore_tmp_files/*; do
-        echo "----------- $f -----------"
-        sudo cat "$f"
-      done
-    fi
-    bb_log_err "mariadb service didn't start properly after installation"
-    exit 1
-  fi
+  bb_log_info "test for systemd service for mariadb.service started"
+  sudo systemctl status mariadb.service --no-pager
 fi
 
 # Due to MDEV-14622 and its effect on Spider installation,
@@ -120,6 +103,8 @@ sudo mariadb --verbose -e "create database test; \
   grant all on *.* to galera;"
 sudo mariadb -e "select @@version"
 bb_log_info "test for MDEV-18563, MDEV-18526"
+
+# disabling -e so save_failure_logs won't have an effect while having setting up for next test
 set +e
 
 control_mariadb_server stop
@@ -134,6 +119,7 @@ for p in /bin /sbin /usr/bin /usr/sbin /usr/local/bin /usr/local/sbin; do
   fi
 done
 sudo mariadb-install-db --no-defaults --user=mysql --plugin-maturity=unknown
+
 set +e
 ## Install mariadb-test for further use
 # sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 apt-get install -y mariadb-test"

scripts/deb-upgrade.sh

@@ -110,32 +110,24 @@ apt_get_update
 # We will wait till they finish, to avoid any clashes with SQL we are going to execute
 wait_for_mariadb_upgrade
 
-if ! sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
-  apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $package_list"; then
-  bb_log_err "Installation of a previous release failed, see the output above"
-  exit 1
-fi
+bb_log_info "Install previous version"
+sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
+  apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $package_list"
 
 wait_for_mariadb_upgrade
 
 if [[ -n $spider_package_list ]]; then
-  if ! sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
-    apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $spider_package_list"; then
-    bb_log_err "Installation of Spider from the previous release failed, see the output above"
-    exit 1
-  fi
+  bb_log_info "Install spider packages"
+  sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
+    apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $spider_package_list"
   wait_for_mariadb_upgrade
 fi
 
 # To avoid confusing errors in further logic, do an explicit check
 # whether the service is up and running
 if [[ $systemdCapability == "yes" ]]; then
-  if ! sudo systemctl status mariadb --no-pager; then
-    sudo journalctl -xe --no-pager
-    get_columnstore_logs
-    bb_log_err "mariadb service didn't start properly after installation"
-    exit 1
-  fi
+  bb_log_info "Ensure mariadb.service is running"
+  sudo systemctl status mariadb --no-pager
 fi
 
 if [[ $test_mode == "all" ]]; then
@@ -164,20 +156,20 @@ deb_setup_bb_galera_artifacts_mirror
 deb_setup_bb_artifacts_mirror
 apt_get_update
 
-# Install the new packages
-if ! sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
-  apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $package_list"; then
-  bb_log_err "installation of the new packages failed, see the output above"
-  exit 1
-fi
+# now we upgrade, this is what we should save
+trap save_failure_logs ERR
+set -e
+
+bb_log_info "Install new packages"
+sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
+  apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $package_list"
+
 wait_for_mariadb_upgrade
 
 if [[ -n $spider_package_list ]]; then
-  if ! sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
-    apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $spider_package_list"; then
-    bb_log_err "Installation of the new Spider packages failed, see the output above"
-    exit 1
-  fi
+  bb_log_info "Install spider packages $spider_package_list"
+  sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \
+    apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $spider_package_list"
   wait_for_mariadb_upgrade
 fi
 if [[ $test_mode == "columnstore" ]]; then

scripts/rpm-install.sh

@@ -44,36 +44,32 @@ rpm_setup_bb_artifacts_mirror
 rpm_pkg_makecache
 
 # install all packages
-pkg_list=$(rpm_repoquery) ||
+read -ra package_array = <<< "$(rpm_repoquery)"
+
+if [ ${#package_array[@]} -eq 0 ]; then
   bb_log_err "Unable to retrieve package list from repository"
+fi
 
-# ID_LIKE may not exist
-set +u
-if [[ $ID_LIKE =~ ^suse* ]]; then
-  echo "$pkg_list" | xargs sudo "$pkg_cmd" -n install
+if [[ "${ID_LIKE:-empty}" =~ ^suse* ]]; then
+  sudo "$pkg_cmd" -n install "${package_array[@]}"
 else
-  echo "$pkg_list" | xargs sudo "$pkg_cmd" -y install
+  sudo "$pkg_cmd" -y install "${package_array[@]}"
 fi
-set -u
 
-sh -c 'g=/usr/lib*/galera*/libgalera_smm.so; echo -e "[galera]\nwsrep_provider=$g"' |
+
+galera=(/usr/lib*/galera*/libgalera_smm.so)
+if [ ${#galera[@]} -ne 1 ]; then
+    bb_log_error "Expected exactly one file, found ${#galera[@]}"
+    exit 1
+fi
+echo -e "[galera]\nwsrep_provider=${galera[0]}" |
   sudo tee /etc/my.cnf.d/galera.cnf
-case "$systemdCapability" in
-  yes)
-    if ! sudo systemctl start mariadb; then
-      sudo journalctl -lxn 500 --no-pager -u mariadb.service
-      sudo systemctl -l status mariadb.service --no-pager
-      exit 1
-    fi
-    ;;
-  no)
-    sudo /etc/init.d/mysql restart
-    ;;
-  *)
-    bb_log_warn "should never happen, check your configuration:"
-    bb_log_warn "(systemdCapability property is not set or is set to a wrong value)"
-    ;;
-esac
+
+# Any of the below steps could fail
+trap save_failure_logs ERR
+set -e
+
+control_mariadb_server start
 
 sudo mariadb -e "drop database if exists test; \
   create database test; \