diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg index 2838d542..90be25f8 100644 --- a/master-libvirt/master.cfg +++ b/master-libvirt/master.cfg @@ -198,12 +198,27 @@ def getPAMTestStep(): command=["./pam-test.sh"], ) +def getSaveArtifactsStep(): + return steps.DirectoryUpload( + name="save mariadb log files", + doStepIf=hasFailed, + workersrc="/home/buildbot/logs/", + masterdest=util.Interpolate( + "/srv/buildbot/packages/" + + "%(prop:tarbuildnum)s" + + "/logs/" + + "%(prop:buildername)s" + ), + ) + + # FACTORY ## f_deb_install f_deb_install = util.BuildFactory() f_deb_install.addStep(getScript("deb-install.sh")) f_deb_install.addStep(getDebInstallStep()) +f_deb_install.addStep(getSaveArtifactsStep()) f_deb_install.addStep(getScript("pam-test.sh")) f_deb_install.addStep(getPAMTestStep()) @@ -212,11 +227,13 @@ f_deb_upgrade = util.BuildFactory() f_deb_upgrade.addStep(getMajorVersionStep()) f_deb_upgrade.addStep(getScript("deb-upgrade.sh")) f_deb_upgrade.addStep(getDebUpgradeStep()) +f_deb_upgrade.addStep(getSaveArtifactsStep()) ## f_rpm_install f_rpm_install = util.BuildFactory() f_rpm_install.addStep(getScript("rpm-install.sh")) f_rpm_install.addStep(getRpmInstallStep()) +f_rpm_install.addStep(getSaveArtifactsStep()) f_rpm_install.addStep(getScript("pam-test.sh")) f_rpm_install.addStep(getPAMTestStep()) @@ -225,6 +242,7 @@ f_rpm_upgrade = util.BuildFactory() f_rpm_upgrade.addStep(getMajorVersionStep()) f_rpm_upgrade.addStep(getScript("rpm-upgrade.sh")) f_rpm_upgrade.addStep(getRpmUpgradeStep()) +f_rpm_upgrade.addStep(getSaveArtifactsStep()) ####### WORKERS and BUILDERS diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh index b0f3bf14..fc816c88 100644 --- a/scripts/bash_lib.sh +++ b/scripts/bash_lib.sh @@ -437,27 +437,37 @@ upgrade_test_type() { esac } -get_columnstore_logs() { +save_failure_logs() { + local logdir=../logs/ + local logfile=$logdir/$test_mode.log + mkdir -p $logdir + bb_log_err "Previous step failed, saving mariadb logs" if [[ $test_mode == "columnstore" ]]; then bb_log_info "storing Columnstore logs in columnstore_logs" set +ex - # It is done in such a weird way, because Columnstore currently makes its logs hard to read - # //TEMP this is fragile and weird (test that /var/log/mariadb/columnstore exist) - for f in $(sudo ls /var/log/mariadb/columnstore | xargs); do - f=/var/log/mariadb/columnstore/$f - echo "----------- $f -----------" >>/home/buildbot/columnstore_logs - sudo cat "$f" 1>>/home/buildbot/columnstore_logs 2>&1 + for f in $(sudo find /tmp/columnstore_tmp_files /var/log/mariadb/columnstore -type f); do + echo "----------- $f -----------" >>"$logfile" + sudo cat "$f" 1>>"$logfile" 2>&1 done - for f in /tmp/columnstore_tmp_files/*; do - echo "----------- $f -----------" >>/home/buildbot/columnstore_logs - sudo cat "$f" | sudo tee -a /home/buildbot/columnstore_logs 2>&1 + for s in mcs-writeengineserver mcs-controllernode.service mcs-ddlproc.service mcs-dmlproc.service \ + mcs-loadbrm.service mcs-primproc.service mcs-workernode@1.service; do + echo "----------- $s -----------" >>"$logfile" + sudo journalctl -u "$s" | tee -a "$logfile" 2>&1 done + if [ -d /var/lib/columnstore ]; then + tar -Jcvf $logdir/columnstore.tar.bz2 /var/lib/columnstore + fi + fi + echo "----------- mariadb.service -----------" >>"$logfile" + sudo journalctl -u mariadb.service | tee -a "$logfile" 2>&1 + if [ -f "$logfile" ]; then + bzip2 "$logfile" fi + sudo find /var/lib/systemd/coredump/ -type -f -exec mv {} $logdir \; } check_mariadb_server_and_create_structures() { # All the commands below should succeed - set -e sudo mariadb -e "CREATE DATABASE db" sudo mariadb -e "CREATE TABLE db.t_innodb(a1 SERIAL, c1 CHAR(8)) ENGINE=InnoDB; INSERT INTO db.t_innodb VALUES (1,'foo'),(2,'bar')" sudo mariadb -e "CREATE TABLE db.t_myisam(a2 SERIAL, c2 CHAR(8)) ENGINE=MyISAM; INSERT INTO db.t_myisam VALUES (1,'foo'),(2,'bar')" @@ -468,19 +478,14 @@ check_mariadb_server_and_create_structures() { sudo mariadb -e "CREATE PROCEDURE db.p() SELECT * FROM db.v_merge" sudo mariadb -e "CREATE FUNCTION db.f() RETURNS INT DETERMINISTIC RETURN 1" if [[ $test_mode == "columnstore" ]]; then - if ! sudo mariadb -e "CREATE TABLE db.t_columnstore(a INT, c VARCHAR(8)) ENGINE=ColumnStore; SHOW CREATE TABLE db.t_columnstore; INSERT INTO db.t_columnstore VALUES (1,'foo'),(2,'bar')"; then - get_columnstore_logs - exit 1 - fi + sudo mariadb -e "CREATE TABLE db.t_columnstore(a INT, c VARCHAR(8)) ENGINE=ColumnStore; SHOW CREATE TABLE db.t_columnstore; INSERT INTO db.t_columnstore VALUES (1,'foo'),(2,'bar')" fi - set +e } check_mariadb_server_and_verify_structures() { # Print "have_xx" capabilitites for the new server sudo mariadb -e "select 'Stat' t, variable_name name, variable_value val from information_schema.global_status where variable_name like '%have%' union select 'Vars' t, variable_name name, variable_value val from information_schema.global_variables where variable_name like '%have%' order by t, name" # All the commands below should succeed - set -e sudo mariadb -e "select @@version, @@version_comment" sudo mariadb -e "SHOW TABLES IN db" sudo mariadb -e "SELECT * FROM db.t_innodb; INSERT INTO db.t_innodb VALUES (3,'foo'),(4,'bar')" @@ -495,12 +500,8 @@ check_mariadb_server_and_verify_structures() { sudo mariadb -e "SELECT db.f()" if [[ $test_mode == "columnstore" ]]; then - if ! sudo mariadb -e "SELECT * FROM db.t_columnstore; INSERT INTO db.t_columnstore VALUES (3,'foo'),(4,'bar')"; then - get_columnstore_logs - exit 1 - fi + sudo mariadb -e "SELECT * FROM db.t_columnstore; INSERT INTO db.t_columnstore VALUES (3,'foo'),(4,'bar')" fi - set +e } control_mariadb_server() { @@ -512,6 +513,11 @@ control_mariadb_server() { no) sudo /etc/init.d/mysql "$1" ;; + *) + bb_log_warn "should never happen, check your configuration:" + bb_log_warn "(systemdCapability property is not set or is set to a wrong value ($systemdCapability))" + ;; + esac } diff --git a/scripts/deb-install.sh b/scripts/deb-install.sh index 1de387be..c94d345d 100755 --- a/scripts/deb-install.sh +++ b/scripts/deb-install.sh @@ -69,6 +69,9 @@ fi # apt get update may be running in the background (Ubuntu start). apt_get_update +# set -e already set at start of script +trap save_failure_logs ERR + sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \ apt-get install -y $package_list $columnstore_package_list" @@ -78,28 +81,8 @@ wait_for_mariadb_upgrade # To avoid confusing errors in further logic, do an explicit check whether the # service is up and running if [[ $systemdCapability == "yes" ]]; then - if ! sudo systemctl status mariadb --no-pager; then - sudo journalctl -xe --no-pager - bb_log_warn "mariadb service isn't running properly after installation" - if echo "$package_list" | grep -q columnstore; then - bb_log_info "It is likely to be caused by ColumnStore" - bb_log_info "problems upon installation, getting the logs" - set +e - # It is done in such a weird way, because Columnstore currently makes its - # logs hard to read - for f in $(sudo ls /var/log/mariadb/columnstore | xargs); do - f=/var/log/mariadb/columnstore/$f - echo "----------- $f -----------" - sudo cat "$f" - done - for f in /tmp/columnstore_tmp_files/*; do - echo "----------- $f -----------" - sudo cat "$f" - done - fi - bb_log_err "mariadb service didn't start properly after installation" - exit 1 - fi + bb_log_info "test for systemd service for mariadb.service started" + sudo systemctl status mariadb.service --no-pager fi # Due to MDEV-14622 and its effect on Spider installation, @@ -120,6 +103,8 @@ sudo mariadb --verbose -e "create database test; \ grant all on *.* to galera;" sudo mariadb -e "select @@version" bb_log_info "test for MDEV-18563, MDEV-18526" + +# disabling -e so save_failure_logs won't have an effect while having setting up for next test set +e control_mariadb_server stop @@ -134,6 +119,7 @@ for p in /bin /sbin /usr/bin /usr/sbin /usr/local/bin /usr/local/sbin; do fi done sudo mariadb-install-db --no-defaults --user=mysql --plugin-maturity=unknown + set +e ## Install mariadb-test for further use # sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 apt-get install -y mariadb-test" diff --git a/scripts/deb-upgrade.sh b/scripts/deb-upgrade.sh index f25f348e..932ac7b2 100755 --- a/scripts/deb-upgrade.sh +++ b/scripts/deb-upgrade.sh @@ -110,32 +110,24 @@ apt_get_update # We will wait till they finish, to avoid any clashes with SQL we are going to execute wait_for_mariadb_upgrade -if ! sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \ - apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $package_list"; then - bb_log_err "Installation of a previous release failed, see the output above" - exit 1 -fi +bb_log_info "Install previous version" +sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \ + apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $package_list" wait_for_mariadb_upgrade if [[ -n $spider_package_list ]]; then - if ! sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \ - apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $spider_package_list"; then - bb_log_err "Installation of Spider from the previous release failed, see the output above" - exit 1 - fi + bb_log_info "Install spider packages" + sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \ + apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $spider_package_list" wait_for_mariadb_upgrade fi # To avoid confusing errors in further logic, do an explicit check # whether the service is up and running if [[ $systemdCapability == "yes" ]]; then - if ! sudo systemctl status mariadb --no-pager; then - sudo journalctl -xe --no-pager - get_columnstore_logs - bb_log_err "mariadb service didn't start properly after installation" - exit 1 - fi + bb_log_info "Ensure mariadb.service is running" + sudo systemctl status mariadb --no-pager fi if [[ $test_mode == "all" ]]; then @@ -164,20 +156,20 @@ deb_setup_bb_galera_artifacts_mirror deb_setup_bb_artifacts_mirror apt_get_update -# Install the new packages -if ! sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \ - apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $package_list"; then - bb_log_err "installation of the new packages failed, see the output above" - exit 1 -fi +# now we upgrade, this is what we should save +trap save_failure_logs ERR +set -e + +bb_log_info "Install new packages" +sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \ + apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $package_list" + wait_for_mariadb_upgrade if [[ -n $spider_package_list ]]; then - if ! sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \ - apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $spider_package_list"; then - bb_log_err "Installation of the new Spider packages failed, see the output above" - exit 1 - fi + bb_log_info "Install spider packages $spider_package_list" + sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \ + apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $spider_package_list" wait_for_mariadb_upgrade fi if [[ $test_mode == "columnstore" ]]; then diff --git a/scripts/rpm-install.sh b/scripts/rpm-install.sh index 0c7a3dec..c6207659 100755 --- a/scripts/rpm-install.sh +++ b/scripts/rpm-install.sh @@ -44,36 +44,32 @@ rpm_setup_bb_artifacts_mirror rpm_pkg_makecache # install all packages -pkg_list=$(rpm_repoquery) || +read -ra package_array = <<< "$(rpm_repoquery)" + +if [ ${#package_array[@]} -eq 0 ]; then bb_log_err "Unable to retrieve package list from repository" +fi -# ID_LIKE may not exist -set +u -if [[ $ID_LIKE =~ ^suse* ]]; then - echo "$pkg_list" | xargs sudo "$pkg_cmd" -n install +if [[ "${ID_LIKE:-empty}" =~ ^suse* ]]; then + sudo "$pkg_cmd" -n install "${package_array[@]}" else - echo "$pkg_list" | xargs sudo "$pkg_cmd" -y install + sudo "$pkg_cmd" -y install "${package_array[@]}" fi -set -u -sh -c 'g=/usr/lib*/galera*/libgalera_smm.so; echo -e "[galera]\nwsrep_provider=$g"' | + +galera=(/usr/lib*/galera*/libgalera_smm.so) +if [ ${#galera[@]} -ne 1 ]; then + bb_log_error "Expected exactly one file, found ${#galera[@]}" + exit 1 +fi +echo -e "[galera]\nwsrep_provider=${galera[0]}" | sudo tee /etc/my.cnf.d/galera.cnf -case "$systemdCapability" in - yes) - if ! sudo systemctl start mariadb; then - sudo journalctl -lxn 500 --no-pager -u mariadb.service - sudo systemctl -l status mariadb.service --no-pager - exit 1 - fi - ;; - no) - sudo /etc/init.d/mysql restart - ;; - *) - bb_log_warn "should never happen, check your configuration:" - bb_log_warn "(systemdCapability property is not set or is set to a wrong value)" - ;; -esac + +# Any of the below steps could fail +trap save_failure_logs ERR +set -e + +control_mariadb_server start sudo mariadb -e "drop database if exists test; \ create database test; \ diff --git a/scripts/rpm-upgrade.sh b/scripts/rpm-upgrade.sh index 991f1ad7..812b7ef9 100755 --- a/scripts/rpm-upgrade.sh +++ b/scripts/rpm-upgrade.sh @@ -119,9 +119,7 @@ bb_log_info "Package_list: $package_list" # /etc/zypp/repos.d/SUSE_Linux_Enterprise_Server_12_SP3_x86_64:SLES12-SP3-Updates.repo # /etc/zypp/repos.d/SUSE_Linux_Enterprise_Server_12_SP3_x86_64:SLES12-SP3-Pool.repo -# ID_LIKE may not exist -set +u -if [[ $ID_LIKE =~ ^suse* ]]; then +if [[ "${ID_LIKE:-empty}" =~ ^suse* ]]; then sudo "$pkg_cmd" clean --all pkg_cmd_options="-n" pkg_cmd_upgrade="update" @@ -132,38 +130,19 @@ else fi set -u +read -ra package_array <<< "$package_list" + # Install previous release -echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_options" install || - bb_log_err "installation of a previous release failed, see the output above" -#fi +bb_log_info "Install previous release" +sudo "$pkg_cmd" "$pkg_cmd_options" install "${package_array[@]}" # Start the server, check that it is working and create some structures -case $(expr "$prev_major_version" '<' "10.1")"$systemdCapability" in - 0yes) - sudo systemctl start mariadb - if [[ $distro != *"sles"* ]] && [[ $distro != *"suse"* ]]; then - sudo systemctl enable mariadb - else - bb_log_warn "due to MDEV-23044 mariadb service won't be enabled in the test" - fi - sudo systemctl status mariadb --no-pager - ;; - *) - sudo /etc/init.d/mysql start - ;; -esac - -# shellcheck disable=SC2181 -if (($? != 0)); then - bb_log_err "Server startup failed" - sudo cat /var/log/messages | grep -iE 'mysqld|mariadb' - sudo cat /var/lib/mysql/*.err - exit 1 -fi +# +control_mariadb_server start check_mariadb_server_and_create_structures -# Store information about the server before upgrade +bb_log_info "Store information about the server before upgrade" collect_dependencies old rpm store_mariadb_server_info old @@ -187,19 +166,25 @@ fi # //TEMP upgrade does not work without this but why? Can't we fix it? if [[ $test_type == "major" ]]; then bb_log_info "remove old packages for major upgrade" - packages_to_remove=$(rpm -qa | grep 'MariaDB-' | awk -F'-' '{print $1"-"$2}') - echo "$packages_to_remove" | xargs sudo "$pkg_cmd" "$pkg_cmd_options" remove + readarray -t package_array <<< "$(rpm -qa | grep 'MariaDB-' | awk -F'-' '{print $1"-"$2}')" + sudo "$pkg_cmd" "$pkg_cmd_options" remove "${package_array[@]}" rpm -qa | grep -iE 'maria|mysql' || true fi rpm_setup_bb_galera_artifacts_mirror rpm_setup_bb_artifacts_mirror + +# Any of the below steps could fail +# This is where the new packages are processed from +trap save_failure_logs ERR +set -e + if [[ $test_type == "major" ]]; then # major upgrade (remove then install) - echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_options" install + sudo "$pkg_cmd" "$pkg_cmd_options" install "${package_array[@]}" else # minor upgrade (upgrade works) - echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_options" "$pkg_cmd_upgrade" + sudo "$pkg_cmd" "$pkg_cmd_options" "$pkg_cmd_upgrade" "${package_array[@]}" fi # set +e @@ -225,25 +210,24 @@ if [[ $test_type == "major" ]] || [[ $test_mode == "columnstore" ]]; then control_mariadb_server restart fi -# Make sure that the new server is running -if sudo mariadb -e "select @@version" | grep "$old_version"; then - bb_log_err "the server was not upgraded or was not restarted after upgrade" - exit 1 -fi +bb_log_info "Make sure that the new server is running" +sudo mariadb -e "select @@version" | grep "$old_version" -# Run mariadb-upgrade for non-GA branches (minor upgrades in GA branches -# shouldn't need it) if [[ $major_version == "$development_branch" ]] || [[ $test_type == "major" ]]; then + bb_log_info "Run MariaDB upgrade" sudo mariadb-upgrade +else + bb_log_info "Skipping mariadb-upgrade for non-GA branches and minor upgrades in GA branchesa - shouldn't need it" fi -# Check that the server is functioning and previously created structures are available +bb_log_info "Check that the server is functioning and previously created structures are available" check_mariadb_server_and_verify_structures -# Store information about the server after upgrade +bb_log_info "Store information about the server after upgrade" collect_dependencies new rpm store_mariadb_server_info new +bb_log_info "Check upgrade versions" check_upgraded_versions bb_log_ok "all done"