MCOL-6022 mtr step refactoring

.drone.jsonnet

@@ -342,36 +342,13 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
     },
     commands: [
       prepareTestStage(getContainerName("mtr"), result, true),
-      "docker cp mysql-test/columnstore mtr$${DRONE_BUILD_NUMBER}:" + mtr_path + "/suite/",
-      execInnerDocker("chown -R mysql:mysql " + mtr_path, getContainerName("mtr")),
-      // disable systemd 'ProtectSystem' (we need to write to /usr/share/)
-      execInnerDocker("bash -c 'sed -i /ProtectSystem/d $(systemctl show --property FragmentPath mariadb | sed s/FragmentPath=//)'", getContainerName("mtr")),
-      execInnerDocker("systemctl daemon-reload", getContainerName("mtr")),
-      execInnerDocker("systemctl start mariadb", getContainerName("mtr")),
-      // Set RAM consumption limits to avoid RAM contention b/w mtr and regression steps.
-      execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", getContainerName("mtr")),
-      execInnerDocker('mariadb -e "create database if not exists test;"', getContainerName("mtr")),
-      execInnerDocker("systemctl restart mariadb-columnstore", getContainerName("mtr")),
-
-      // delay mtr for manual debugging on live instance
-      "sleep $${MTR_DELAY_SECONDS:-1s}",
       'MTR_SUITE_LIST=$([ "$MTR_FULL_SUITE" == true ] && echo "' + mtr_full_set + '" || echo "$MTR_SUITE_LIST")',
-      if (event == "custom" || event == "cron") then
-        execInnerDocker('bash -c "wget -qO- https://cspkg.s3.amazonaws.com/mtr-test-data.tar.lz4 | lz4 -dc - | tar xf - -C /"',
-                        getContainerName("mtr")),
-      if (event == "custom" || event == "cron") then
-        execInnerDocker('bash -c "cd ' + mtr_path + " && ./mtr --extern socket=" + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/setup"',
-                        getContainerName("mtr")),
-
-      if (event == "cron") then
-        execInnerDocker('bash -c "cd ' + mtr_path + " && ./mtr --extern socket=" + socket_path +
-                        " --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite="
-                        + std.join(",", std.map(function(x) "columnstore/" + x, std.split(mtr_full_set, ","))),
-                        getContainerName("mtr")) + '"'
-      else
-        execInnerDocker('bash -c "cd ' + mtr_path + " && ./mtr --extern socket=" + socket_path +
-                        ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/$${MTR_SUITE_LIST//,/,columnstore/}"',
-                        getContainerName("mtr")),
+
+      'bash /mdb/' + builddir + '/storage/columnstore/columnstore/build/run_mtr.sh' +
+      ' --container-name ' + getContainerName("mtr") +
+      ' --distro ' + platform +
+      ' --suite-list $${MTR_SUITE_LIST}' +
+      ' --triggering-event ' + event,
     ],
   },
   mtrlog:: {

build/prepare_test_stage.sh

@@ -118,7 +118,7 @@ echo "Installing columnstore..."
 if [[ "$RESULT" == *rocky* ]]; then
     execInnerDockerWithRetry "$CONTAINER_NAME" 'yum install -y MariaDB-columnstore-engine MariaDB-test'
 else
-    execInnerDockerWithRetry "$CONTAINER_NAME" 'apt update -y && apt install -y mariadb-plugin-columnstore mariadb-test'
+    execInnerDockerWithRetry "$CONTAINER_NAME" 'apt update -y && apt install -y mariadb-plugin-columnstore mariadb-test mariadb-test-data'
 fi
 
 sleep 5

build/run_mtr.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+set -eo pipefail
+
+SCRIPT_LOCATION=$(dirname "$0")
+source "$SCRIPT_LOCATION"/utils.sh
+
+optparse.define short=c long=container-name desc="Name of the Docker container where mtr tests will run" variable=CONTAINER_NAME
+optparse.define short=d long=distro desc="Linux distro for which mtr is runned" variable=DISTRO
+optparse.define short=s long=suite-list desc="Comma-separated list of test suites to run" variable=MTR_SUITE_LIST
+optparse.define short=e long=triggering-event desc="Event that triggers testrun" variable=EVENT
+source $(optparse.build)
+
+MTR_FULL_SET="basic,bugfixes,devregression,autopilot,extended,multinode,oracle,1pmonly"
+
+echo "Arguments received: $@"
+
+if [[ "$EUID" -ne 0 ]]; then
+    error "Please run script as root"
+    exit 1
+fi
+
+if [[ -z "${CONTAINER_NAME}" ]]; then
+    echo "Please provide mtr container name as a parameter, e.g. ./run_mtr.sh -c mtr183"
+    exit 1
+fi
+
+if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then
+    error "Container '${CONTAINER_NAME}' is not running."
+    exit 1
+fi
+
+if [[ "$DISTRO" == *rocky* ]]; then
+    SOCKET_PATH="/var/lib/mysql/mysql.sock"
+    MTR_PATH="/usr/share/mysql-test"
+else
+    SOCKET_PATH="/run/mysqld/mysqld.sock"
+    MTR_PATH="/usr/share/mysql/mysql-test"
+fi
+
+message "Running mtr tests..."
+
+execInnerDocker "${CONTAINER_NAME}" "chown -R mysql:mysql ${MTR_PATH}"
+
+# disable systemd 'ProtectSystem' (we need to write to /usr/share/)
+execInnerDocker "${CONTAINER_NAME}" "sed -i /ProtectSystem/d \$(systemctl show --property FragmentPath mariadb | sed s/FragmentPath=//) || true"
+execInnerDocker "${CONTAINER_NAME}" "systemctl daemon-reload"
+execInnerDocker "${CONTAINER_NAME}" "systemctl start mariadb"
+
+# Set RAM consumption limits to avoid RAM contention b/w mtr and regression steps.
+execInnerDocker "${CONTAINER_NAME}" "/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local"
+execInnerDocker "${CONTAINER_NAME}" "mariadb -e \"create database if not exists test;\""
+execInnerDocker "${CONTAINER_NAME}" "systemctl restart mariadb-columnstore"
+
+if [[ "${EVENT}" == "custom" || "${EVENT}" == "cron" ]]; then
+    execInnerDocker "${CONTAINER_NAME}" "cd ${MTR_PATH} && ./mtr --extern socket=${SOCKET_PATH} --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/setup"
+    execInnerDocker "${CONTAINER_NAME}" "wget -qO- https://cspkg.s3.amazonaws.com/mtr-test-data.tar.lz4 | lz4 -dc - | tar xf - -C /"
+fi
+
+if [[ "${EVENT}" == "cron" ]]; then
+    execInnerDocker "${CONTAINER_NAME}" "cd ${MTR_PATH} && ./mtr --extern socket=${SOCKET_PATH} --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/${MTR_FULL_SET//,/,columnstore/}"
+else
+    execInnerDocker "${CONTAINER_NAME}" "cd ${MTR_PATH} && ./mtr --extern socket=${SOCKET_PATH} --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/${MTR_SUITE_LIST//,/,columnstore/}"
+fi

mysql-test/columnstore/basic/r/mcol-5005.result

@@ -58,7 +58,8 @@ t4	CREATE TABLE `t4` (
   `a` varchar(15) DEFAULT NULL
 ) ENGINE=Columnstore DEFAULT CHARSET=latin2 COLLATE=latin2_general_ci
 SELECT `schema`, tablename, columnname, charsetnum FROM calpontsys.syscolumn
-WHERE `schema`='mcol5005' AND tablename in ('t1', 't2', 't3', 't4');
+WHERE `schema`='mcol5005' AND tablename in ('t1', 't2', 't3', 't4')
+ORDER BY tablename, columnname, charsetnum;
 schema	tablename	columnname	charsetnum
 mcol5005	t1	a	227
 mcol5005	t1	b	33

mysql-test/columnstore/basic/r/mcol641-create.result

@@ -11,7 +11,17 @@ CREATE TABLE cs3(d1 DECIMAL(38) UNSIGNED ZEROFILL, d2 DECIMAL(38,10) UNSIGNED ZE
 Warnings:
 Warning	1618	ZEROFILL is ignored in ColumnStore
 CREATE TABLE cs4(d1 DECIMAL(18), d2 DECIMAL(18,10), d3 DECIMAL(18,18)) ENGINE=columnstore;
-SELECT TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,DATA_TYPE,COLUMN_LENGTH,COLUMN_POSITION,COLUMN_DEFAULT,NUMERIC_PRECISION,NUMERIC_SCALE FROM information_schema.columnstore_columns WHERE table_name = 'cs1' OR table_name = 'cs2' OR table_name = 'cs3' OR table_name = 'cs4' ORDER BY table_name,column_name ASC;
+SELECT TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,DATA_TYPE,COLUMN_LENGTH,COLUMN_POSITION,COLUMN_DEFAULT,NUMERIC_PRECISION,NUMERIC_SCALE FROM information_schema.columnstore_columns WHERE table_name = 'cs1' OR table_name = 'cs2' OR table_name = 'cs3' OR table_name = 'cs4'
+ORDER BY
+TABLE_SCHEMA,
+TABLE_NAME,
+COLUMN_NAME,
+DATA_TYPE,
+COLUMN_LENGTH,
+COLUMN_POSITION,
+COLUMN_DEFAULT,
+NUMERIC_PRECISION,
+NUMERIC_SCALE;
 TABLE_SCHEMA	TABLE_NAME	COLUMN_NAME	DATA_TYPE	COLUMN_LENGTH	COLUMN_POSITION	COLUMN_DEFAULT	NUMERIC_PRECISION	NUMERIC_SCALE
 test_mcol641_create	cs1	d1	decimal	16	0	NULL	38	0
 test_mcol641_create	cs1	d2	decimal	16	1	NULL	38	10
@@ -40,7 +50,17 @@ ALTER TABLE cs2 ADD COLUMN (d5 DECIMAL(38,5), d6 DECIMAL(35,15));
 ERROR 42000: The storage engine for the table doesn't support Multiple actions in alter table statement is currently not supported by Columnstore.
 ALTER TABLE cs3 MODIFY d1 DECIMAL(38) SIGNED;
 ERROR HY000: Internal error: CAL0001: Alter table Failed:  Altertable: Error in the action type  
-SELECT TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,DATA_TYPE,COLUMN_LENGTH,COLUMN_POSITION,COLUMN_DEFAULT,NUMERIC_PRECISION,NUMERIC_SCALE FROM information_schema.columnstore_columns WHERE table_schema = 'test_mcol641_create' ORDER BY table_name,column_name ASC;
+SELECT TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,DATA_TYPE,COLUMN_LENGTH,COLUMN_POSITION,COLUMN_DEFAULT,NUMERIC_PRECISION,NUMERIC_SCALE FROM information_schema.columnstore_columns WHERE table_schema = 'test_mcol641_create'
+ORDER BY
+TABLE_SCHEMA,
+TABLE_NAME,
+COLUMN_NAME,
+DATA_TYPE,
+COLUMN_LENGTH,
+COLUMN_POSITION,
+COLUMN_DEFAULT,
+NUMERIC_PRECISION,
+NUMERIC_SCALE;
 TABLE_SCHEMA	TABLE_NAME	COLUMN_NAME	DATA_TYPE	COLUMN_LENGTH	COLUMN_POSITION	COLUMN_DEFAULT	NUMERIC_PRECISION	NUMERIC_SCALE
 test_mcol641_create	cs1	d1	decimal	16	0	NULL	38	0
 test_mcol641_create	cs1	d2	decimal	16	1	NULL	38	10

mysql-test/columnstore/basic/r/mcs229_data_compression_type.result

@@ -37,16 +37,16 @@ t2	CREATE TABLE `t2` (
   `t2_DATE` date DEFAULT NULL,
   `t2_TIME` time DEFAULT NULL
 ) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci
-SELECT `schema`, tablename, columnname, compressiontype FROM calpontsys.syscolumn WHERE `schema`='mcs229_db' ORDER BY 2;
+SELECT `schema`, tablename, columnname, compressiontype FROM calpontsys.syscolumn WHERE `schema`='mcs229_db' ORDER BY 2,3,4;
 schema	tablename	columnname	compressiontype
+mcs229_db	t1	t1_date	2
+mcs229_db	t1	t1_decimal	2
 mcs229_db	t1	t1_int	2
 mcs229_db	t1	t1_text	2
-mcs229_db	t1	t1_date	2
 mcs229_db	t1	t1_time	2
-mcs229_db	t1	t1_decimal	2
 mcs229_db	t2	t2_date	2
+mcs229_db	t2	t2_decimal	2
 mcs229_db	t2	t2_int	2
 mcs229_db	t2	t2_text	2
 mcs229_db	t2	t2_time	2
-mcs229_db	t2	t2_decimal	2
 DROP DATABASE mcs229_db;

mysql-test/columnstore/basic/t/mcol-5005.test

@@ -52,7 +52,8 @@ SHOW CREATE TABLE t3;
 SHOW CREATE TABLE t4;
 
 SELECT `schema`, tablename, columnname, charsetnum FROM calpontsys.syscolumn
-WHERE `schema`='mcol5005' AND tablename in ('t1', 't2', 't3', 't4');
+WHERE `schema`='mcol5005' AND tablename in ('t1', 't2', 't3', 't4')
+ORDER BY tablename, columnname, charsetnum;
 
 DROP DATABASE mcol5005;
 

mysql-test/columnstore/basic/t/mcol641-create.test

@@ -19,7 +19,17 @@ CREATE TABLE cs2(d1 DECIMAL(38) SIGNED, d2 DECIMAL(38,10) SIGNED, d3 DECIMAL(38,
 #This must raise a warning
 CREATE TABLE cs3(d1 DECIMAL(38) UNSIGNED ZEROFILL, d2 DECIMAL(38,10) UNSIGNED ZEROFILL, d3 DECIMAL(38,38) UNSIGNED ZEROFILL) ENGINE=columnstore;
 CREATE TABLE cs4(d1 DECIMAL(18), d2 DECIMAL(18,10), d3 DECIMAL(18,18)) ENGINE=columnstore;
-SELECT TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,DATA_TYPE,COLUMN_LENGTH,COLUMN_POSITION,COLUMN_DEFAULT,NUMERIC_PRECISION,NUMERIC_SCALE FROM information_schema.columnstore_columns WHERE table_name = 'cs1' OR table_name = 'cs2' OR table_name = 'cs3' OR table_name = 'cs4' ORDER BY table_name,column_name ASC;
+SELECT TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,DATA_TYPE,COLUMN_LENGTH,COLUMN_POSITION,COLUMN_DEFAULT,NUMERIC_PRECISION,NUMERIC_SCALE FROM information_schema.columnstore_columns WHERE table_name = 'cs1' OR table_name = 'cs2' OR table_name = 'cs3' OR table_name = 'cs4'
+ORDER BY
+TABLE_SCHEMA,
+TABLE_NAME,
+COLUMN_NAME,
+DATA_TYPE,
+COLUMN_LENGTH,
+COLUMN_POSITION,
+COLUMN_DEFAULT,
+NUMERIC_PRECISION,
+NUMERIC_SCALE;
 
 ALTER TABLE cs1 DROP COLUMN d4;
 ALTER TABLE cs1 ADD COLUMN d7 DECIMAL(38,37);
@@ -34,7 +44,17 @@ ALTER TABLE cs2 ADD COLUMN (d5 DECIMAL(38,5), d6 DECIMAL(35,15));
 # Bug. This must work fine.
 --error 1815
 ALTER TABLE cs3 MODIFY d1 DECIMAL(38) SIGNED;
-SELECT TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,DATA_TYPE,COLUMN_LENGTH,COLUMN_POSITION,COLUMN_DEFAULT,NUMERIC_PRECISION,NUMERIC_SCALE FROM information_schema.columnstore_columns WHERE table_schema = 'test_mcol641_create' ORDER BY table_name,column_name ASC;
+SELECT TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,DATA_TYPE,COLUMN_LENGTH,COLUMN_POSITION,COLUMN_DEFAULT,NUMERIC_PRECISION,NUMERIC_SCALE FROM information_schema.columnstore_columns WHERE table_schema = 'test_mcol641_create'
+ORDER BY
+TABLE_SCHEMA,
+TABLE_NAME,
+COLUMN_NAME,
+DATA_TYPE,
+COLUMN_LENGTH,
+COLUMN_POSITION,
+COLUMN_DEFAULT,
+NUMERIC_PRECISION,
+NUMERIC_SCALE;
 
 # This must return an error
 --error 1815

mysql-test/columnstore/basic/t/mcs229_data_compression_type.test

@@ -49,7 +49,7 @@ CREATE TABLE t2
 --replace_regex /( COLLATE=latin1_swedish_ci)//
 SHOW CREATE TABLE t2;
 
-SELECT `schema`, tablename, columnname, compressiontype FROM calpontsys.syscolumn WHERE `schema`='mcs229_db' ORDER BY 2;
+SELECT `schema`, tablename, columnname, compressiontype FROM calpontsys.syscolumn WHERE `schema`='mcs229_db' ORDER BY 2,3,4;
 
 # Clean UP
 DROP DATABASE mcs229_db;

tests/scripts/fullmtr.sh

@@ -4,7 +4,7 @@ SCRIPT_LOCATION=$(dirname "$0")
 MARIADB_SOURCE_PATH=$(realpath $SCRIPT_LOCATION/../../../../../)
 COLUMNSTORE_MTR_SOURCE=$(realpath $SCRIPT_LOCATION/../../mysql-test/columnstore)
 INSTALLED_MTR_PATH='/usr/share/mysql/mysql-test'
-COLUMSNTORE_MTR_INSTALLED=${INSTALLED_MTR_PATH}/suite/columnstore
+COLUMSNTORE_MTR_INSTALLED=${INSTALLED_MTR_PATH}/plugin/columnstore/columnstore/
 PATCHNAME=$(realpath $SCRIPT_LOCATION)/mtr_warn.patch
 CURRENT_DIR=`pwd`
 mysql -e "create database if not exists test;"