Skip to content

Commit 800e679

Browse files
mariadb-SergeyZefirovmariadb-SergeyZefirov
committed
revert changes in drone.jsonnet
1 parent 39c28ba commit 800e679

File tree

1 file changed

+224
-1
lines changed

1 file changed

+224
-1
lines changed

‎.drone.jsonnet

Lines changed: 224 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,19 @@
11
local events = ['pull_request', 'cron'];
22

33
local servers = {
4+
develop: ['10.6-enterprise'],
45
'stable-23.10': ['10.6-enterprise'],
56
};
67

78
local platforms = {
8-
'stable-23.10': ['rockylinux:8', 'rockylinux:9'],
9+
develop: ['rockylinux:8', 'rockylinux:9', 'debian:11', 'debian:12', 'ubuntu:20.04', 'ubuntu:22.04', 'ubuntu:24.04'],
10+
'stable-23.10': ['rockylinux:8', 'rockylinux:9', 'debian:11', 'debian:12', 'ubuntu:20.04', 'ubuntu:22.04','ubuntu:24.04'],
911
};
1012

13+
local platforms_arm = {
14+
develop: ['rockylinux:8', 'rockylinux:9', 'debian:11', 'debian:12', 'ubuntu:20.04', 'ubuntu:22.04', 'ubuntu:24.04'],
15+
'stable-23.10': ['rockylinux:8', 'rockylinux:9', 'debian:11', 'debian:12', 'ubuntu:20.04', 'ubuntu:22.04', 'ubuntu:24.04'],
16+
};
1117

1218
local any_branch = '**';
1319
local platforms_custom = platforms.develop;
@@ -18,20 +24,49 @@ local platforms_mtr = platforms.develop;
1824
local builddir = 'verylongdirnameforverystrangecpackbehavior';
1925

2026
local mtr_suite_list = 'basic,bugfixes';
27+
local mtr_full_set = 'basic,bugfixes,devregression,autopilot,extended,multinode,oracle,1pmonly';
2128

2229
local upgrade_test_lists = {
2330
"rockylinux8": {
31+
"arm64": ["10.6.4-1", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.15-10"],
2432
"amd64": ["10.6.4-1", "10.6.5-2", "10.6.7-3", "10.6.8-4", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"]
2533
},
2634
"rockylinux9": {
35+
"arm64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"],
2736
"amd64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"]
2837
},
38+
"debian11": {
39+
"arm64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"],
40+
"amd64": ["10.6.5-2", "10.6.7-3", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"]
41+
},
42+
"debian12": {
43+
"arm64": [],
44+
"amd64": []
45+
},
46+
"ubuntu20.04": {
47+
"arm64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"],
48+
"amd64": ["10.6.4-1", "10.6.5-2", "10.6.7-3", "10.6.8-4", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"]
49+
},
50+
"ubuntu22.04": {
51+
"arm64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"],
52+
"amd64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"]
53+
},
54+
"ubuntu24.04":
55+
{
56+
"arm64": [],
57+
"amd64": []
58+
},
2959
};
3060

3161
local testRun(platform) =
3262
local platform_map = {
3363
'rockylinux:8': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure',
3464
'rockylinux:9': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure',
65+
'debian:11': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure',
66+
'debian:12': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure',
67+
'ubuntu:20.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure',
68+
'ubuntu:22.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure',
69+
'ubuntu:24.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure',
3570

3671
};
3772
platform_map[platform];
@@ -49,11 +84,17 @@ local rockylinux9_deps = "dnf install -y 'dnf-command(config-manager)' " +
4984

5085
local rockylinux_common_deps = ' && dnf install -y git lz4 lz4-devel cppunit-devel cmake3 boost-devel snappy-devel pcre2-devel';
5186

87+
local deb_deps = 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev';
5288

5389
local testPreparation(platform) =
5490
local platform_map = {
5591
'rockylinux:8': rockylinux8_deps + rockylinux_common_deps,
5692
'rockylinux:9': rockylinux9_deps + rockylinux_common_deps,
93+
'debian:11': deb_deps,
94+
'debian:12': deb_deps,
95+
'ubuntu:20.04': deb_deps,
96+
'ubuntu:22.04': deb_deps,
97+
'ubuntu:24.04': deb_deps,
5798

5899
};
59100
platform_map[platform];
@@ -264,6 +305,162 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
264305
execInnerDocker('mariadb -e "insert into test.t1 values (2); select * from test.t1"', dockerImage("smoke")),
265306
],
266307
},
308+
upgrade(version):: {
309+
name: 'upgrade-test from ' + version,
310+
depends_on: ['regressionlog'],
311+
image: 'docker',
312+
volumes: [pipeline._volumes.docker],
313+
environment: {
314+
UPGRADE_TOKEN: {
315+
from_secret: 'es_token',
316+
},
317+
},
318+
commands: [
319+
// why do we mount cgroups here, but miss it on other steps?
320+
'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --env UCF_FORCE_CONFNEW=1 --name upgrade$${DRONE_BUILD_NUMBER}' + version + ' --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target']
321+
+ prepareTestStage(dockerImage('upgrade') + version, pkg_format, result, false) + [
322+
if (pkg_format == 'deb')
323+
then execInnerDocker('bash -c "./upgrade_setup_deb.sh '+ version + ' ' + result + ' ' + arch + ' ' + repo_pkg_url_no_res +' $${UPGRADE_TOKEN}"',
324+
dockerImage('upgrade') + version),
325+
if (std.split(platform, ':')[0] == 'rockylinux')
326+
then execInnerDocker('bash -c "./upgrade_setup_rpm.sh '+ version + ' ' + result + ' ' + arch + ' ' + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"',
327+
dockerImage('upgrade') + version),
328+
],
329+
},
330+
upgradelog:: {
331+
name: 'upgradelog',
332+
depends_on: std.map(function(p) 'upgrade-test from ' + p, mdb_server_versions),
333+
image: 'docker',
334+
volumes: [pipeline._volumes.docker],
335+
commands: [
336+
'echo',
337+
] + std.flatMap(function(ver) reportTestStage(dockerImage('upgrade') + ver, result, "upgrade_"+ver), mdb_server_versions),
338+
when: {
339+
status: ['success', 'failure'],
340+
},
341+
},
342+
mtr:: {
343+
name: 'mtr',
344+
depends_on: ['smoke'],
345+
image: 'docker:git',
346+
volumes: [pipeline._volumes.docker],
347+
environment: {
348+
MTR_SUITE_LIST: '${MTR_SUITE_LIST:-' + mtr_suite_list + '}',
349+
MTR_FULL_SUITE: '${MTR_FULL_SUITE:-false}',
350+
},
351+
commands: [
352+
'docker run --shm-size=500m --memory 8g --env MYSQL_TEST_DIR=' + mtr_path + ' --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target']
353+
+ prepareTestStage('mtr$${DRONE_BUILD_NUMBER}', pkg_format, result, true) + [
354+
installEngine(dockerImage("mtr"), pkg_format),
355+
'docker cp mysql-test/columnstore mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/suite/',
356+
execInnerDocker('chown -R mysql:mysql ' + mtr_path, dockerImage("mtr")),
357+
// disable systemd 'ProtectSystem' (we need to write to /usr/share/)
358+
execInnerDocker("bash -c 'sed -i /ProtectSystem/d $(systemctl show --property FragmentPath mariadb | sed s/FragmentPath=//)'", dockerImage('mtr')),
359+
execInnerDocker('systemctl daemon-reload', dockerImage("mtr")),
360+
execInnerDocker('systemctl start mariadb', dockerImage("mtr")),
361+
// Set RAM consumption limits to avoid RAM contention b/w mtr and regression steps.
362+
execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", dockerImage("mtr")),
363+
execInnerDocker('mariadb -e "create database if not exists test;"', dockerImage("mtr")),
364+
execInnerDocker('systemctl restart mariadb-columnstore', dockerImage("mtr")),
365+
366+
// delay mtr for manual debugging on live instance
367+
'sleep $${MTR_DELAY_SECONDS:-1s}',
368+
'MTR_SUITE_LIST=$([ "$MTR_FULL_SUITE" == true ] && echo "' + mtr_full_set + '" || echo "$MTR_SUITE_LIST")',
369+
if (event == 'custom' || event == 'cron') then
370+
execInnerDocker('bash -c "wget -qO- https://cspkg.s3.amazonaws.com/mtr-test-data.tar.lz4 | lz4 -dc - | tar xf - -C /"',
371+
dockerImage('mtr')),
372+
if (event == 'custom' || event == 'cron') then
373+
execInnerDocker('bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/setup"',
374+
dockerImage('mtr')),
375+
376+
if (event == 'cron') then
377+
execInnerDocker('bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path +
378+
' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite='
379+
+ std.join(',', std.map(function(x) 'columnstore/' + x, std.split(mtr_full_set, ','))),
380+
dockerImage('mtr')) + '"'
381+
else
382+
execInnerDocker('bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path +
383+
' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/$${MTR_SUITE_LIST//,/,columnstore/}"',
384+
dockerImage('mtr')),
385+
],
386+
},
387+
mtrlog:: {
388+
name: 'mtrlog',
389+
depends_on: ['mtr'],
390+
image: 'docker',
391+
volumes: [pipeline._volumes.docker],
392+
commands: [
393+
'echo "---------- start mariadb service logs ----------"',
394+
execInnerDocker('journalctl -u mariadb --no-pager || echo "mariadb service failure"', dockerImage('mtr')),
395+
'echo "---------- end mariadb service logs ----------"',
396+
'echo',
397+
'echo "---------- start columnstore debug log ----------"',
398+
execInnerDocker('cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', dockerImage('mtr')),
399+
'echo "---------- end columnstore debug log ----------"',
400+
'echo "---------- end columnstore debug log ----------"',
401+
'docker cp mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/var/log /drone/src/' + result + '/mtr-logs || echo "missing ' + mtr_path + '/var/log"'
402+
] + reportTestStage(dockerImage('mtr'), result, "mtr"),
403+
when: {
404+
status: ['success', 'failure'],
405+
},
406+
},
407+
prepare_regression:: {
408+
name: 'prepare regression',
409+
depends_on: ['mtr', 'publish pkg', 'publish cmapi build'],
410+
when: {
411+
status: ['success', 'failure'],
412+
},
413+
image: 'docker:git',
414+
volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
415+
environment: {
416+
REGRESSION_BRANCH_REF: '${DRONE_SOURCE_BRANCH}',
417+
REGRESSION_REF_AUX: branch_ref,
418+
},
419+
commands: [
420+
// compute branch.
421+
'echo "$$REGRESSION_REF"',
422+
'echo "$$REGRESSION_BRANCH_REF"',
423+
// if REGRESSION_REF is empty, try to see whether regression repository has a branch named as one we PR.
424+
'export REGRESSION_REF=$${REGRESSION_REF:-$$(git ls-remote https://github.com/mariadb-corporation/mariadb-columnstore-regression-test --h --sort origin "refs/heads/$$REGRESSION_BRANCH_REF" | grep -E -o "[^/]+$$")}',
425+
'echo "$$REGRESSION_REF"',
426+
// REGRESSION_REF can be empty if there is no appropriate branch in regression repository.
427+
// assign what is appropriate by default.
428+
'export REGRESSION_REF=$${REGRESSION_REF:-$$REGRESSION_REF_AUX}',
429+
'echo "$$REGRESSION_REF"',
430+
// clone regression test repo
431+
'git clone --recurse-submodules --branch $$REGRESSION_REF --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-regression-test',
432+
// where are we now?
433+
'cd mariadb-columnstore-regression-test',
434+
'git rev-parse --abbrev-ref HEAD && git rev-parse HEAD',
435+
'cd ..',
436+
'docker run --shm-size=500m --memory 10g --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name regression$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target']
437+
+ prepareTestStage(dockerImage('regression'), pkg_format, result, true) + [
438+
439+
'docker cp mariadb-columnstore-regression-test regression$${DRONE_BUILD_NUMBER}:/',
440+
// list storage manager binary
441+
'ls -la /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager',
442+
'docker cp /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager regression$${DRONE_BUILD_NUMBER}:/',
443+
// check storage-manager unit test binary file
444+
execInnerDocker('ls -l /storage-manager',dockerImage('regression')),
445+
// copy test data for regression test suite
446+
execInnerDocker('bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"',dockerImage('regression')),
447+
installEngine(dockerImage('regression'), pkg_format),
448+
449+
// set mariadb lower_case_table_names=1 config option
450+
execInnerDocker('sed -i "/^.mariadb.$/a lower_case_table_names=1" ' + config_path_prefix + 'server.cnf', dockerImage('regression')),
451+
// set default client character set to utf-8
452+
execInnerDocker('sed -i "/^.client.$/a default-character-set=utf8" ' + config_path_prefix + 'client.cnf',dockerImage('regression')),
453+
454+
// Set RAM consumption limits to avoid RAM contention b/w mtr andregression steps.
455+
execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", dockerImage("regression")),
456+
457+
execInnerDocker('systemctl start mariadb',dockerImage('regression')),
458+
execInnerDocker('systemctl restart mariadb-columnstore',dockerImage('regression')),
459+
// delay regression for manual debugging on live instance
460+
'sleep $${REGRESSION_DELAY_SECONDS:-1s}',
461+
execInnerDocker('/usr/bin/g++ /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp -O2 -o /mariadb-columnstore-regression-test/mysql/queries/queryTester',dockerImage('regression')),
462+
],
463+
},
267464
smokelog:: {
268465
name: 'smokelog',
269466
depends_on: ['smoke'],
@@ -303,6 +500,32 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
303500
status: ['success', 'failure'],
304501
},
305502
},
503+
regressionlog:: {
504+
name: 'regressionlog',
505+
depends_on: [regression_tests[std.length(regression_tests) - 1]],
506+
image: 'docker',
507+
volumes: [pipeline._volumes.docker],
508+
commands: [
509+
'echo "---------- start columnstore regression short report ----------"',
510+
execInnerDocker('cat go.log || echo "missing go.log"',
511+
dockerImage('regression'),
512+
'--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest'),
513+
514+
'echo "---------- end columnstore regression short report ----------"',
515+
'echo',
516+
'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/ /drone/src/' + result + '/',
517+
'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs.tgz /drone/src/' + result + '/ || echo "missing testErrorLogs.tgz"',
518+
execInnerDocker('bash -c "tar czf regressionQueries.tgz /mariadb-columnstore-regression-test/mysql/queries/"',dockerImage('regression')),
519+
execInnerDocker('bash -c "tar czf testErrorLogs2.tgz *.log /var/log/mariadb/columnstore" || echo "failed to grab regression results"',
520+
dockerImage('regression'),
521+
'--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest'),
522+
'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs2.tgz /drone/src/' + result + '/ || echo "missing testErrorLogs.tgz"',
523+
'docker cp regression$${DRONE_BUILD_NUMBER}:regressionQueries.tgz /drone/src/' + result + '/'
524+
] + reportTestStage(dockerImage('regression'), result, "regression"),
525+
when: {
526+
status: ['success', 'failure'],
527+
},
528+
},
306529
dockerfile:: {
307530
name: 'dockerfile',
308531
depends_on: ['publish pkg', 'publish cmapi build'],

0 commit comments

Comments
 (0)