Skip to content

Commit ad00c6f

Browse files
mariadb-SergeyZefirovmariadb-SergeyZefirov
committed
Debug logs; trim drone.jsonnet
1 parent db91612 commit ad00c6f

File tree

2 files changed

+19
-224
lines changed

2 files changed

+19
-224
lines changed

‎.drone.jsonnet

Lines changed: 1 addition & 224 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,13 @@
11
local events = ['pull_request', 'cron'];
22

33
local servers = {
4-
develop: ['10.6-enterprise'],
54
'stable-23.10': ['10.6-enterprise'],
65
};
76

87
local platforms = {
9-
develop: ['rockylinux:8', 'rockylinux:9', 'debian:11', 'debian:12', 'ubuntu:20.04', 'ubuntu:22.04', 'ubuntu:24.04'],
10-
'stable-23.10': ['rockylinux:8', 'rockylinux:9', 'debian:11', 'debian:12', 'ubuntu:20.04', 'ubuntu:22.04','ubuntu:24.04'],
8+
'stable-23.10': ['rockylinux:8', 'rockylinux:9'],
119
};
1210

13-
local platforms_arm = {
14-
develop: ['rockylinux:8', 'rockylinux:9', 'debian:11', 'debian:12', 'ubuntu:20.04', 'ubuntu:22.04', 'ubuntu:24.04'],
15-
'stable-23.10': ['rockylinux:8', 'rockylinux:9', 'debian:11', 'debian:12', 'ubuntu:20.04', 'ubuntu:22.04', 'ubuntu:24.04'],
16-
};
1711

1812
local any_branch = '**';
1913
local platforms_custom = platforms.develop;
@@ -24,49 +18,20 @@ local platforms_mtr = platforms.develop;
2418
local builddir = 'verylongdirnameforverystrangecpackbehavior';
2519

2620
local mtr_suite_list = 'basic,bugfixes';
27-
local mtr_full_set = 'basic,bugfixes,devregression,autopilot,extended,multinode,oracle,1pmonly';
2821

2922
local upgrade_test_lists = {
3023
"rockylinux8": {
31-
"arm64": ["10.6.4-1", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.15-10"],
3224
"amd64": ["10.6.4-1", "10.6.5-2", "10.6.7-3", "10.6.8-4", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"]
3325
},
3426
"rockylinux9": {
35-
"arm64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"],
3627
"amd64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"]
3728
},
38-
"debian11": {
39-
"arm64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"],
40-
"amd64": ["10.6.5-2", "10.6.7-3", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"]
41-
},
42-
"debian12": {
43-
"arm64": [],
44-
"amd64": []
45-
},
46-
"ubuntu20.04": {
47-
"arm64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"],
48-
"amd64": ["10.6.4-1", "10.6.5-2", "10.6.7-3", "10.6.8-4", "10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"]
49-
},
50-
"ubuntu22.04": {
51-
"arm64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"],
52-
"amd64": ["10.6.9-5", "10.6.11-6", "10.6.12-7", "10.6.14-9", "10.6.15-10"]
53-
},
54-
"ubuntu24.04":
55-
{
56-
"arm64": [],
57-
"amd64": []
58-
},
5929
};
6030

6131
local testRun(platform) =
6232
local platform_map = {
6333
'rockylinux:8': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure',
6434
'rockylinux:9': 'ctest3 -R columnstore: -j $(nproc) --output-on-failure',
65-
'debian:11': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure',
66-
'debian:12': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure',
67-
'ubuntu:20.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure',
68-
'ubuntu:22.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure',
69-
'ubuntu:24.04': 'cd builddir; ctest -R columnstore: -j $(nproc) --output-on-failure',
7035

7136
};
7237
platform_map[platform];
@@ -84,17 +49,11 @@ local rockylinux9_deps = "dnf install -y 'dnf-command(config-manager)' " +
8449

8550
local rockylinux_common_deps = ' && dnf install -y git lz4 lz4-devel cppunit-devel cmake3 boost-devel snappy-devel pcre2-devel';
8651

87-
local deb_deps = 'apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev';
8852

8953
local testPreparation(platform) =
9054
local platform_map = {
9155
'rockylinux:8': rockylinux8_deps + rockylinux_common_deps,
9256
'rockylinux:9': rockylinux9_deps + rockylinux_common_deps,
93-
'debian:11': deb_deps,
94-
'debian:12': deb_deps,
95-
'ubuntu:20.04': deb_deps,
96-
'ubuntu:22.04': deb_deps,
97-
'ubuntu:24.04': deb_deps,
9857

9958
};
10059
platform_map[platform];
@@ -305,162 +264,6 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
305264
execInnerDocker('mariadb -e "insert into test.t1 values (2); select * from test.t1"', dockerImage("smoke")),
306265
],
307266
},
308-
upgrade(version):: {
309-
name: 'upgrade-test from ' + version,
310-
depends_on: ['regressionlog'],
311-
image: 'docker',
312-
volumes: [pipeline._volumes.docker],
313-
environment: {
314-
UPGRADE_TOKEN: {
315-
from_secret: 'es_token',
316-
},
317-
},
318-
commands: [
319-
// why do we mount cgroups here, but miss it on other steps?
320-
'docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --env UCF_FORCE_CONFNEW=1 --name upgrade$${DRONE_BUILD_NUMBER}' + version + ' --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target']
321-
+ prepareTestStage(dockerImage('upgrade') + version, pkg_format, result, false) + [
322-
if (pkg_format == 'deb')
323-
then execInnerDocker('bash -c "./upgrade_setup_deb.sh '+ version + ' ' + result + ' ' + arch + ' ' + repo_pkg_url_no_res +' $${UPGRADE_TOKEN}"',
324-
dockerImage('upgrade') + version),
325-
if (std.split(platform, ':')[0] == 'rockylinux')
326-
then execInnerDocker('bash -c "./upgrade_setup_rpm.sh '+ version + ' ' + result + ' ' + arch + ' ' + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"',
327-
dockerImage('upgrade') + version),
328-
],
329-
},
330-
upgradelog:: {
331-
name: 'upgradelog',
332-
depends_on: std.map(function(p) 'upgrade-test from ' + p, mdb_server_versions),
333-
image: 'docker',
334-
volumes: [pipeline._volumes.docker],
335-
commands: [
336-
'echo',
337-
] + std.flatMap(function(ver) reportTestStage(dockerImage('upgrade') + ver, result, "upgrade_"+ver), mdb_server_versions),
338-
when: {
339-
status: ['success', 'failure'],
340-
},
341-
},
342-
mtr:: {
343-
name: 'mtr',
344-
depends_on: ['smoke'],
345-
image: 'docker:git',
346-
volumes: [pipeline._volumes.docker],
347-
environment: {
348-
MTR_SUITE_LIST: '${MTR_SUITE_LIST:-' + mtr_suite_list + '}',
349-
MTR_FULL_SUITE: '${MTR_FULL_SUITE:-false}',
350-
},
351-
commands: [
352-
'docker run --shm-size=500m --memory 8g --env MYSQL_TEST_DIR=' + mtr_path + ' --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target']
353-
+ prepareTestStage('mtr$${DRONE_BUILD_NUMBER}', pkg_format, result, true) + [
354-
installEngine(dockerImage("mtr"), pkg_format),
355-
'docker cp mysql-test/columnstore mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/suite/',
356-
execInnerDocker('chown -R mysql:mysql ' + mtr_path, dockerImage("mtr")),
357-
// disable systemd 'ProtectSystem' (we need to write to /usr/share/)
358-
execInnerDocker("bash -c 'sed -i /ProtectSystem/d $(systemctl show --property FragmentPath mariadb | sed s/FragmentPath=//)'", dockerImage('mtr')),
359-
execInnerDocker('systemctl daemon-reload', dockerImage("mtr")),
360-
execInnerDocker('systemctl start mariadb', dockerImage("mtr")),
361-
// Set RAM consumption limits to avoid RAM contention b/w mtr and regression steps.
362-
execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", dockerImage("mtr")),
363-
execInnerDocker('mariadb -e "create database if not exists test;"', dockerImage("mtr")),
364-
execInnerDocker('systemctl restart mariadb-columnstore', dockerImage("mtr")),
365-
366-
// delay mtr for manual debugging on live instance
367-
'sleep $${MTR_DELAY_SECONDS:-1s}',
368-
'MTR_SUITE_LIST=$([ "$MTR_FULL_SUITE" == true ] && echo "' + mtr_full_set + '" || echo "$MTR_SUITE_LIST")',
369-
if (event == 'custom' || event == 'cron') then
370-
execInnerDocker('bash -c "wget -qO- https://cspkg.s3.amazonaws.com/mtr-test-data.tar.lz4 | lz4 -dc - | tar xf - -C /"',
371-
dockerImage('mtr')),
372-
if (event == 'custom' || event == 'cron') then
373-
execInnerDocker('bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/setup"',
374-
dockerImage('mtr')),
375-
376-
if (event == 'cron') then
377-
execInnerDocker('bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path +
378-
' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite='
379-
+ std.join(',', std.map(function(x) 'columnstore/' + x, std.split(mtr_full_set, ','))),
380-
dockerImage('mtr')) + '"'
381-
else
382-
execInnerDocker('bash -c "cd ' + mtr_path + ' && ./mtr --extern socket=' + socket_path +
383-
' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/$${MTR_SUITE_LIST//,/,columnstore/}"',
384-
dockerImage('mtr')),
385-
],
386-
},
387-
mtrlog:: {
388-
name: 'mtrlog',
389-
depends_on: ['mtr'],
390-
image: 'docker',
391-
volumes: [pipeline._volumes.docker],
392-
commands: [
393-
'echo "---------- start mariadb service logs ----------"',
394-
execInnerDocker('journalctl -u mariadb --no-pager || echo "mariadb service failure"', dockerImage('mtr')),
395-
'echo "---------- end mariadb service logs ----------"',
396-
'echo',
397-
'echo "---------- start columnstore debug log ----------"',
398-
execInnerDocker('cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', dockerImage('mtr')),
399-
'echo "---------- end columnstore debug log ----------"',
400-
'echo "---------- end columnstore debug log ----------"',
401-
'docker cp mtr$${DRONE_BUILD_NUMBER}:' + mtr_path + '/var/log /drone/src/' + result + '/mtr-logs || echo "missing ' + mtr_path + '/var/log"'
402-
] + reportTestStage(dockerImage('mtr'), result, "mtr"),
403-
when: {
404-
status: ['success', 'failure'],
405-
},
406-
},
407-
prepare_regression:: {
408-
name: 'prepare regression',
409-
depends_on: ['mtr', 'publish pkg', 'publish cmapi build'],
410-
when: {
411-
status: ['success', 'failure'],
412-
},
413-
image: 'docker:git',
414-
volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
415-
environment: {
416-
REGRESSION_BRANCH_REF: '${DRONE_SOURCE_BRANCH}',
417-
REGRESSION_REF_AUX: branch_ref,
418-
},
419-
commands: [
420-
// compute branch.
421-
'echo "$$REGRESSION_REF"',
422-
'echo "$$REGRESSION_BRANCH_REF"',
423-
// if REGRESSION_REF is empty, try to see whether regression repository has a branch named as one we PR.
424-
'export REGRESSION_REF=$${REGRESSION_REF:-$$(git ls-remote https://github.com/mariadb-corporation/mariadb-columnstore-regression-test --h --sort origin "refs/heads/$$REGRESSION_BRANCH_REF" | grep -E -o "[^/]+$$")}',
425-
'echo "$$REGRESSION_REF"',
426-
// REGRESSION_REF can be empty if there is no appropriate branch in regression repository.
427-
// assign what is appropriate by default.
428-
'export REGRESSION_REF=$${REGRESSION_REF:-$$REGRESSION_REF_AUX}',
429-
'echo "$$REGRESSION_REF"',
430-
// clone regression test repo
431-
'git clone --recurse-submodules --branch $$REGRESSION_REF --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-regression-test',
432-
// where are we now?
433-
'cd mariadb-columnstore-regression-test',
434-
'git rev-parse --abbrev-ref HEAD && git rev-parse HEAD',
435-
'cd ..',
436-
'docker run --shm-size=500m --memory 10g --env OS=' + result + ' --env PACKAGES_URL=' + packages_url + ' --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name regression$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach ' + img + ' ' + init + ' --unit=basic.target']
437-
+ prepareTestStage(dockerImage('regression'), pkg_format, result, true) + [
438-
439-
'docker cp mariadb-columnstore-regression-test regression$${DRONE_BUILD_NUMBER}:/',
440-
// list storage manager binary
441-
'ls -la /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager',
442-
'docker cp /mdb/' + builddir + '/storage/columnstore/columnstore/storage-manager regression$${DRONE_BUILD_NUMBER}:/',
443-
// check storage-manager unit test binary file
444-
execInnerDocker('ls -l /storage-manager',dockerImage('regression')),
445-
// copy test data for regression test suite
446-
execInnerDocker('bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"',dockerImage('regression')),
447-
installEngine(dockerImage('regression'), pkg_format),
448-
449-
// set mariadb lower_case_table_names=1 config option
450-
execInnerDocker('sed -i "/^.mariadb.$/a lower_case_table_names=1" ' + config_path_prefix + 'server.cnf', dockerImage('regression')),
451-
// set default client character set to utf-8
452-
execInnerDocker('sed -i "/^.client.$/a default-character-set=utf8" ' + config_path_prefix + 'client.cnf',dockerImage('regression')),
453-
454-
// Set RAM consumption limits to avoid RAM contention b/w mtr andregression steps.
455-
execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", dockerImage("regression")),
456-
457-
execInnerDocker('systemctl start mariadb',dockerImage('regression')),
458-
execInnerDocker('systemctl restart mariadb-columnstore',dockerImage('regression')),
459-
// delay regression for manual debugging on live instance
460-
'sleep $${REGRESSION_DELAY_SECONDS:-1s}',
461-
execInnerDocker('/usr/bin/g++ /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp -O2 -o /mariadb-columnstore-regression-test/mysql/queries/queryTester',dockerImage('regression')),
462-
],
463-
},
464267
smokelog:: {
465268
name: 'smokelog',
466269
depends_on: ['smoke'],
@@ -500,32 +303,6 @@ local Pipeline(branch, platform, event, arch='amd64', server='10.6-enterprise')
500303
status: ['success', 'failure'],
501304
},
502305
},
503-
regressionlog:: {
504-
name: 'regressionlog',
505-
depends_on: [regression_tests[std.length(regression_tests) - 1]],
506-
image: 'docker',
507-
volumes: [pipeline._volumes.docker],
508-
commands: [
509-
'echo "---------- start columnstore regression short report ----------"',
510-
execInnerDocker('cat go.log || echo "missing go.log"',
511-
dockerImage('regression'),
512-
'--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest'),
513-
514-
'echo "---------- end columnstore regression short report ----------"',
515-
'echo',
516-
'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/ /drone/src/' + result + '/',
517-
'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs.tgz /drone/src/' + result + '/ || echo "missing testErrorLogs.tgz"',
518-
execInnerDocker('bash -c "tar czf regressionQueries.tgz /mariadb-columnstore-regression-test/mysql/queries/"',dockerImage('regression')),
519-
execInnerDocker('bash -c "tar czf testErrorLogs2.tgz *.log /var/log/mariadb/columnstore" || echo "failed to grab regression results"',
520-
dockerImage('regression'),
521-
'--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest'),
522-
'docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs2.tgz /drone/src/' + result + '/ || echo "missing testErrorLogs.tgz"',
523-
'docker cp regression$${DRONE_BUILD_NUMBER}:regressionQueries.tgz /drone/src/' + result + '/'
524-
] + reportTestStage(dockerImage('regression'), result, "regression"),
525-
when: {
526-
status: ['success', 'failure'],
527-
},
528-
},
529306
dockerfile:: {
530307
name: 'dockerfile',
531308
depends_on: ['publish pkg', 'publish cmapi build'],

‎utils/loggingcpp/exceptclasses.h

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,6 +283,24 @@ class ProtocolError : public std::logic_error
283283
} \
284284
} while (0)
285285

286+
#define idblog(x) \
287+
do \
288+
{ \
289+
{ \
290+
std::ostringstream os; \
291+
\
292+
os << __FILE__ << "@" << __LINE__ << ": \'" << x << "\'"; \
293+
std::cerr << os.str() << std::endl; \
294+
logging::MessageLog logger((logging::LoggingID())); \
295+
logging::Message message; \
296+
logging::Message::Args args; \
297+
\
298+
args.add(os.str()); \
299+
message.format(args); \
300+
logger.logErrorMessage(message); \
301+
} \
302+
} while (0)
303+
286304
#define idbassert_s(x, s) \
287305
do \
288306
{ \

0 commit comments

Comments
 (0)