Skip to content

Commit 2d8e16f

Browse files
mariadb-LeonidFedorovmariadb-LeonidFedorov
committed
chore(build): better build pipeline
1 parent ddfbeb1 commit 2d8e16f

File tree

1 file changed

+143
-98
lines changed

1 file changed

+143
-98
lines changed

‎build/bootstrap_mcs.sh

Lines changed: 143 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ install_deps() {
9797
bison cmake libncurses5-dev python3 libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
9898
libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \
9999
libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \
100-
graphviz devscripts ccache equivs eatmydata curl"
100+
graphviz devscripts ccache equivs eatmydata curl python3"
101101

102102
if [[ "$OS" == *"rockylinux:8"* || "$OS" == *"rocky:8"* ]]; then
103103
command="dnf install -y curl 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools && \
@@ -144,6 +144,9 @@ if [[ ${BRANCH_NAME_TO_OUTDIR} = true ]]; then
144144
fi
145145

146146
disable_git_restore_frozen_revision() {
147+
if [[ $DO_NOT_FREEZE_REVISION = true ]]; then
148+
return
149+
fi
147150
cd $MDB_SOURCE_PATH
148151
git config submodule.storage/columnstore/columnstore.update none
149152
cd - >/dev/null
@@ -152,6 +155,15 @@ disable_git_restore_frozen_revision() {
152155
DEP_GRAPH_PATH="$MARIA_BUILD_PATH/dependency_graph/mariadb.dot"
153156

154157
stop_service() {
158+
if [[ $RESTART_SERVICES = false || $RECOMPILE_ONLY = true ]]; then
159+
return
160+
fi
161+
162+
if [ "$EUID" -ne 0 ]; then
163+
error "Please run script as root to be able to manage services"
164+
exit 1
165+
fi
166+
155167
message_split
156168
message "Stopping MariaDB services"
157169
systemctl stop mariadb
@@ -169,6 +181,15 @@ check_service() {
169181
}
170182

171183
start_service() {
184+
if [[ $RESTART_SERVICES = false || $RECOMPILE_ONLY = true ]]; then
185+
return
186+
fi
187+
188+
if [ "$EUID" -ne 0 ]; then
189+
error "Please run script as root to be able to manage services"
190+
exit 1
191+
fi
192+
172193
message_split
173194
message "Starting MariaDB services"
174195
systemctl start mariadb-columnstore
@@ -185,16 +206,26 @@ start_service() {
185206
}
186207

187208
start_storage_manager_if_needed() {
188-
if [[ $CLOUD_STORAGE_ENABLED = true ]]; then
189-
export MCS_USE_S3_STORAGE=1
190-
message_split
191-
message "Starting Storage Manager service"
192-
systemctl start mcs-storagemanager
193-
check_service mcs-storagemanager
209+
if [[ $CLOUD_STORAGE_ENABLED = false ]]; then
210+
return
194211
fi
212+
213+
export MCS_USE_S3_STORAGE=1
214+
message_split
215+
message "Starting Storage Manager service"
216+
systemctl start mcs-storagemanager
217+
check_service mcs-storagemanager
195218
}
196219

197220
clean_old_installation() {
221+
if [[ $NO_CLEAN = true || $RECOMPILE_ONLY = true ]]; then
222+
return
223+
fi
224+
225+
if [ "$EUID" -ne 0 ]; then
226+
error "Please run script as root to be able to clean installations"
227+
exit 1
228+
fi
198229
message_split
199230
message "Cleaning old installation"
200231
rm -rf /var/lib/columnstore/data1/*
@@ -479,19 +510,28 @@ build_binary() {
479510
message "Configuring cmake silently"
480511
${CMAKE_BIN_NAME} "${MDB_CMAKE_FLAGS[@]}" -S"$MDB_SOURCE_PATH" -B"$MARIA_BUILD_PATH" | spinner
481512
message_split
482-
check_debian_install_file
513+
# check_debian_install_file // will be uncommented later
483514
generate_svgs
484515

485-
${CMAKE_BIN_NAME} --build "$MARIA_BUILD_PATH" -j "$CPUS" | onelinearizator &&
486-
message "Installing silently" &&
487-
${CMAKE_BIN_NAME} --install "$MARIA_BUILD_PATH" | spinner 30
488-
516+
${CMAKE_BIN_NAME} --build "$MARIA_BUILD_PATH" -j "$CPUS" | onelinearizator
489517
check_errorcode
490518

491519
message "Adding symbol link to compile_commands.json to the source root"
492520
ln -sf "$MARIA_BUILD_PATH/compile_commands.json" "$MDB_SOURCE_PATH"
493521
}
494522

523+
install_binary() {
524+
if [[ $RECOMPILE_ONLY = true ]]; then
525+
warn "No binary installation done"
526+
return
527+
fi
528+
529+
message "Installing silently"
530+
cd $MDB_SOURCE_PATH
531+
${CMAKE_BIN_NAME} --install "$MARIA_BUILD_PATH" | spinner 30
532+
check_errorcode
533+
}
534+
495535
check_user_and_group() {
496536
user=$1
497537
if [ -z "$(grep "$user" /etc/passwd)" ]; then
@@ -510,24 +550,26 @@ run_unit_tests() {
510550
message_split
511551
if [[ $SKIP_UNIT_TESTS = true ]]; then
512552
warn "Skipping unittests"
513-
else
514-
message "Running unittests"
515-
cd $MARIA_BUILD_PATH
516-
${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress --output-on-failure
517-
cd - >/dev/null
553+
return
518554
fi
555+
556+
message "Running unittests"
557+
cd $MARIA_BUILD_PATH
558+
${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress --output-on-failure
559+
cd - >/dev/null
519560
}
520561

521562
run_microbenchmarks_tests() {
522563
message_split
523564
if [[ $RUN_BENCHMARKS = false ]]; then
524565
warn "Skipping microbenchmarks"
525-
else
526-
message "Runnning microbenchmarks"
527-
cd $MARIA_BUILD_PATH
528-
${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc) --progress
529-
cd - >/dev/null
566+
return
530567
fi
568+
569+
message "Runnning microbenchmarks"
570+
cd $MARIA_BUILD_PATH
571+
${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc) --progress
572+
cd - >/dev/null
531573
}
532574

533575
disable_plugins_for_bootstrap() {
@@ -593,95 +635,99 @@ make_dir() {
593635
}
594636

595637
install() {
596-
if [[ $RECOMPILE_ONLY = false ]]; then
638+
if [[ $RECOMPILE_ONLY = true ]]; then
639+
warn "No install configuration done"
640+
return
641+
fi
597642

598-
if [ "$EUID" -ne 0 ]; then
599-
error "Please run script as root to install MariaDb to system paths"
600-
exit 1
601-
fi
643+
if [ "$EUID" -ne 0 ]; then
644+
error "Please run script as root to install MariaDb to system paths"
645+
exit 1
646+
fi
602647

603-
message_split
604-
message "Installing MariaDB"
605-
disable_plugins_for_bootstrap
648+
message_split
649+
message "Installing MariaDB"
650+
disable_plugins_for_bootstrap
606651

607-
make_dir "$REPORT_PATH"
608-
chmod 777 "$REPORT_PATH"
652+
make_dir "$REPORT_PATH"
653+
chmod 777 "$REPORT_PATH"
609654

610-
check_user_and_group mysql
611-
check_user_and_group syslog
655+
check_user_and_group mysql
656+
check_user_and_group syslog
612657

613-
make_dir $CONFIG_DIR
658+
make_dir $CONFIG_DIR
614659

615-
echo "[client-server]
660+
echo "[client-server]
616661
socket=/run/mysqld/mysqld.sock" >$CONFIG_DIR/socket.cnf
617662

618-
make_dir /var/lib/mysql
663+
make_dir /var/lib/mysql
619664

620-
message "Running mysql_install_db"
621-
sudo -u mysql mysql_install_db --rpm --user=mysql >/dev/null
665+
message "Running mariadb-install-db"
666+
sudo -u mysql mariadb-install-db --rpm --user=mysql >/dev/null
622667

623-
enable_columnstore_back
668+
enable_columnstore_back
624669

625-
make_dir /etc/columnstore
670+
make_dir /etc/columnstore
626671

627-
if [[ "$NO_CLEAN" == false ]]; then
628-
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml
629-
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf
630-
fi
672+
if [[ "$NO_CLEAN" == false ]]; then
673+
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml
674+
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf
675+
fi
631676

632-
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/
677+
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/
633678

634-
if [[ "$OS" = *"ubuntu"* || "$OS" = *"debian"* ]]; then
635-
make_dir /usr/share/mysql
636-
make_dir /etc/mysql/
637-
cp "$MDB_SOURCE_PATH"/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh
638-
cp "$MDB_SOURCE_PATH"/debian/additions/debian-start /etc/mysql/debian-start
639-
>/etc/mysql/debian.cnf
640-
fi
679+
if [[ "$OS" = *"ubuntu"* || "$OS" = *"debian"* ]]; then
680+
make_dir /usr/share/mysql
681+
make_dir /etc/mysql/
682+
cp "$MDB_SOURCE_PATH"/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh
683+
cp "$MDB_SOURCE_PATH"/debian/additions/debian-start /etc/mysql/debian-start
684+
>/etc/mysql/debian.cnf
685+
fi
641686

642-
fix_config_files
687+
fix_config_files
643688

644-
if [ -d "$DEBCONFIG_DIR" ]; then
645-
message "Copying configs from $DEBCONFIG_DIR to $CONFIG_DIR"
646-
cp -rp "$DEBCONFIG_DIR"/* "$CONFIG_DIR"
647-
fi
689+
if [ -d "$DEBCONFIG_DIR" ]; then
690+
message "Copying configs from $DEBCONFIG_DIR to $CONFIG_DIR"
691+
cp -rp "$DEBCONFIG_DIR"/* "$CONFIG_DIR"
692+
fi
648693

649-
make_dir /var/lib/columnstore/data1
650-
make_dir /var/lib/columnstore/data1/systemFiles
651-
make_dir /var/lib/columnstore/data1/systemFiles/dbrm
652-
make_dir /run/mysqld/
653-
make_dir $DATA_DIR
694+
make_dir /var/lib/columnstore/data1
695+
make_dir /var/lib/columnstore/data1/systemFiles
696+
make_dir /var/lib/columnstore/data1/systemFiles/dbrm
697+
make_dir /run/mysqld/
698+
make_dir $DATA_DIR
654699

655-
chmod +x $INSTALL_PREFIX/bin/mariadb*
700+
chmod +x $INSTALL_PREFIX/bin/mariadb*
656701

657-
ldconfig
702+
ldconfig
658703

659-
start_storage_manager_if_needed
704+
start_storage_manager_if_needed
660705

661-
message "Running columnstore-post-install"
662-
make_dir /var/lib/columnstore/local
663-
columnstore-post-install --rpmmode=install
664-
message "Running install_mcs_mysql"
665-
install_mcs_mysql.sh
666-
fi
706+
message "Running columnstore-post-install"
707+
make_dir /var/lib/columnstore/local
708+
columnstore-post-install --rpmmode=install
709+
message "Running install_mcs_mysql"
710+
install_mcs_mysql.sh
667711

668712
chown -R syslog:syslog /var/log/mariadb/
669713
chmod 777 /var/log/mariadb/
670714
chmod 777 /var/log/mariadb/columnstore
671715
}
672716

673717
smoke() {
674-
if [[ $SKIP_SMOKE = false ]]; then
675-
message_split
676-
message "Creating test database"
677-
mariadb -e "create database if not exists test;"
678-
message "Selecting magic numbers"
679-
MAGIC=$(mysql -N test <"$MDB_SOURCE_PATH"/storage/columnstore/columnstore/tests/scripts/smoke.sql)
680-
if [[ $MAGIC == '42' ]]; then
681-
message "Great answer correct!"
682-
else
683-
warn "Smoke failed, answer is '$MAGIC'"
684-
fi
718+
if [[ $RECOMPILE_ONLY = true || $SKIP_SMOKE = true || $RESTART_SERVICES = false ]]; then
719+
return
720+
fi
721+
722+
message_split
723+
message "Creating test database"
724+
mariadb -e "create database if not exists test;"
725+
message "Selecting magic numbers"
726+
MAGIC=$(mariadb -N test <"$MDB_SOURCE_PATH"/storage/columnstore/columnstore/tests/scripts/smoke.sql)
727+
if [[ $MAGIC == '42' ]]; then
728+
message "Great answer correct!"
729+
else
730+
warn "Smoke failed, answer is '$MAGIC'"
685731
fi
686732
}
687733

@@ -692,23 +738,22 @@ fi
692738
construct_cmake_flags
693739
init_submodules
694740

695-
if [[ $BUILD_PACKAGES = false ]]; then
696-
stop_service
697-
698-
if [[ $NO_CLEAN = false ]]; then
699-
clean_old_installation
700-
fi
701-
build_binary
702-
run_unit_tests
703-
run_microbenchmarks_tests
704-
install
705-
if [[ $RESTART_SERVICES = true ]]; then
706-
start_service
707-
smoke
708-
fi
709-
else
741+
if [[ $BUILD_PACKAGES = true ]]; then
710742
modify_packaging
711743
build_package
744+
message_splitted "PACKAGES BUILD FINISHED"
745+
746+
return 0
712747
fi
713748

749+
stop_service
750+
clean_old_installation
751+
build_binary
752+
install_binary
753+
run_unit_tests
754+
run_microbenchmarks_tests
755+
install
756+
start_service
757+
smoke
758+
714759
message_splitted "FINISHED"

0 commit comments

Comments
 (0)