Skip to content

chore(build): better build pipeline #3584

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 9, 2025
Merged

chore(build): better build pipeline #3584

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
241 changes: 143 additions & 98 deletions build/bootstrap_mcs.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ install_deps() {
bison cmake libncurses5-dev python3 libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \
libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \
graphviz devscripts ccache equivs eatmydata curl"
graphviz devscripts ccache equivs eatmydata curl python3"

if [[ "$OS" == *"rockylinux:8"* || "$OS" == *"rocky:8"* ]]; then
command="dnf install -y curl 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools && \
Expand Down Expand Up @@ -144,6 +144,9 @@ if [[ ${BRANCH_NAME_TO_OUTDIR} = true ]]; then
fi

disable_git_restore_frozen_revision() {
if [[ $DO_NOT_FREEZE_REVISION = true ]]; then
return
fi
cd $MDB_SOURCE_PATH
git config submodule.storage/columnstore/columnstore.update none
cd - >/dev/null
Expand All @@ -152,6 +155,15 @@ disable_git_restore_frozen_revision() {
DEP_GRAPH_PATH="$MARIA_BUILD_PATH/dependency_graph/mariadb.dot"

stop_service() {
if [[ $RESTART_SERVICES = false || $RECOMPILE_ONLY = true ]]; then
return
fi

if [ "$EUID" -ne 0 ]; then
error "Please run script as root to be able to manage services"
exit 1
fi

message_split
message "Stopping MariaDB services"
systemctl stop mariadb
Expand All @@ -169,6 +181,15 @@ check_service() {
}

start_service() {
if [[ $RESTART_SERVICES = false || $RECOMPILE_ONLY = true ]]; then
return
fi

if [ "$EUID" -ne 0 ]; then
error "Please run script as root to be able to manage services"
exit 1
fi

message_split
message "Starting MariaDB services"
systemctl start mariadb-columnstore
Expand All @@ -185,16 +206,26 @@ start_service() {
}

start_storage_manager_if_needed() {
if [[ $CLOUD_STORAGE_ENABLED = true ]]; then
export MCS_USE_S3_STORAGE=1
message_split
message "Starting Storage Manager service"
systemctl start mcs-storagemanager
check_service mcs-storagemanager
if [[ $CLOUD_STORAGE_ENABLED = false ]]; then
return
fi

export MCS_USE_S3_STORAGE=1
message_split
message "Starting Storage Manager service"
systemctl start mcs-storagemanager
check_service mcs-storagemanager
}

clean_old_installation() {
if [[ $NO_CLEAN = true || $RECOMPILE_ONLY = true ]]; then
return
fi

if [ "$EUID" -ne 0 ]; then
error "Please run script as root to be able to clean installations"
exit 1
fi
message_split
message "Cleaning old installation"
rm -rf /var/lib/columnstore/data1/*
Expand Down Expand Up @@ -479,19 +510,28 @@ build_binary() {
message "Configuring cmake silently"
${CMAKE_BIN_NAME} "${MDB_CMAKE_FLAGS[@]}" -S"$MDB_SOURCE_PATH" -B"$MARIA_BUILD_PATH" | spinner
message_split
check_debian_install_file
# check_debian_install_file // will be uncommented later
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did you forget to uncomment it? Or by later you mean sometime after merging this?

generate_svgs

${CMAKE_BIN_NAME} --build "$MARIA_BUILD_PATH" -j "$CPUS" | onelinearizator &&
message "Installing silently" &&
${CMAKE_BIN_NAME} --install "$MARIA_BUILD_PATH" | spinner 30

${CMAKE_BIN_NAME} --build "$MARIA_BUILD_PATH" -j "$CPUS" | onelinearizator
check_errorcode

message "Adding symbol link to compile_commands.json to the source root"
ln -sf "$MARIA_BUILD_PATH/compile_commands.json" "$MDB_SOURCE_PATH"
}

install_binary() {
if [[ $RECOMPILE_ONLY = true ]]; then
warn "No binary installation done"
return
fi

message "Installing silently"
cd $MDB_SOURCE_PATH
${CMAKE_BIN_NAME} --install "$MARIA_BUILD_PATH" | spinner 30
check_errorcode
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe it is better use trap command to catch exit signal and call build failure error printing? This way you won't need to call check_errorcode every time
See build_cmapi.sh:39-46

}

check_user_and_group() {
user=$1
if [ -z "$(grep "$user" /etc/passwd)" ]; then
Expand All @@ -510,24 +550,26 @@ run_unit_tests() {
message_split
if [[ $SKIP_UNIT_TESTS = true ]]; then
warn "Skipping unittests"
else
message "Running unittests"
cd $MARIA_BUILD_PATH
${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress --output-on-failure
cd - >/dev/null
return
fi

message "Running unittests"
cd $MARIA_BUILD_PATH
${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress --output-on-failure
cd - >/dev/null
}

run_microbenchmarks_tests() {
message_split
if [[ $RUN_BENCHMARKS = false ]]; then
warn "Skipping microbenchmarks"
else
message "Runnning microbenchmarks"
cd $MARIA_BUILD_PATH
${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc) --progress
cd - >/dev/null
return
fi

message "Runnning microbenchmarks"
cd $MARIA_BUILD_PATH
${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc) --progress
cd - >/dev/null
}

disable_plugins_for_bootstrap() {
Expand Down Expand Up @@ -593,95 +635,99 @@ make_dir() {
}

install() {
if [[ $RECOMPILE_ONLY = false ]]; then
if [[ $RECOMPILE_ONLY = true ]]; then
warn "No install configuration done"
return
fi

if [ "$EUID" -ne 0 ]; then
error "Please run script as root to install MariaDb to system paths"
exit 1
fi
if [ "$EUID" -ne 0 ]; then
error "Please run script as root to install MariaDb to system paths"
exit 1
fi

message_split
message "Installing MariaDB"
disable_plugins_for_bootstrap
message_split
message "Installing MariaDB"
disable_plugins_for_bootstrap

make_dir "$REPORT_PATH"
chmod 777 "$REPORT_PATH"
make_dir "$REPORT_PATH"
chmod 777 "$REPORT_PATH"

check_user_and_group mysql
check_user_and_group syslog
check_user_and_group mysql
check_user_and_group syslog

make_dir $CONFIG_DIR
make_dir $CONFIG_DIR

echo "[client-server]
echo "[client-server]
socket=/run/mysqld/mysqld.sock" >$CONFIG_DIR/socket.cnf

make_dir /var/lib/mysql
make_dir /var/lib/mysql

message "Running mysql_install_db"
sudo -u mysql mysql_install_db --rpm --user=mysql >/dev/null
message "Running mariadb-install-db"
sudo -u mysql mariadb-install-db --rpm --user=mysql >/dev/null

enable_columnstore_back
enable_columnstore_back

make_dir /etc/columnstore
make_dir /etc/columnstore

if [[ "$NO_CLEAN" == false ]]; then
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf
fi
if [[ "$NO_CLEAN" == false ]]; then
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf
fi

cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/

if [[ "$OS" = *"ubuntu"* || "$OS" = *"debian"* ]]; then
make_dir /usr/share/mysql
make_dir /etc/mysql/
cp "$MDB_SOURCE_PATH"/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh
cp "$MDB_SOURCE_PATH"/debian/additions/debian-start /etc/mysql/debian-start
>/etc/mysql/debian.cnf
fi
if [[ "$OS" = *"ubuntu"* || "$OS" = *"debian"* ]]; then
make_dir /usr/share/mysql
make_dir /etc/mysql/
cp "$MDB_SOURCE_PATH"/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh
cp "$MDB_SOURCE_PATH"/debian/additions/debian-start /etc/mysql/debian-start
>/etc/mysql/debian.cnf
fi

fix_config_files
fix_config_files

if [ -d "$DEBCONFIG_DIR" ]; then
message "Copying configs from $DEBCONFIG_DIR to $CONFIG_DIR"
cp -rp "$DEBCONFIG_DIR"/* "$CONFIG_DIR"
fi
if [ -d "$DEBCONFIG_DIR" ]; then
message "Copying configs from $DEBCONFIG_DIR to $CONFIG_DIR"
cp -rp "$DEBCONFIG_DIR"/* "$CONFIG_DIR"
fi

make_dir /var/lib/columnstore/data1
make_dir /var/lib/columnstore/data1/systemFiles
make_dir /var/lib/columnstore/data1/systemFiles/dbrm
make_dir /run/mysqld/
make_dir $DATA_DIR
make_dir /var/lib/columnstore/data1
make_dir /var/lib/columnstore/data1/systemFiles
make_dir /var/lib/columnstore/data1/systemFiles/dbrm
make_dir /run/mysqld/
make_dir $DATA_DIR

chmod +x $INSTALL_PREFIX/bin/mariadb*
chmod +x $INSTALL_PREFIX/bin/mariadb*

ldconfig
ldconfig

start_storage_manager_if_needed
start_storage_manager_if_needed

message "Running columnstore-post-install"
make_dir /var/lib/columnstore/local
columnstore-post-install --rpmmode=install
message "Running install_mcs_mysql"
install_mcs_mysql.sh
fi
message "Running columnstore-post-install"
make_dir /var/lib/columnstore/local
columnstore-post-install --rpmmode=install
message "Running install_mcs_mysql"
install_mcs_mysql.sh

chown -R syslog:syslog /var/log/mariadb/
chmod 777 /var/log/mariadb/
chmod 777 /var/log/mariadb/columnstore
}

smoke() {
if [[ $SKIP_SMOKE = false ]]; then
message_split
message "Creating test database"
mariadb -e "create database if not exists test;"
message "Selecting magic numbers"
MAGIC=$(mysql -N test <"$MDB_SOURCE_PATH"/storage/columnstore/columnstore/tests/scripts/smoke.sql)
if [[ $MAGIC == '42' ]]; then
message "Great answer correct!"
else
warn "Smoke failed, answer is '$MAGIC'"
fi
if [[ $RECOMPILE_ONLY = true || $SKIP_SMOKE = true || $RESTART_SERVICES = false ]]; then
return
fi

message_split
message "Creating test database"
mariadb -e "create database if not exists test;"
message "Selecting magic numbers"
MAGIC=$(mariadb -N test <"$MDB_SOURCE_PATH"/storage/columnstore/columnstore/tests/scripts/smoke.sql)
if [[ $MAGIC == '42' ]]; then
message "Great answer correct!"
else
warn "Smoke failed, answer is '$MAGIC'"
fi
}

Expand All @@ -692,23 +738,22 @@ fi
construct_cmake_flags
init_submodules

if [[ $BUILD_PACKAGES = false ]]; then
stop_service

if [[ $NO_CLEAN = false ]]; then
clean_old_installation
fi
build_binary
run_unit_tests
run_microbenchmarks_tests
install
if [[ $RESTART_SERVICES = true ]]; then
start_service
smoke
fi
else
if [[ $BUILD_PACKAGES = true ]]; then
modify_packaging
build_package
message_splitted "PACKAGES BUILD FINISHED"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe put this inside of build_package?


return 0
fi

stop_service
clean_old_installation
build_binary
install_binary
run_unit_tests
run_microbenchmarks_tests
install
start_service
smoke

message_splitted "FINISHED"