From 74e1da65a3a06b1632ad95cfedecd93cb9c714f5 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov Date: Mon, 9 Jun 2025 09:37:16 +0000 Subject: [PATCH] chore(build): better build pipeline --- build/bootstrap_mcs.sh | 241 ++++++++++++++++++++++++----------------- 1 file changed, 143 insertions(+), 98 deletions(-) diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 680eec342..dff4aea4d 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -97,7 +97,7 @@ install_deps() { bison cmake libncurses5-dev python3 libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \ libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \ libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \ - graphviz devscripts ccache equivs eatmydata curl" + graphviz devscripts ccache equivs eatmydata curl python3" if [[ "$OS" == *"rockylinux:8"* || "$OS" == *"rocky:8"* ]]; then command="dnf install -y curl 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools && \ @@ -144,6 +144,9 @@ if [[ ${BRANCH_NAME_TO_OUTDIR} = true ]]; then fi disable_git_restore_frozen_revision() { + if [[ $DO_NOT_FREEZE_REVISION = true ]]; then + return + fi cd $MDB_SOURCE_PATH git config submodule.storage/columnstore/columnstore.update none cd - >/dev/null @@ -152,6 +155,15 @@ disable_git_restore_frozen_revision() { DEP_GRAPH_PATH="$MARIA_BUILD_PATH/dependency_graph/mariadb.dot" stop_service() { + if [[ $RESTART_SERVICES = false || $RECOMPILE_ONLY = true ]]; then + return + fi + + if [ "$EUID" -ne 0 ]; then + error "Please run script as root to be able to manage services" + exit 1 + fi + message_split message "Stopping MariaDB services" systemctl stop mariadb @@ -169,6 +181,15 @@ check_service() { } start_service() { + if [[ $RESTART_SERVICES = false || $RECOMPILE_ONLY = true ]]; then + return + fi + + if [ "$EUID" -ne 0 ]; then + error "Please run script as root to be able to manage services" + exit 1 + fi + message_split message "Starting MariaDB services" systemctl start mariadb-columnstore @@ -185,16 +206,26 @@ start_service() { } start_storage_manager_if_needed() { - if [[ $CLOUD_STORAGE_ENABLED = true ]]; then - export MCS_USE_S3_STORAGE=1 - message_split - message "Starting Storage Manager service" - systemctl start mcs-storagemanager - check_service mcs-storagemanager + if [[ $CLOUD_STORAGE_ENABLED = false ]]; then + return fi + + export MCS_USE_S3_STORAGE=1 + message_split + message "Starting Storage Manager service" + systemctl start mcs-storagemanager + check_service mcs-storagemanager } clean_old_installation() { + if [[ $NO_CLEAN = true || $RECOMPILE_ONLY = true ]]; then + return + fi + + if [ "$EUID" -ne 0 ]; then + error "Please run script as root to be able to clean installations" + exit 1 + fi message_split message "Cleaning old installation" rm -rf /var/lib/columnstore/data1/* @@ -479,19 +510,28 @@ build_binary() { message "Configuring cmake silently" ${CMAKE_BIN_NAME} "${MDB_CMAKE_FLAGS[@]}" -S"$MDB_SOURCE_PATH" -B"$MARIA_BUILD_PATH" | spinner message_split - check_debian_install_file + # check_debian_install_file // will be uncommented later generate_svgs - ${CMAKE_BIN_NAME} --build "$MARIA_BUILD_PATH" -j "$CPUS" | onelinearizator && - message "Installing silently" && - ${CMAKE_BIN_NAME} --install "$MARIA_BUILD_PATH" | spinner 30 - + ${CMAKE_BIN_NAME} --build "$MARIA_BUILD_PATH" -j "$CPUS" | onelinearizator check_errorcode message "Adding symbol link to compile_commands.json to the source root" ln -sf "$MARIA_BUILD_PATH/compile_commands.json" "$MDB_SOURCE_PATH" } +install_binary() { + if [[ $RECOMPILE_ONLY = true ]]; then + warn "No binary installation done" + return + fi + + message "Installing silently" + cd $MDB_SOURCE_PATH + ${CMAKE_BIN_NAME} --install "$MARIA_BUILD_PATH" | spinner 30 + check_errorcode +} + check_user_and_group() { user=$1 if [ -z "$(grep "$user" /etc/passwd)" ]; then @@ -510,24 +550,26 @@ run_unit_tests() { message_split if [[ $SKIP_UNIT_TESTS = true ]]; then warn "Skipping unittests" - else - message "Running unittests" - cd $MARIA_BUILD_PATH - ${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress --output-on-failure - cd - >/dev/null + return fi + + message "Running unittests" + cd $MARIA_BUILD_PATH + ${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress --output-on-failure + cd - >/dev/null } run_microbenchmarks_tests() { message_split if [[ $RUN_BENCHMARKS = false ]]; then warn "Skipping microbenchmarks" - else - message "Runnning microbenchmarks" - cd $MARIA_BUILD_PATH - ${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc) --progress - cd - >/dev/null + return fi + + message "Runnning microbenchmarks" + cd $MARIA_BUILD_PATH + ${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc) --progress + cd - >/dev/null } disable_plugins_for_bootstrap() { @@ -593,77 +635,79 @@ make_dir() { } install() { - if [[ $RECOMPILE_ONLY = false ]]; then + if [[ $RECOMPILE_ONLY = true ]]; then + warn "No install configuration done" + return + fi - if [ "$EUID" -ne 0 ]; then - error "Please run script as root to install MariaDb to system paths" - exit 1 - fi + if [ "$EUID" -ne 0 ]; then + error "Please run script as root to install MariaDb to system paths" + exit 1 + fi - message_split - message "Installing MariaDB" - disable_plugins_for_bootstrap + message_split + message "Installing MariaDB" + disable_plugins_for_bootstrap - make_dir "$REPORT_PATH" - chmod 777 "$REPORT_PATH" + make_dir "$REPORT_PATH" + chmod 777 "$REPORT_PATH" - check_user_and_group mysql - check_user_and_group syslog + check_user_and_group mysql + check_user_and_group syslog - make_dir $CONFIG_DIR + make_dir $CONFIG_DIR - echo "[client-server] + echo "[client-server] socket=/run/mysqld/mysqld.sock" >$CONFIG_DIR/socket.cnf - make_dir /var/lib/mysql + make_dir /var/lib/mysql - message "Running mysql_install_db" - sudo -u mysql mysql_install_db --rpm --user=mysql >/dev/null + message "Running mariadb-install-db" + sudo -u mysql mariadb-install-db --rpm --user=mysql >/dev/null - enable_columnstore_back + enable_columnstore_back - make_dir /etc/columnstore + make_dir /etc/columnstore - if [[ "$NO_CLEAN" == false ]]; then - cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml - cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf - fi + if [[ "$NO_CLEAN" == false ]]; then + cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml + cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf + fi - cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/ + cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/ - if [[ "$OS" = *"ubuntu"* || "$OS" = *"debian"* ]]; then - make_dir /usr/share/mysql - make_dir /etc/mysql/ - cp "$MDB_SOURCE_PATH"/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh - cp "$MDB_SOURCE_PATH"/debian/additions/debian-start /etc/mysql/debian-start - >/etc/mysql/debian.cnf - fi + if [[ "$OS" = *"ubuntu"* || "$OS" = *"debian"* ]]; then + make_dir /usr/share/mysql + make_dir /etc/mysql/ + cp "$MDB_SOURCE_PATH"/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh + cp "$MDB_SOURCE_PATH"/debian/additions/debian-start /etc/mysql/debian-start + >/etc/mysql/debian.cnf + fi - fix_config_files + fix_config_files - if [ -d "$DEBCONFIG_DIR" ]; then - message "Copying configs from $DEBCONFIG_DIR to $CONFIG_DIR" - cp -rp "$DEBCONFIG_DIR"/* "$CONFIG_DIR" - fi + if [ -d "$DEBCONFIG_DIR" ]; then + message "Copying configs from $DEBCONFIG_DIR to $CONFIG_DIR" + cp -rp "$DEBCONFIG_DIR"/* "$CONFIG_DIR" + fi - make_dir /var/lib/columnstore/data1 - make_dir /var/lib/columnstore/data1/systemFiles - make_dir /var/lib/columnstore/data1/systemFiles/dbrm - make_dir /run/mysqld/ - make_dir $DATA_DIR + make_dir /var/lib/columnstore/data1 + make_dir /var/lib/columnstore/data1/systemFiles + make_dir /var/lib/columnstore/data1/systemFiles/dbrm + make_dir /run/mysqld/ + make_dir $DATA_DIR - chmod +x $INSTALL_PREFIX/bin/mariadb* + chmod +x $INSTALL_PREFIX/bin/mariadb* - ldconfig + ldconfig - start_storage_manager_if_needed + start_storage_manager_if_needed - message "Running columnstore-post-install" - make_dir /var/lib/columnstore/local - columnstore-post-install --rpmmode=install - message "Running install_mcs_mysql" - install_mcs_mysql.sh - fi + message "Running columnstore-post-install" + make_dir /var/lib/columnstore/local + columnstore-post-install --rpmmode=install + message "Running install_mcs_mysql" + install_mcs_mysql.sh chown -R syslog:syslog /var/log/mariadb/ chmod 777 /var/log/mariadb/ @@ -671,17 +715,19 @@ install() { } smoke() { - if [[ $SKIP_SMOKE = false ]]; then - message_split - message "Creating test database" - mariadb -e "create database if not exists test;" - message "Selecting magic numbers" - MAGIC=$(mysql -N test <"$MDB_SOURCE_PATH"/storage/columnstore/columnstore/tests/scripts/smoke.sql) - if [[ $MAGIC == '42' ]]; then - message "Great answer correct!" - else - warn "Smoke failed, answer is '$MAGIC'" - fi + if [[ $RECOMPILE_ONLY = true || $SKIP_SMOKE = true || $RESTART_SERVICES = false ]]; then + return + fi + + message_split + message "Creating test database" + mariadb -e "create database if not exists test;" + message "Selecting magic numbers" + MAGIC=$(mariadb -N test <"$MDB_SOURCE_PATH"/storage/columnstore/columnstore/tests/scripts/smoke.sql) + if [[ $MAGIC == '42' ]]; then + message "Great answer correct!" + else + warn "Smoke failed, answer is '$MAGIC'" fi } @@ -692,23 +738,22 @@ fi construct_cmake_flags init_submodules -if [[ $BUILD_PACKAGES = false ]]; then - stop_service - - if [[ $NO_CLEAN = false ]]; then - clean_old_installation - fi - build_binary - run_unit_tests - run_microbenchmarks_tests - install - if [[ $RESTART_SERVICES = true ]]; then - start_service - smoke - fi -else +if [[ $BUILD_PACKAGES = true ]]; then modify_packaging build_package + message_splitted "PACKAGES BUILD FINISHED" + + return 0 fi +stop_service +clean_old_installation +build_binary +install_binary +run_unit_tests +run_microbenchmarks_tests +install +start_service +smoke + message_splitted "FINISHED"