diff --git a/.github/workflows/pr-pipeline.yaml b/.github/workflows/pr-pipeline.yaml index 44bdf61..6dbc0f8 100644 --- a/.github/workflows/pr-pipeline.yaml +++ b/.github/workflows/pr-pipeline.yaml @@ -7,6 +7,9 @@ on: jobs: build: + env: + TTY_ARG: "" # Disable interactive build mode in cicd since it does not contain a tty device. + strategy: matrix: build_type: ["build", "build-with-python"] diff --git a/.github/workflows/release-pipeline.yaml b/.github/workflows/release-pipeline.yaml index 671b05a..0219596 100644 --- a/.github/workflows/release-pipeline.yaml +++ b/.github/workflows/release-pipeline.yaml @@ -9,6 +9,9 @@ on: # simply build the files in the same job they are released. jobs: build_and_publish: + env: + TTY_ARG: "" # Disable interactive build mode in cicd since it does not contain a tty device. + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -27,4 +30,4 @@ jobs: - name: Publish release uses: softprops/action-gh-release@v2 with: - files: build/artifacts/gdb-static*.tar.gz \ No newline at end of file + files: build/artifacts/gdb-static*.tar.gz diff --git a/.gitmodules b/.gitmodules index cb981e5..1e29ea0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -12,3 +12,6 @@ [submodule "src/submodule_packages/libexpat"] path = src/submodule_packages/libexpat url = git@github.com:guyush1/libexpat.git +[submodule "xz"] + path = src/submodule_packages/xz + url = https://github.com/tukaani-project/xz.git diff --git a/Dockerfile b/Dockerfile index 43f8daa..a84d3c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ FROM ubuntu:24.04 -# Install dependencies RUN apt update && apt install -y \ + autopoint \ + binutils-multiarch \ bison \ file \ flex \ @@ -18,16 +19,22 @@ RUN apt update && apt install -y \ gcc-mipsel-linux-gnu \ gcc-powerpc-linux-gnu \ git \ - libncurses-dev \ libtool \ m4 \ make \ patch \ pkg-config \ python3.12 \ + python3-requests \ libpython3-dev \ texinfo \ wget \ xz-utils +COPY src/docker_utils/download_musl_toolchains.py . +RUN python3.12 -u download_musl_toolchains.py + WORKDIR /app/gdb + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["bash"] diff --git a/Makefile b/Makefile index 5791380..4efc9e1 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,10 @@ ALL_PACK_TARGETS := $(PACK_TARGETS) $(PYTHON_PACK_TARGETS) SUBMODULE_PACKAGES := $(wildcard src/submodule_packages/*) BUILD_PACKAGES_DIR := "build/packages" +# We would like to run in interactive mode when avaliable (non-ci usually). +# This is disabled by the ci automation manually. +TTY_ARG ?= -it + .PHONY: clean help download_packages build build-docker-image $(ALL_TARGETS) $(ALL_PACK_TARGETS) .NOTPARALLEL: build pack @@ -27,7 +31,7 @@ help: @echo "" @echo " make clean" -build/build-docker-image.stamp: Dockerfile +build/build-docker-image.stamp: Dockerfile src/docker_utils/download_musl_toolchains.py mkdir -p build docker buildx build --tag gdb-static . touch build/build-docker-image.stamp @@ -36,7 +40,7 @@ build-docker-image: build/build-docker-image.stamp build/download-packages.stamp: build/build-docker-image.stamp src/compilation/download_packages.sh mkdir -p $(BUILD_PACKAGES_DIR) - docker run --user $(shell id -u):$(shell id -g) \ + docker run $(TTY_ARG) --user $(shell id -u):$(shell id -g) \ --rm --volume .:/app/gdb gdb-static env TERM=xterm-256color \ /app/gdb/src/compilation/download_packages.sh /app/gdb/$(BUILD_PACKAGES_DIR)/ touch build/download-packages.stamp @@ -59,7 +63,7 @@ $(PYTHON_TARGETS): build-with-python-%: _build-%: symlink-git-packages download-packages build-docker-image mkdir -p build - docker run --user $(shell id -u):$(shell id -g) \ + docker run $(TTY_ARG) --user $(shell id -u):$(shell id -g) \ --rm --volume .:/app/gdb gdb-static env TERM=xterm-256color \ /app/gdb/src/compilation/build.sh $* /app/gdb/build/ /app/gdb/src $(WITH_PYTHON) diff --git a/README.md b/README.md index 4579082..bdd63cd 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ This is where `gdb-static` comes in! We provide static builds of `gdb` (and `gdb
- **Static Builds**: No dependencies, no installation, just download and run! +- **Musl Based**: We use Musl in order to create distribution-independant binaries that can work anywhere. - **Latest Versions**: We keep our builds up-to-date with the latest versions of GDB. - **Builtin Python (Optional)**: We provide builds with Python support built-in. - **XML Support**: Our builds come with XML support built-in, which is useful for some GDB commands. diff --git a/compilation.md b/compilation.md index 6a39dc2..dbbcaf3 100644 --- a/compilation.md +++ b/compilation.md @@ -14,13 +14,15 @@ Please note that when specifying a compilation dir throughout the compilation pr Instead, always use absolute paths. Examples to the throughout the script: -- - arm-linux-gnueabi-gcc -- - arm-linux-gnueabi-g++ -- - arm-linux-gnueabi +- - arm-linux-musleabi-gcc +- - arm-linux-musleabi-g++ +- - arm-linux-musleabi - - /home/username/projects/libgmp-x.y.z/build-arm/ -Environment info: -- glibc version: 2.39-0ubuntu8.3 (NOTE: When i compiled gdb using an older glibc, such as the one i had in my ubuntu-20.04 machine, i received a segfault in gdb, so the libc version is important!). +## Choosing the right compiler + +It is recommended to use a musl-based compiler. Unlike glibc, musl is not dependant on your distribution. +Using regular gcc that is glibc based should work most of the time, but we had some people reporting crashes mainly in gdbserver, so we switched to musl-based toolchains. # Compiling gdb statically to the host platform diff --git a/src/compilation/build.sh b/src/compilation/build.sh index d08d3a7..dcc6d13 100755 --- a/src/compilation/build.sh +++ b/src/compilation/build.sh @@ -28,23 +28,23 @@ function set_compliation_variables() { >&2 fancy_title "Setting compilation variables for $target_arch" if [[ "$target_arch" == "arm" ]]; then - CROSS=arm-linux-gnueabi- - export HOST=arm-linux-gnueabi + CROSS=arm-linux-musleabi- + export HOST=arm-linux-musleabi elif [[ "$target_arch" == "aarch64" ]]; then - CROSS=aarch64-linux-gnu- - export HOST=aarch64-linux-gnu + CROSS=aarch64-linux-musl- + export HOST=aarch64-linux-musl elif [[ "$target_arch" == "powerpc" ]]; then - CROSS=powerpc-linux-gnu- - export HOST=powerpc-linux-gnu + CROSS=powerpc-linux-musl- + export HOST=powerpc-linux-musl elif [[ "$target_arch" == "mips" ]]; then - CROSS=mips-linux-gnu- - export HOST=mips-linux-gnu + CROSS=mips-linux-musl- + export HOST=mips-linux-musl elif [[ "$target_arch" == "mipsel" ]]; then - CROSS=mipsel-linux-gnu- - export HOST=mipsel-linux-gnu + CROSS=mipsel-linux-musl- + export HOST=mipsel-linux-musl elif [[ "$target_arch" == "x86_64" ]]; then - CROSS=x86_64-linux-gnu- - export HOST=x86_64-linux-gnu + CROSS=x86_64-linux-musl- + export HOST=x86_64-linux-musl fi export CC="${CROSS}gcc" @@ -61,14 +61,25 @@ function set_up_lib_search_paths() { # Set up library-related linker search paths. # # Parameters: - # $1: ncursesw build dir - # $2: libexpat build dir - local ncursesw_build_dir="$1" - local libexpat_build_dir="$2" + # $1: iconv build dir + # $2: gmp build dir + # $3: mpfr build dir + # $4: ncursesw build dir + # $5: libexpat build dir + local iconv_build_dir="$1" + local gmp_build_dir="$2" + local mpfr_build_dir="$3" + local ncursesw_build_dir="$4" + local libexpat_build_dir="$5" # I) Allow tui mode by adding our custom built static ncursesw library to the linker search path. # II) Allow parsing xml files by adding libexpat library to the linker search path. - export LDFLAGS="-L$ncursesw_build_dir/lib -L$libexpat_build_dir/lib/.libs $LDFLAGS" + export LDFLAGS="-L$ncursesw_build_dir/lib -L$libexpat_build_dir/lib/ $LDFLAGS" + + # Add library standard headers to the CC / CXX flags. + export INCLUDE_PATHS="-I$iconv_build_dir/include -I$gmp_build_dir/include -I$mpfr_build_dir/include -I$ncursesw_build_dir/include -I$libexpat_build_dir/include" + export CC="$CC $INCLUDE_PATHS" + export CXX="$CXX $INCLUDE_PATHS" } function build_iconv() { @@ -92,7 +103,7 @@ function build_iconv() { echo "$iconv_build_dir" mkdir -p "$iconv_build_dir" - if [[ -f "$iconv_build_dir/lib/.libs/libiconv.a" ]]; then + if [[ -f "$iconv_build_dir/lib/libiconv.a" ]]; then >&2 echo "Skipping build: iconv already built for $target_arch" return 0 fi @@ -102,7 +113,7 @@ function build_iconv() { >&2 fancy_title "Building libiconv for $target_arch" ../configure --enable-static "CC=$CC" "CXX=$CXX" "--host=$HOST" \ - "CFLAGS=$CFLAGS" "CXXFLAGS=$CXXFLAGS" 1>&2 + "CFLAGS=$CFLAGS" "CXXFLAGS=$CXXFLAGS" --prefix="$(realpath .)" 1>&2 if [[ $? -ne 0 ]]; then return 1 fi @@ -112,9 +123,10 @@ function build_iconv() { return 1 fi - cp -r ./include ./lib/.libs/ - mkdir -p ./lib/.libs/lib/ - cp ./lib/.libs/libiconv.a ./lib/.libs/lib/ + make -j$(nproc) install 1>&2 + if [[ $? -ne 0 ]]; then + return 1 + fi >&2 fancy_title "Finished building libiconv for $target_arch" @@ -142,7 +154,7 @@ function build_lzma() { echo "$lzma_build_dir" mkdir -p "$lzma_build_dir" - if [[ -f "$lzma_build_dir/usr/local/lib/liblzma.a" ]]; then + if [[ -f "$lzma_build_dir/lib/liblzma.a" ]]; then >&2 echo "Skipping build: lzma already built for $target_arch" return 0 fi @@ -151,8 +163,16 @@ function build_lzma() { >&2 fancy_title "Building liblzma for $target_arch" + # Make sure configure exists by running autogen.sh + ( + cd .. && ./autogen.sh 1>&2 + ) + + # lzma's autoconf contains a bug, it's instal prefix is relative + # to the current build directory. + # Hence, we set the prefix here to "/" instead of realpath . . ../configure --enable-static "CC=$CC" "CXX=$CXX" "--host=$HOST" \ - "CFLAGS=$CFLAGS" "CXXFLAGS=$CXXFLAGS" 1>&2 + "CFLAGS=$CFLAGS" "CXXFLAGS=$CXXFLAGS" --prefix="/" 1>&2 if [[ $? -ne 0 ]]; then return 1 fi @@ -194,7 +214,7 @@ function build_libgmp() { echo "$gmp_build_dir" mkdir -p "$gmp_build_dir" - if [[ -f "$gmp_build_dir/.libs/lib/libgmp.a" ]]; then + if [[ -f "$gmp_build_dir/lib/libgmp.a" ]]; then >&2 echo "Skipping build: libgmp already built for $target_arch" return 0 fi @@ -204,7 +224,7 @@ function build_libgmp() { >&2 fancy_title "Building libgmp for $target_arch" ../configure --enable-static "CC=$CC" "CXX=$CXX" "--host=$HOST" \ - "CFLAGS=$CFLAGS" "CXXFLAGS=$CXXFLAGS" 1>&2 + "CFLAGS=$CFLAGS" "CXXFLAGS=$CXXFLAGS" --prefix="$(realpath .)" 1>&2 if [[ $? -ne 0 ]]; then return 1 fi @@ -214,10 +234,10 @@ function build_libgmp() { return 1 fi - mkdir -p ./.libs/include/ - cp gmp.h ./.libs/include/ - mkdir -p ./.libs/lib/ - cp ./.libs/libgmp.a ./.libs/lib/ + make -j$(nproc) install 1>&2 + if [[ $? -ne 0 ]]; then + return 1 + fi >&2 fancy_title "Finished building libgmp for $target_arch" @@ -241,10 +261,13 @@ function build_ncurses() { local target_arch="$2" local ncurses_build_dir="$(realpath "$ncurses_dir/build-$target_arch")" - echo "$ncurses_build_dir" - mkdir -p "$ncurses_build_dir" + # ncurses needs a custom install dir due to it's non-standard compilation directories. + local ncurses_install_dir="$ncurses_build_dir/output" + + echo "$ncurses_install_dir" + mkdir -p "$ncurses_install_dir" - if [[ -f "$ncurses_build_dir/lib/libncursesw.a" ]]; then + if [[ -f "$ncurses_install_dir/lib/libncursesw.a" ]]; then >&2 echo "Skipping build: libncursesw already built for $target_arch" return 0 fi @@ -254,7 +277,7 @@ function build_ncurses() { >&2 fancy_title "Building libncursesw for $target_arch" ../configure --enable-static "CC=$CC" "CXX=$CXX" "--host=$HOST" \ - "CFLAGS=$CFLAGS" "CXXFLAGS=$CXXFLAGS" "--enable-widec" 1>&2 + "CFLAGS=$CFLAGS" "CXXFLAGS=$CXXFLAGS" --prefix="$ncurses_install_dir" "--enable-widec" 1>&2 if [[ $? -ne 0 ]]; then return 1 fi @@ -264,6 +287,11 @@ function build_ncurses() { return 1 fi + make -j$(nproc) install 1>&2 + if [[ $? -ne 0 ]]; then + return 1 + fi + >&2 fancy_title "Finished building libncursesw for $target_arch" popd > /dev/null @@ -289,7 +317,7 @@ function build_libexpat() { echo "$libexpat_build_dir" mkdir -p "$libexpat_build_dir" - if [[ -f "$libexpat_build_dir/lib/.libs/libexpat.a" ]]; then + if [[ -f "$libexpat_build_dir/lib/libexpat.a" ]]; then >&2 echo "Skipping build: libexpat already built for $target_arch" return 0 fi @@ -304,7 +332,7 @@ function build_libexpat() { fi ../expat/configure --enable-static "CC=$CC" "CXX=$CXX" "--host=$HOST" \ - "CFLAGS=$CFLAGS" "CXXFLAGS=$CXXFLAGS" 1>&2 + "CFLAGS=$CFLAGS" "CXXFLAGS=$CXXFLAGS" --prefix="$(realpath .)" 1>&2 if [[ $? -ne 0 ]]; then return 1 fi @@ -314,6 +342,11 @@ function build_libexpat() { return 1 fi + make -j$(nproc) install 1>&2 + if [[ $? -ne 0 ]]; then + return 1 + fi + >&2 fancy_title "Finished building libexpat for $target_arch" popd > /dev/null @@ -356,7 +389,7 @@ function build_python() { export MODULE_BUILDTYPE="static" export CONFIG_SITE="$python_dir/config.site-static" >&2 CFLAGS="-static" LDFLAGS="-static" ../configure \ - --prefix=$(realpath .) \ + --prefix="$(realpath .)" \ --disable-test-modules \ --with-ensurepip=no \ --without-decimal-contextvar \ @@ -415,7 +448,7 @@ function build_libmpfr() { mkdir -p "$mpfr_build_dir" echo "$mpfr_build_dir" - if [[ -f "$mpfr_build_dir/src/.libs/lib/libmpfr.a" ]]; then + if [[ -f "$mpfr_build_dir/lib/libmpfr.a" ]]; then >&2 echo "Skipping build: libmpfr already built for $target_arch" return 0 fi @@ -424,7 +457,7 @@ function build_libmpfr() { >&2 fancy_title "Building libmpfr for $target_arch" - ../configure --enable-static "--with-gmp-build=$libgmp_build_dir" \ + ../configure --enable-static --prefix="$(realpath .)" "--with-gmp=$libgmp_build_dir" \ "CC=$CC" "CXX=$CXX" "--host=$HOST" \ "CFLAGS=$CFLAGS" "CXXFLAGS=$CXXFLAGS" 1>&2 if [[ $? -ne 0 ]]; then @@ -436,10 +469,10 @@ function build_libmpfr() { return 1 fi - mkdir -p ./src/.libs/include - cp ../src/mpfr.h ./src/.libs/include/ - mkdir -p ./src/.libs/lib - cp ./src/.libs/libmpfr.a ./src/.libs/lib/ + make -j$(nproc) install 1>&2 + if [[ $? -ne 0 ]]; then + return 1 + fi >&2 fancy_title "Finished building libmpfr for $target_arch" @@ -493,7 +526,7 @@ function build_gdb() { >&2 fancy_title "Building gdb for $target_arch" - ../configure -C --enable-static --with-static-standard-libraries --disable-inprocess-agent \ + ../configure --enable-static --with-static-standard-libraries --disable-inprocess-agent \ --enable-tui "$python_flag" \ --with-expat --with-libexpat-type="static" \ --with-gdb-datadir="/usr/share/gdb" --with-separate-debug-dir="/usr/lib/debug" \ @@ -552,7 +585,7 @@ function install_gdb() { mkdir -p "$artifacts_location" - make -C "$gdb_build_dir" install "DESTDIR=$temp_artifacts_dir" 1>&2 + make -j$(nproc) -C "$gdb_build_dir" install "DESTDIR=$temp_artifacts_dir" 1>&2 if [[ $? -ne 0 ]]; then rm -rf "$temp_artifacts_dir" return 1 @@ -655,7 +688,11 @@ function build_gdb_with_dependencies() { return 1 fi - set_up_lib_search_paths "$ncursesw_build_dir" "$libexpat_build_dir" + set_up_lib_search_paths "$iconv_build_dir" \ + "$gmp_build_dir" \ + "$mpfr_build_dir" \ + "$ncursesw_build_dir" \ + "$libexpat_build_dir" if [[ "$with_python" == "yes" ]]; then local gdb_python_dir="$packages_dir/binutils-gdb/gdb/python/lib/" @@ -667,10 +704,10 @@ function build_gdb_with_dependencies() { fi build_and_install_gdb "$packages_dir/binutils-gdb" \ - "$iconv_build_dir/lib/.libs/" \ - "$lzma_build_dir/usr/local/" \ - "$gmp_build_dir/.libs/" \ - "$mpfr_build_dir/src/.libs/" \ + "$iconv_build_dir" \ + "$lzma_build_dir" \ + "$gmp_build_dir" \ + "$mpfr_build_dir" \ "$with_python" \ "$artifacts_dir" \ "$target_arch" diff --git a/src/compilation/download_packages.sh b/src/compilation/download_packages.sh index a212710..ff630cb 100755 --- a/src/compilation/download_packages.sh +++ b/src/compilation/download_packages.sh @@ -10,7 +10,6 @@ SOURCE_URLS=( "https://ftp.gnu.org/pub/gnu/gmp/gmp-6.3.0.tar.xz" "https://ftp.gnu.org/pub/gnu/mpfr/mpfr-4.2.1.tar.xz" "https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.5.tar.gz" - "https://github.com/tukaani-project/xz/releases/download/v5.8.1/xz-5.8.1.tar.xz" ) function unpack_tarball() { diff --git a/src/docker_utils/download_musl_toolchains.py b/src/docker_utils/download_musl_toolchains.py new file mode 100755 index 0000000..b7b3376 --- /dev/null +++ b/src/docker_utils/download_musl_toolchains.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3.12 + +import requests +import tarfile +import os +import shutil + +from pathlib import Path + +ARCHS = { + "x86_64" : "https://more.musl.cc/11/x86_64-linux-musl/x86_64-linux-musl-cross.tgz", + "arm" : "https://more.musl.cc/10/x86_64-linux-musl/arm-linux-musleabi-cross.tgz", + "aarch64" : "https://more.musl.cc/11/x86_64-linux-musl/aarch64-linux-musl-cross.tgz", + "powerpc" : "https://more.musl.cc/11/x86_64-linux-musl/powerpc-linux-musl-cross.tgz", + "mips" : "https://more.musl.cc/11/x86_64-linux-musl/mips-linux-musl-cross.tgz", + "mipsel" : "https://more.musl.cc/11/x86_64-linux-musl/mipsel-linux-musl-cross.tgz", +} +CHUNK_SIZE = 65536 +MUSL_TOOLCHAINS_DIR = Path("/musl-toolchains") +ENTRYPOINT = "/entrypoint.sh" + +def download_file(url: str, filename: str): + print(f"Downloading {filename}") + with requests.get(url, stream=True) as r: + r.raise_for_status() + with open(filename, "wb") as f: + for chunk in r.iter_content(chunk_size=CHUNK_SIZE): + f.write(chunk) + print(f"{filename} downloaded.") + +def extract_tarball(filename: str, dst: Path): + print(f"Extracting {filename}") + with tarfile.open(filename, "r:gz") as tar: + tar.extractall(path=dst) + print(f"{filename} extracted") + +def add_to_path(curr_path: str, package_path: Path): + new_path = str((package_path / "bin").resolve()) + if curr_path != "": + return new_path + ":" + curr_path + return new_path + + +def main(): + os.mkdir(MUSL_TOOLCHAINS_DIR) + + updated_path = "" + for arch, url in ARCHS.items(): + filename = url.split("/")[-1] + download_file(url, filename) + extract_tarball(filename, MUSL_TOOLCHAINS_DIR) + updated_path = add_to_path(updated_path, MUSL_TOOLCHAINS_DIR / filename.removesuffix(".tgz")) + + # Fix the x86_64 dynamic loader if needed: + # Unfortunately, the internal gdb build scripts builds some binaries (that generate documentation) + # in a dynamic manner. + # + # Because we may use a musl-based toolchain, this means that we need to set-up the dynamic loader. + # The fix may seem a little hacky, but it is simple, and is the best we can do. + if "x86_64" in ARCHS: + x86_toolchain_name = ARCHS["x86_64"].split("/")[-1].removesuffix(".tgz") + x86_toolchain_path = MUSL_TOOLCHAINS_DIR / x86_toolchain_name + x86_loader_path = x86_toolchain_path / "x86_64-linux-musl" / "lib" / "libc.so" + shutil.copy2(x86_loader_path, "/lib/ld-musl-x86_64.so.1") + + # Create the entrypoint with the updated path. + with open(ENTRYPOINT, mode="w") as f: + f.write( +f"""#!/usr/bin/env bash +export PATH="$PATH:{updated_path}" +exec "$@" +""") + + # Make sure we can execute the entrypoint. + os.chmod(ENTRYPOINT, 0o755) + + # Append the path to bash.bashrc so that other users will have these paths. + with open("/etc/bash.bashrc", mode="a") as f: + f.write(f"\nexport PATH=\"$PATH:{updated_path}\"") + + +if __name__ == "__main__": + main() diff --git a/src/submodule_packages/binutils-gdb b/src/submodule_packages/binutils-gdb index 25f97cc..6103f49 160000 --- a/src/submodule_packages/binutils-gdb +++ b/src/submodule_packages/binutils-gdb @@ -1 +1 @@ -Subproject commit 25f97ccb7ff1a9d7bd0e25535d749acf6e1a87be +Subproject commit 6103f4984ba5824d340dd6801ab8244785fb8c53 diff --git a/src/submodule_packages/xz b/src/submodule_packages/xz new file mode 160000 index 0000000..56aa9a0 --- /dev/null +++ b/src/submodule_packages/xz @@ -0,0 +1 @@ +Subproject commit 56aa9a07968a6a73fada6f9b96122c5165829f3b