From ad4cf366cbcef0498b9e149840e51a3ad6977fd6 Mon Sep 17 00:00:00 2001 From: William Tan <1284324+Ninja3047@users.noreply.github.com> Date: Wed, 1 Feb 2023 16:56:33 -0500 Subject: [PATCH 01/35] look for files that are executable --- tool_run_scripts/anvill.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tool_run_scripts/anvill.py b/tool_run_scripts/anvill.py index 5ac0c03..143b219 100755 --- a/tool_run_scripts/anvill.py +++ b/tool_run_scripts/anvill.py @@ -267,10 +267,10 @@ def anvill_python_main(args, source_path, dest_path): # get all the bitcode log.info(f"Listing files in {str(source_path)}") - sources = list(source_path.rglob("*.elf")) - # Sometimes we forget the .elf suffix - sources.extend(list(source_path.rglob("*.o"))) - log.info(f"Found {len(sources)} ELF files") + # Filter for files that are executable + sources = [source for source in source_path.rglob("*") if os.path.isfile(source_path) and os.access(source_path, os.X_OK)] + + log.info(f"Found {len(sources)} Executable files") # load test to ignore anvill_stats.set_stat("start_time", str(datetime.now())) From d37be4e1d7a7865a6850bd71937eb320f972db92 Mon Sep 17 00:00:00 2001 From: William Tan <1284324+Ninja3047@users.noreply.github.com> Date: Wed, 1 Feb 2023 17:06:41 -0500 Subject: [PATCH 02/35] use yes for gpg --- datasets/fetch_amp_challengebins.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasets/fetch_amp_challengebins.sh b/datasets/fetch_amp_challengebins.sh index 6beb2e1..0b26a52 100755 --- a/datasets/fetch_amp_challengebins.sh +++ b/datasets/fetch_amp_challengebins.sh @@ -37,7 +37,7 @@ while [[ $# -gt 0 ]] ; do done curl -LO https://tob-amp-share.nyc3.digitaloceanspaces.com/challenge-binaries-latest.tar.xz.gpg -gpg --no-tty --batch --pinentry-mode loopback --passphrase "${TOB_AMP_PASSPHRASE}" \ +gpg --no-tty --batch --yes --pinentry-mode loopback --passphrase "${TOB_AMP_PASSPHRASE}" \ -o challenge-binaries-latest.tar.xz \ --decrypt challenge-binaries-latest.tar.xz.gpg rm -rf challenge-binaries-latest.tar.xz.gpg From d1295483ca37a54255c74cf3b0078396b2a82028 Mon Sep 17 00:00:00 2001 From: William Tan <1284324+Ninja3047@users.noreply.github.com> Date: Wed, 1 Feb 2023 17:11:45 -0500 Subject: [PATCH 03/35] fix typo --- tool_run_scripts/anvill.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool_run_scripts/anvill.py b/tool_run_scripts/anvill.py index 143b219..4de9041 100755 --- a/tool_run_scripts/anvill.py +++ b/tool_run_scripts/anvill.py @@ -268,7 +268,7 @@ def anvill_python_main(args, source_path, dest_path): # get all the bitcode log.info(f"Listing files in {str(source_path)}") # Filter for files that are executable - sources = [source for source in source_path.rglob("*") if os.path.isfile(source_path) and os.access(source_path, os.X_OK)] + sources = [source for source in source_path.rglob("*") if os.path.isfile(source) and os.access(source, os.X_OK)] log.info(f"Found {len(sources)} Executable files") From 263e6b65e7213476717d382ba58d5c8d4972bbdd Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 2 Feb 2023 13:15:12 -0500 Subject: [PATCH 04/35] add langid overrides to options --- tool_run_scripts/anvill.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tool_run_scripts/anvill.py b/tool_run_scripts/anvill.py index 4de9041..3911a57 100755 --- a/tool_run_scripts/anvill.py +++ b/tool_run_scripts/anvill.py @@ -35,8 +35,14 @@ class AnvillGhidraCmd(ToolCmd): + def __init__(self, tool, infile, outdir, source_base, index, stats, language_overrides): + super().__init__(tool, infile, outdir, source_base, index, stats) + self.lang_overrides = language_overrides + def make_tool_cmd(self): f = self.infile.stem + fullname = self.infile.name + jsonfile = f"{self.index}-{f}.pb" self.tmpout = self.outdir.joinpath("work").joinpath(jsonfile) @@ -48,8 +54,9 @@ def make_tool_cmd(self): "/tmp", f"dummy_ghidra_proj{self.index}-{f}", "-readOnly", - "-deleteProject", - "-import", + "-deleteProject"] + + (["-processor", self.lang_overrides[fullname]] if fullname in self.lang_overrides else []) + +["-import", str(self.infile), "-postScript", "anvillHeadlessExportScript", @@ -202,10 +209,10 @@ def save(self): reprofile.write("\n") -def run_anvill_ghidra(ghidra_dir, output_dir, failonly, source_path, stats, input_and_idx): +def run_anvill_ghidra(ghidra_dir, output_dir, failonly, source_path, stats, language_id_overrides, input_and_idx): idx, input_file = input_and_idx cmd = AnvillGhidraCmd(ghidra_dir, input_file, output_dir, - source_path, idx, stats) + source_path, idx, stats, language_id_overrides) retcode = cmd.run() log.debug(f"Anvill run returned {retcode}") @@ -261,9 +268,14 @@ def anvill_python_main(args, source_path, dest_path): num_cpus = os.cpu_count() anvill_stats = Stats() + + language_id_overrides = {} + if args.test_options: with open(args.test_options, "r") as rf: anvill_stats.load_rules(rf) + if "language_id_overrides" in anvill_stats.rules: + language_id_overrides = anvill_stats.rules['language_id_overrides'] # get all the bitcode log.info(f"Listing files in {str(source_path)}") @@ -279,7 +291,7 @@ def anvill_python_main(args, source_path, dest_path): # workspace for anvill-python apply_anvill_ghidra = partial( - run_anvill_ghidra, os.path.expanduser(args.ghidra_install_dir), dest_path, args.only_fails, source_path, anvill_stats) + run_anvill_ghidra, os.path.expanduser(args.ghidra_install_dir), dest_path, args.only_fails, source_path, anvill_stats, language_id_overrides) with ThreadPool(num_cpus) as p: with tqdm(total=max_items_python) as pbar: From 401fe03c93135f6e808241fb1c1d2065108d80ef Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 2 Feb 2023 14:47:19 -0500 Subject: [PATCH 05/35] fix order of init --- tool_run_scripts/anvill.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool_run_scripts/anvill.py b/tool_run_scripts/anvill.py index 3911a57..c79084d 100755 --- a/tool_run_scripts/anvill.py +++ b/tool_run_scripts/anvill.py @@ -36,8 +36,8 @@ class AnvillGhidraCmd(ToolCmd): def __init__(self, tool, infile, outdir, source_base, index, stats, language_overrides): - super().__init__(tool, infile, outdir, source_base, index, stats) self.lang_overrides = language_overrides + super().__init__(tool, infile, outdir, source_base, index, stats) def make_tool_cmd(self): f = self.infile.stem From 6dd89b4ca3ee8300aab672657a3b9b502076d53c Mon Sep 17 00:00:00 2001 From: William Tan <1284324+Ninja3047@users.noreply.github.com> Date: Fri, 3 Feb 2023 16:16:10 -0500 Subject: [PATCH 06/35] initialize ghidra script compilation cache --- tool_run_scripts/anvill.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tool_run_scripts/anvill.py b/tool_run_scripts/anvill.py index c79084d..6ae5bd3 100755 --- a/tool_run_scripts/anvill.py +++ b/tool_run_scripts/anvill.py @@ -208,6 +208,30 @@ def save(self): reprofile.write(" ".join(self.cmd)) reprofile.write("\n") +# Run the script with no input to trigger script compilation so it gets saved in the cache +def initialize_ghidra_cache(ghidra_dir): + try: + args = [os.path.join(ghidra_dir, "support", "analyzeHeadless")] + args.extend([ + "/tmp", + "dummy_ghidra_proj_init", + "-readOnly", + "-deleteProject" + "-preScript", + "anvillHeadlessExportScript", + ]) + + subprocess.run(args=args) + except OSError as oe: + log.error(f"Could not initialize ghidra: {oe}") + sys.exit(1) + except subprocess.CalledProcessError as cpe: + log.error(f"Could not initialize: {cpe}") + sys.exit(1) + except subprocess.TimeoutExpired as tme: + log.error(f"Could not initialize ghidra: timeout exception") + sys.exit(1) + def run_anvill_ghidra(ghidra_dir, output_dir, failonly, source_path, stats, language_id_overrides, input_and_idx): idx, input_file = input_and_idx @@ -289,6 +313,9 @@ def anvill_python_main(args, source_path, dest_path): max_items_python = len(sources) + # initialize ghidra cache to pre-compile the script + initialize_ghidra_cache(os.path.expanduser(args.ghidra_install_dir)) + # workspace for anvill-python apply_anvill_ghidra = partial( run_anvill_ghidra, os.path.expanduser(args.ghidra_install_dir), dest_path, args.only_fails, source_path, anvill_stats, language_id_overrides) From 06363f1eb18de02c7b15c69371ccacea56bef400 Mon Sep 17 00:00:00 2001 From: William Tan <1284324+Ninja3047@users.noreply.github.com> Date: Fri, 3 Feb 2023 16:16:54 -0500 Subject: [PATCH 07/35] fix typo --- tool_run_scripts/anvill.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool_run_scripts/anvill.py b/tool_run_scripts/anvill.py index 6ae5bd3..8f27836 100755 --- a/tool_run_scripts/anvill.py +++ b/tool_run_scripts/anvill.py @@ -282,7 +282,7 @@ def get_anvill_version(cmd): log.error(f"Could not get anvill version: {cpe}") sys.exit(1) except subprocess.TimeoutExpired as tme: - log.error(f"Could not get anvill version: timeout execption") + log.error(f"Could not get anvill version: timeout exception") sys.exit(1) return rt.stdout.decode("utf-8") From 6d4732f23bad41c6c3a3b6a45eb129b36ed36b97 Mon Sep 17 00:00:00 2001 From: William Tan <1284324+Ninja3047@users.noreply.github.com> Date: Fri, 3 Feb 2023 16:17:38 -0500 Subject: [PATCH 08/35] use abstract class --- tool_run_scripts/toolcmd.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tool_run_scripts/toolcmd.py b/tool_run_scripts/toolcmd.py index 4310642..ac2ad8b 100644 --- a/tool_run_scripts/toolcmd.py +++ b/tool_run_scripts/toolcmd.py @@ -1,3 +1,4 @@ +from abc import ABC, abstractmethod import re import logging import signal @@ -13,7 +14,7 @@ ASAN_ERROR_RE = re.compile('AddressSanitizer: [a-zA-Z\-]+ .*/([^:]+:[\d]+)') CLANG_ERROR_RE = re.compile("error: ([\w']+) *([\w']*) *([\w']+) *([\w']+)") -class ToolCmd: +class ToolCmd(ABC): def __init__(self, tool, infile, outdir, source_base, index, stats): self.source_base = source_base self.index = index @@ -32,8 +33,9 @@ def set_output(self, rc, out, err): self.out = out self.err = err + @abstractmethod def make_tool_cmd(self): - raise RuntimeError("Please override make_tool_cmd") + pass def clang_traceback(self, msg): if not msg: From 734826c063c8cde68d7e064b90372f764f1e2cd8 Mon Sep 17 00:00:00 2001 From: William Tan <1284324+Ninja3047@users.noreply.github.com> Date: Fri, 3 Feb 2023 16:19:18 -0500 Subject: [PATCH 09/35] fix python lints --- tool_run_scripts/toolcmd.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tool_run_scripts/toolcmd.py b/tool_run_scripts/toolcmd.py index ac2ad8b..624a053 100644 --- a/tool_run_scripts/toolcmd.py +++ b/tool_run_scripts/toolcmd.py @@ -9,10 +9,10 @@ log.addHandler(logging.StreamHandler()) log.setLevel(logging.DEBUG) -FILE_NAME_RE = re.compile("([^/\s]+\.[^/\s]+:\d+)") -PYTHON_ERROR_RE = re.compile('([^/\s]+\.py)", line (\d+)') -ASAN_ERROR_RE = re.compile('AddressSanitizer: [a-zA-Z\-]+ .*/([^:]+:[\d]+)') -CLANG_ERROR_RE = re.compile("error: ([\w']+) *([\w']*) *([\w']+) *([\w']+)") +FILE_NAME_RE = re.compile(r"([^/\s]+\.[^/\s]+:\d+)") +PYTHON_ERROR_RE = re.compile(r'([^/\s]+\.py)", line (\d+)') +ASAN_ERROR_RE = re.compile(r'AddressSanitizer: [a-zA-Z\-]+ .*/([^:]+:[\d]+)') +CLANG_ERROR_RE = re.compile(r"error: ([\w']+) *([\w']*) *([\w']+) *([\w']+)") class ToolCmd(ABC): def __init__(self, tool, infile, outdir, source_base, index, stats): @@ -126,7 +126,7 @@ def __del__(self): log.debug(f"Unlinking on delete {self.tmpout}") try: os.unlink(self.tmpout) - except FileNotFoundError as fnf: + except FileNotFoundError: log.debug(f"Tried to delete a file that doesn't exist: {self.tmpout}") def run(self): From e11f9e5f8404615442a0ee01480b14b0d26b4b11 Mon Sep 17 00:00:00 2001 From: William Tan <1284324+Ninja3047@users.noreply.github.com> Date: Fri, 3 Feb 2023 16:34:08 -0500 Subject: [PATCH 10/35] fix missing comma --- tool_run_scripts/anvill.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool_run_scripts/anvill.py b/tool_run_scripts/anvill.py index 8f27836..f124e29 100755 --- a/tool_run_scripts/anvill.py +++ b/tool_run_scripts/anvill.py @@ -216,7 +216,7 @@ def initialize_ghidra_cache(ghidra_dir): "/tmp", "dummy_ghidra_proj_init", "-readOnly", - "-deleteProject" + "-deleteProject", "-preScript", "anvillHeadlessExportScript", ]) From c114ae6f41d89120b80575b6246a5d016ac518bf Mon Sep 17 00:00:00 2001 From: William Tan <1284324+Ninja3047@users.noreply.github.com> Date: Mon, 6 Feb 2023 14:25:26 -0500 Subject: [PATCH 11/35] skip hidden files --- tool_run_scripts/anvill.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool_run_scripts/anvill.py b/tool_run_scripts/anvill.py index f124e29..2269470 100755 --- a/tool_run_scripts/anvill.py +++ b/tool_run_scripts/anvill.py @@ -304,7 +304,7 @@ def anvill_python_main(args, source_path, dest_path): # get all the bitcode log.info(f"Listing files in {str(source_path)}") # Filter for files that are executable - sources = [source for source in source_path.rglob("*") if os.path.isfile(source) and os.access(source, os.X_OK)] + sources = [source for source in source_path.rglob("*") if source.is_file() and os.access(source, os.X_OK) and not source.name.startswith(".")] log.info(f"Found {len(sources)} Executable files") From 2f5ae380e78f2288ead5f4c1c6aef30c68c9d721 Mon Sep 17 00:00:00 2001 From: 2over12 Date: Sat, 11 Feb 2023 10:50:33 -0500 Subject: [PATCH 12/35] make time outs pass for now --- tool_run_scripts/stats.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tool_run_scripts/stats.py b/tool_run_scripts/stats.py index 26df3ee..8b24038 100644 --- a/tool_run_scripts/stats.py +++ b/tool_run_scripts/stats.py @@ -80,10 +80,11 @@ def print_fails(self, fail_count=5, output=None, verbose=True): def get_fail_count(self): success_runs = len(self.stats.get("output.success", [])) + timed_out_runs = self.stats.get("program_timeouts", 0) program_runs = self.stats.get("program_runs", 0) ignored_outputs = len(self.stats.get("outputignore_success", [])) ignored_outputs += len(self.stats.get("outputignore_fail", [])) - return (program_runs - ignored_outputs) - success_runs + return (program_runs - ignored_outputs) - (success_runs + timed_out_runs) def print_stats(self, output=None): # emit start/end time From f0e090dea0cdb322f80774bddc417cd07af3cd56 Mon Sep 17 00:00:00 2001 From: William Tan <1284324+Ninja3047@users.noreply.github.com> Date: Thu, 16 Feb 2023 16:45:16 -0500 Subject: [PATCH 13/35] add FixGlobalRegisters as a postScript (#11) --- tool_run_scripts/anvill.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tool_run_scripts/anvill.py b/tool_run_scripts/anvill.py index 2269470..9bd574b 100755 --- a/tool_run_scripts/anvill.py +++ b/tool_run_scripts/anvill.py @@ -59,6 +59,8 @@ def make_tool_cmd(self): +["-import", str(self.infile), "-postScript", + "FixGlobalRegister" + "-postScript", "anvillHeadlessExportScript", str(self.tmpout), ]) From da206215f92228162f1249f277839caf0080b1a4 Mon Sep 17 00:00:00 2001 From: William Tan <1284324+Ninja3047@users.noreply.github.com> Date: Thu, 16 Feb 2023 17:51:20 -0500 Subject: [PATCH 14/35] fix missing comma --- tool_run_scripts/anvill.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool_run_scripts/anvill.py b/tool_run_scripts/anvill.py index 9bd574b..4610464 100755 --- a/tool_run_scripts/anvill.py +++ b/tool_run_scripts/anvill.py @@ -59,7 +59,7 @@ def make_tool_cmd(self): +["-import", str(self.infile), "-postScript", - "FixGlobalRegister" + "FixGlobalRegister", "-postScript", "anvillHeadlessExportScript", str(self.tmpout), From 3bf1ea36c73ad35cbf8faa44f8b9bd8c8f911f97 Mon Sep 17 00:00:00 2001 From: 2over12 Date: Fri, 17 Feb 2023 10:42:23 -0500 Subject: [PATCH 15/35] add enable opt flag --- tool_run_scripts/anvill.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tool_run_scripts/anvill.py b/tool_run_scripts/anvill.py index 4610464..74b7d23 100755 --- a/tool_run_scripts/anvill.py +++ b/tool_run_scripts/anvill.py @@ -155,6 +155,7 @@ def make_tool_cmd(self): str(self.tmpout), "-stats_out", str(self.stats_file), + "-remove_next_pc_assignments", "-logtostderr", ]) From 07ff24154d8bae0a7082286d072b79910bf5dd9c Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 08:45:57 -0400 Subject: [PATCH 16/35] add draft action --- .github/workflows/build.yml | 50 +++++++++++++++++++++++++++++ actions/build-cxx-common/action.yml | 43 +++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 .github/workflows/build.yml create mode 100644 actions/build-cxx-common/action.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..1ad236e --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,50 @@ +# +# Copyright (c) 2019-present, Trail of Bits, Inc. +# All rights reserved. +# +# This source code is licensed in accordance with the terms specified in +# the LICENSE file found in the root directory of this source tree. +# + +name: Build + +on: + # Run this workflow once every 6 hours against the master branch + #schedule: + # - cron: "0 */6 * * *" + + push: + branches: + - "*" + + tags: + - "*" + + pull_request: + branches: + - "*" + +jobs: + build_linux: + needs: cleanup_stale_workflows + + strategy: + fail-fast: false + matrix: + image: + - { name: "ubuntu", tag: "22.04" } + llvm: ["16"] + cxxcommon_version: ["v0.3.2"] + + runs-on: + labels: gha-ubuntu-32 + container: + image: docker.pkg.github.com/lifting-bits/cxx-common/vcpkg-builder-${{ matrix.image.name }}:${{ matrix.image.tag }} + + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + steps: + - uses: actions/checkout@v3 + - uses: ./actions/build-cxx-common diff --git a/actions/build-cxx-common/action.yml b/actions/build-cxx-common/action.yml new file mode 100644 index 0000000..8b18c38 --- /dev/null +++ b/actions/build-cxx-common/action.yml @@ -0,0 +1,43 @@ +name: "Lifting tools CXX-Common" +description: "Build cxx-common latest for lifting-tools" +inputs: + target-export-path: # id of input + description: "where to export the cxx-common build" + required: true +outputs: + random-number: + description: "Random number" + value: ${{ steps.random-number-generator.outputs.random-number }} + +runs: + using: "composite" + steps: + - uses: actions/checkout@v3 + with: + submodules: true + repository: "https://github.com/lifting-bits/cxx-common.git" + - name: "Bootstrap vcpkg" + shell: "bash" + run: | + ./vcpkg/bootstrap-vcpkg.sh + - name: "Setup NuGet Credentials" + shell: "bash" + # Replace with your organization name + run: | + mono `./vcpkg/vcpkg fetch nuget | tail -n 1` \ + sources add \ + -source "https://nuget.pkg.github.com//index.json" \ + -storepasswordincleartext \ + -name "GitHub" \ + -username "lifting-bits" \ + -password "${{ secrets.GITHUB_TOKEN }}" + mono `./vcpkg/vcpkg fetch nuget | tail -n 1` \ + setapikey "${{ secrets.GITHUB_TOKEN }}" \ + -source "https://nuget.pkg.github.com//index.json" + # Omit this step if you're using manifests + - name: "vcpkg package restore" + shell: "bash" + run: > + ./vcpkg/vcpkg install sqlite3 cpprestsdk + - run: ./build_dependencies.sh --release --export-dir ./${{ env.ARTIFACT_NAME }} ${{ matrix.llvm }} --clean-after-build --debug + shell: bash From b4720f9fd887fbede6b2c174c2cc8b93e311a771 Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 08:48:08 -0400 Subject: [PATCH 17/35] remove stale dep --- .github/workflows/build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1ad236e..022ee5c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,8 +26,6 @@ on: jobs: build_linux: - needs: cleanup_stale_workflows - strategy: fail-fast: false matrix: From f4d5351bf5bc9dfe0ce7fe383100cb47753c127a Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 08:51:10 -0400 Subject: [PATCH 18/35] fix runner --- .github/workflows/build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 022ee5c..dabe87e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,8 +34,7 @@ jobs: llvm: ["16"] cxxcommon_version: ["v0.3.2"] - runs-on: - labels: gha-ubuntu-32 + runs-on: ubuntu-22.04 container: image: docker.pkg.github.com/lifting-bits/cxx-common/vcpkg-builder-${{ matrix.image.name }}:${{ matrix.image.tag }} From 6f96d98f7db320f85d170d1e9daeeb6783222096 Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 09:06:29 -0400 Subject: [PATCH 19/35] secrets input --- .github/workflows/build.yml | 4 ---- actions/build-cxx-common/action.yml | 18 ++++++++---------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dabe87e..d4fcb5a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,10 +38,6 @@ jobs: container: image: docker.pkg.github.com/lifting-bits/cxx-common/vcpkg-builder-${{ matrix.image.name }}:${{ matrix.image.tag }} - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - steps: - uses: actions/checkout@v3 - uses: ./actions/build-cxx-common diff --git a/actions/build-cxx-common/action.yml b/actions/build-cxx-common/action.yml index 8b18c38..499be69 100644 --- a/actions/build-cxx-common/action.yml +++ b/actions/build-cxx-common/action.yml @@ -4,11 +4,9 @@ inputs: target-export-path: # id of input description: "where to export the cxx-common build" required: true -outputs: - random-number: - description: "Random number" - value: ${{ steps.random-number-generator.outputs.random-number }} - + gh-token: + description: "token for the target nuget cache" + required: true runs: using: "composite" steps: @@ -26,18 +24,18 @@ runs: run: | mono `./vcpkg/vcpkg fetch nuget | tail -n 1` \ sources add \ - -source "https://nuget.pkg.github.com//index.json" \ + -source "https://nuget.pkg.github.com/lifting-bits/index.json" \ -storepasswordincleartext \ -name "GitHub" \ -username "lifting-bits" \ - -password "${{ secrets.GITHUB_TOKEN }}" + -password "${{ inputs.gh-token }}" mono `./vcpkg/vcpkg fetch nuget | tail -n 1` \ - setapikey "${{ secrets.GITHUB_TOKEN }}" \ - -source "https://nuget.pkg.github.com//index.json" + setapikey "${{ inputs.gh-token }}" \ + -source "https://nuget.pkg.github.com/lifting-bits/index.json" # Omit this step if you're using manifests - name: "vcpkg package restore" shell: "bash" run: > ./vcpkg/vcpkg install sqlite3 cpprestsdk - - run: ./build_dependencies.sh --release --export-dir ./${{ env.ARTIFACT_NAME }} ${{ matrix.llvm }} --clean-after-build --debug + - run: ./build_dependencies.sh --release --export-dir ${{ inputs.target-export-path }} ${{ matrix.llvm }} --debug shell: bash From 542e1ac65f6dc859571e17cf7fff084de68ce9ec Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 09:09:12 -0400 Subject: [PATCH 20/35] add creds for image back --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d4fcb5a..4057283 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,6 +37,9 @@ jobs: runs-on: ubuntu-22.04 container: image: docker.pkg.github.com/lifting-bits/cxx-common/vcpkg-builder-${{ matrix.image.name }}:${{ matrix.image.tag }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v3 From 896ddfc4f252d5abc6b8e393df7786bfe0fe3f83 Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 09:11:46 -0400 Subject: [PATCH 21/35] fix ref --- actions/build-cxx-common/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/build-cxx-common/action.yml b/actions/build-cxx-common/action.yml index 499be69..6736b24 100644 --- a/actions/build-cxx-common/action.yml +++ b/actions/build-cxx-common/action.yml @@ -13,7 +13,7 @@ runs: - uses: actions/checkout@v3 with: submodules: true - repository: "https://github.com/lifting-bits/cxx-common.git" + repository: "lifting-bits/cxx-common" - name: "Bootstrap vcpkg" shell: "bash" run: | From 2d4c9d2a5d67ecc56d0015a202078868576caf52 Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 09:22:25 -0400 Subject: [PATCH 22/35] working dir --- .github/workflows/build.yml | 1 - actions/build-cxx-common/action.yml | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4057283..62f0f89 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,6 @@ on: jobs: build_linux: strategy: - fail-fast: false matrix: image: - { name: "ubuntu", tag: "22.04" } diff --git a/actions/build-cxx-common/action.yml b/actions/build-cxx-common/action.yml index 6736b24..ecafb0e 100644 --- a/actions/build-cxx-common/action.yml +++ b/actions/build-cxx-common/action.yml @@ -13,14 +13,16 @@ runs: - uses: actions/checkout@v3 with: submodules: true + path: cxx-common repository: "lifting-bits/cxx-common" - name: "Bootstrap vcpkg" shell: "bash" + working-directory: cxx-common run: | ./vcpkg/bootstrap-vcpkg.sh - name: "Setup NuGet Credentials" shell: "bash" - # Replace with your organization name + working-directory: cxx-common run: | mono `./vcpkg/vcpkg fetch nuget | tail -n 1` \ sources add \ @@ -34,6 +36,7 @@ runs: -source "https://nuget.pkg.github.com/lifting-bits/index.json" # Omit this step if you're using manifests - name: "vcpkg package restore" + working-directory: cxx-common shell: "bash" run: > ./vcpkg/vcpkg install sqlite3 cpprestsdk From 5b1fc10f64df1eb9badd12bb1714f0128128d2bb Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 09:37:43 -0400 Subject: [PATCH 23/35] reuse initilization code --- actions/build-cxx-common/action.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/actions/build-cxx-common/action.yml b/actions/build-cxx-common/action.yml index ecafb0e..992eb0c 100644 --- a/actions/build-cxx-common/action.yml +++ b/actions/build-cxx-common/action.yml @@ -15,6 +15,15 @@ runs: submodules: true path: cxx-common repository: "lifting-bits/cxx-common" + + - name: Initialize vcpkg + shell: bash + working-directory: cxx-common + run: | + { read -r vcpkg_repo_url && read -r vcpkg_commit; } <./vcpkg_info.txt || exit 1 + git clone "${vcpkg_repo_url}" + git -C vcpkg checkout "${vcpkg_commit}" + echo "VCPKG_ROOT=$(pwd)/vcpkg" >> $GITHUB_ENV - name: "Bootstrap vcpkg" shell: "bash" working-directory: cxx-common From ef4c8039893623fdf38910abe1c49a73482a9814 Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 09:45:43 -0400 Subject: [PATCH 24/35] add token --- .github/workflows/build.yml | 2 ++ actions/build-cxx-common/action.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 62f0f89..8f89c69 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,3 +43,5 @@ jobs: steps: - uses: actions/checkout@v3 - uses: ./actions/build-cxx-common + with: + gh-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/actions/build-cxx-common/action.yml b/actions/build-cxx-common/action.yml index 992eb0c..149eb01 100644 --- a/actions/build-cxx-common/action.yml +++ b/actions/build-cxx-common/action.yml @@ -24,6 +24,7 @@ runs: git clone "${vcpkg_repo_url}" git -C vcpkg checkout "${vcpkg_commit}" echo "VCPKG_ROOT=$(pwd)/vcpkg" >> $GITHUB_ENV + echo "VCPKG_DISABLE_METRICS=1 >> $GITHUB_ENV" - name: "Bootstrap vcpkg" shell: "bash" working-directory: cxx-common @@ -50,4 +51,5 @@ runs: run: > ./vcpkg/vcpkg install sqlite3 cpprestsdk - run: ./build_dependencies.sh --release --export-dir ${{ inputs.target-export-path }} ${{ matrix.llvm }} --debug + working-directory: cxx-common shell: bash From 68207f3ceaa6f050472981803c4ed43dc84d81b2 Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 10:58:25 -0400 Subject: [PATCH 25/35] actually install, probably need this to be an input? --- actions/build-cxx-common/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/build-cxx-common/action.yml b/actions/build-cxx-common/action.yml index 149eb01..844010f 100644 --- a/actions/build-cxx-common/action.yml +++ b/actions/build-cxx-common/action.yml @@ -50,6 +50,6 @@ runs: shell: "bash" run: > ./vcpkg/vcpkg install sqlite3 cpprestsdk - - run: ./build_dependencies.sh --release --export-dir ${{ inputs.target-export-path }} ${{ matrix.llvm }} --debug + - run: ./build_dependencies.sh --release --export-dir ${{ inputs.target-export-path }} llvm-${{ matrix.llvm }} --debug working-directory: cxx-common shell: bash From 2fb5858bcc66662bc6698176520d3cac31abbc8d Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 11:37:22 -0400 Subject: [PATCH 26/35] add use cache env var --- actions/build-cxx-common/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/actions/build-cxx-common/action.yml b/actions/build-cxx-common/action.yml index 844010f..1aa4d46 100644 --- a/actions/build-cxx-common/action.yml +++ b/actions/build-cxx-common/action.yml @@ -25,6 +25,7 @@ runs: git -C vcpkg checkout "${vcpkg_commit}" echo "VCPKG_ROOT=$(pwd)/vcpkg" >> $GITHUB_ENV echo "VCPKG_DISABLE_METRICS=1 >> $GITHUB_ENV" + echo "VCPKG_USE_NUGET_CACHE=1 >> $GITHUB_ENV" - name: "Bootstrap vcpkg" shell: "bash" working-directory: cxx-common From 5e88d8d229fce3607bbb421bb6bc2fad53a2e6b6 Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 12:10:29 -0400 Subject: [PATCH 27/35] add more flags --- actions/build-cxx-common/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/actions/build-cxx-common/action.yml b/actions/build-cxx-common/action.yml index 1aa4d46..bc75488 100644 --- a/actions/build-cxx-common/action.yml +++ b/actions/build-cxx-common/action.yml @@ -26,6 +26,7 @@ runs: echo "VCPKG_ROOT=$(pwd)/vcpkg" >> $GITHUB_ENV echo "VCPKG_DISABLE_METRICS=1 >> $GITHUB_ENV" echo "VCPKG_USE_NUGET_CACHE=1 >> $GITHUB_ENV" + echo "VCPKG_BINARY_SOURCES='clear;nuget,GitHub,readwrite;nugettimeout,3601' >> $GITHUB_ENV" - name: "Bootstrap vcpkg" shell: "bash" working-directory: cxx-common From ee82839fb394109d6a15c65866193373d4427e84 Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 12:25:17 -0400 Subject: [PATCH 28/35] fix quotes --- actions/build-cxx-common/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/actions/build-cxx-common/action.yml b/actions/build-cxx-common/action.yml index bc75488..26fcb88 100644 --- a/actions/build-cxx-common/action.yml +++ b/actions/build-cxx-common/action.yml @@ -24,9 +24,9 @@ runs: git clone "${vcpkg_repo_url}" git -C vcpkg checkout "${vcpkg_commit}" echo "VCPKG_ROOT=$(pwd)/vcpkg" >> $GITHUB_ENV - echo "VCPKG_DISABLE_METRICS=1 >> $GITHUB_ENV" - echo "VCPKG_USE_NUGET_CACHE=1 >> $GITHUB_ENV" - echo "VCPKG_BINARY_SOURCES='clear;nuget,GitHub,readwrite;nugettimeout,3601' >> $GITHUB_ENV" + echo "VCPKG_DISABLE_METRICS=1" >> $GITHUB_ENV + echo "VCPKG_USE_NUGET_CACHE=1" >> $GITHUB_ENV + echo "VCPKG_BINARY_SOURCES='clear;nuget,GitHub,readwrite;nugettimeout,3601'" >> $GITHUB_ENV - name: "Bootstrap vcpkg" shell: "bash" working-directory: cxx-common From 97aca57b350a20dc003075f6a4de1c281496a4bf Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 12:27:45 -0400 Subject: [PATCH 29/35] is it supposed to parse like this --- actions/build-cxx-common/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/build-cxx-common/action.yml b/actions/build-cxx-common/action.yml index 26fcb88..e1eee45 100644 --- a/actions/build-cxx-common/action.yml +++ b/actions/build-cxx-common/action.yml @@ -26,7 +26,7 @@ runs: echo "VCPKG_ROOT=$(pwd)/vcpkg" >> $GITHUB_ENV echo "VCPKG_DISABLE_METRICS=1" >> $GITHUB_ENV echo "VCPKG_USE_NUGET_CACHE=1" >> $GITHUB_ENV - echo "VCPKG_BINARY_SOURCES='clear;nuget,GitHub,readwrite;nugettimeout,3601'" >> $GITHUB_ENV + echo "VCPKG_BINARY_SOURCES=clear;nuget,GitHub,readwrite;nugettimeout,3601" >> $GITHUB_ENV - name: "Bootstrap vcpkg" shell: "bash" working-directory: cxx-common From 45a8c0c4150ba62fa4dbcbd870e8ee7d720cf810 Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 12:48:30 -0400 Subject: [PATCH 30/35] try in single shell --- actions/build-cxx-common/action.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/actions/build-cxx-common/action.yml b/actions/build-cxx-common/action.yml index e1eee45..6029f4f 100644 --- a/actions/build-cxx-common/action.yml +++ b/actions/build-cxx-common/action.yml @@ -46,12 +46,7 @@ runs: mono `./vcpkg/vcpkg fetch nuget | tail -n 1` \ setapikey "${{ inputs.gh-token }}" \ -source "https://nuget.pkg.github.com/lifting-bits/index.json" - # Omit this step if you're using manifests - - name: "vcpkg package restore" - working-directory: cxx-common - shell: "bash" - run: > + ./vcpkg/vcpkg install sqlite3 cpprestsdk - - run: ./build_dependencies.sh --release --export-dir ${{ inputs.target-export-path }} llvm-${{ matrix.llvm }} --debug - working-directory: cxx-common - shell: bash + + ./build_dependencies.sh --release --export-dir ${{ inputs.target-export-path }} llvm-${{ matrix.llvm }} --debug From 97babea1c9442ceb224aa77be15a71cea2e12653 Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 12:51:31 -0400 Subject: [PATCH 31/35] add debug --- actions/build-cxx-common/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/build-cxx-common/action.yml b/actions/build-cxx-common/action.yml index 6029f4f..c92433b 100644 --- a/actions/build-cxx-common/action.yml +++ b/actions/build-cxx-common/action.yml @@ -47,6 +47,6 @@ runs: setapikey "${{ inputs.gh-token }}" \ -source "https://nuget.pkg.github.com/lifting-bits/index.json" - ./vcpkg/vcpkg install sqlite3 cpprestsdk + ./vcpkg/vcpkg install sqlite3 cpprestsdk --debug ./build_dependencies.sh --release --export-dir ${{ inputs.target-export-path }} llvm-${{ matrix.llvm }} --debug From 8bdb9135abd055f251c63e155b438bae700b900c Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 14:22:30 -0400 Subject: [PATCH 32/35] try replace env var --- actions/build-cxx-common/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/actions/build-cxx-common/action.yml b/actions/build-cxx-common/action.yml index c92433b..3b0add4 100644 --- a/actions/build-cxx-common/action.yml +++ b/actions/build-cxx-common/action.yml @@ -36,6 +36,7 @@ runs: shell: "bash" working-directory: cxx-common run: | + export GITHUB_REPOSITORY=lifting-bits/cxx-common mono `./vcpkg/vcpkg fetch nuget | tail -n 1` \ sources add \ -source "https://nuget.pkg.github.com/lifting-bits/index.json" \ From 8b75ceec3891510aaea3b55d10fa567de015efec Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 14:30:20 -0400 Subject: [PATCH 33/35] test if we can unset the repo --- actions/build-cxx-common/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/build-cxx-common/action.yml b/actions/build-cxx-common/action.yml index 3b0add4..e3e5a65 100644 --- a/actions/build-cxx-common/action.yml +++ b/actions/build-cxx-common/action.yml @@ -36,7 +36,7 @@ runs: shell: "bash" working-directory: cxx-common run: | - export GITHUB_REPOSITORY=lifting-bits/cxx-common + export GITHUB_REPOSITORY="" mono `./vcpkg/vcpkg fetch nuget | tail -n 1` \ sources add \ -source "https://nuget.pkg.github.com/lifting-bits/index.json" \ From 50953160b4a60cbd155292a958af3111a077836b Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 14:47:39 -0400 Subject: [PATCH 34/35] unset both vars --- actions/build-cxx-common/action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/actions/build-cxx-common/action.yml b/actions/build-cxx-common/action.yml index e3e5a65..7f8f602 100644 --- a/actions/build-cxx-common/action.yml +++ b/actions/build-cxx-common/action.yml @@ -36,7 +36,9 @@ runs: shell: "bash" working-directory: cxx-common run: | - export GITHUB_REPOSITORY="" + unset GITHUB_REPOSITORY + unset GITHUB_SERVER_URL + mono `./vcpkg/vcpkg fetch nuget | tail -n 1` \ sources add \ -source "https://nuget.pkg.github.com/lifting-bits/index.json" \ From 6c275422c1f297a7647a6039622ab2b1c38b69c7 Mon Sep 17 00:00:00 2001 From: 2over12 Date: Thu, 1 Jun 2023 15:14:14 -0400 Subject: [PATCH 35/35] try to use supplied env var --- actions/build-cxx-common/action.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/actions/build-cxx-common/action.yml b/actions/build-cxx-common/action.yml index 7f8f602..173eca7 100644 --- a/actions/build-cxx-common/action.yml +++ b/actions/build-cxx-common/action.yml @@ -27,6 +27,7 @@ runs: echo "VCPKG_DISABLE_METRICS=1" >> $GITHUB_ENV echo "VCPKG_USE_NUGET_CACHE=1" >> $GITHUB_ENV echo "VCPKG_BINARY_SOURCES=clear;nuget,GitHub,readwrite;nugettimeout,3601" >> $GITHUB_ENV + echo "VCPKG_NUGET_REPOSITORY=''" >> $GITHUB_ENV - name: "Bootstrap vcpkg" shell: "bash" working-directory: cxx-common @@ -36,9 +37,6 @@ runs: shell: "bash" working-directory: cxx-common run: | - unset GITHUB_REPOSITORY - unset GITHUB_SERVER_URL - mono `./vcpkg/vcpkg fetch nuget | tail -n 1` \ sources add \ -source "https://nuget.pkg.github.com/lifting-bits/index.json" \