From f8b5e7bdd34c10d7e7ce31ed8ed4772bb249c83d Mon Sep 17 00:00:00 2001 From: mayantaylor Date: Mon, 17 Jun 2024 07:52:03 -0500 Subject: [PATCH 01/22] resolving arch flag issues for mac --- setup.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index de217b53..70eb8874 100644 --- a/setup.py +++ b/setup.py @@ -23,9 +23,14 @@ def get_build_machine(): machine = platform.machine() if machine == 'arm64' or machine == 'aarch64': - return 'arm64' + return 'arm8' return machine +def get_archflag_machine(): + machine = platform.machine() + if machine == 'arm64' or machine == 'aarch64': + return 'arm64' + return machine def get_build_os(): os = platform.system() @@ -53,7 +58,7 @@ def get_build_triple(build_mpi): libcharm_filename2 = 'charm.lib' charmrun_filename = 'charmrun.exe' elif system == 'darwin': - os.environ['ARCHFLAGS'] = f'-arch {machine}' + os.environ['ARCHFLAGS'] = f'-arch {get_archflag_machine()}' libcharm_filename = 'libcharm.dylib' charmrun_filename = 'charmrun' if 'CPPFLAGS' in os.environ: @@ -169,7 +174,7 @@ def build_libcharm(charm_src_dir, build_dir): raise DistutilsSetupError('An error occured while building charm library') if system == 'darwin': - old_file_path = os.path.join(charm_src_dir, 'charm', 'lib', 'libcharm.so') + old_file_path = os.path.join(charm_src_dir, 'charm', 'lib', 'libcharm.dylib') new_file_path = os.path.join(charm_src_dir, 'charm', 'lib', libcharm_filename) shutil.move(old_file_path, new_file_path) cmd = ['install_name_tool', '-id', '@rpath/../.libs/' + libcharm_filename, new_file_path] From 5e0b5b21a0d5e38c55d1a353330404c106d0e043 Mon Sep 17 00:00:00 2001 From: mayantaylor Date: Mon, 17 Jun 2024 11:42:24 -0500 Subject: [PATCH 02/22] adding mac build to test suite --- .github/workflows/charm4py.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index 96657c79..3437a573 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -9,7 +9,7 @@ on: - cron: '5 0 * * *' # Runs at 00:05 UTC every day. jobs: - build: + ubuntu-latest: runs-on: ubuntu-latest steps: @@ -28,3 +28,23 @@ jobs: export PYTHONPATH="$PWD" export CHARM4PY_TEST_NUM_PROCESSES=2 python3 auto_test.py + + macos-latest: + runs-on: macos-latest + + steps: + - uses: actions/checkout@v3 + - name: build-charm4py + run: | + git fetch --unshallow # Need full repo for 'git describe' used by setup.py + pip3 install setuptools cython cffi greenlet numpy torch torchvision + git clone https://github.com/UIUC-PPL/charm charm_src/charm + export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" + export CHARM_BUILD_PROCESSES=2 + export CHARM4PY_BUILD_CFFI=1 + python3 setup.py build_ext --inplace + - name: test-charm4py + run: | + export PYTHONPATH="$PWD" + export CHARM4PY_TEST_NUM_PROCESSES=2 + python3 auto_test.py From d98b8f73bdde03d2b061377300a8aa56e482d203 Mon Sep 17 00:00:00 2001 From: mayantaylor Date: Mon, 17 Jun 2024 11:48:18 -0500 Subject: [PATCH 03/22] adding macos-12 to actions --- .github/workflows/charm4py.yml | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index 3437a573..cd6fbecc 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -29,8 +29,8 @@ jobs: export CHARM4PY_TEST_NUM_PROCESSES=2 python3 auto_test.py - macos-latest: - runs-on: macos-latest + macos-arm: + runs-on: macos-14 steps: - uses: actions/checkout@v3 @@ -48,3 +48,24 @@ jobs: export PYTHONPATH="$PWD" export CHARM4PY_TEST_NUM_PROCESSES=2 python3 auto_test.py + + macos-arm: + runs-on: macos-12 + + steps: + - uses: actions/checkout@v3 + - name: build-charm4py + run: | + git fetch --unshallow # Need full repo for 'git describe' used by setup.py + pip3 install setuptools cython cffi greenlet numpy torch torchvision + git clone https://github.com/UIUC-PPL/charm charm_src/charm + export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" + export CHARM_BUILD_PROCESSES=2 + export CHARM4PY_BUILD_CFFI=1 + python3 setup.py build_ext --inplace + - name: test-charm4py + run: | + export PYTHONPATH="$PWD" + export CHARM4PY_TEST_NUM_PROCESSES=2 + python3 auto_test.py + From 5586bcc759b7d8c3a6e033f81f9319f2930d967f Mon Sep 17 00:00:00 2001 From: mayantaylor Date: Mon, 17 Jun 2024 11:52:05 -0500 Subject: [PATCH 04/22] x86 test name --- .github/workflows/charm4py.yml | 95 +++++++++++++++++----------------- 1 file changed, 47 insertions(+), 48 deletions(-) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index cd6fbecc..929356ed 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -6,66 +6,65 @@ on: - master pull_request: schedule: - - cron: '5 0 * * *' # Runs at 00:05 UTC every day. + - cron: "5 0 * * *" # Runs at 00:05 UTC every day. jobs: ubuntu-latest: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: build-charm4py - run: | - git fetch --unshallow # Need full repo for 'git describe' used by setup.py - pip3 install setuptools cython cffi greenlet numpy torch torchvision - git clone https://github.com/UIUC-PPL/charm charm_src/charm - export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" - export CHARM_BUILD_PROCESSES=2 - export CHARM4PY_BUILD_CFFI=1 - python3 setup.py build_ext --inplace - - name: test-charm4py - run: | - export PYTHONPATH="$PWD" - export CHARM4PY_TEST_NUM_PROCESSES=2 - python3 auto_test.py + - uses: actions/checkout@v3 + - name: build-charm4py + run: | + git fetch --unshallow # Need full repo for 'git describe' used by setup.py + pip3 install setuptools cython cffi greenlet numpy torch torchvision + git clone https://github.com/UIUC-PPL/charm charm_src/charm + export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" + export CHARM_BUILD_PROCESSES=2 + export CHARM4PY_BUILD_CFFI=1 + python3 setup.py build_ext --inplace + - name: test-charm4py + run: | + export PYTHONPATH="$PWD" + export CHARM4PY_TEST_NUM_PROCESSES=2 + python3 auto_test.py macos-arm: runs-on: macos-14 steps: - - uses: actions/checkout@v3 - - name: build-charm4py - run: | - git fetch --unshallow # Need full repo for 'git describe' used by setup.py - pip3 install setuptools cython cffi greenlet numpy torch torchvision - git clone https://github.com/UIUC-PPL/charm charm_src/charm - export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" - export CHARM_BUILD_PROCESSES=2 - export CHARM4PY_BUILD_CFFI=1 - python3 setup.py build_ext --inplace - - name: test-charm4py - run: | - export PYTHONPATH="$PWD" - export CHARM4PY_TEST_NUM_PROCESSES=2 - python3 auto_test.py + - uses: actions/checkout@v3 + - name: build-charm4py + run: | + git fetch --unshallow # Need full repo for 'git describe' used by setup.py + pip3 install setuptools cython cffi greenlet numpy torch torchvision + git clone https://github.com/UIUC-PPL/charm charm_src/charm + export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" + export CHARM_BUILD_PROCESSES=2 + export CHARM4PY_BUILD_CFFI=1 + python3 setup.py build_ext --inplace + - name: test-charm4py + run: | + export PYTHONPATH="$PWD" + export CHARM4PY_TEST_NUM_PROCESSES=2 + python3 auto_test.py - macos-arm: + macos-x86: runs-on: macos-12 steps: - - uses: actions/checkout@v3 - - name: build-charm4py - run: | - git fetch --unshallow # Need full repo for 'git describe' used by setup.py - pip3 install setuptools cython cffi greenlet numpy torch torchvision - git clone https://github.com/UIUC-PPL/charm charm_src/charm - export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" - export CHARM_BUILD_PROCESSES=2 - export CHARM4PY_BUILD_CFFI=1 - python3 setup.py build_ext --inplace - - name: test-charm4py - run: | - export PYTHONPATH="$PWD" - export CHARM4PY_TEST_NUM_PROCESSES=2 - python3 auto_test.py - + - uses: actions/checkout@v3 + - name: build-charm4py + run: | + git fetch --unshallow # Need full repo for 'git describe' used by setup.py + pip3 install setuptools cython cffi greenlet numpy torch torchvision + git clone https://github.com/UIUC-PPL/charm charm_src/charm + export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" + export CHARM_BUILD_PROCESSES=2 + export CHARM4PY_BUILD_CFFI=1 + python3 setup.py build_ext --inplace + - name: test-charm4py + run: | + export PYTHONPATH="$PWD" + export CHARM4PY_TEST_NUM_PROCESSES=2 + python3 auto_test.py From c8c9971b4e78f09bc190d0f24e8b57e60d7d4136 Mon Sep 17 00:00:00 2001 From: mayantaylor Date: Mon, 17 Jun 2024 12:02:21 -0500 Subject: [PATCH 05/22] cleanup and python venv setup --- .github/workflows/charm4py.yml | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index 929356ed..c7196f19 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -29,31 +29,18 @@ jobs: export CHARM4PY_TEST_NUM_PROCESSES=2 python3 auto_test.py - macos-arm: + macos: runs-on: macos-14 + strategy: + matrix: + os: [macos-14, macos-12] steps: - uses: actions/checkout@v3 - - name: build-charm4py + - name: venv setup run: | - git fetch --unshallow # Need full repo for 'git describe' used by setup.py - pip3 install setuptools cython cffi greenlet numpy torch torchvision - git clone https://github.com/UIUC-PPL/charm charm_src/charm - export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" - export CHARM_BUILD_PROCESSES=2 - export CHARM4PY_BUILD_CFFI=1 - python3 setup.py build_ext --inplace - - name: test-charm4py - run: | - export PYTHONPATH="$PWD" - export CHARM4PY_TEST_NUM_PROCESSES=2 - python3 auto_test.py - - macos-x86: - runs-on: macos-12 - - steps: - - uses: actions/checkout@v3 + python3 -m venv .venv + source .venv/bin/activate - name: build-charm4py run: | git fetch --unshallow # Need full repo for 'git describe' used by setup.py From 99cad8f54c2e2823226a6393f6cedc090ce43005 Mon Sep 17 00:00:00 2001 From: mayantaylor Date: Mon, 17 Jun 2024 12:04:16 -0500 Subject: [PATCH 06/22] venv update --- .github/workflows/charm4py.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index c7196f19..9218e95a 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -37,12 +37,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: venv setup + - name: build-charm4py run: | python3 -m venv .venv source .venv/bin/activate - - name: build-charm4py - run: | git fetch --unshallow # Need full repo for 'git describe' used by setup.py pip3 install setuptools cython cffi greenlet numpy torch torchvision git clone https://github.com/UIUC-PPL/charm charm_src/charm From 3001ee85a4a33db331463f4c9ad7ad8b8d32fde3 Mon Sep 17 00:00:00 2001 From: mayantaylor Date: Mon, 17 Jun 2024 12:06:41 -0500 Subject: [PATCH 07/22] build matrix --- .github/workflows/charm4py.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index 9218e95a..3de20f78 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -30,7 +30,7 @@ jobs: python3 auto_test.py macos: - runs-on: macos-14 + runs-on: ${{ matrix.os }} strategy: matrix: os: [macos-14, macos-12] From d963a077ee75649576ff9724c9afc8bff20aeec3 Mon Sep 17 00:00:00 2001 From: mayantaylor Date: Mon, 17 Jun 2024 12:12:25 -0500 Subject: [PATCH 08/22] adding venv source to test block --- .github/workflows/charm4py.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index 3de20f78..6836fe4c 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -50,6 +50,7 @@ jobs: python3 setup.py build_ext --inplace - name: test-charm4py run: | + source .venv/bin/activate export PYTHONPATH="$PWD" export CHARM4PY_TEST_NUM_PROCESSES=2 python3 auto_test.py From ef1b2faeadcd27ba1cfa41bc43a3f93eb72843e1 Mon Sep 17 00:00:00 2001 From: ritvikrao Date: Mon, 17 Jun 2024 12:32:57 -0500 Subject: [PATCH 09/22] lower numpy version for testing --- .github/workflows/charm4py.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index 6836fe4c..4123bf27 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -42,7 +42,7 @@ jobs: python3 -m venv .venv source .venv/bin/activate git fetch --unshallow # Need full repo for 'git describe' used by setup.py - pip3 install setuptools cython cffi greenlet numpy torch torchvision + pip3 install setuptools cython cffi greenlet numpy<2 torch torchvision git clone https://github.com/UIUC-PPL/charm charm_src/charm export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" export CHARM_BUILD_PROCESSES=2 From e2506883cab8e8aef34f6e00cb6d958b4bb721cb Mon Sep 17 00:00:00 2001 From: ritvikrao Date: Mon, 17 Jun 2024 12:42:07 -0500 Subject: [PATCH 10/22] setup-python action --- .github/workflows/charm4py.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index 4123bf27..8f86a012 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -37,12 +37,17 @@ jobs: steps: - uses: actions/checkout@v3 + - name: setup-python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + pip3 install -r requirements.txt - name: build-charm4py run: | - python3 -m venv .venv - source .venv/bin/activate git fetch --unshallow # Need full repo for 'git describe' used by setup.py - pip3 install setuptools cython cffi greenlet numpy<2 torch torchvision git clone https://github.com/UIUC-PPL/charm charm_src/charm export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" export CHARM_BUILD_PROCESSES=2 @@ -50,7 +55,6 @@ jobs: python3 setup.py build_ext --inplace - name: test-charm4py run: | - source .venv/bin/activate export PYTHONPATH="$PWD" export CHARM4PY_TEST_NUM_PROCESSES=2 python3 auto_test.py From ae84eb2be743b419eadbbfcc94b59ed38ab392f5 Mon Sep 17 00:00:00 2001 From: ritvikrao Date: Mon, 17 Jun 2024 12:45:15 -0500 Subject: [PATCH 11/22] pip3 additions --- .github/workflows/charm4py.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index 8f86a012..debfe9c8 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -45,6 +45,7 @@ jobs: run: | python3 -m pip install --upgrade pip pip3 install -r requirements.txt + pip3 install setuptools cffi torch torchvision - name: build-charm4py run: | git fetch --unshallow # Need full repo for 'git describe' used by setup.py From 859a5bc6be42f6246e2f9b4e9e274d3764ed7c85 Mon Sep 17 00:00:00 2001 From: ritvikrao Date: Mon, 17 Jun 2024 13:02:01 -0500 Subject: [PATCH 12/22] numpy version for x86 build --- .github/workflows/charm4py.yml | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index debfe9c8..f6356554 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -30,10 +30,7 @@ jobs: python3 auto_test.py macos: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [macos-14, macos-12] + runs-on: macos-14 steps: - uses: actions/checkout@v3 @@ -59,3 +56,32 @@ jobs: export PYTHONPATH="$PWD" export CHARM4PY_TEST_NUM_PROCESSES=2 python3 auto_test.py + + macos-x86: + runs-on: macos-12 + + steps: + - uses: actions/checkout@v3 + - name: setup-python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + pip3 install setuptools cython cffi greenlet numpy=1.26.4 torch torchvision + - name: build-charm4py + run: | + git fetch --unshallow # Need full repo for 'git describe' used by setup.py + git clone https://github.com/UIUC-PPL/charm charm_src/charm + export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" + export CHARM_BUILD_PROCESSES=2 + export CHARM4PY_BUILD_CFFI=1 + python3 setup.py build_ext --inplace + - name: test-charm4py + run: | + export PYTHONPATH="$PWD" + export CHARM4PY_TEST_NUM_PROCESSES=2 + python3 auto_test.py + + From fdf49e48ddc0a00be4dde2818df6483384ea7ef4 Mon Sep 17 00:00:00 2001 From: ritvikrao Date: Mon, 17 Jun 2024 13:04:14 -0500 Subject: [PATCH 13/22] typo --- .github/workflows/charm4py.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index f6356554..f6454ae6 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -69,7 +69,7 @@ jobs: - name: Install dependencies run: | python3 -m pip install --upgrade pip - pip3 install setuptools cython cffi greenlet numpy=1.26.4 torch torchvision + pip3 install setuptools cython cffi greenlet numpy==1.26.4 torch torchvision - name: build-charm4py run: | git fetch --unshallow # Need full repo for 'git describe' used by setup.py From ee1a55a2cf827dd1014bfda402e82a6c68919960 Mon Sep 17 00:00:00 2001 From: ritvikrao Date: Mon, 17 Jun 2024 13:52:19 -0500 Subject: [PATCH 14/22] python 3.11 for x86 --- .github/workflows/charm4py.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index f6454ae6..a463e01c 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -65,11 +65,11 @@ jobs: - name: setup-python uses: actions/setup-python@v5 with: - python-version: '3.x' + python-version: '3.11' - name: Install dependencies run: | python3 -m pip install --upgrade pip - pip3 install setuptools cython cffi greenlet numpy==1.26.4 torch torchvision + pip3 install setuptools cython cffi greenlet numpy torch torchvision - name: build-charm4py run: | git fetch --unshallow # Need full repo for 'git describe' used by setup.py From 682f4ce633256be7249c80bcf5d77f69d052beba Mon Sep 17 00:00:00 2001 From: ritvikrao Date: Mon, 17 Jun 2024 14:15:40 -0500 Subject: [PATCH 15/22] numpy version reduction --- .github/workflows/charm4py.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index a463e01c..3c2fd56b 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -69,7 +69,7 @@ jobs: - name: Install dependencies run: | python3 -m pip install --upgrade pip - pip3 install setuptools cython cffi greenlet numpy torch torchvision + pip3 install setuptools cython cffi greenlet numpy==1.23.2 torch torchvision - name: build-charm4py run: | git fetch --unshallow # Need full repo for 'git describe' used by setup.py From 49da52b33d425c82e24fba741cdaf11caecaec09 Mon Sep 17 00:00:00 2001 From: ritvikrao Date: Mon, 17 Jun 2024 14:35:09 -0500 Subject: [PATCH 16/22] disable fork safety for test on x86 --- .github/workflows/charm4py.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index 3c2fd56b..bc66f5a1 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -82,6 +82,7 @@ jobs: run: | export PYTHONPATH="$PWD" export CHARM4PY_TEST_NUM_PROCESSES=2 + export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES python3 auto_test.py From cfd1cab227e17f0e55090ddf0aaedf1e7e788662 Mon Sep 17 00:00:00 2001 From: Matthias Diener Date: Mon, 17 Jun 2024 15:43:10 -0500 Subject: [PATCH 17/22] set python_requires just a guess, but the code uses f-strings, which were introduced in 3.6 --- setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 70eb8874..afcdc8ab 100644 --- a/setup.py +++ b/setup.py @@ -197,7 +197,7 @@ def build_libcharm(charm_src_dir, build_dir): for output_dir in lib_output_dirs: log.info('copying ' + os.path.relpath(lib_src_path) + ' to ' + os.path.relpath(output_dir)) shutil.copy(lib_src_path, output_dir) - + # ---- copy charmrun ---- charmrun_src_path = os.path.join(charm_src_dir, 'charm', 'bin', charmrun_filename) @@ -273,7 +273,7 @@ def run(self): class _renameInstalled(_install_lib): def __init__(self, *args, **kwargs): _install_lib.__init__(self, *args, **kwargs) - + def install(self): log.info("Renaming libraries") outfiles = _install_lib.install(self) @@ -329,7 +329,7 @@ def install(self): extra_link_args=["-Wl,-rpath,@loader_path/../.libs"] else: extra_link_args=["-Wl,-rpath,$ORIGIN/../.libs"] - + cobject_extra_args = [] log.info("Extra object args for object store") if os.name != 'nt': @@ -346,7 +346,7 @@ def install(self): extra_compile_args=['-g0', '-O3'], extra_link_args=extra_link_args, ), compile_time_env={'HAVE_NUMPY': haveNumpy})) - + extensions.extend(cythonize(setuptools.Extension('charm4py.c_object_store', sources=['charm4py/c_object_store.pyx'], include_dirs=['charm_src/charm/include'] + my_include_dirs, @@ -392,7 +392,7 @@ def install(self): ], }, install_requires=['numpy>=1.10.0', 'greenlet>=3.0.0', 'cython>=3.0.0', 'cmake'], - #python_requires='>=2.7, ~=3.4', + python_requires='~=3.6', classifiers=[ 'Intended Audience :: Developers', 'License :: Free for non-commercial use', From 10afc1f0ba21acd06bca12e252a0b35c3d6e13c4 Mon Sep 17 00:00:00 2001 From: Matthias Diener Date: Mon, 17 Jun 2024 15:50:01 -0500 Subject: [PATCH 18/22] update URLs --- .github/workflows/charm4py.yml | 10 +++++----- README.rst | 8 ++++---- docs/contact.rst | 4 ++-- docs/examples.rst | 2 +- setup.py | 7 ++----- 5 files changed, 14 insertions(+), 17 deletions(-) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index bc66f5a1..d3ba17a7 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -18,7 +18,7 @@ jobs: run: | git fetch --unshallow # Need full repo for 'git describe' used by setup.py pip3 install setuptools cython cffi greenlet numpy torch torchvision - git clone https://github.com/UIUC-PPL/charm charm_src/charm + git clone https://github.com/charmplusplus/charm charm_src/charm export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" export CHARM_BUILD_PROCESSES=2 export CHARM4PY_BUILD_CFFI=1 @@ -46,7 +46,7 @@ jobs: - name: build-charm4py run: | git fetch --unshallow # Need full repo for 'git describe' used by setup.py - git clone https://github.com/UIUC-PPL/charm charm_src/charm + git clone https://github.com/charmplusplus/charm charm_src/charm export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" export CHARM_BUILD_PROCESSES=2 export CHARM4PY_BUILD_CFFI=1 @@ -73,7 +73,7 @@ jobs: - name: build-charm4py run: | git fetch --unshallow # Need full repo for 'git describe' used by setup.py - git clone https://github.com/UIUC-PPL/charm charm_src/charm + git clone https://github.com/charmplusplus/charm charm_src/charm export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" export CHARM_BUILD_PROCESSES=2 export CHARM4PY_BUILD_CFFI=1 @@ -84,5 +84,5 @@ jobs: export CHARM4PY_TEST_NUM_PROCESSES=2 export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES python3 auto_test.py - - + + diff --git a/README.rst b/README.rst index 028f2238..7c6ff895 100644 --- a/README.rst +++ b/README.rst @@ -3,8 +3,8 @@ Charm4py ======== -.. image:: https://github.com/UIUC-PPL/charm4py/actions/workflows/charm4py.yml/badge.svg?event=push - :target: https://github.com/UIUC-PPL/charm4py/actions/workflows/charm4py.yml +.. image:: https://github.com/charmplusplus/charm4py/actions/workflows/charm4py.yml/badge.svg?event=push + :target: https://github.com/charmplusplus/charm4py/actions/workflows/charm4py.yml .. image:: http://readthedocs.org/projects/charm4py/badge/?version=latest :target: https://charm4py.readthedocs.io/ @@ -83,8 +83,8 @@ or email us at . Main author at -.. _Charm++: https://github.com/UIUC-PPL/charm +.. _Charm++: https://github.com/charmplusplus/charm .. _Documentation: https://charm4py.readthedocs.io -.. _discussion page: https://github.com/UIUC-PPL/charm4py/discussions +.. _discussion page: https://github.com/charmplusplus/charm4py/discussions diff --git a/docs/contact.rst b/docs/contact.rst index 5ddfc3ce..804fe546 100644 --- a/docs/contact.rst +++ b/docs/contact.rst @@ -5,7 +5,7 @@ Contact You can contact us on the `discussion page`_ for discussion of any topics related to Charm4py or Charm++, or in our mailing list at -.. _discussion page: https://github.com/UIUC-PPL/charm4py/discussions +.. _discussion page: https://github.com/charmplusplus/charm4py/discussions -Please use the GitHub page to report and track issues (https://github.com/UIUC-PPL/charm4py) +Please use the GitHub page to report and track issues (https://github.com/charmplusplus/charm4py) diff --git a/docs/examples.rst b/docs/examples.rst index 881d47d1..308d56a9 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -3,7 +3,7 @@ Examples ======== There are several examples in the source code repository, with documentation -and comments: https://github.com/UIUC-PPL/charm4py/tree/master/examples +and comments: https://github.com/charmplusplus/charm4py/tree/master/examples These include: diff --git a/setup.py b/setup.py index afcdc8ab..2d9df824 100644 --- a/setup.py +++ b/setup.py @@ -380,7 +380,7 @@ def install(self): author_email='jjgalvez@illinois.edu', description='Charm4py Parallel Programming Framework', long_description=long_description, - url='https://github.com/UIUC-PPL/charm4py', + url='https://github.com/charmplusplus/charm4py', keywords='parallel parallel-programming distributed distributed-computing hpc HPC runtime', packages=setuptools.find_packages(), package_data={ @@ -401,10 +401,7 @@ def install(self): 'Operating System :: POSIX :: Linux', 'Operating System :: Microsoft :: Windows', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3', 'Topic :: System :: Distributed Computing', 'Topic :: System :: Clustering', ], From e20a469039cd33d366c35b94852b250c0e05c883 Mon Sep 17 00:00:00 2001 From: Matthias Diener Date: Mon, 17 Jun 2024 15:58:03 -0500 Subject: [PATCH 19/22] restructure as matrix --- .github/workflows/charm4py.yml | 106 +++++++++++++++++---------------- 1 file changed, 55 insertions(+), 51 deletions(-) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index d3ba17a7..5f910525 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -9,43 +9,47 @@ on: - cron: "5 0 * * *" # Runs at 00:05 UTC every day. jobs: - ubuntu-latest: - runs-on: ubuntu-latest + # ubuntu-latest: + # runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: build-charm4py - run: | - git fetch --unshallow # Need full repo for 'git describe' used by setup.py - pip3 install setuptools cython cffi greenlet numpy torch torchvision - git clone https://github.com/charmplusplus/charm charm_src/charm - export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" - export CHARM_BUILD_PROCESSES=2 - export CHARM4PY_BUILD_CFFI=1 - python3 setup.py build_ext --inplace - - name: test-charm4py - run: | - export PYTHONPATH="$PWD" - export CHARM4PY_TEST_NUM_PROCESSES=2 - python3 auto_test.py + # steps: + # - uses: actions/checkout@v4 + # - name: build-charm4py + # run: | + # git fetch --unshallow # Need full repo for 'git describe' used by setup.py + # pip3 install setuptools cython cffi greenlet numpy torch torchvision + # git clone https://github.com/charmplusplus/charm charm_src/charm + # export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" + # export CHARM_BUILD_PROCESSES=2 + # export CHARM4PY_BUILD_CFFI=1 + # python3 setup.py build_ext --inplace + # - name: test-charm4py + # run: | + # export PYTHONPATH="$PWD" + # export CHARM4PY_TEST_NUM_PROCESSES=2 + # python3 auto_test.py - macos: - runs-on: macos-14 + auto_test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + python-version: ["3.7", "3.12", "3.13-dev"] + os: [ubuntu-latest, macos-13, macos-14] + fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Need full repo for 'git describe' used by setup.py - name: setup-python uses: actions/setup-python@v5 with: - python-version: '3.x' + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python3 -m pip install --upgrade pip - pip3 install -r requirements.txt pip3 install setuptools cffi torch torchvision - name: build-charm4py run: | - git fetch --unshallow # Need full repo for 'git describe' used by setup.py git clone https://github.com/charmplusplus/charm charm_src/charm export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" export CHARM_BUILD_PROCESSES=2 @@ -57,32 +61,32 @@ jobs: export CHARM4PY_TEST_NUM_PROCESSES=2 python3 auto_test.py - macos-x86: - runs-on: macos-12 + # macos-x86: + # runs-on: macos-12 - steps: - - uses: actions/checkout@v3 - - name: setup-python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Install dependencies - run: | - python3 -m pip install --upgrade pip - pip3 install setuptools cython cffi greenlet numpy==1.23.2 torch torchvision - - name: build-charm4py - run: | - git fetch --unshallow # Need full repo for 'git describe' used by setup.py - git clone https://github.com/charmplusplus/charm charm_src/charm - export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" - export CHARM_BUILD_PROCESSES=2 - export CHARM4PY_BUILD_CFFI=1 - python3 setup.py build_ext --inplace - - name: test-charm4py - run: | - export PYTHONPATH="$PWD" - export CHARM4PY_TEST_NUM_PROCESSES=2 - export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES - python3 auto_test.py + # steps: + # - uses: actions/checkout@v3 + # - name: setup-python + # uses: actions/setup-python@v5 + # with: + # python-version: '3.11' + # - name: Install dependencies + # run: | + # python3 -m pip install --upgrade pip + # pip3 install setuptools cython cffi greenlet numpy==1.23.2 torch torchvision + # - name: build-charm4py + # run: | + # git fetch --unshallow # Need full repo for 'git describe' used by setup.py + # git clone https://github.com/charmplusplus/charm charm_src/charm + # export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" + # export CHARM_BUILD_PROCESSES=2 + # export CHARM4PY_BUILD_CFFI=1 + # python3 setup.py build_ext --inplace + # - name: test-charm4py + # run: | + # export PYTHONPATH="$PWD" + # export CHARM4PY_TEST_NUM_PROCESSES=2 + # export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES + # python3 auto_test.py From b0f49da34bfbdb86155f3f9fb91358e39293bcbd Mon Sep 17 00:00:00 2001 From: Matthias Diener Date: Mon, 17 Jun 2024 16:03:12 -0500 Subject: [PATCH 20/22] install cython --- .github/workflows/charm4py.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index 5f910525..b4c0535d 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -47,19 +47,19 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - pip3 install setuptools cffi torch torchvision + pip install setuptools cython cffi greenlet numpy torch torchvision - name: build-charm4py run: | git clone https://github.com/charmplusplus/charm charm_src/charm export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" export CHARM_BUILD_PROCESSES=2 export CHARM4PY_BUILD_CFFI=1 - python3 setup.py build_ext --inplace + python setup.py build_ext --inplace - name: test-charm4py run: | export PYTHONPATH="$PWD" export CHARM4PY_TEST_NUM_PROCESSES=2 - python3 auto_test.py + python auto_test.py # macos-x86: # runs-on: macos-12 From de4db71e4ba1bbc190332349ee01f121962c6d88 Mon Sep 17 00:00:00 2001 From: Matthias Diener Date: Mon, 17 Jun 2024 16:24:09 -0500 Subject: [PATCH 21/22] disable configs that can not work, install filelock package for ray test --- .github/workflows/charm4py.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index b4c0535d..c6e528dc 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -33,8 +33,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.7", "3.12", "3.13-dev"] + python-version: ["3.7", "3.12"] + # macos-13 is x86_64, macos-14 is arm64 os: [ubuntu-latest, macos-13, macos-14] + exclude: + - os: macos-14 # Python 3.7 does not exist for macos-14 + python-version: "3.7" fail-fast: false steps: @@ -47,15 +51,15 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - pip install setuptools cython cffi greenlet numpy torch torchvision - - name: build-charm4py + pip install setuptools cython cffi greenlet numpy torch torchvision filelock + - name: Build charm++/charm4py run: | git clone https://github.com/charmplusplus/charm charm_src/charm export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" export CHARM_BUILD_PROCESSES=2 export CHARM4PY_BUILD_CFFI=1 python setup.py build_ext --inplace - - name: test-charm4py + - name: Run auto_test.py run: | export PYTHONPATH="$PWD" export CHARM4PY_TEST_NUM_PROCESSES=2 From a9e994bba8741e4422315085c38a7566d99065b4 Mon Sep 17 00:00:00 2001 From: Matthias Diener Date: Mon, 17 Jun 2024 16:47:25 -0500 Subject: [PATCH 22/22] macos-13 build fixes --- .github/workflows/charm4py.yml | 60 +++++----------------------------- 1 file changed, 9 insertions(+), 51 deletions(-) diff --git a/.github/workflows/charm4py.yml b/.github/workflows/charm4py.yml index c6e528dc..763d2a14 100644 --- a/.github/workflows/charm4py.yml +++ b/.github/workflows/charm4py.yml @@ -9,26 +9,6 @@ on: - cron: "5 0 * * *" # Runs at 00:05 UTC every day. jobs: - # ubuntu-latest: - # runs-on: ubuntu-latest - - # steps: - # - uses: actions/checkout@v4 - # - name: build-charm4py - # run: | - # git fetch --unshallow # Need full repo for 'git describe' used by setup.py - # pip3 install setuptools cython cffi greenlet numpy torch torchvision - # git clone https://github.com/charmplusplus/charm charm_src/charm - # export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" - # export CHARM_BUILD_PROCESSES=2 - # export CHARM4PY_BUILD_CFFI=1 - # python3 setup.py build_ext --inplace - # - name: test-charm4py - # run: | - # export PYTHONPATH="$PWD" - # export CHARM4PY_TEST_NUM_PROCESSES=2 - # python3 auto_test.py - auto_test: runs-on: ${{ matrix.os }} strategy: @@ -52,7 +32,11 @@ jobs: - name: Install dependencies run: | pip install setuptools cython cffi greenlet numpy torch torchvision filelock - - name: Build charm++/charm4py + if [ ${{ matrix.os }} == 'macos-13' ]; then + # pypi only distributes torch packages w/ numpy v1 for macos-x86_64 + pip install 'numpy<2' + fi + - name: Build Charm++/Charm4py run: | git clone https://github.com/charmplusplus/charm charm_src/charm export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" @@ -63,34 +47,8 @@ jobs: run: | export PYTHONPATH="$PWD" export CHARM4PY_TEST_NUM_PROCESSES=2 + if [ ${{ matrix.os }} == 'macos-13' ]; then + # Crashes in ray test otherwise + export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES + fi python auto_test.py - - # macos-x86: - # runs-on: macos-12 - - # steps: - # - uses: actions/checkout@v3 - # - name: setup-python - # uses: actions/setup-python@v5 - # with: - # python-version: '3.11' - # - name: Install dependencies - # run: | - # python3 -m pip install --upgrade pip - # pip3 install setuptools cython cffi greenlet numpy==1.23.2 torch torchvision - # - name: build-charm4py - # run: | - # git fetch --unshallow # Need full repo for 'git describe' used by setup.py - # git clone https://github.com/charmplusplus/charm charm_src/charm - # export CHARM_EXTRA_BUILD_OPTS="--enable-error-checking" - # export CHARM_BUILD_PROCESSES=2 - # export CHARM4PY_BUILD_CFFI=1 - # python3 setup.py build_ext --inplace - # - name: test-charm4py - # run: | - # export PYTHONPATH="$PWD" - # export CHARM4PY_TEST_NUM_PROCESSES=2 - # export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES - # python3 auto_test.py - -