Merge pull request #915 from PowerGridModel/feature/conda-build-cmake

Use cmake to build the DLL for conda package

Author: TonyXiang8787

.github/conda_pgm_env.yml

@@ -15,6 +15,8 @@ dependencies:
   - nlohmann_json
   - msgpack-cxx
   - numpy
+  - cmake
+  - ninja
   # test deps
   - pytest
   - pytest-cov

.github/workflows/main.yml

@@ -235,10 +235,18 @@ jobs:
     strategy:
       matrix:
         os: ["ubuntu", "windows"] # We do not test conda for MacOS
-  
+        include:
+          - os: ubuntu
+            shell: bash -el {0}
+          - os: windows
+            shell: powershell
+    
+    env:
+      POWER_GRID_MODEL_NO_BINARY_BUILD: 1
+
     defaults:
       run:
-        shell: bash -el {0}
+        shell: ${{ matrix.shell }}
     steps:
       - uses: actions/checkout@v4
 
@@ -253,8 +261,26 @@ jobs:
         run: |
           conda info
           conda list
-      
-      - name: Build
+
+      - name: Build and install cmake target for Windows
+        if: matrix.os == 'windows'
+        run: |          
+          $vsPath = &(Join-Path ${env:ProgramFiles(x86)} '\Microsoft Visual Studio\Installer\vswhere.exe') -property installationpath
+          Import-Module (Join-Path $vsPath 'Common7\Tools\Microsoft.VisualStudio.DevShell.dll')
+          Enter-VsDevShell -VsInstallPath $vsPath -SkipAutomaticLocation -DevCmdArguments '-arch=x64 -host_arch=x64'
+
+          cmake -GNinja -DCMAKE_BUILD_TYPE=Release -B build/ -S .
+          cmake --build build/ --verbose -j1
+          cmake --install build/ --prefix ${env:CONDA_PREFIX}/Library
+
+      - name: Build and install cmake target for Linux
+        if: matrix.os == 'ubuntu'
+        run: |
+          cmake -GNinja -DCMAKE_BUILD_TYPE=Release -B build/ -S . -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DCMAKE_PREFIX_PATH=$CONDA_PREFIX
+          cmake --build build/ --verbose -j1
+          cmake --install build/      
+
+      - name: Build python
         run: python -m pip install . -vv --no-build-isolation --no-deps   
 
       - name: Test

setup.py

@@ -55,28 +55,6 @@ def get_pre_installed_header_include() -> list[str]:
         return []
 
 
-def get_conda_include() -> list[str]:
-    """
-    Get conda include path, if we are inside conda environment
-
-    Returns:
-        either empty list or a list of header paths
-    """
-    include_paths = []
-    # in the conda build system the system root is defined in CONDA_PREFIX or BUILD_PREFIX
-    for prefix in ["CONDA_PREFIX", "BUILD_PREFIX"]:
-        if prefix in os.environ:
-            conda_path = os.environ[prefix]
-            if if_win:
-                # windows has Library folder prefix
-                include_paths.append(os.path.join(conda_path, "Library", "include"))
-                include_paths.append(os.path.join(conda_path, "Library", "include", "eigen3"))
-            else:
-                include_paths.append(os.path.join(conda_path, "include"))
-                include_paths.append(os.path.join(conda_path, "include", "eigen3"))
-    return include_paths
-
-
 # custom class for ctypes
 class CTypesExtension(Extension):
     pass
@@ -150,6 +128,23 @@ def generate_build_ext(pkg_dir: Path, pkg_name: str):
     Returns:
 
     """
+    pkg_bin_dir = pkg_dir / "src" / pkg_name
+    # remove old extension build
+    build_dir = pkg_dir / "build"
+    if build_dir.exists():
+        shutil.rmtree(build_dir)
+    # remove binary
+    bin_files = list(chain(pkg_bin_dir.rglob("*.so"), pkg_bin_dir.rglob("*.dll"), pkg_bin_dir.rglob("*.dylib")))
+    for bin_file in bin_files:
+        print(f"Remove binary file: {bin_file}")
+        bin_file.unlink()
+
+    # By setting POWER_GRID_MODEL_NO_BINARY_BUILD we do not build the extension.
+    # This is usually set in conda-build recipe, so conda build process only wraps the pure Python package.
+    # As a user or developer, DO NOT set this environment variable unless you really know what you are doing.
+    if "POWER_GRID_MODEL_NO_BINARY_BUILD" in os.environ:
+        return {}
+
     # fetch dependent headers
     pgm = Path("power_grid_model")
     pgm_c = Path("power_grid_model_c")
@@ -161,7 +156,6 @@ def generate_build_ext(pkg_dir: Path, pkg_name: str):
     ]
     include_dirs += get_required_dependency_include()
     include_dirs += get_pre_installed_header_include()
-    include_dirs += get_conda_include()
     # compiler and link flag
     cflags: list[str] = []
     lflags: list[str] = []
@@ -179,17 +173,6 @@ def generate_build_ext(pkg_dir: Path, pkg_name: str):
     define_macros = [
         ("EIGEN_MPL2_ONLY", "1"),  # only MPL-2 part of eigen3
     ]
-    pkg_bin_dir = pkg_dir / "src" / pkg_name
-
-    # remove old extension build
-    build_dir = pkg_dir / "build"
-    if build_dir.exists():
-        shutil.rmtree(build_dir)
-    # remove binary
-    bin_files = list(chain(pkg_bin_dir.rglob("*.so"), pkg_bin_dir.rglob("*.dll")))
-    for bin_file in bin_files:
-        print(f"Remove binary file: {bin_file}")
-        bin_file.unlink()
 
     # build steps for Windows and Linux
     # different treat for windows and linux

src/power_grid_model/_core/power_grid_core.py

@@ -6,6 +6,7 @@
 Loader for the dynamic library
 """
 
+import os
 import platform
 from ctypes import CDLL, POINTER, c_char, c_char_p, c_double, c_size_t, c_void_p
 from inspect import signature
@@ -125,11 +126,27 @@ def _load_core() -> CDLL:
     Returns: DLL/SO object
 
     """
+    # first try to find the DLL local
     if platform.system() == "Windows":
         dll_file = "_power_grid_core.dll"
     else:
         dll_file = "_power_grid_core.so"
-    cdll = CDLL(str(Path(__file__).parent / dll_file))
+    dll_path = Path(__file__).parent / dll_file
+
+    # if local DLL is not found, try to find the DLL from conda environment
+    if (not dll_path.exists()) and ("CONDA_PREFIX" in os.environ):
+        if platform.system() == "Windows":
+            dll_file = "power_grid_model_c.dll"
+        elif platform.system() == "Darwin":
+            dll_file = "libpower_grid_model_c.dylib"
+        elif platform.system() == "Linux":
+            dll_file = "libpower_grid_model_c.so"
+        else:
+            raise NotImplementedError(f"Unsupported platform: {platform.system()}")
+        # the dll will be found through conda environment
+        dll_path = Path(dll_file)
+
+    cdll = CDLL(str(dll_path))
     # assign return types
     # handle
     cdll.PGM_create_handle.argtypes = []