Merge branch 'main' into nerfstudio

Author: jeongseok-meta

.ci_support/build_all.py

@@ -26,8 +26,10 @@
 EXAMPLE_RECIPE_FOLDERS = ["example", "example-v1"]
 LOCAL_CHANNELS = os.environ.get("CONDA_BLD_PATH", "local").split(",")
 
+
 def get_host_platform():
     from sys import platform
+
     if platform == "linux" or platform == "linux2":
         return "linux"
     elif platform == "darwin":
@@ -54,7 +56,9 @@ def build_all(recipes_dir, arch):
 
     platform = get_host_platform()
     script_dir = os.path.dirname(os.path.realpath(__file__))
-    variant_config_file = os.path.join(script_dir, "{}.yaml".format(get_config_name(arch)))
+    variant_config_file = os.path.join(
+        script_dir, "{}.yaml".format(get_config_name(arch))
+    )
 
     has_meta_yaml = False
     has_recipe_yaml = False
@@ -65,11 +69,11 @@ def build_all(recipes_dir, arch):
         meta_yaml = os.path.join(recipes_dir, folder, "meta.yaml")
         if os.path.exists(meta_yaml):
             has_meta_yaml = True
-            with(open(meta_yaml, "r", encoding="utf-8")) as f:
-                text = ''.join(f.readlines())
-                if 'cuda' in text:
+            with open(meta_yaml, "r", encoding="utf-8") as f:
+                text = "".join(f.readlines())
+                if "cuda" in text:
                     found_cuda = True
-                if 'sysroot_linux-64' in text:
+                if "sysroot_linux-64" in text:
                     found_centos7 = True
 
         recipe_yaml = os.path.join(recipes_dir, folder, "recipe.yaml")
@@ -86,16 +90,18 @@ def build_all(recipes_dir, arch):
         if os.path.exists(cbc):
             with open(cbc, "r") as f:
                 lines = f.readlines()
-            pat = re.compile(r"^([^\#]*?)\s+\#\s\[.*(not\s(linux|unix)|(?<!not\s)(osx|win)).*\]\s*$")
+            pat = re.compile(
+                r"^([^\#]*?)\s+\#\s\[.*(not\s(linux|unix)|(?<!not\s)(osx|win)).*\]\s*$"
+            )
             # remove lines with selectors that don't apply to linux, i.e. if they contain
             # "not linux", "not unix", "osx" or "win"; this also removes trailing newlines
             lines = [pat.sub("", x) for x in lines]
             text = "\n".join(lines)
-            if platform == 'linux' and ('c_stdlib_version' in text):
+            if platform == "linux" and ("c_stdlib_version" in text):
                 config = load(text, Loader=BaseLoader)
-                if 'c_stdlib_version' in config:
-                    for version in config['c_stdlib_version']:
-                        version = tuple([int(x) for x in version.split('.')])
+                if "c_stdlib_version" in config:
+                    for version in config["c_stdlib_version"]:
+                        version = tuple([int(x) for x in version.split(".")])
                         print(f"Found c_stdlib_version for linux: {version=}")
                         found_centos7 |= version == (2, 17)
 
@@ -105,7 +111,7 @@ def build_all(recipes_dir, arch):
         raise ValueError("Neither a meta.yaml or a recipe.yaml recipes was found")
 
     if found_cuda:
-        print('##vso[task.setvariable variable=NEED_CUDA;isOutput=true]1')
+        print("##vso[task.setvariable variable=NEED_CUDA;isOutput=true]1")
     if found_centos7:
         os.environ["DEFAULT_LINUX_VERSION"] = "cos7"
         print("Overriding DEFAULT_LINUX_VERSION to be cos7")
@@ -118,70 +124,78 @@ def build_all(recipes_dir, arch):
         if os.path.exists(cbc):
             with open(cbc, "r") as f:
                 lines = f.readlines()
-            pat = re.compile(r"^([^\#]*?)\s+\#\s\[.*(not\s(osx|unix)|(?<!not\s)(linux|win)).*\]\s*$")
+            pat = re.compile(
+                r"^([^\#]*?)\s+\#\s\[.*(not\s(osx|unix)|(?<!not\s)(linux|win)).*\]\s*$"
+            )
             # remove lines with selectors that don't apply to osx, i.e. if they contain
             # "not osx", "not unix", "linux" or "win"; this also removes trailing newlines
             lines = [pat.sub("", x) for x in lines]
             text = "\n".join(lines)
-            if platform == 'osx' and (
-                    'MACOSX_DEPLOYMENT_TARGET' in text or
-                    'MACOSX_SDK_VERSION' in text or
-                    'c_stdlib_version' in text):
+            if platform == "osx" and (
+                "MACOSX_DEPLOYMENT_TARGET" in text
+                or "MACOSX_SDK_VERSION" in text
+                or "c_stdlib_version" in text
+            ):
                 config = load(text, Loader=BaseLoader)
 
-                if 'MACOSX_DEPLOYMENT_TARGET' in config:
-                    for version in config['MACOSX_DEPLOYMENT_TARGET']:
-                        version = tuple([int(x) for x in version.split('.')])
+                if "MACOSX_DEPLOYMENT_TARGET" in config:
+                    for version in config["MACOSX_DEPLOYMENT_TARGET"]:
+                        version = tuple([int(x) for x in version.split(".")])
                         deployment_version = max(deployment_version, version)
-                if 'c_stdlib_version' in config:
-                    for version in config['c_stdlib_version']:
-                        version = tuple([int(x) for x in version.split('.')])
+                if "c_stdlib_version" in config:
+                    for version in config["c_stdlib_version"]:
+                        version = tuple([int(x) for x in version.split(".")])
                         print(f"Found c_stdlib_version for osx: {version=}")
                         deployment_version = max(deployment_version, version)
-                if 'MACOSX_SDK_VERSION' in config:
-                    for version in config['MACOSX_SDK_VERSION']:
-                        version = tuple([int(x) for x in version.split('.')])
+                if "MACOSX_SDK_VERSION" in config:
+                    for version in config["MACOSX_SDK_VERSION"]:
+                        version = tuple([int(x) for x in version.split(".")])
                         sdk_version = max(sdk_version, deployment_version, version)
 
-            if 'channel_sources' not in text:
-                new_channel_urls = [*LOCAL_CHANNELS, 'conda-forge']
+            if "channel_sources" not in text:
+                new_channel_urls = [*LOCAL_CHANNELS, "conda-forge"]
             else:
                 config = load(text, Loader=BaseLoader)
-                new_channel_urls = [*LOCAL_CHANNELS, *config['channel_sources'][0].split(',')]
+                new_channel_urls = [
+                    *LOCAL_CHANNELS,
+                    *config["channel_sources"][0].split(","),
+                ]
             if channel_urls is None:
                 channel_urls = new_channel_urls
             elif channel_urls != new_channel_urls:
-                raise ValueError(f'Detected different channel_sources in the recipes: {channel_urls} vs. {new_channel_urls}. Consider submitting them in separate PRs')
+                raise ValueError(
+                    f"Detected different channel_sources in the recipes: {channel_urls} vs. {new_channel_urls}. Consider submitting them in separate PRs"
+                )
 
     if channel_urls is None:
-        channel_urls = [*LOCAL_CHANNELS, 'conda-forge']
+        channel_urls = [*LOCAL_CHANNELS, "conda-forge"]
 
-    with open(variant_config_file, 'r') as f:
-        variant_text = ''.join(f.readlines())
+    with open(variant_config_file, "r") as f:
+        variant_text = "".join(f.readlines())
 
     if deployment_version != (0, 0):
-        deployment_version = '.'.join([str(x) for x in deployment_version])
+        deployment_version = ".".join([str(x) for x in deployment_version])
         print("Overriding MACOSX_DEPLOYMENT_TARGET to be ", deployment_version)
-        variant_text += '\nMACOSX_DEPLOYMENT_TARGET:\n'
+        variant_text += "\nMACOSX_DEPLOYMENT_TARGET:\n"
         variant_text += f'- "{deployment_version}"\n'
 
     if sdk_version != (0, 0):
-        sdk_version = '.'.join([str(x) for x in sdk_version])
+        sdk_version = ".".join([str(x) for x in sdk_version])
         print("Overriding MACOSX_SDK_VERSION to be ", sdk_version)
-        variant_text += '\nMACOSX_SDK_VERSION:\n'
+        variant_text += "\nMACOSX_SDK_VERSION:\n"
         variant_text += f'- "{sdk_version}"\n'
 
-    with open(variant_config_file, 'w') as f:
+    with open(variant_config_file, "w") as f:
         f.write(variant_text)
 
     if platform == "osx" and (sdk_version != (0, 0) or deployment_version != (0, 0)):
         subprocess.run("run_conda_forge_build_setup", shell=True, check=True)
 
-    if 'conda-forge' not in channel_urls:
-        raise ValueError('conda-forge needs to be part of channel_sources')
-    
+    if "conda-forge" not in channel_urls:
+        raise ValueError("conda-forge needs to be part of channel_sources")
+
     if has_meta_yaml:
-        print("Building {} with {}".format(','.join(folders), ','.join(channel_urls)))
+        print("Building {} with {}".format(",".join(folders), ",".join(channel_urls)))
         build_folders(recipes_dir, folders, arch, channel_urls)
     elif has_recipe_yaml:
         print(
@@ -193,28 +207,31 @@ def build_all(recipes_dir, arch):
 
 
 def get_config(arch, channel_urls):
-    exclusive_config_files = [os.path.join(conda.base.context.context.root_prefix,
-                                           'conda_build_config.yaml')]
+    exclusive_config_files = [
+        os.path.join(conda.base.context.context.root_prefix, "conda_build_config.yaml")
+    ]
     script_dir = os.path.dirname(os.path.realpath(__file__))
     # since variant builds override recipe/conda_build_config.yaml, see
     # https://github.com/conda/conda-build/blob/3.21.8/conda_build/variants.py#L175-L181
     # we need to make sure not to use variant_configs here, otherwise
     # staged-recipes PRs cannot override anything using the recipe-cbc.
-    exclusive_config_file = os.path.join(script_dir, '{}.yaml'.format(
-        get_config_name(arch)))
+    exclusive_config_file = os.path.join(
+        script_dir, "{}.yaml".format(get_config_name(arch))
+    )
     if os.path.exists(exclusive_config_file):
         exclusive_config_files.append(exclusive_config_file)
 
     config = conda_build.api.Config(
-        arch=arch, exclusive_config_files=exclusive_config_files,
-        channel_urls=channel_urls, error_overlinking=True,
+        arch=arch,
+        exclusive_config_files=exclusive_config_files,
+        channel_urls=channel_urls,
+        error_overlinking=True,
     )
     return config
 
 
 def build_folders(recipes_dir, folders, arch, channel_urls):
-
-    index_path = os.path.join(sys.exec_prefix, 'conda-bld')
+    index_path = os.path.join(sys.exec_prefix, "conda-bld")
     os.makedirs(index_path, exist_ok=True)
     conda_index.api.update_index(index_path)
     index = conda.core.index.get_index(channel_urls=channel_urls)
@@ -223,26 +240,38 @@ def build_folders(recipes_dir, folders, arch, channel_urls):
     config = get_config(arch, channel_urls)
     platform = get_host_platform()
 
-    worker = {'platform': platform, 'arch': arch,
-              'label': '{}-{}'.format(platform, arch)}
+    worker = {
+        "platform": platform,
+        "arch": arch,
+        "label": "{}-{}".format(platform, arch),
+    }
 
-    G = construct_graph(recipes_dir, worker=worker, run='build',
-                        conda_resolve=conda_resolve, folders=folders,
-                        config=config, finalize=False)
+    G = construct_graph(
+        recipes_dir,
+        worker=worker,
+        run="build",
+        conda_resolve=conda_resolve,
+        folders=folders,
+        config=config,
+        finalize=False,
+    )
     order = list(nx.topological_sort(G))
     order.reverse()
 
-    print('Computed that there are {} distributions to build from {} recipes'
-          .format(len(order), len(folders)))
+    print(
+        "Computed that there are {} distributions to build from {} recipes".format(
+            len(order), len(folders)
+        )
+    )
     if not order:
-        print('Nothing to do')
+        print("Nothing to do")
         return
     print("Resolved dependencies, will be built in the following order:")
-    print('    '+'\n    '.join(order))
+    print("    " + "\n    ".join(order))
 
     d = OrderedDict()
     for node in order:
-        d[G.nodes[node]['meta'].meta_path] = 1
+        d[G.nodes[node]["meta"].meta_path] = 1
 
     for recipe in d.keys():
         conda_build.api.build([recipe], config=get_config(arch, channel_urls))
@@ -285,7 +314,7 @@ def build_folders_rattler_build(
     # Construct a temporary file where we write the combined variant config. We can then pass that
     # to rattler-build.
     with tempfile.NamedTemporaryFile(delete=False) as fp:
-        fp.write(variant_config.encode("utf-8"))   
+        fp.write(variant_config.encode("utf-8"))
         atexit.register(os.unlink, fp.name)
 
     # Execute rattler-build.
@@ -295,17 +324,21 @@ def build_folders_rattler_build(
 def check_recipes_in_correct_dir(root_dir, correct_dir):
     for path in Path(root_dir).glob("*"):
         path = Path(path)
-        if path.is_dir() and path.name.lower() in ('.pixi', 'build_artifacts', 'miniforge3'):
+        if path.is_dir() and path.name.lower() in (
+            ".pixi",
+            "build_artifacts",
+            "miniforge3",
+        ):
             # ignore pkg_cache in build_artifacts
             continue
-        for recipe_path in path.rglob('*.yaml'):
+        for recipe_path in path.rglob("*.yaml"):
             if recipe_path.name not in ("meta.yaml", "recipe.yaml"):
                 continue
             recipe_path = recipe_path.absolute().relative_to(root_dir)
             if (
-                (recipe_path.parts[0] != correct_dir and recipe_path.parts[0] != "broken-recipes")
-                or len(recipe_path.parts) != 3
-            ):
+                recipe_path.parts[0] != correct_dir
+                and recipe_path.parts[0] != "broken-recipes"
+            ) or len(recipe_path.parts) != 3:
                 raise RuntimeError(
                     f"recipe {recipe_path} in wrong directory; "
                     f"must be under {correct_dir}/<name>/"
@@ -336,21 +369,22 @@ def read_mambabuild(recipes_dir):
 
 def use_mambabuild():
     from boa.cli.mambabuild import prepare
+
     prepare()
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        '--arch',
-        default='64',
-        help='target architecture (second component of a subdir; e.g. 64, arm64, ppc64le)'
+        "--arch",
+        default="64",
+        help="target architecture (second component of a subdir; e.g. 64, arm64, ppc64le)",
     )
     args = parser.parse_args()
     root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
     check_recipes_in_correct_dir(root_dir, "recipes")
     use_mamba = read_mambabuild(os.path.join(root_dir, "recipes"))
     if use_mamba:
-      use_mambabuild()
-      subprocess.run("conda clean --all --yes", shell=True, check=True)
+        use_mambabuild()
+        subprocess.run("conda clean --all --yes", shell=True, check=True)
     build_all(os.path.join(root_dir, "recipes"), args.arch)