Merge branch 'branch-25.10' into test/json_stream_failure

Author: vyasr

.github/labeler.yml

@@ -1,34 +1,34 @@
 # Documentation for config - https://github.com/actions/labeler#common-examples
 
 Python:
-  - changed-files:
-      any-glob-to-any-file:
-        - 'python/**'
-        - 'notebooks/**'
+- changed-files:
+  - any-glob-to-any-file:
+    - 'python/**'
+    - 'notebooks/**'
 cudf.pandas:
-  - changed-files:
-      any-glob-to-any-file:
-        - 'python/cudf/cudf/pandas/**'
-        - 'python/cudf/cudf_pandas_tests/**'
+- changed-files:
+  - any-glob-to-any-file:
+    - 'python/cudf/cudf/pandas/**'
+    - 'python/cudf/cudf_pandas_tests/**'
 cudf-polars:
-  - changed-files:
-      any-glob-to-any-file:
-        - 'python/cudf_polars/**'
+- changed-files:
+  - any-glob-to-any-file:
+    - 'python/cudf_polars/**'
 pylibcudf:
-  - changed-files:
-      any-glob-to-any-file:
-        - 'python/pylibcudf/**'
+- changed-files:
+  - any-glob-to-any-file:
+    - 'python/pylibcudf/**'
 libcudf:
-  - changed-files:
-      any-glob-to-any-file:
-        - 'cpp/**'
+- changed-files:
+  - any-glob-to-any-file:
+    - 'cpp/**'
 CMake:
-  - changed-files:
-      any-glob-to-any-file:
-        - '**/CMakeLists.txt'
-        - '**/cmake/**'
-        - '**/*.cmake'
+- changed-files:
+  - any-glob-to-any-file:
+    - '**/CMakeLists.txt'
+    - '**/cmake/**'
+    - '**/*.cmake'
 Java:
-  - changed-files:
-      any-glob-to-any-file:
-        - 'java/**'
+- changed-files:
+  - any-glob-to-any-file:
+    - 'java/**'

.github/workflows/labeler.yml

@@ -5,6 +5,9 @@ on:
 
 jobs:
   triage:
+    permissions:
+      contents: read
+      pull-requests: write
     runs-on: ubuntu-latest
     steps:
     - name: Checkout code

dependencies.yaml

@@ -765,7 +765,7 @@ dependencies:
     common:
       - output_types: conda
         packages:
-          - cmake>=3.30.4,<4.0.0
+          - *cmake_ver
           - ninja
           - maven
           - openjdk=8.*

python/cudf/cudf/pandas/_wrappers/pandas.py

@@ -429,6 +429,22 @@ def Index__setattr__(self, name, value):
     },
 )
 
+# Special caseing `ArrowDtype` as it is not yet added to `cudf` namespace
+# both fast and slow paths are `pd.ArrowDtype`
+ArrowDtype = make_final_proxy_type(
+    "ArrowDtype",
+    pd.ArrowDtype,
+    pd.ArrowDtype,
+    bases=(pd.api.extensions.ExtensionDtype,),
+    fast_to_slow=lambda fast: fast,
+    slow_to_fast=lambda slow: slow,
+    additional_attributes={
+        "__from_arrow__": _FastSlowAttribute("__from_arrow__"),
+        "__hash__": _FastSlowAttribute("__hash__"),
+        "pyarrow_dtype": _FastSlowAttribute("pyarrow_dtype"),
+    },
+)
+
 SparseArray = make_final_proxy_type(
     "SparseDtype",
     _Unusable,
@@ -1109,20 +1125,46 @@ def Index__setattr__(self, name, value):
 
 try:
     from pandas.io.formats.style import Styler as pd_Styler  # isort: skip
+    from pandas.io.formats.style import StylerRenderer as pd_StylerRenderer
+
+    StylerRenderer = make_final_proxy_type(
+        "StylerRenderer",
+        _Unusable,
+        pd_StylerRenderer,
+        fast_to_slow=_Unusable(),
+        slow_to_fast=_Unusable(),
+    )
 
     Styler = make_final_proxy_type(
         "Styler",
         _Unusable,
         pd_Styler,
+        bases=(StylerRenderer,),
         fast_to_slow=_Unusable(),
         slow_to_fast=_Unusable(),
         additional_attributes={
             "css": _FastSlowAttribute("css"),
             "ctx": _FastSlowAttribute("ctx"),
-            "index": _FastSlowAttribute("ctx"),
+            "index": _FastSlowAttribute("index"),
             "data": _FastSlowAttribute("data"),
-            "_display_funcs": _FastSlowAttribute("_display_funcs"),
+            "_display_funcs": _FastSlowAttribute(
+                "_display_funcs", private=True
+            ),
             "table_styles": _FastSlowAttribute("table_styles"),
+            "columns": _FastSlowAttribute("columns"),
+            "caption": _FastSlowAttribute("caption"),
+            "_todo": _FastSlowAttribute("_todo", private=True),
+            "ctx_columns": _FastSlowAttribute("ctx_columns"),
+            "ctx_index": _FastSlowAttribute("ctx_index"),
+            "_display_funcs_index": _FastSlowAttribute(
+                "_display_funcs_index", private=True
+            ),
+            "uuid": _FastSlowAttribute("uuid"),
+            "hide_index_": _FastSlowAttribute("hide_index_"),
+            "hide_index_names": _FastSlowAttribute("hide_index_names"),
+            "hide_columns_": _FastSlowAttribute("hide_columns_"),
+            "hide_column_names": _FastSlowAttribute("hide_column_names"),
+            "table_attributes": _FastSlowAttribute("table_attributes"),
         },
     )
 except ImportError:

python/cudf/cudf/pandas/scripts/conftest-patch.py

@@ -2754,46 +2754,6 @@ def pytest_unconfigure(config):
     "tests/extension/test_arrow.py::TestArrowArray::test_arith_series_with_scalar[uint8-__rmod__]",
     "tests/extension/test_arrow.py::TestArrowArray::test_array_interface[duration[ns]]",
     "tests/extension/test_arrow.py::TestArrowArray::test_array_interface[timestamp[ns]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[binary]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[bool]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[date32[day]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[date64[ms]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[decimal128(7, 3)]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[double]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[duration[ms]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[duration[ns]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[duration[s]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[duration[us]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[float]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[int16]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[int32]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[int64]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[int8]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[string]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[time32[ms]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[time32[s]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[time64[ns]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[time64[us]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[timestamp[ms, tz=US/Eastern]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[timestamp[ms, tz=US/Pacific]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[timestamp[ms, tz=UTC]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[timestamp[ms]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[timestamp[ns, tz=US/Eastern]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[timestamp[ns, tz=US/Pacific]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[timestamp[ns, tz=UTC]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[timestamp[ns]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[timestamp[s, tz=US/Eastern]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[timestamp[s, tz=US/Pacific]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[timestamp[s, tz=UTC]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[timestamp[s]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[timestamp[us, tz=US/Eastern]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[timestamp[us, tz=US/Pacific]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[timestamp[us, tz=UTC]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[timestamp[us]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[uint16]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[uint32]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[uint64]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_array_type[uint8]",
     "tests/extension/test_arrow.py::TestArrowArray::test_astype_own_type[binary-False]",
     "tests/extension/test_arrow.py::TestArrowArray::test_astype_own_type[bool-False]",
     "tests/extension/test_arrow.py::TestArrowArray::test_astype_own_type[date32[day]-False]",
@@ -3030,8 +2990,6 @@ def pytest_unconfigure(config):
     "tests/extension/test_arrow.py::TestArrowArray::test_container_shift[uint8-2-indices2-True]",
     "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[duration[ms]]",
     "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[duration[ms]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[duration[ns]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[duration[ns]]",
     "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[duration[s]]",
     "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[duration[s]]",
     "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[duration[us]]",
@@ -3044,14 +3002,6 @@ def pytest_unconfigure(config):
     "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[timestamp[ms, tz=UTC]]",
     "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[timestamp[ms]]",
     "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[timestamp[ms]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[timestamp[ns, tz=US/Eastern]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[timestamp[ns, tz=US/Eastern]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[timestamp[ns, tz=US/Pacific]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[timestamp[ns, tz=US/Pacific]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[timestamp[ns, tz=UTC]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[timestamp[ns, tz=UTC]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[timestamp[ns]]",
-    "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[timestamp[ns]]",
     "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[timestamp[s, tz=US/Eastern]]",
     "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[timestamp[s, tz=US/Eastern]]",
     "tests/extension/test_arrow.py::TestArrowArray::test_getitem_scalar[timestamp[s, tz=US/Pacific]]",
@@ -9833,41 +9783,14 @@ def pytest_unconfigure(config):
     "tests/io/excel/test_writers.py::TestExcelWriterEngineTests::test_register_writer",
     "tests/io/excel/test_writers.py::TestFSPath::test_excelfile_fspath",
     "tests/io/formats/style/test_format.py::test_format_clear[format_index-_display_funcs_columns-kwargs2]",
-    "tests/io/formats/style/test_format.py::test_format_clear[format_index-_display_funcs_index-kwargs1]",
     "tests/io/formats/style/test_html.py::test_from_custom_template_style",
     "tests/io/formats/style/test_html.py::test_from_custom_template_table",
     "tests/io/formats/style/test_html.py::test_html_template_extends_options",
     "tests/io/formats/style/test_html.py::test_replaced_css_class_names",
-    "tests/io/formats/style/test_html.py::test_sticky_basic[True-False-False]",
-    "tests/io/formats/style/test_html.py::test_sticky_basic[True-False-True]",
-    "tests/io/formats/style/test_html.py::test_sticky_basic[True-True-False]",
-    "tests/io/formats/style/test_html.py::test_sticky_basic[True-True-True]",
-    "tests/io/formats/style/test_html.py::test_sticky_levels[levels0-False-False]",
-    "tests/io/formats/style/test_html.py::test_sticky_levels[levels0-False-True]",
-    "tests/io/formats/style/test_html.py::test_sticky_levels[levels0-True-False]",
-    "tests/io/formats/style/test_html.py::test_sticky_levels[levels0-True-True]",
-    "tests/io/formats/style/test_html.py::test_sticky_levels[levels1-False-False]",
-    "tests/io/formats/style/test_html.py::test_sticky_levels[levels1-False-True]",
-    "tests/io/formats/style/test_html.py::test_sticky_levels[levels1-True-False]",
-    "tests/io/formats/style/test_html.py::test_sticky_levels[levels1-True-True]",
-    "tests/io/formats/style/test_html.py::test_sticky_levels[one-False-False]",
-    "tests/io/formats/style/test_html.py::test_sticky_levels[one-False-True]",
-    "tests/io/formats/style/test_html.py::test_sticky_levels[one-True-False]",
-    "tests/io/formats/style/test_html.py::test_sticky_levels[one-True-True]",
-    "tests/io/formats/style/test_style.py::TestStyler::test_apply_axis",
     "tests/io/formats/style/test_style.py::TestStyler::test_caption",
-    "tests/io/formats/style/test_style.py::TestStyler::test_export",
     "tests/io/formats/style/test_style.py::TestStyler::test_non_reducing_multi_slice_on_multiindex[slice_7]",
     "tests/io/formats/style/test_style.py::TestStyler::test_table_styles",
     "tests/io/formats/style/test_style.py::TestStyler::test_uuid",
-    "tests/io/formats/style/test_style.py::test_apply_map_header[columns-apply]",
-    "tests/io/formats/style/test_style.py::test_apply_map_header[columns-map]",
-    "tests/io/formats/style/test_style.py::test_apply_map_header[index-apply]",
-    "tests/io/formats/style/test_style.py::test_apply_map_header[index-map]",
-    "tests/io/formats/style/test_style.py::test_apply_map_header_mi[columns-apply]",
-    "tests/io/formats/style/test_style.py::test_apply_map_header_mi[columns-map]",
-    "tests/io/formats/style/test_style.py::test_apply_map_header_mi[index-apply]",
-    "tests/io/formats/style/test_style.py::test_apply_map_header_mi[index-map]",
     "tests/io/formats/style/test_style.py::test_clear",
     "tests/io/formats/style/test_style.py::test_copy[False-False-False]",
     "tests/io/formats/style/test_style.py::test_copy[False-False-True]",
@@ -9877,22 +9800,6 @@ def pytest_unconfigure(config):
     "tests/io/formats/style/test_style.py::test_copy[True-False-True]",
     "tests/io/formats/style/test_style.py::test_copy[True-True-False]",
     "tests/io/formats/style/test_style.py::test_copy[True-True-True]",
-    "tests/io/formats/style/test_style.py::test_export",
-    "tests/io/formats/style/test_style.py::test_hide_columns_level[False-1]",
-    "tests/io/formats/style/test_style.py::test_hide_columns_level[False-level2]",
-    "tests/io/formats/style/test_style.py::test_hide_columns_level[False-level3]",
-    "tests/io/formats/style/test_style.py::test_hide_columns_level[False-one]",
-    "tests/io/formats/style/test_style.py::test_hide_columns_level[True-1]",
-    "tests/io/formats/style/test_style.py::test_hide_columns_level[True-level2]",
-    "tests/io/formats/style/test_style.py::test_hide_columns_level[True-level3]",
-    "tests/io/formats/style/test_style.py::test_hide_columns_level[True-one]",
-    "tests/io/formats/style/test_style.py::test_hide_index_level[1]",
-    "tests/io/formats/style/test_style.py::test_hide_index_level[level2]",
-    "tests/io/formats/style/test_style.py::test_hide_index_level[level3]",
-    "tests/io/formats/style/test_style.py::test_hide_index_level[one]",
-    "tests/io/formats/style/test_to_latex.py::test_css_convert_apply_index[columns]",
-    "tests/io/formats/style/test_to_latex.py::test_parse_latex_table_wrapping",
-    "tests/io/formats/style/test_to_latex.py::test_styler_object_after_render",
     "tests/io/formats/test_eng_formatting.py::TestEngFormatter::test_nan",
     "tests/io/formats/test_format.py::TestDataFrameFormatting::test_info_repr",
     "tests/io/formats/test_format.py::TestDataFrameFormatting::test_info_repr_max_cols",

python/pylibcudf/pylibcudf/column.pxd

@@ -6,6 +6,7 @@ from libc.stdint cimport uint64_t
 
 from rmm.librmm.device_buffer cimport device_buffer
 from rmm.pylibrmm.stream cimport Stream
+from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource
 from pylibcudf.libcudf.column.column cimport column
 from pylibcudf.libcudf.column.column_view cimport (
     column_view,
@@ -59,6 +60,7 @@ cdef class Column:
     cdef Column from_libcudf(
         unique_ptr[column] libcudf_col,
         Stream stream,
+        DeviceMemoryResource mr=*
     )
 
     @staticmethod

python/pylibcudf/pylibcudf/column.pyx

@@ -36,6 +36,7 @@ from pylibcudf.libcudf.copying cimport get_element
 
 from rmm.pylibrmm.device_buffer cimport DeviceBuffer
 from rmm.pylibrmm.stream cimport Stream
+from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource
 
 from .gpumemoryview cimport gpumemoryview
 from .filling cimport sequence
@@ -52,7 +53,7 @@ from ._interop_helpers cimport (
     _release_device_array,
     _metadata_to_libcudf,
 )
-from .utils cimport _get_stream
+from .utils cimport _get_stream, _get_memory_resource
 
 from .gpumemoryview import _datatype_from_dtype_desc
 from ._interop_helpers import ArrowLike, ColumnMetadata
@@ -562,6 +563,7 @@ cdef class Column:
     cdef Column from_libcudf(
         unique_ptr[column] libcudf_col,
         Stream stream,
+        DeviceMemoryResource mr=None
     ):
         """Create a Column from a libcudf column.
 
@@ -577,23 +579,24 @@ cdef class Column:
         cdef column_contents contents = libcudf_col.get().release()
 
         stream = _get_stream(stream)
+        mr = _get_memory_resource(mr)
         # Note that when converting to cudf Column objects we'll need to pull
         # out the base object.
         cdef gpumemoryview data = gpumemoryview(
-            DeviceBuffer.c_from_unique_ptr(move(contents.data), stream)
+            DeviceBuffer.c_from_unique_ptr(move(contents.data), stream, mr)
         )
 
         cdef gpumemoryview mask = None
         if null_count > 0:
             mask = gpumemoryview(
-                DeviceBuffer.c_from_unique_ptr(move(contents.null_mask), stream)
+                DeviceBuffer.c_from_unique_ptr(move(contents.null_mask), stream, mr)
             )
 
         children = []
         if contents.children.size() != 0:
             for i in range(contents.children.size()):
                 children.append(
-                    Column.from_libcudf(move(contents.children[i]), stream)
+                    Column.from_libcudf(move(contents.children[i]), stream, mr)
                 )
 
         return Column(

python/pylibcudf/pylibcudf/concatenate.pxd

@@ -3,10 +3,11 @@
 from .table cimport Table
 
 from rmm.pylibrmm.stream cimport Stream
+from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource
 
 
 # There is no way to define a fused type that is a list of other objects, so we cannot
 # unify the column and table paths without using runtime dispatch instead. In this case
 # we choose to prioritize API consistency over performance, so we use the same function
 # with a bit of runtime dispatch overhead.
-cpdef concatenate(list objects, Stream stream=*)
+cpdef concatenate(list objects, Stream stream=*, DeviceMemoryResource mr=*)

python/pylibcudf/pylibcudf/concatenate.pyi

@@ -1,5 +1,6 @@
 # Copyright (c) 2024, NVIDIA CORPORATION.
 
+from rmm.pylibrmm.memory_resource import DeviceMemoryResource
 from rmm.pylibrmm.stream import Stream
 
 from pylibcudf.column import Column
@@ -8,4 +9,5 @@ from pylibcudf.table import Table
 def concatenate[ColumnOrTable: (Column, Table)](
     objects: list[ColumnOrTable],
     stream: Stream | None = None,
+    mr: DeviceMemoryResource | None = None,
 ) -> ColumnOrTable: ...