File tree 1 file changed +16
-3
lines changed
python/ray/data/preprocessors
1 file changed +16
-3
lines changed Original file line number Diff line number Diff line change 1
- from typing import TYPE_CHECKING
1
+ from typing import TYPE_CHECKING , Optional
2
2
3
3
from ray .air .util .data_batch_conversion import BatchFormat
4
4
from ray .data import Dataset
@@ -79,9 +79,22 @@ def fit_transform(self, ds: Dataset) -> Dataset:
79
79
ds = preprocessor .fit_transform (ds )
80
80
return ds
81
81
82
- def _transform (self , ds : Dataset ) -> Dataset :
82
+ def _transform (
83
+ self ,
84
+ ds : Dataset ,
85
+ batch_size : Optional [int ],
86
+ num_cpus : Optional [float ] = None ,
87
+ memory : Optional [float ] = None ,
88
+ concurrency : Optional [int ] = None ,
89
+ ) -> Dataset :
83
90
for preprocessor in self .preprocessors :
84
- ds = preprocessor .transform (ds )
91
+ ds = preprocessor .transform (
92
+ ds ,
93
+ batch_size = batch_size ,
94
+ num_cpus = num_cpus ,
95
+ memory = memory ,
96
+ concurrency = concurrency ,
97
+ )
85
98
return ds
86
99
87
100
def _transform_batch (self , df : "DataBatchType" ) -> "DataBatchType" :
You can’t perform that action at this time.
0 commit comments