Skip to content

Commit f391218

Browse files
YifanShenSZYifan Shen
and
Yifan Shen
authored
8.0b1 Release (#2232)
* 8.0b1 release * auto rerun flaky tests --------- Co-authored-by: Yifan Shen <yifan_shen3@apple.com>
1 parent 9883e8d commit f391218

File tree

325 files changed

+41191
-6710
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

325 files changed

+41191
-6710
lines changed

NOTICE.txt

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,3 +23,26 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
2323
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
2424
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
2525
SOFTWARE.
26+
27+
28+
29+
This project contains content in the files coremltools/optimize/torch/layerwise_compression/_quant.py,
30+
coremltools/optimize/torch/layerwise_compression/algorithms.py,
31+
and coremltools/optimize/torch/layerwise_compression/layerwise_compressor.py which are adapted from
32+
gtpq (https://github.com/IST-DASLab/gptq/). It also contains content in the file coremltools/optimize/torch/layerwise_compression/algorithms.py which is adapted from sparsegpt (https://github.com/IST-DASLab/sparsegpt). The license for these follows:
33+
34+
Apache License 2.0
35+
36+
Copyright 2023 IST Austria Distributed Algorithms and Systems Lab
37+
38+
Licensed under the Apache License, Version 2.0 (the "License");
39+
you may not use this file except in compliance with the License.
40+
You may obtain a copy of the License at
41+
42+
http://www.apache.org/licenses/LICENSE-2.0
43+
44+
Unless required by applicable law or agreed to in writing, software
45+
distributed under the License is distributed on an "AS IS" BASIS,
46+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
47+
See the License for the specific language governing permissions and
48+
limitations under the License.

coremlpython/CoreMLPython.h

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,33 @@
1515

1616
#import <CoreML/CoreML.h>
1717

18+
19+
#ifndef BUILT_WITH_MACOS15_SDK
20+
#define BUILT_WITH_MACOS15_SDK \
21+
!(TARGET_OS_OSX && (!defined(__MAC_15_0) || __MAC_OS_X_VERSION_MAX_ALLOWED < __MAC_15_0))
22+
#endif
23+
24+
// Print BUILT_WITH_MACOS15_SDK value
25+
#if BUILT_WITH_MACOS15_SDK
26+
#pragma message ("Building with macOS 15+ SDK")
27+
#else
28+
#pragma message ("Building without macOS 15 SDK")
29+
#endif
30+
31+
1832
namespace py = pybind11;
1933

2034
namespace CoreML {
2135
namespace Python {
2236

37+
38+
struct State {
39+
#if BUILT_WITH_MACOS15_SDK
40+
// MLState must be wrapped in a C++ class for PyBind.
41+
MLState* m_state = nil;
42+
#endif
43+
};
44+
2345
class Model {
2446
private:
2547
MLModel *m_model = nil;
@@ -35,13 +57,19 @@ namespace CoreML {
3557
Model(const Model&) = delete;
3658
Model& operator=(const Model&) = delete;
3759
~Model();
38-
explicit Model(const std::string& urlStr, const std::string& computeUnits);
60+
explicit Model(const std::string& urlStr, const std::string& computeUnits, const std::string& functionName);
3961
explicit Model(MLModel* m_model, NSURL* compiledUrl, bool deleteCompiledModelOnExit);
4062

41-
py::dict predict(const py::dict& input) const;
4263
py::list batchPredict(const py::list& batch) const;
4364

4465
py::str getCompiledModelPath() const;
66+
67+
py::dict predict(const py::dict& input, State* state=NULL) const;
68+
69+
#if BUILT_WITH_MACOS15_SDK
70+
State newState() const;
71+
#endif
72+
4573
};
4674
}
4775
}

coremlpython/CoreMLPython.mm

Lines changed: 41 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ bool usingMacOS13OrHigher() {
4242
}
4343
}
4444

45-
Model::Model(const std::string& urlStr, const std::string& computeUnits) {
45+
Model::Model(const std::string& urlStr, const std::string& computeUnits, const std::string& functionName) {
4646
@autoreleasepool {
4747
NSError *error = nil;
4848

@@ -80,6 +80,12 @@ bool usingMacOS13OrHigher() {
8080
MLModelConfiguration *configuration = [MLModelConfiguration new];
8181
setComputeUnit(configuration, computeUnits);
8282

83+
if (!functionName.empty()) {
84+
#if BUILT_WITH_MACOS15_SDK
85+
configuration.functionName = [NSString stringWithUTF8String:functionName.c_str()];
86+
#endif
87+
}
88+
8389
// Create MLModel
8490
m_model = [MLModel modelWithContentsOfURL:compiledUrl configuration:configuration error:&error];
8591
Utils::handleError(error);
@@ -94,13 +100,28 @@ bool usingMacOS13OrHigher() {
94100
{
95101
}
96102

97-
py::dict Model::predict(const py::dict& input) const {
103+
104+
py::dict Model::predict(const py::dict& input, State* state) const {
98105
@autoreleasepool {
99106
NSError *error = nil;
100107
MLDictionaryFeatureProvider *inFeatures = Utils::dictToFeatures(input, &error);
101108
Utils::handleError(error);
102-
id<MLFeatureProvider> outFeatures = [m_model predictionFromFeatures:static_cast<MLDictionaryFeatureProvider * _Nonnull>(inFeatures)
103-
error:&error];
109+
110+
id<MLFeatureProvider> outFeatures;
111+
#if BUILT_WITH_MACOS15_SDK
112+
if (state == NULL) {
113+
outFeatures = [m_model predictionFromFeatures:static_cast<MLDictionaryFeatureProvider * _Nonnull>(inFeatures)
114+
error:&error];
115+
} else {
116+
outFeatures = [m_model predictionFromFeatures:static_cast<MLDictionaryFeatureProvider * _Nonnull>(inFeatures)
117+
usingState:state->m_state
118+
error:&error];
119+
}
120+
#else
121+
outFeatures = [m_model predictionFromFeatures:static_cast<MLDictionaryFeatureProvider * _Nonnull>(inFeatures)
122+
error:&error];
123+
#endif
124+
104125
Utils::handleError(error);
105126
return Utils::featuresToDict(outFeatures);
106127
}
@@ -163,6 +184,15 @@ bool usingMacOS13OrHigher() {
163184
}
164185

165186

187+
#if BUILT_WITH_MACOS15_SDK
188+
State Model::newState() const {
189+
State result;
190+
result.m_state = [m_model newState];
191+
return result;
192+
}
193+
#endif
194+
195+
166196
py::bytes Model::autoSetSpecificationVersion(const py::bytes& modelBytes) {
167197

168198
CoreML::Specification::Model model;
@@ -207,14 +237,20 @@ bool usingMacOS13OrHigher() {
207237
py::module m("libcoremlpython", "CoreML.Framework Python bindings");
208238

209239
py::class_<Model>(m, "_MLModelProxy")
210-
.def(py::init<const std::string&, const std::string&>())
240+
.def(py::init<const std::string&, const std::string&, const std::string&>())
211241
.def("predict", &Model::predict)
212242
.def("batchPredict", &Model::batchPredict)
213243
.def("get_compiled_model_path", &Model::getCompiledModelPath)
214244
.def_static("auto_set_specification_version", &Model::autoSetSpecificationVersion)
215245
.def_static("maximum_supported_specification_version", &Model::maximumSupportedSpecificationVersion)
246+
#if BUILT_WITH_MACOS15_SDK
247+
.def("newState", &Model::newState)
248+
#endif
216249
.def_static("compileModel", &Model::compileModel);
217250

251+
252+
py::class_<State>(m, "_State", py::module_local());
253+
218254
return m.ptr();
219255
}
220256

coremltools/__init__.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,9 @@
6464
# New versions for iOS 17.0
6565
_SPECIFICATION_VERSION_IOS_17 = 8
6666

67+
# New versions for iOS 18.0
68+
_SPECIFICATION_VERSION_IOS_18 = 9
69+
6770

6871
class ComputeUnit(_Enum):
6972
'''
@@ -82,6 +85,7 @@ class ComputeUnit(_Enum):
8285
_SPECIFICATION_VERSION_IOS_15: "CoreML5",
8386
_SPECIFICATION_VERSION_IOS_16: "CoreML6",
8487
_SPECIFICATION_VERSION_IOS_17: "CoreML7",
88+
_SPECIFICATION_VERSION_IOS_18: "CoreML8",
8589
}
8690

8791
# Default specification version for each backend
@@ -94,7 +98,7 @@ class ComputeUnit(_Enum):
9498
# expose unified converter in coremltools package level
9599
from .converters import ClassifierConfig
96100
from .converters import ColorLayout as colorlayout
97-
from .converters import EnumeratedShapes, ImageType, RangeDim, Shape, TensorType, convert
101+
from .converters import EnumeratedShapes, ImageType, RangeDim, Shape, StateType, TensorType, convert
98102
from .converters.mil._deployment_compatibility import AvailableTarget as target
99103
from .converters.mil.mil.passes.defs import quantization as transform
100104
from .converters.mil.mil.passes.defs.quantization import ComputePrecision as precision

coremltools/_deps/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ def __get_sklearn_version(version):
153153

154154
# ---------------------------------------------------------------------------------------
155155
_HAS_TORCH = True
156-
_TORCH_MAX_VERSION = "2.2.0"
156+
_TORCH_MAX_VERSION = "2.3.0"
157157
_HAS_TORCH_EXPORT_API = False
158158
try:
159159
import torch

coremltools/converters/__init__.py

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,15 @@
44
# found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
55

66
# expose directories as imports
7-
from . import libsvm
8-
from . import sklearn
9-
from . import xgboost
7+
from . import libsvm, sklearn, xgboost
108
from ._converters_entry import convert
119
from .mil import (
1210
ClassifierConfig,
1311
ColorLayout,
14-
TensorType,
12+
EnumeratedShapes,
1513
ImageType,
1614
RangeDim,
1715
Shape,
18-
EnumeratedShapes,
16+
StateType,
17+
TensorType,
1918
)

coremltools/converters/_converters_entry.py

Lines changed: 74 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
InputType,
3030
RangeDim,
3131
Shape,
32+
StateType,
3233
TensorType,
3334
)
3435
from coremltools.converters.mil.mil import Program, types
@@ -73,6 +74,7 @@ def convert(
7374
package_dir=None,
7475
debug=False,
7576
pass_pipeline: Optional[PassPipeline] = None,
77+
states=None,
7678
):
7779
"""
7880
Convert a TensorFlow or PyTorch model to the Core ML model format as either
@@ -403,7 +405,7 @@ def skip_real_div_ops(op):
403405
returned.
404406
405407
An enum with the following possible values:
406-
408+
407409
* ``coremltools.ComputeUnit.ALL``: Use all compute units available, including the neural engine.
408410
* ``coremltools.ComputeUnit.CPU_ONLY``: Limit the model to only use the CPU.
409411
* ``coremltools.ComputeUnit.CPU_AND_GPU``: Use both the CPU and GPU, but not the neural engine.
@@ -477,6 +479,50 @@ def skip_real_div_ops(op):
477479
478480
mlmodel = ct.convert(model, pass_pipeline=ct.PassPipeline.DEFAULT_PALETTIZATION)
479481
482+
states:
483+
Create a stateful ``mlprogram`` model
484+
by providing the ``StateType`` in the ``states`` argument (for details see `MIL Input Types <https://apple.github.io/coremltools/source/coremltools.converters.mil.input_types.html>`_).
485+
The stateful model is useful when converting a large language model with KV-Cache.
486+
The name of ``StateType`` must match the key of the PyTorch ``named_buffers()`` method in the source traced model.
487+
488+
The following example converts a torch model with a buffer called ``state_1``.
489+
490+
.. sourcecode:: python
491+
492+
class UpdateBufferModel(torch.nn.Module):
493+
def __init__(self):
494+
super(UpdateBufferModel, self).__init__()
495+
self.register_buffer(
496+
"state_1", torch.tensor(np.array([0, 0, 0], dtype=np.float32))
497+
)
498+
499+
def forward(self, x):
500+
# In place update of the model state
501+
self.state_1.add_(x)
502+
return self.state_1
503+
504+
505+
model = UpdateBufferModel()
506+
traced_model = torch.jit.trace(model, torch.tensor([1, 2, 3], dtype=torch.float32))
507+
508+
inputs = [
509+
ct.TensorType(shape=(1, 2)),
510+
]
511+
states = [
512+
ct.StateType(
513+
wrapped_type=ct.TensorType(
514+
shape=(1, 2),
515+
),
516+
name="state_1",
517+
),
518+
]
519+
mlmodel = ct.convert(
520+
traced_model,
521+
inputs=inputs,
522+
states=states,
523+
minimum_deployment_target=ct.target.iOS18,
524+
)
525+
480526
Returns
481527
-------
482528
@@ -526,8 +572,7 @@ def skip_real_div_ops(op):
526572
>>> results = mlmodel.predict({"input": example_input.numpy()})
527573
>>> print(results['1651']) # 1651 is the node name given by PyTorch's JIT
528574
529-
See `Conversion Options <https://apple.github.io/coremltools/docs-guides/source/conversion-options.html>`_ for
530-
more advanced options.
575+
For more options see `Conversion Options <https://apple.github.io/coremltools/docs-guides/source/conversion-options.html>`_.
531576
"""
532577
_check_deployment_target(minimum_deployment_target)
533578
outputs_as_strings, outputs_as_tensor_or_image_types = _validate_outputs_argument(outputs)
@@ -578,6 +623,15 @@ def skip_real_div_ops(op):
578623
and need_fp16_cast_pass
579624
)
580625

626+
# Verify the inputs cannot contains state
627+
if states is None:
628+
states = []
629+
_verify_inputs_doesnot_contains_states(inputs)
630+
631+
# states can only passed if the source is pytorch
632+
if len(states) > 0 and exact_source != "pytorch":
633+
raise ValueError("'states' can only be passed with pytorch source model.")
634+
581635
mlmodel = mil_convert(
582636
model,
583637
convert_from=exact_source,
@@ -592,6 +646,7 @@ def skip_real_div_ops(op):
592646
specification_version=specification_version,
593647
main_pipeline=pass_pipeline,
594648
use_default_fp16_io=use_default_fp16_io,
649+
states=states,
595650
)
596651

597652
if exact_target == "mlprogram" and mlmodel._input_has_infinite_upper_bound():
@@ -658,6 +713,20 @@ def _check_deployment_target(minimum_deployment_target):
658713
raise TypeError(msg.format(minimum_deployment_target))
659714

660715

716+
def _verify_inputs_doesnot_contains_states(
717+
inputs: List[InputType],
718+
) -> None:
719+
"""
720+
Verify that StateType is not present in the inputs.
721+
"""
722+
if inputs is None:
723+
return
724+
725+
for val in inputs:
726+
if isinstance(val, StateType):
727+
raise ValueError("'inputs' cannot contain an instance of StateType.")
728+
729+
661730
def _validate_outputs_argument(outputs):
662731
"""
663732
- validate properties that the "outputs" argument must satisfy, for instance, it should either be a list
@@ -848,9 +917,9 @@ def _flatten_list(_inputs):
848917

849918
elif exact_source == "pytorch":
850919
if _HAS_TORCH_EXPORT_API and isinstance(model, ExportedProgram):
851-
if model.dialect != "EDGE":
920+
if model.dialect not in ("ATEN", "EDGE"):
852921
raise NotImplementedError(
853-
f"Conversion for models with only EDGE dialect is supported/tested. Provided Dialect: {model.dialect}"
922+
f"Conversion for models with only ATEN or EDGE dialect is supported/tested. Provided Dialect: {model.dialect}"
854923
)
855924

856925
# TODO: rdar://115845792 ([Executorch] Handle user provided inputs/outputs in the convert API)

coremltools/converters/mil/Makefile

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ SRC_PACKAGES=.
88

99
TF_IOS13_TEST=../tensorflow/test
1010
MIL_TEST="."
11-
MIL_TEST_INTERNAL="../../../../coremltools-internal/coremltools_internal/converters/mil"
1211

1312
.PHONY: all lint test style checkstyle
1413

@@ -26,7 +25,7 @@ lint:
2625
${PYTHON} -m pylint -j 0 ${SRC_PACKAGES}
2726

2827
test:
29-
${PYTHON} -m pytest -W ignore::DeprecationWarning ${MIL_TEST} ${MIL_TEST_INTERNAL}
28+
${PYTHON} -m pytest -W ignore::DeprecationWarning ${MIL_TEST}
3029

3130
test_ref:
3231
${PYTHON} -m pytest -W ignore::DeprecationWarning ${TF_IOS13_TEST}

coremltools/converters/mil/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,6 @@
1111
get_existing_symbol, get_new_symbol, get_new_variadic_symbol,
1212
mil_list, register_op)
1313
from .input_types import (ClassifierConfig, ColorLayout, EnumeratedShapes,
14-
ImageType, InputType, RangeDim, Shape, TensorType)
14+
ImageType, InputType, RangeDim, Shape, TensorType, StateType)
1515
from .frontend.tensorflow.tf_op_registry import register_tf_op
1616
from .frontend.torch import register_torch_op

0 commit comments

Comments
 (0)