Merge branch 'main' into free-prepare_model_for_gradient_checkpointing

Author: qgallouedec

src/peft/peft_model.py

@@ -1317,9 +1317,17 @@ def load_adapter(
         ):
             device_map = kwargs.get("device_map", "auto")
             max_memory = kwargs.get("max_memory", None)
-            offload_dir = kwargs.get("offload_folder", None)
+            offload_folder = kwargs.get("offload_folder", None)
+            offload_dir = kwargs.get("offload_dir", None)
             offload_index = kwargs.get("offload_index", None)
 
+            if offload_dir is not None and offload_folder is not None:
+                # see https://github.com/huggingface/peft/issues/2541
+                raise ValueError("Cannot use `offload_folder` when `offload_dir` is specified.")
+            elif offload_dir is None:
+                # to keep backwards compatibility
+                offload_dir = offload_folder
+
             dispatch_model_kwargs = {}
             # Safety checker for previous `accelerate` versions
             # `offload_index` was introduced in https://github.com/huggingface/accelerate/pull/873/

src/peft/tuners/lora/model.py

@@ -675,6 +675,8 @@ def add_weighted_adapter(
             r=new_rank,
             lora_alpha=new_rank,
             target_modules=new_target_modules,
+            alpha_pattern={},
+            rank_pattern={},
         )
         self.inject_adapter(self.model, adapter_name)
 

src/peft/tuners/tuners_utils.py

@@ -114,6 +114,28 @@ def onload_layer(layer):
         layer.base_layer._hf_hook.post_forward(layer.base_layer, torch.tensor([]))
 
 
+def _check_lora_target_modules_mamba(peft_config: PeftConfig, model: nn.Module, target_name: str):
+    """
+    Prevent applying LoRA to incompatible modules in specific architectures (e.g., Mamba).
+    """
+
+    lora_like_types = {"LORA", "ADALORA", "XLORA", "RANDLORA"}
+    incompatible_modules = {"out_proj", "conv1d"}
+    mamba_model_types = {"falcon_h1", "mamba", "mamba2", "falcon_mamba"}
+
+    if (
+        peft_config.peft_type in lora_like_types
+        and hasattr(model, "config")
+        and getattr(model.config, "model_type", None) in mamba_model_types
+    ):
+        if target_name in incompatible_modules:
+            raise ValueError(
+                f"[PEFT:{peft_config.peft_type}] Module '{target_name}' is incompatible with Mamba-based models "
+                f"(model_type='{model.config.model_type}'). Incompatible modules: {incompatible_modules}. "
+                "Please remove it from `target_modules` to avoid compatibility issues."
+            )
+
+
 class BaseTuner(nn.Module, ABC):
     r"""
     A base tuner model that provides the common methods and attributes for all tuners that are injectable into a
@@ -398,6 +420,12 @@ def _check_merge_allowed(self):
                 + example_code
             )
 
+    def _check_target_module_compatiblity(self, peft_config: PeftConfig, model: nn.Module, target_name: str):
+        """
+        Prevent applying LoRA to incompatible modules in specific architectures (e.g., Mamba).
+        """
+        _check_lora_target_modules_mamba(peft_config, model, target_name)
+
     def inject_adapter(
         self, model: nn.Module, adapter_name: str, autocast_adapter_dtype: bool = True, low_cpu_mem_usage: bool = False
     ) -> None:
@@ -497,6 +525,7 @@ def inject_adapter(
             else:
                 self.targeted_module_names.append(key)
                 parent, target, target_name = _get_submodules(model, key)
+                self._check_target_module_compatiblity(peft_config, model, target_name)
                 ctx = init_empty_weights if low_cpu_mem_usage else nullcontext
                 with ctx():
                     self._create_and_replace(peft_config, adapter_name, target, target_name, parent, current_key=key)