added support for Conv1d for DoRA (#2531)

DoRA now supports Conv1d layers and, notably, the check for how to deal with other than linear layers was softened from checking for 4 dimensions to now 3 dimensions since `Conv1d` layers have 3 elements instead of 4.

Author: EskildAndersen

method_comparison/MetaMathQA/run.py

@@ -31,6 +31,9 @@
 from typing import Any, Callable, ContextManager, Literal, Optional
 
 import torch
+from data import (
+    get_train_valid_test_datasets,
+)
 from torch import nn
 from torch.amp import GradScaler, autocast
 from tqdm import tqdm
@@ -52,9 +55,6 @@
     validate_experiment_path,
 )
 
-from data import (
-    get_train_valid_test_datasets,
-)
 from peft import AdaLoraConfig, PeftConfig
 from peft.utils import SAFETENSORS_WEIGHTS_NAME
 

src/peft/tuners/lora/dora.py

@@ -48,7 +48,7 @@ def update_layer(self, *, base_layer, lora_A, lora_B, scaling, place_on_cpu=Fals
                 base_layer = deepcopy(base_layer)
 
             weight = dequantize_module_weight(base_layer)
-            if weight.data.ndim >= 4:  # For handling LoRAs applied to Conv layers.
+            if weight.data.ndim >= 3:  # For handling LoRAs applied to Conv layers.
                 lora_weight = torch.mm(lora_B.flatten(start_dim=1), lora_A.flatten(start_dim=1))
                 lora_weight = lora_weight.reshape(weight.shape)
             else:
@@ -183,6 +183,12 @@ def __repr__(self) -> str:
         return "lora.dora." + rep
 
 
+class DoraConv1dLayer(_DoraConvNdLayer):
+    def __init__(self, fan_in_fan_out):
+        super().__init__(fan_in_fan_out)
+        self.conv_fn = F.conv1d
+
+
 class DoraConv2dLayer(_DoraConvNdLayer):
     def __init__(self, fan_in_fan_out):
         super().__init__(fan_in_fan_out)

src/peft/tuners/lora/variants.py

@@ -21,8 +21,8 @@
 
 from peft.utils.other import transpose
 
-from .dora import DoraConv2dLayer, DoraConv3dLayer, DoraEmbeddingLayer, DoraLinearLayer
-from .layer import Conv2d, Conv3d, Embedding, Linear, LoraVariant, _ConvNd
+from .dora import DoraConv1dLayer, DoraConv2dLayer, DoraConv3dLayer, DoraEmbeddingLayer, DoraLinearLayer
+from .layer import Conv1d, Conv2d, Conv3d, Embedding, Linear, LoraVariant, _ConvNd
 
 
 class DoraLinearVariant(LoraVariant):
@@ -296,6 +296,13 @@ def forward(module: _ConvNd, active_adapter: str, x: torch.Tensor, result: torch
         return result
 
 
+class DoraConv1dVariant(_DoraConvNdVariant):
+    @staticmethod
+    def init(module: Conv1d, adapter_name: str, **kwargs: Any) -> None:
+        dora_layer = DoraConv1dLayer(fan_in_fan_out=False)
+        _DoraConvNdVariant.init_convd_variant(module, adapter_name, dora_layer=dora_layer)
+
+
 class DoraConv2dVariant(_DoraConvNdVariant):
     @staticmethod
     def init(module: Conv2d, adapter_name: str, **kwargs: Any) -> None:

tests/test_custom_models.py

@@ -113,6 +113,7 @@
         {"target_modules": ["conv1d"], "trainable_token_indices": {"emb": [0, 10]}},
     ),
     ("Conv1d LoRA", "Conv1d", LoraConfig, {"target_modules": ["conv1d"]}),
+    ("Conv1d LoRA with DoRA", "Conv1d", LoraConfig, {"target_modules": ["conv1d"], "use_dora": True}),
     ("Conv2d 1 LoRA", "Conv2d", LoraConfig, {"target_modules": ["conv2d"]}),
     ("Conv2d 2 LoRA", "Conv2d", LoraConfig, {"target_modules": ["conv2d", "lin0"]}),
     ("Conv2d 1 LoRA with DoRA", "Conv2d", LoraConfig, {"target_modules": ["conv2d"], "use_dora": True}),