Updated LoRA scaling, and learning rates.

Author: leofillioux

src/thunder/config/adaptation/lora.yaml

@@ -5,6 +5,6 @@ lora_alpha: 16
 # Hyperparameters
 batch_size: 2
 num_workers: 2
-lr: [1e-5, 1e-6, 1e-7]
+lr: [1e-3, 1e-4, 1e-5]
 weight_decay: [0.0, 1e-3, 1e-4]
 epochs: 20

src/thunder/models/adapters.py

@@ -114,7 +114,7 @@ def __init__(
         std = torch.sqrt(torch.tensor(rank).float())
         self.A = torch.nn.Parameter(torch.randn(in_dim, rank) / std)
         self.B = torch.nn.Parameter(torch.zeros(rank, out_dim))
-        self.alpha = alpha
+        self.alpha = alpha / rank
 
     def forward(self, src: torch.Tensor) -> torch.Tensor:
         """