End of training

README.md

@@ -66,7 +66,7 @@ lora_r: 8
 lora_target_linear: true
 lr_scheduler: cosine
 max_steps: 10
-micro_batch_size: 1
+micro_batch_size: 2
 mlflow_experiment_name: /tmp/db70e57b42c5fff2_train_data.json
 model_type: AutoModelForCausalLM
 num_epochs: 4
@@ -92,7 +92,7 @@ wandb_name: 3cfb546e-4735-4a27-9a50-f4d92ba35258
 wandb_project: Gradients-On-Demand
 wandb_run: your_name
 wandb_runid: 3cfb546e-4735-4a27-9a50-f4d92ba35258
-warmup_steps: 5
+warmup_steps: 10
 weight_decay: 0.0
 xformers_attention: null
 
@@ -104,7 +104,7 @@ xformers_attention: null
 
 This model is a fine-tuned version of [NousResearch/CodeLlama-7b-hf-flash](https://huggingface.co/NousResearch/CodeLlama-7b-hf-flash) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.3627
+- Loss: 0.4006
 
 ## Model description
 
@@ -124,30 +124,30 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 0.0002
-- train_batch_size: 1
-- eval_batch_size: 1
+- train_batch_size: 2
+- eval_batch_size: 2
 - seed: 42
 - gradient_accumulation_steps: 16
-- total_train_batch_size: 16
+- total_train_batch_size: 32
 - optimizer: Use OptimizerNames.ADAMW_TORCH_4BIT with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: cosine
-- lr_scheduler_warmup_steps: 5
+- lr_scheduler_warmup_steps: 10
 - training_steps: 10
 
 ### Training results
 
 | Training Loss | Epoch  | Step | Validation Loss |
 |:-------------:|:------:|:----:|:---------------:|
-| 14.6764       | 0.0048 | 1    | 1.0043          |
-| 16.0244       | 0.0096 | 2    | 0.9961          |
-| 17.5417       | 0.0144 | 3    | 0.9555          |
-| 17.311        | 0.0192 | 4    | 0.8481          |
-| 12.0214       | 0.0240 | 5    | 0.6824          |
-| 13.3294       | 0.0288 | 6    | 0.5323          |
-| 8.38          | 0.0337 | 7    | 0.4619          |
-| 5.5358        | 0.0385 | 8    | 0.3950          |
-| 6.8075        | 0.0433 | 9    | 0.3684          |
-| 5.2666        | 0.0481 | 10   | 0.3627          |
+| 14.6966       | 0.0096 | 1    | 0.9651          |
+| 17.2403       | 0.0192 | 2    | 0.9630          |
+| 15.8582       | 0.0288 | 3    | 0.9526          |
+| 14.0816       | 0.0385 | 4    | 0.9209          |
+| 14.2308       | 0.0481 | 5    | 0.8555          |
+| 14.0635       | 0.0577 | 6    | 0.7542          |
+| 12.24         | 0.0673 | 7    | 0.6446          |
+| 9.584         | 0.0769 | 8    | 0.5435          |
+| 7.7745        | 0.0865 | 9    | 0.4615          |
+| 5.8718        | 0.0962 | 10   | 0.4006          |
 
 
 ### Framework versions

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:52a9004528fef0fedc5a7c55c1ce2548f6b138ffd8ca89a686ff9fcda8cb84b3
+oid sha256:72948994e561007d62010d55858b35ff4eb46ae5275169f7fd80b33c4aeb1f29
 size 80115210