Upload Odran LoRA adapter from checkpoint-400

.gitattributes

@@ -33,3 +33,13 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+optimizer_0/.metadata filter=lfs diff=lfs merge=lfs -text
+optimizer_0/__0_0.distcp filter=lfs diff=lfs merge=lfs -text
+optimizer_0/__1_0.distcp filter=lfs diff=lfs merge=lfs -text
+optimizer_0/__2_0.distcp filter=lfs diff=lfs merge=lfs -text
+optimizer_0/__3_0.distcp filter=lfs diff=lfs merge=lfs -text
+pytorch_model_fsdp_0/.metadata filter=lfs diff=lfs merge=lfs -text
+pytorch_model_fsdp_0/__0_0.distcp filter=lfs diff=lfs merge=lfs -text
+pytorch_model_fsdp_0/__1_0.distcp filter=lfs diff=lfs merge=lfs -text
+pytorch_model_fsdp_0/__2_0.distcp filter=lfs diff=lfs merge=lfs -text
+pytorch_model_fsdp_0/__3_0.distcp filter=lfs diff=lfs merge=lfs -text

adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.3-70B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": false,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "v_proj",
+    "down_proj",
+    "o_proj",
+    "up_proj",
+    "q_proj",
+    "gate_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80c40321c5a253bc103edccfb039d67d61131205b9f778eaf7d76f93f3fe78bf
+size 3313653448

optimizer_0/.metadata

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64102e617d4661169425a1eb9099a87c4c85d929dcdcd2f37a1ea9ba56fa9114
+size 3150477

optimizer_0/__0_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ac3767cbb3bbbefb5123e6913a380a3b3840a7b75a8a9202a69a29669e4b774
+size 1660282560

optimizer_0/__1_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bea521e304811b3d29548ccf505f5ddc1fab77a1c620f3b71e498b9ece64bcd
+size 1660879924

optimizer_0/__2_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:456bd3c6c013a6caf3074cbe1d78b5720043fd8dcff846c6234c235c9c0fc971
+size 1660971455

optimizer_0/__3_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1efba0245759e6302f796e2b5103c317ad28e95a5c04bb2906acacd1d88311e
+size 1660883391

pytorch_model_fsdp_0/.metadata

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6209ef27fbac7993ab30411d98173adb3224b177cbaf741e474b0ae6005c32a4
+size 1168104

pytorch_model_fsdp_0/__0_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38ba202148af62fc18f9d458d9ecd20888e76845b370b8383b2f17f6ad6cfa5b
+size 830141280

pytorch_model_fsdp_0/__1_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:378db2596c9edcef01f84acd5617a72e33b18d1fa648368fee7df3eea1dd511a
+size 830141280

pytorch_model_fsdp_0/__2_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17e568a0c7f2e4b6e5142d746618699aff120285a93e5b4fa2132871c3b58b5d
+size 830141280

pytorch_model_fsdp_0/__3_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd0904da9756201f4dc6d64d44a433f298fc4f4a509405a4fc2821bc2def975f
+size 830141280

rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fd1bb2644abcf50c0ce4c4061a86a0fe7e7faf33cbcee057ad56c2a3f76ae9c
+size 15429

rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83ed0ce5cab10013ab07588ecaa5d6b10a4a89fa2038a6736ac6b0c81856217c
+size 15429

rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec38ee0bc967d3f569507a136e3a4987c172d113b2fce0b961487add77525096
+size 15429

rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7fc6aa996e6c160c19fe4bd731880671dc2160b7b11730b48b6a56b60e15ccc
+size 15429

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7f16f0344a2a18efc9191bb5c94435a1f6aedcd9fd51f2cd37b5387038f2ff3
+size 1465

trainer_state.json

@@ -0,0 +1,474 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.43106351451471675,
+  "eval_steps": 50,
+  "global_step": 400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01077658786286792,
+      "grad_norm": 0.7500938773155212,
+      "learning_rate": 1.6071428571428572e-05,
+      "loss": 0.3934,
+      "mean_token_accuracy": 0.9211844336241484,
+      "num_tokens": 860689.0,
+      "step": 10
+    },
+    {
+      "epoch": 0.02155317572573584,
+      "grad_norm": 0.1784556359052658,
+      "learning_rate": 3.392857142857143e-05,
+      "loss": 0.3003,
+      "mean_token_accuracy": 0.9324536714702845,
+      "num_tokens": 1727975.0,
+      "step": 20
+    },
+    {
+      "epoch": 0.03232976358860376,
+      "grad_norm": 0.17185558378696442,
+      "learning_rate": 4.9999351014638695e-05,
+      "loss": 0.2695,
+      "mean_token_accuracy": 0.9337602369487286,
+      "num_tokens": 2596794.0,
+      "step": 30
+    },
+    {
+      "epoch": 0.04310635145147168,
+      "grad_norm": 0.21529479324817657,
+      "learning_rate": 4.992151353344481e-05,
+      "loss": 0.2706,
+      "mean_token_accuracy": 0.9353477034717799,
+      "num_tokens": 3463293.0,
+      "step": 40
+    },
+    {
+      "epoch": 0.053882939314339594,
+      "grad_norm": 0.3255652189254761,
+      "learning_rate": 4.971434188480771e-05,
+      "loss": 0.2387,
+      "mean_token_accuracy": 0.9378637354820967,
+      "num_tokens": 4305832.0,
+      "step": 50
+    },
+    {
+      "epoch": 0.053882939314339594,
+      "eval_loss": NaN,
+      "eval_mean_token_accuracy": 0.9415765093891534,
+      "eval_num_tokens": 4305832.0,
+      "eval_runtime": 810.7208,
+      "eval_samples_per_second": 1.495,
+      "eval_steps_per_second": 0.374,
+      "step": 50
+    },
+    {
+      "epoch": 0.06465952717720752,
+      "grad_norm": 0.20627367496490479,
+      "learning_rate": 4.9378911219025094e-05,
+      "loss": 0.2021,
+      "mean_token_accuracy": 0.9466543391346931,
+      "num_tokens": 5153764.0,
+      "step": 60
+    },
+    {
+      "epoch": 0.07543611504007544,
+      "grad_norm": 0.20740991830825806,
+      "learning_rate": 4.891696230701104e-05,
+      "loss": 0.2537,
+      "mean_token_accuracy": 0.9357574578374624,
+      "num_tokens": 6003237.0,
+      "step": 70
+    },
+    {
+      "epoch": 0.08621270290294336,
+      "grad_norm": 0.2172331064939499,
+      "learning_rate": 4.833089250628786e-05,
+      "loss": 0.2203,
+      "mean_token_accuracy": 0.9420852873474359,
+      "num_tokens": 6851328.0,
+      "step": 80
+    },
+    {
+      "epoch": 0.09698929076581128,
+      "grad_norm": 0.28385892510414124,
+      "learning_rate": 4.762374331951703e-05,
+      "loss": 0.1894,
+      "mean_token_accuracy": 0.9459741819649935,
+      "num_tokens": 7711868.0,
+      "step": 90
+    },
+    {
+      "epoch": 0.10776587862867919,
+      "grad_norm": 0.24213817715644836,
+      "learning_rate": 4.679918461013628e-05,
+      "loss": 0.2403,
+      "mean_token_accuracy": 0.9376974884420634,
+      "num_tokens": 8558775.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.10776587862867919,
+      "eval_loss": NaN,
+      "eval_mean_token_accuracy": 0.94512011332087,
+      "eval_num_tokens": 8558775.0,
+      "eval_runtime": 808.9526,
+      "eval_samples_per_second": 1.498,
+      "eval_steps_per_second": 0.375,
+      "step": 100
+    },
+    {
+      "epoch": 0.11854246649154711,
+      "grad_norm": 0.18466246128082275,
+      "learning_rate": 4.5861495557018205e-05,
+      "loss": 0.2379,
+      "mean_token_accuracy": 0.9391791693866253,
+      "num_tokens": 9415930.0,
+      "step": 110
+    },
+    {
+      "epoch": 0.12931905435441504,
+      "grad_norm": 0.14023905992507935,
+      "learning_rate": 4.4815542446989376e-05,
+      "loss": 0.2174,
+      "mean_token_accuracy": 0.938202116638422,
+      "num_tokens": 10264693.0,
+      "step": 120
+    },
+    {
+      "epoch": 0.14009564221728296,
+      "grad_norm": 0.14716534316539764,
+      "learning_rate": 4.366675342045903e-05,
+      "loss": 0.1919,
+      "mean_token_accuracy": 0.949906887114048,
+      "num_tokens": 11131720.0,
+      "step": 130
+    },
+    {
+      "epoch": 0.15087223008015088,
+      "grad_norm": 0.1462382823228836,
+      "learning_rate": 4.242109030121908e-05,
+      "loss": 0.1989,
+      "mean_token_accuracy": 0.9474684063345193,
+      "num_tokens": 11994844.0,
+      "step": 140
+    },
+    {
+      "epoch": 0.1616488179430188,
+      "grad_norm": 0.18550986051559448,
+      "learning_rate": 4.10850176566091e-05,
+      "loss": 0.2512,
+      "mean_token_accuracy": 0.9342048281803728,
+      "num_tokens": 12853647.0,
+      "step": 150
+    },
+    {
+      "epoch": 0.1616488179430188,
+      "eval_loss": NaN,
+      "eval_mean_token_accuracy": 0.9469890043680423,
+      "eval_num_tokens": 12853647.0,
+      "eval_runtime": 805.1581,
+      "eval_samples_per_second": 1.505,
+      "eval_steps_per_second": 0.376,
+      "step": 150
+    },
+    {
+      "epoch": 0.17242540580588672,
+      "grad_norm": 0.15864673256874084,
+      "learning_rate": 3.966546924861362e-05,
+      "loss": 0.2514,
+      "mean_token_accuracy": 0.9352497901767493,
+      "num_tokens": 13698484.0,
+      "step": 160
+    },
+    {
+      "epoch": 0.18320199366875464,
+      "grad_norm": 0.18158355355262756,
+      "learning_rate": 3.816981204999882e-05,
+      "loss": 0.1831,
+      "mean_token_accuracy": 0.9467186477035284,
+      "num_tokens": 14562607.0,
+      "step": 170
+    },
+    {
+      "epoch": 0.19397858153162256,
+      "grad_norm": 0.1639261543750763,
+      "learning_rate": 3.660580801223301e-05,
+      "loss": 0.2392,
+      "mean_token_accuracy": 0.9383120592683554,
+      "num_tokens": 15431962.0,
+      "step": 180
+    },
+    {
+      "epoch": 0.20475516939449048,
+      "grad_norm": 0.1809382289648056,
+      "learning_rate": 3.498157378360205e-05,
+      "loss": 0.2411,
+      "mean_token_accuracy": 0.9381648041307926,
+      "num_tokens": 16292904.0,
+      "step": 190
+    },
+    {
+      "epoch": 0.21553175725735838,
+      "grad_norm": 0.17740917205810547,
+      "learning_rate": 3.3305538586569115e-05,
+      "loss": 0.2272,
+      "mean_token_accuracy": 0.9419581316411495,
+      "num_tokens": 17159109.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.21553175725735838,
+      "eval_loss": NaN,
+      "eval_mean_token_accuracy": 0.948202914924118,
+      "eval_num_tokens": 17159109.0,
+      "eval_runtime": 807.2101,
+      "eval_samples_per_second": 1.501,
+      "eval_steps_per_second": 0.375,
+      "step": 200
+    },
+    {
+      "epoch": 0.2263083451202263,
+      "grad_norm": 0.17101362347602844,
+      "learning_rate": 3.158640047298099e-05,
+      "loss": 0.22,
+      "mean_token_accuracy": 0.940649115294218,
+      "num_tokens": 18024050.0,
+      "step": 210
+    },
+    {
+      "epoch": 0.23708493298309422,
+      "grad_norm": 0.20847490429878235,
+      "learning_rate": 2.9833081184141316e-05,
+      "loss": 0.2464,
+      "mean_token_accuracy": 0.9387204153463244,
+      "num_tokens": 18878935.0,
+      "step": 220
+    },
+    {
+      "epoch": 0.24786152084596214,
+      "grad_norm": 0.15264753997325897,
+      "learning_rate": 2.8054679850011827e-05,
+      "loss": 0.2191,
+      "mean_token_accuracy": 0.9407731007784605,
+      "num_tokens": 19736346.0,
+      "step": 230
+    },
+    {
+      "epoch": 0.2586381087088301,
+      "grad_norm": 0.18150846660137177,
+      "learning_rate": 2.6260425767826875e-05,
+      "loss": 0.2076,
+      "mean_token_accuracy": 0.944101719558239,
+      "num_tokens": 20585921.0,
+      "step": 240
+    },
+    {
+      "epoch": 0.269414696571698,
+      "grad_norm": 0.14254361391067505,
+      "learning_rate": 2.4459630505184034e-05,
+      "loss": 0.2322,
+      "mean_token_accuracy": 0.9364934735000133,
+      "num_tokens": 21445814.0,
+      "step": 250
+    },
+    {
+      "epoch": 0.269414696571698,
+      "eval_loss": NaN,
+      "eval_mean_token_accuracy": 0.9491029914849662,
+      "eval_num_tokens": 21445814.0,
+      "eval_runtime": 807.3804,
+      "eval_samples_per_second": 1.501,
+      "eval_steps_per_second": 0.375,
+      "step": 250
+    },
+    {
+      "epoch": 0.2801912844345659,
+      "grad_norm": 0.15035845339298248,
+      "learning_rate": 2.2661639576179678e-05,
+      "loss": 0.2346,
+      "mean_token_accuracy": 0.9386640608310699,
+      "num_tokens": 22309490.0,
+      "step": 260
+    },
+    {
+      "epoch": 0.2909678722974338,
+      "grad_norm": 0.15642204880714417,
+      "learning_rate": 2.087578394137369e-05,
+      "loss": 0.2508,
+      "mean_token_accuracy": 0.934289775043726,
+      "num_tokens": 23187912.0,
+      "step": 270
+    },
+    {
+      "epoch": 0.30174446016030176,
+      "grad_norm": 0.14067009091377258,
+      "learning_rate": 1.9111331583282106e-05,
+      "loss": 0.2967,
+      "mean_token_accuracy": 0.929693965613842,
+      "num_tokens": 24048802.0,
+      "step": 280
+    },
+    {
+      "epoch": 0.31252104802316966,
+      "grad_norm": 0.14918063580989838,
+      "learning_rate": 1.7377439408704392e-05,
+      "loss": 0.2311,
+      "mean_token_accuracy": 0.9404968038201332,
+      "num_tokens": 24904585.0,
+      "step": 290
+    },
+    {
+      "epoch": 0.3232976358860376,
+      "grad_norm": 0.174724742770195,
+      "learning_rate": 1.5683105727495778e-05,
+      "loss": 0.2107,
+      "mean_token_accuracy": 0.9439530853182078,
+      "num_tokens": 25773494.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.3232976358860376,
+      "eval_loss": NaN,
+      "eval_mean_token_accuracy": 0.9496467538399271,
+      "eval_num_tokens": 25773494.0,
+      "eval_runtime": 805.4422,
+      "eval_samples_per_second": 1.505,
+      "eval_steps_per_second": 0.376,
+      "step": 300
+    },
+    {
+      "epoch": 0.3340742237489055,
+      "grad_norm": 0.18415142595767975,
+      "learning_rate": 1.4037123554403783e-05,
+      "loss": 0.2651,
+      "mean_token_accuracy": 0.934630136191845,
+      "num_tokens": 26626532.0,
+      "step": 310
+    },
+    {
+      "epoch": 0.34485081161177344,
+      "grad_norm": 0.17664654552936554,
+      "learning_rate": 1.2448034976316394e-05,
+      "loss": 0.2156,
+      "mean_token_accuracy": 0.9397824894636869,
+      "num_tokens": 27489072.0,
+      "step": 320
+    },
+    {
+      "epoch": 0.35562739947464134,
+      "grad_norm": 0.18474246561527252,
+      "learning_rate": 1.0924086821740436e-05,
+      "loss": 0.253,
+      "mean_token_accuracy": 0.9348474022001028,
+      "num_tokens": 28356866.0,
+      "step": 330
+    },
+    {
+      "epoch": 0.3664039873375093,
+      "grad_norm": 0.17681080102920532,
+      "learning_rate": 9.47318786257029e-06,
+      "loss": 0.2299,
+      "mean_token_accuracy": 0.9430933259427547,
+      "num_tokens": 29201710.0,
+      "step": 340
+    },
+    {
+      "epoch": 0.3771805752003772,
+      "grad_norm": 0.13366708159446716,
+      "learning_rate": 8.102867770255338e-06,
+      "loss": 0.1968,
+      "mean_token_accuracy": 0.9477964635938406,
+      "num_tokens": 30051104.0,
+      "step": 350
+    },
+    {
+      "epoch": 0.3771805752003772,
+      "eval_loss": NaN,
+      "eval_mean_token_accuracy": 0.9500517268778861,
+      "eval_num_tokens": 30051104.0,
+      "eval_runtime": 804.9943,
+      "eval_samples_per_second": 1.506,
+      "eval_steps_per_second": 0.376,
+      "step": 350
+    },
+    {
+      "epoch": 0.3879571630632451,
+      "grad_norm": 0.16616250574588776,
+      "learning_rate": 6.820238039369648e-06,
+      "loss": 0.2548,
+      "mean_token_accuracy": 0.9345032583922148,
+      "num_tokens": 30917570.0,
+      "step": 360
+    },
+    {
+      "epoch": 0.398733750926113,
+      "grad_norm": 0.15461881458759308,
+      "learning_rate": 5.6319550813771105e-06,
+      "loss": 0.2211,
+      "mean_token_accuracy": 0.9464546106755733,
+      "num_tokens": 31790142.0,
+      "step": 370
+    },
+    {
+      "epoch": 0.40951033878898097,
+      "grad_norm": 0.17302246391773224,
+      "learning_rate": 4.544185680123053e-06,
+      "loss": 0.2306,
+      "mean_token_accuracy": 0.9386022459715605,
+      "num_tokens": 32636883.0,
+      "step": 380
+    },
+    {
+      "epoch": 0.42028692665184886,
+      "grad_norm": 0.1838221251964569,
+      "learning_rate": 3.562574988326342e-06,
+      "loss": 0.1777,
+      "mean_token_accuracy": 0.9493914619088173,
+      "num_tokens": 33496196.0,
+      "step": 390
+    },
+    {
+      "epoch": 0.43106351451471675,
+      "grad_norm": 0.1729215532541275,
+      "learning_rate": 2.6922172311593885e-06,
+      "loss": 0.2097,
+      "mean_token_accuracy": 0.9471536636352539,
+      "num_tokens": 34365750.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.43106351451471675,
+      "eval_loss": NaN,
+      "eval_mean_token_accuracy": 0.9508995474761862,
+      "eval_num_tokens": 34365750.0,
+      "eval_runtime": 827.3579,
+      "eval_samples_per_second": 1.465,
+      "eval_steps_per_second": 0.366,
+      "step": 400
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 464,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.5713262092602573e+18,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}