Training in progress, epoch 0

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

KETI_b1_s4_e1_training_log.log

@@ -0,0 +1,28 @@
+03/28/2025 03:26:22 - INFO  - Output Directory: output/gemma-3-27b-pt/20250328_032606_gemma-3-27b-pt_LoRA
+03/28/2025 03:26:22 - INFO  - Experiment name: KETI_b1_s4_e1
+03/28/2025 03:26:22 - INFO  - Using 6 GPU(s): NVIDIA A100-SXM4-80GB
+03/28/2025 03:26:22 - INFO  - torch_dtype: torch.bfloat16
+03/28/2025 03:26:22 - INFO  - ✅ FFT or LoRA 모드로 학습합니다.
+03/28/2025 03:26:45 - INFO  - Initializing LORA model...
+03/28/2025 03:27:05 - INFO  - gcc -pthread -B /root/pai/envs/llm-finetuning/compiler_compat -DNDEBUG -fwrapv -O2 -Wall -fPIC -O2 -isystem /root/pai/envs/llm-finetuning/include -fPIC -O2 -isystem /root/pai/envs/llm-finetuning/include -fPIC -c /tmp/tmpvhsu4dwj/test.c -o /tmp/tmpvhsu4dwj/test.o
+03/28/2025 03:27:06 - INFO  - gcc -pthread -B /root/pai/envs/llm-finetuning/compiler_compat -DNDEBUG -fwrapv -O2 -Wall -fPIC -O2 -isystem /root/pai/envs/llm-finetuning/include -fPIC -O2 -isystem /root/pai/envs/llm-finetuning/include -fPIC -c /tmp/tmp_hmdziqb/test.c -o /tmp/tmp_hmdziqb/test.o
+03/28/2025 03:27:07 - INFO  - Start Training !
+03/28/2025 03:27:39 - INFO  - [Epoch 0.11] [Step 10] loss: 3.5848
+03/28/2025 03:28:04 - INFO  - [Epoch 0.22] [Step 20] loss: 3.0443
+03/28/2025 03:28:30 - INFO  - [Epoch 0.33] [Step 30] loss: 2.9392
+03/28/2025 03:28:55 - INFO  - [Epoch 0.44] [Step 40] loss: 2.9002
+03/28/2025 03:29:18 - INFO  - [Epoch 0.55] [Step 50] loss: 2.8820
+03/28/2025 03:29:42 - INFO  - [Epoch 0.66] [Step 60] loss: 2.8730
+03/28/2025 03:30:06 - INFO  - [Epoch 0.77] [Step 70] loss: 2.8597
+03/28/2025 03:30:29 - INFO  - [Epoch 0.88] [Step 80] loss: 2.8434
+03/28/2025 03:30:52 - INFO  - [Epoch 0.99] [Step 90] loss: 2.8448
+03/28/2025 03:31:18 - INFO  - ✅ Training complete. Logging system usage...
+03/28/2025 03:31:18 - INFO  - >> System Usage - CPU: 3.3%, RAM: 3.6%, SSD: 72.60GB / 1888.43GB
+03/28/2025 03:31:18 - INFO  - >> GPU 0: 78.23 GB used
+03/28/2025 03:31:18 - INFO  - >> GPU 1: 78.33 GB used
+03/28/2025 03:31:18 - INFO  - >> GPU 2: 77.85 GB used
+03/28/2025 03:31:18 - INFO  - >> GPU 3: 78.40 GB used
+03/28/2025 03:31:18 - INFO  - >> GPU 4: 79.06 GB used
+03/28/2025 03:31:18 - INFO  - >> GPU 5: 77.98 GB used
+03/28/2025 03:31:18 - INFO  - >> Total GPU Memory Used: 469.84 GB
+03/28/2025 03:31:18 - INFO  - >> Total GPU Power Consumption: 550.97 W

adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "google/gemma-3-27b-pt",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "o_proj",
+    "k_proj",
+    "v_proj",
+    "up_proj",
+    "gate_proj",
+    "down_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d739e6bea9aae45c1075780480ca0529129c6dfbcf132fad9d64c7438d1b3f2
+size 454183400

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<image_soft_token>": 262144
+}

runs/Mar28_03-26-49_llm-server-779876f58-9zzqd/events.out.tfevents.1743132428.llm-server-779876f58-9zzqd.3709446.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:344898f4c07483ecae5a3309a9d944675734bcb48ea2efc020b66a32f4de64fb
+size 9244

special_tokens_map.json

@@ -0,0 +1,33 @@
+{
+  "boi_token": "<start_of_image>",
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eoi_token": "<end_of_image>",
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "image_token": "<image_soft_token>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4667f2089529e8e7657cfb6d1c19910ae71ff5f28aa7ab2ff2763330affad795
+size 33384568

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c
+size 4689074

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8eda9c7d5c08f1a593c6b3b830efa8c6fce2401cf36c721eab924cc2482b9cea
+size 5816