Upload checkpoint with token

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,202 @@
+---
+base_model: hai2131/sailor2-sft
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.15.2

adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "hai2131/sailor2-sft",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "gate_proj",
+    "k_proj",
+    "up_proj",
+    "q_proj",
+    "down_proj",
+    "v_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3d9817cf262c8f4b78c429ef51037a3c417c0177d938d5ff276103587fd6d06
+size 140860656

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.jinja

@@ -0,0 +1,6 @@
+{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are an AI assistant named Sailor2, created by Sea AI Lab. As an AI assistant, you can answer questions in English, Chinese, and Southeast Asian languages such as Burmese, Cebuano, Ilocano, Indonesian, Javanese, Khmer, Lao, Malay, Sundanese, Tagalog, Thai, Vietnamese, and Waray. Your responses should be friendly, unbiased, informative, detailed, and faithful.<|im_end|>
+' }}{% endif %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9efe7dc75bf541479a10348dc64d800a34b4e599e08c662bf7c176b933e0fc2
+size 72272293

rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cc3307602102ef5b4699c5d0d0877f4df4e9a8c6ed16159e0eb5f7c205b7a82
+size 15365

rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a564fd29fdaea985856b48f282cabacb0dffa1dd1a79d98c9e6177e1de774cf
+size 15365

rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c1fbc115b0e5dd3df1a482bef8a39fc8b8fc0f501ef419324e60d082c1133bc
+size 15365

rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:476ddfa1fd95c624fc018bdacac3fd79e42329b309ca4076c70c58da0694461a
+size 15365

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc3bfdba43e140aa8e5cd5fc947cfb23559f9daad7f487c0e1a19d5a1110e00e
+size 1465

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

tokenizer_config.json

@@ -0,0 +1,214 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "max_length": 3848,
+  "model_max_length": 4096,
+  "pad_to_multiple_of": null,
+  "pad_token": "<|endoftext|>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "stride": 0,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,1234 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 3750,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.04,
+      "grad_norm": 39.1501579284668,
+      "learning_rate": 1.3066666666666667e-06,
+      "logits/chosen": 2.069579839706421,
+      "logits/rejected": 2.1242542266845703,
+      "logps/chosen": -73.9653091430664,
+      "logps/rejected": -88.28418731689453,
+      "loss": 4.8463,
+      "nll_loss": 0.46437376737594604,
+      "rewards/accuracies": 0.7049999833106995,
+      "rewards/chosen": -7.396530628204346,
+      "rewards/margins": 1.431888461112976,
+      "rewards/rejected": -8.828418731689453,
+      "step": 50
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 23.26362419128418,
+      "learning_rate": 2.64e-06,
+      "logits/chosen": 2.039077043533325,
+      "logits/rejected": 2.1060538291931152,
+      "logps/chosen": -57.755859375,
+      "logps/rejected": -72.76531982421875,
+      "loss": 3.8951,
+      "nll_loss": 0.35950180888175964,
+      "rewards/accuracies": 0.7112500071525574,
+      "rewards/chosen": -5.77558708190918,
+      "rewards/margins": 1.5009456872940063,
+      "rewards/rejected": -7.276532173156738,
+      "step": 100
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 29.71898078918457,
+      "learning_rate": 3.973333333333333e-06,
+      "logits/chosen": 2.017021894454956,
+      "logits/rejected": 2.063059091567993,
+      "logps/chosen": -52.110809326171875,
+      "logps/rejected": -63.39885711669922,
+      "loss": 3.5484,
+      "nll_loss": 0.3122263252735138,
+      "rewards/accuracies": 0.6600000262260437,
+      "rewards/chosen": -5.211080551147461,
+      "rewards/margins": 1.1288050413131714,
+      "rewards/rejected": -6.339885711669922,
+      "step": 150
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 21.129802703857422,
+      "learning_rate": 5.306666666666667e-06,
+      "logits/chosen": 2.0352795124053955,
+      "logits/rejected": 2.09993052482605,
+      "logps/chosen": -46.72736740112305,
+      "logps/rejected": -59.81150436401367,
+      "loss": 3.1762,
+      "nll_loss": 0.2814980447292328,
+      "rewards/accuracies": 0.7087500095367432,
+      "rewards/chosen": -4.672736644744873,
+      "rewards/margins": 1.308414340019226,
+      "rewards/rejected": -5.9811506271362305,
+      "step": 200
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 25.128114700317383,
+      "learning_rate": 6.640000000000001e-06,
+      "logits/chosen": 2.0161619186401367,
+      "logits/rejected": 2.080204725265503,
+      "logps/chosen": -46.92048263549805,
+      "logps/rejected": -59.91488265991211,
+      "loss": 3.0645,
+      "nll_loss": 0.2713809609413147,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.6920485496521,
+      "rewards/margins": 1.2994396686553955,
+      "rewards/rejected": -5.991488456726074,
+      "step": 250
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 22.990211486816406,
+      "learning_rate": 7.973333333333334e-06,
+      "logits/chosen": 2.006951332092285,
+      "logits/rejected": 2.0681934356689453,
+      "logps/chosen": -43.61228942871094,
+      "logps/rejected": -58.394710540771484,
+      "loss": 2.8512,
+      "nll_loss": 0.2565268874168396,
+      "rewards/accuracies": 0.7512500286102295,
+      "rewards/chosen": -4.361229419708252,
+      "rewards/margins": 1.4782426357269287,
+      "rewards/rejected": -5.839471817016602,
+      "step": 300
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 17.802993774414062,
+      "learning_rate": 9.306666666666667e-06,
+      "logits/chosen": 1.9981739521026611,
+      "logits/rejected": 2.056286334991455,
+      "logps/chosen": -42.770790100097656,
+      "logps/rejected": -53.638092041015625,
+      "loss": 3.0075,
+      "nll_loss": 0.25887373089790344,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.2770795822143555,
+      "rewards/margins": 1.0867302417755127,
+      "rewards/rejected": -5.363808631896973,
+      "step": 350
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 21.133516311645508,
+      "learning_rate": 9.998752338940612e-06,
+      "logits/chosen": 1.876654028892517,
+      "logits/rejected": 1.9367033243179321,
+      "logps/chosen": -38.542076110839844,
+      "logps/rejected": -48.9306755065918,
+      "loss": 2.6798,
+      "nll_loss": 0.22952309250831604,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -3.8542075157165527,
+      "rewards/margins": 1.0388593673706055,
+      "rewards/rejected": -4.893066883087158,
+      "step": 400
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 23.868408203125,
+      "learning_rate": 9.988142751731797e-06,
+      "logits/chosen": 1.7912520170211792,
+      "logits/rejected": 1.8582316637039185,
+      "logps/chosen": -38.205177307128906,
+      "logps/rejected": -51.677650451660156,
+      "loss": 2.6625,
+      "nll_loss": 0.23552022874355316,
+      "rewards/accuracies": 0.7412499785423279,
+      "rewards/chosen": -3.8205177783966064,
+      "rewards/margins": 1.3472480773925781,
+      "rewards/rejected": -5.167766094207764,
+      "step": 450
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 19.167991638183594,
+      "learning_rate": 9.966729958067638e-06,
+      "logits/chosen": 1.7662477493286133,
+      "logits/rejected": 1.8194670677185059,
+      "logps/chosen": -38.83460998535156,
+      "logps/rejected": -50.7568359375,
+      "loss": 2.7149,
+      "nll_loss": 0.23406948149204254,
+      "rewards/accuracies": 0.7137500047683716,
+      "rewards/chosen": -3.8834612369537354,
+      "rewards/margins": 1.1922229528427124,
+      "rewards/rejected": -5.075684070587158,
+      "step": 500
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 22.746564865112305,
+      "learning_rate": 9.934560333291077e-06,
+      "logits/chosen": 1.6905887126922607,
+      "logits/rejected": 1.756126046180725,
+      "logps/chosen": -38.07200241088867,
+      "logps/rejected": -50.685115814208984,
+      "loss": 2.6286,
+      "nll_loss": 0.22550682723522186,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -3.8072004318237305,
+      "rewards/margins": 1.2613120079040527,
+      "rewards/rejected": -5.068512439727783,
+      "step": 550
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 19.4133358001709,
+      "learning_rate": 9.89170354964594e-06,
+      "logits/chosen": 1.7051059007644653,
+      "logits/rejected": 1.7545236349105835,
+      "logps/chosen": -35.82815933227539,
+      "logps/rejected": -47.575111389160156,
+      "loss": 2.5393,
+      "nll_loss": 0.21860088407993317,
+      "rewards/accuracies": 0.7112500071525574,
+      "rewards/chosen": -3.5828158855438232,
+      "rewards/margins": 1.174695611000061,
+      "rewards/rejected": -4.757511138916016,
+      "step": 600
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 16.378013610839844,
+      "learning_rate": 9.83825242538238e-06,
+      "logits/chosen": 1.6559092998504639,
+      "logits/rejected": 1.7158703804016113,
+      "logps/chosen": -36.47942352294922,
+      "logps/rejected": -49.875587463378906,
+      "loss": 2.5252,
+      "nll_loss": 0.21864959597587585,
+      "rewards/accuracies": 0.7337499856948853,
+      "rewards/chosen": -3.647942304611206,
+      "rewards/margins": 1.3396165370941162,
+      "rewards/rejected": -4.987558364868164,
+      "step": 650
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 15.000406265258789,
+      "learning_rate": 9.774322723733216e-06,
+      "logits/chosen": 1.6370524168014526,
+      "logits/rejected": 1.6942112445831299,
+      "logps/chosen": -34.71821212768555,
+      "logps/rejected": -48.40507507324219,
+      "loss": 2.4278,
+      "nll_loss": 0.21341566741466522,
+      "rewards/accuracies": 0.7549999952316284,
+      "rewards/chosen": -3.471820592880249,
+      "rewards/margins": 1.368686556816101,
+      "rewards/rejected": -4.8405070304870605,
+      "step": 700
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 17.42306900024414,
+      "learning_rate": 9.700052902196541e-06,
+      "logits/chosen": 1.606693148612976,
+      "logits/rejected": 1.6712034940719604,
+      "logps/chosen": -34.60812759399414,
+      "logps/rejected": -48.792850494384766,
+      "loss": 2.4102,
+      "nll_loss": 0.21256333589553833,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -3.460812568664551,
+      "rewards/margins": 1.4184722900390625,
+      "rewards/rejected": -4.879284858703613,
+      "step": 750
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 18.26099967956543,
+      "learning_rate": 9.615603812667618e-06,
+      "logits/chosen": 1.574639916419983,
+      "logits/rejected": 1.6343886852264404,
+      "logps/chosen": -34.159088134765625,
+      "logps/rejected": -48.842376708984375,
+      "loss": 2.3734,
+      "nll_loss": 0.20658308267593384,
+      "rewards/accuracies": 0.7512500286102295,
+      "rewards/chosen": -3.4159088134765625,
+      "rewards/margins": 1.4683291912078857,
+      "rewards/rejected": -4.884237766265869,
+      "step": 800
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 19.284168243408203,
+      "learning_rate": 9.521158353069494e-06,
+      "logits/chosen": 1.5315083265304565,
+      "logits/rejected": 1.5860241651535034,
+      "logps/chosen": -35.90006637573242,
+      "logps/rejected": -51.2423095703125,
+      "loss": 2.4185,
+      "nll_loss": 0.2139773815870285,
+      "rewards/accuracies": 0.7487499713897705,
+      "rewards/chosen": -3.5900063514709473,
+      "rewards/margins": 1.5342237949371338,
+      "rewards/rejected": -5.124230861663818,
+      "step": 850
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 28.010725021362305,
+      "learning_rate": 9.416921071236821e-06,
+      "logits/chosen": 1.562249779701233,
+      "logits/rejected": 1.6213380098342896,
+      "logps/chosen": -34.88312530517578,
+      "logps/rejected": -49.12727355957031,
+      "loss": 2.4402,
+      "nll_loss": 0.20900359749794006,
+      "rewards/accuracies": 0.7362499833106995,
+      "rewards/chosen": -3.4883129596710205,
+      "rewards/margins": 1.4244145154953003,
+      "rewards/rejected": -4.912726879119873,
+      "step": 900
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 22.14881134033203,
+      "learning_rate": 9.303117721910801e-06,
+      "logits/chosen": 1.5611748695373535,
+      "logits/rejected": 1.613703966140747,
+      "logps/chosen": -33.08067321777344,
+      "logps/rejected": -46.956138610839844,
+      "loss": 2.2957,
+      "nll_loss": 0.19786672294139862,
+      "rewards/accuracies": 0.7524999976158142,
+      "rewards/chosen": -3.3080673217773438,
+      "rewards/margins": 1.3875468969345093,
+      "rewards/rejected": -4.695613861083984,
+      "step": 950
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 18.468719482421875,
+      "learning_rate": 9.179994777804677e-06,
+      "logits/chosen": 1.532340407371521,
+      "logits/rejected": 1.595700979232788,
+      "logps/chosen": -33.659690856933594,
+      "logps/rejected": -48.1495475769043,
+      "loss": 2.3428,
+      "nll_loss": 0.2023465782403946,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.365969657897949,
+      "rewards/margins": 1.4489855766296387,
+      "rewards/rejected": -4.81495475769043,
+      "step": 1000
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 19.125810623168945,
+      "learning_rate": 9.04781889579873e-06,
+      "logits/chosen": 1.4953616857528687,
+      "logits/rejected": 1.5516945123672485,
+      "logps/chosen": -33.457584381103516,
+      "logps/rejected": -48.823360443115234,
+      "loss": 2.2845,
+      "nll_loss": 0.2007952332496643,
+      "rewards/accuracies": 0.7612500190734863,
+      "rewards/chosen": -3.3457586765289307,
+      "rewards/margins": 1.5365774631500244,
+      "rewards/rejected": -4.882336139678955,
+      "step": 1050
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 18.376596450805664,
+      "learning_rate": 8.906876339420851e-06,
+      "logits/chosen": 1.452122449874878,
+      "logits/rejected": 1.519229531288147,
+      "logps/chosen": -31.664844512939453,
+      "logps/rejected": -47.539306640625,
+      "loss": 2.1647,
+      "nll_loss": 0.19097968935966492,
+      "rewards/accuracies": 0.7925000190734863,
+      "rewards/chosen": -3.1664845943450928,
+      "rewards/margins": 1.5874464511871338,
+      "rewards/rejected": -4.753930568695068,
+      "step": 1100
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 20.535789489746094,
+      "learning_rate": 8.757472358863481e-06,
+      "logits/chosen": 1.4528400897979736,
+      "logits/rejected": 1.5169428586959839,
+      "logps/chosen": -33.37004089355469,
+      "logps/rejected": -47.855865478515625,
+      "loss": 2.2511,
+      "nll_loss": 0.19547508656978607,
+      "rewards/accuracies": 0.7574999928474426,
+      "rewards/chosen": -3.337003707885742,
+      "rewards/margins": 1.4485827684402466,
+      "rewards/rejected": -4.785586357116699,
+      "step": 1150
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 22.283950805664062,
+      "learning_rate": 8.599930529879669e-06,
+      "logits/chosen": 1.4502493143081665,
+      "logits/rejected": 1.5124887228012085,
+      "logps/chosen": -31.987987518310547,
+      "logps/rejected": -47.01323699951172,
+      "loss": 2.1816,
+      "nll_loss": 0.19316434860229492,
+      "rewards/accuracies": 0.7774999737739563,
+      "rewards/chosen": -3.198798418045044,
+      "rewards/margins": 1.5025256872177124,
+      "rewards/rejected": -4.701324462890625,
+      "step": 1200
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 19.24060821533203,
+      "learning_rate": 8.434592052990044e-06,
+      "logits/chosen": 1.4031856060028076,
+      "logits/rejected": 1.462039589881897,
+      "logps/chosen": -30.157163619995117,
+      "logps/rejected": -45.0657844543457,
+      "loss": 2.1647,
+      "nll_loss": 0.18531297147274017,
+      "rewards/accuracies": 0.7662500143051147,
+      "rewards/chosen": -3.015716314315796,
+      "rewards/margins": 1.4908628463745117,
+      "rewards/rejected": -4.5065789222717285,
+      "step": 1250
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 16.792695999145508,
+      "learning_rate": 8.261815014518465e-06,
+      "logits/chosen": 1.334591031074524,
+      "logits/rejected": 1.3955891132354736,
+      "logps/chosen": -29.710214614868164,
+      "logps/rejected": -49.487152099609375,
+      "loss": 1.9604,
+      "nll_loss": 0.17893625795841217,
+      "rewards/accuracies": 0.8450000286102295,
+      "rewards/chosen": -2.9710214138031006,
+      "rewards/margins": 1.977694034576416,
+      "rewards/rejected": -4.948715686798096,
+      "step": 1300
+    },
+    {
+      "epoch": 1.08,
+      "grad_norm": 21.147741317749023,
+      "learning_rate": 8.081973611056784e-06,
+      "logits/chosen": 1.257203221321106,
+      "logits/rejected": 1.3112421035766602,
+      "logps/chosen": -28.43024444580078,
+      "logps/rejected": -48.68648910522461,
+      "loss": 1.872,
+      "nll_loss": 0.16993314027786255,
+      "rewards/accuracies": 0.8662499785423279,
+      "rewards/chosen": -2.843024492263794,
+      "rewards/margins": 2.0256245136260986,
+      "rewards/rejected": -4.868649482727051,
+      "step": 1350
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 18.095643997192383,
+      "learning_rate": 7.89545733903834e-06,
+      "logits/chosen": 1.268505573272705,
+      "logits/rejected": 1.3283963203430176,
+      "logps/chosen": -29.838397979736328,
+      "logps/rejected": -48.807472229003906,
+      "loss": 1.9653,
+      "nll_loss": 0.18002687394618988,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -2.983839988708496,
+      "rewards/margins": 1.8969072103500366,
+      "rewards/rejected": -4.880747318267822,
+      "step": 1400
+    },
+    {
+      "epoch": 1.16,
+      "grad_norm": 20.254247665405273,
+      "learning_rate": 7.702670151175435e-06,
+      "logits/chosen": 1.3044127225875854,
+      "logits/rejected": 1.368518352508545,
+      "logps/chosen": -29.120044708251953,
+      "logps/rejected": -47.06825637817383,
+      "loss": 1.9557,
+      "nll_loss": 0.1742786169052124,
+      "rewards/accuracies": 0.8387500047683716,
+      "rewards/chosen": -2.9120047092437744,
+      "rewards/margins": 1.7948211431503296,
+      "rewards/rejected": -4.706826210021973,
+      "step": 1450
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 15.50015640258789,
+      "learning_rate": 7.5040295815877e-06,
+      "logits/chosen": 1.2754545211791992,
+      "logits/rejected": 1.33790922164917,
+      "logps/chosen": -28.515518188476562,
+      "logps/rejected": -49.134769439697266,
+      "loss": 1.8568,
+      "nll_loss": 0.16948945820331573,
+      "rewards/accuracies": 0.8737499713897705,
+      "rewards/chosen": -2.8515517711639404,
+      "rewards/margins": 2.061924934387207,
+      "rewards/rejected": -4.913476943969727,
+      "step": 1500
+    },
+    {
+      "epoch": 1.24,
+      "grad_norm": 18.263025283813477,
+      "learning_rate": 7.299965841516164e-06,
+      "logits/chosen": 1.2721493244171143,
+      "logits/rejected": 1.3371402025222778,
+      "logps/chosen": -29.20949935913086,
+      "logps/rejected": -48.70906066894531,
+      "loss": 1.9402,
+      "nll_loss": 0.17609195411205292,
+      "rewards/accuracies": 0.8550000190734863,
+      "rewards/chosen": -2.920949935913086,
+      "rewards/margins": 1.9499554634094238,
+      "rewards/rejected": -4.87090539932251,
+      "step": 1550
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 17.24570655822754,
+      "learning_rate": 7.090920887581507e-06,
+      "logits/chosen": 1.2389811277389526,
+      "logits/rejected": 1.2887934446334839,
+      "logps/chosen": -28.364063262939453,
+      "logps/rejected": -46.97511291503906,
+      "loss": 1.8952,
+      "nll_loss": 0.1697738617658615,
+      "rewards/accuracies": 0.8362500071525574,
+      "rewards/chosen": -2.8364062309265137,
+      "rewards/margins": 1.8611053228378296,
+      "rewards/rejected": -4.697511196136475,
+      "step": 1600
+    },
+    {
+      "epoch": 1.32,
+      "grad_norm": 20.073883056640625,
+      "learning_rate": 6.877347464604446e-06,
+      "logits/chosen": 1.2145293951034546,
+      "logits/rejected": 1.2653400897979736,
+      "logps/chosen": -28.30023765563965,
+      "logps/rejected": -47.83637619018555,
+      "loss": 1.8857,
+      "nll_loss": 0.16985949873924255,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.830024003982544,
+      "rewards/margins": 1.9536142349243164,
+      "rewards/rejected": -4.7836384773254395,
+      "step": 1650
+    },
+    {
+      "epoch": 1.3599999999999999,
+      "grad_norm": 20.44426727294922,
+      "learning_rate": 6.659708125061242e-06,
+      "logits/chosen": 1.210806131362915,
+      "logits/rejected": 1.2653027772903442,
+      "logps/chosen": -28.112794876098633,
+      "logps/rejected": -47.89250183105469,
+      "loss": 1.8333,
+      "nll_loss": 0.16796012222766876,
+      "rewards/accuracies": 0.8637499809265137,
+      "rewards/chosen": -2.811279535293579,
+      "rewards/margins": 1.9779702425003052,
+      "rewards/rejected": -4.789250373840332,
+      "step": 1700
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 15.871622085571289,
+      "learning_rate": 6.438474227298065e-06,
+      "logits/chosen": 1.2420759201049805,
+      "logits/rejected": 1.296012282371521,
+      "logps/chosen": -28.223691940307617,
+      "logps/rejected": -47.52497863769531,
+      "loss": 1.8911,
+      "nll_loss": 0.1713765263557434,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -2.82236909866333,
+      "rewards/margins": 1.9301284551620483,
+      "rewards/rejected": -4.752498149871826,
+      "step": 1750
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 17.20113182067871,
+      "learning_rate": 6.2141249146737545e-06,
+      "logits/chosen": 1.2283880710601807,
+      "logits/rejected": 1.2828854322433472,
+      "logps/chosen": -26.78079605102539,
+      "logps/rejected": -45.25880813598633,
+      "loss": 1.8039,
+      "nll_loss": 0.15980492532253265,
+      "rewards/accuracies": 0.8450000286102295,
+      "rewards/chosen": -2.678079605102539,
+      "rewards/margins": 1.8478012084960938,
+      "rewards/rejected": -4.525880813598633,
+      "step": 1800
+    },
+    {
+      "epoch": 1.48,
+      "grad_norm": 20.9070987701416,
+      "learning_rate": 5.987146077842015e-06,
+      "logits/chosen": 1.2112573385238647,
+      "logits/rejected": 1.2712033987045288,
+      "logps/chosen": -28.28427505493164,
+      "logps/rejected": -48.554771423339844,
+      "loss": 1.8766,
+      "nll_loss": 0.1684311032295227,
+      "rewards/accuracies": 0.8387500047683716,
+      "rewards/chosen": -2.828427314758301,
+      "rewards/margins": 2.0270493030548096,
+      "rewards/rejected": -4.855477333068848,
+      "step": 1850
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 16.220539093017578,
+      "learning_rate": 5.7580293024204455e-06,
+      "logits/chosen": 1.2373614311218262,
+      "logits/rejected": 1.2950869798660278,
+      "logps/chosen": -27.206884384155273,
+      "logps/rejected": -46.864036560058594,
+      "loss": 1.7983,
+      "nll_loss": 0.16223596036434174,
+      "rewards/accuracies": 0.8712499737739563,
+      "rewards/chosen": -2.720688819885254,
+      "rewards/margins": 1.9657151699066162,
+      "rewards/rejected": -4.686403751373291,
+      "step": 1900
+    },
+    {
+      "epoch": 1.56,
+      "grad_norm": 19.87531852722168,
+      "learning_rate": 5.5272708043255605e-06,
+      "logits/chosen": 1.2204946279525757,
+      "logits/rejected": 1.266890048980713,
+      "logps/chosen": -26.85431671142578,
+      "logps/rejected": -45.090606689453125,
+      "loss": 1.8634,
+      "nll_loss": 0.1644485890865326,
+      "rewards/accuracies": 0.8387500047683716,
+      "rewards/chosen": -2.685431718826294,
+      "rewards/margins": 1.8236281871795654,
+      "rewards/rejected": -4.509060382843018,
+      "step": 1950
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 20.69000244140625,
+      "learning_rate": 5.295370355079615e-06,
+      "logits/chosen": 1.1966464519500732,
+      "logits/rejected": 1.2531131505966187,
+      "logps/chosen": -26.715871810913086,
+      "logps/rejected": -46.64927673339844,
+      "loss": 1.7931,
+      "nll_loss": 0.16148775815963745,
+      "rewards/accuracies": 0.8424999713897705,
+      "rewards/chosen": -2.6715869903564453,
+      "rewards/margins": 1.993341088294983,
+      "rewards/rejected": -4.664927959442139,
+      "step": 2000
+    },
+    {
+      "epoch": 1.6400000000000001,
+      "grad_norm": 20.352148056030273,
+      "learning_rate": 5.062830199416764e-06,
+      "logits/chosen": 1.2095577716827393,
+      "logits/rejected": 1.2642172574996948,
+      "logps/chosen": -27.680116653442383,
+      "logps/rejected": -47.270565032958984,
+      "loss": 1.8336,
+      "nll_loss": 0.1655382215976715,
+      "rewards/accuracies": 0.8487499952316284,
+      "rewards/chosen": -2.768012046813965,
+      "rewards/margins": 1.9590446949005127,
+      "rewards/rejected": -4.727056980133057,
+      "step": 2050
+    },
+    {
+      "epoch": 1.6800000000000002,
+      "grad_norm": 20.977998733520508,
+      "learning_rate": 4.8301539675328205e-06,
+      "logits/chosen": 1.1827168464660645,
+      "logits/rejected": 1.2348774671554565,
+      "logps/chosen": -26.652387619018555,
+      "logps/rejected": -46.84341812133789,
+      "loss": 1.758,
+      "nll_loss": 0.15646891295909882,
+      "rewards/accuracies": 0.8512499928474426,
+      "rewards/chosen": -2.665238857269287,
+      "rewards/margins": 2.0191030502319336,
+      "rewards/rejected": -4.684341907501221,
+      "step": 2100
+    },
+    {
+      "epoch": 1.72,
+      "grad_norm": 18.503433227539062,
+      "learning_rate": 4.597845584334387e-06,
+      "logits/chosen": 1.1708920001983643,
+      "logits/rejected": 1.2160160541534424,
+      "logps/chosen": -27.61713409423828,
+      "logps/rejected": -47.54500961303711,
+      "loss": 1.8257,
+      "nll_loss": 0.1635328233242035,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -2.7617132663726807,
+      "rewards/margins": 1.9927879571914673,
+      "rewards/rejected": -4.7545013427734375,
+      "step": 2150
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 17.69391441345215,
+      "learning_rate": 4.366408178049728e-06,
+      "logits/chosen": 1.184067964553833,
+      "logits/rejected": 1.2318971157073975,
+      "logps/chosen": -25.899795532226562,
+      "logps/rejected": -45.311283111572266,
+      "loss": 1.7548,
+      "nll_loss": 0.15687091648578644,
+      "rewards/accuracies": 0.8550000190734863,
+      "rewards/chosen": -2.589979648590088,
+      "rewards/margins": 1.9411489963531494,
+      "rewards/rejected": -4.531128406524658,
+      "step": 2200
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 21.093082427978516,
+      "learning_rate": 4.136342990565055e-06,
+      "logits/chosen": 1.165001630783081,
+      "logits/rejected": 1.2157142162322998,
+      "logps/chosen": -26.217182159423828,
+      "logps/rejected": -47.43375015258789,
+      "loss": 1.7791,
+      "nll_loss": 0.16027042269706726,
+      "rewards/accuracies": 0.8675000071525574,
+      "rewards/chosen": -2.621718406677246,
+      "rewards/margins": 2.121656894683838,
+      "rewards/rejected": -4.743375301361084,
+      "step": 2250
+    },
+    {
+      "epoch": 1.8399999999999999,
+      "grad_norm": 22.38056182861328,
+      "learning_rate": 3.908148291846225e-06,
+      "logits/chosen": 1.1481218338012695,
+      "logits/rejected": 1.1908549070358276,
+      "logps/chosen": -24.328386306762695,
+      "logps/rejected": -43.8892707824707,
+      "loss": 1.683,
+      "nll_loss": 0.1506115198135376,
+      "rewards/accuracies": 0.8700000047683716,
+      "rewards/chosen": -2.4328386783599854,
+      "rewards/margins": 1.9560893774032593,
+      "rewards/rejected": -4.388927459716797,
+      "step": 2300
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 38.030303955078125,
+      "learning_rate": 3.6823183007969375e-06,
+      "logits/chosen": 1.1535931825637817,
+      "logits/rejected": 1.2029131650924683,
+      "logps/chosen": -25.072856903076172,
+      "logps/rejected": -44.75934982299805,
+      "loss": 1.7215,
+      "nll_loss": 0.15159070491790771,
+      "rewards/accuracies": 0.8424999713897705,
+      "rewards/chosen": -2.5072858333587646,
+      "rewards/margins": 1.9686492681503296,
+      "rewards/rejected": -4.475934982299805,
+      "step": 2350
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 16.922834396362305,
+      "learning_rate": 3.4593421148906523e-06,
+      "logits/chosen": 1.150818943977356,
+      "logits/rejected": 1.20121431350708,
+      "logps/chosen": -27.409215927124023,
+      "logps/rejected": -46.66661071777344,
+      "loss": 1.7911,
+      "nll_loss": 0.15817226469516754,
+      "rewards/accuracies": 0.8450000286102295,
+      "rewards/chosen": -2.740921974182129,
+      "rewards/margins": 1.9257392883300781,
+      "rewards/rejected": -4.666661262512207,
+      "step": 2400
+    },
+    {
+      "epoch": 1.96,
+      "grad_norm": 15.894891738891602,
+      "learning_rate": 3.239702650894364e-06,
+      "logits/chosen": 1.1605645418167114,
+      "logits/rejected": 1.2133049964904785,
+      "logps/chosen": -25.806852340698242,
+      "logps/rejected": -46.503700256347656,
+      "loss": 1.739,
+      "nll_loss": 0.15509046614170074,
+      "rewards/accuracies": 0.8550000190734863,
+      "rewards/chosen": -2.5806851387023926,
+      "rewards/margins": 2.0696845054626465,
+      "rewards/rejected": -4.650369644165039,
+      "step": 2450
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 17.814403533935547,
+      "learning_rate": 3.023875598978419e-06,
+      "logits/chosen": 1.1574615240097046,
+      "logits/rejected": 1.2072219848632812,
+      "logps/chosen": -25.269817352294922,
+      "logps/rejected": -47.318443298339844,
+      "loss": 1.6896,
+      "nll_loss": 0.15325431525707245,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -2.5269815921783447,
+      "rewards/margins": 2.2048630714416504,
+      "rewards/rejected": -4.731843948364258,
+      "step": 2500
+    },
+    {
+      "epoch": 2.04,
+      "grad_norm": 17.23746109008789,
+      "learning_rate": 2.812328392477536e-06,
+      "logits/chosen": 1.106400489807129,
+      "logits/rejected": 1.1545926332473755,
+      "logps/chosen": -23.484037399291992,
+      "logps/rejected": -48.1876106262207,
+      "loss": 1.499,
+      "nll_loss": 0.14059405028820038,
+      "rewards/accuracies": 0.9449999928474426,
+      "rewards/chosen": -2.3484036922454834,
+      "rewards/margins": 2.470357656478882,
+      "rewards/rejected": -4.818761825561523,
+      "step": 2550
+    },
+    {
+      "epoch": 2.08,
+      "grad_norm": 14.634954452514648,
+      "learning_rate": 2.6055191955342886e-06,
+      "logits/chosen": 1.0940419435501099,
+      "logits/rejected": 1.1334995031356812,
+      "logps/chosen": -24.663402557373047,
+      "logps/rejected": -49.43788146972656,
+      "loss": 1.5717,
+      "nll_loss": 0.1473802775144577,
+      "rewards/accuracies": 0.9275000095367432,
+      "rewards/chosen": -2.4663405418395996,
+      "rewards/margins": 2.477447509765625,
+      "rewards/rejected": -4.943788051605225,
+      "step": 2600
+    },
+    {
+      "epoch": 2.12,
+      "grad_norm": 17.942777633666992,
+      "learning_rate": 2.403895910817593e-06,
+      "logits/chosen": 1.0923943519592285,
+      "logits/rejected": 1.1312522888183594,
+      "logps/chosen": -23.995264053344727,
+      "logps/rejected": -47.21249008178711,
+      "loss": 1.5767,
+      "nll_loss": 0.14557117223739624,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -2.399526357650757,
+      "rewards/margins": 2.3217225074768066,
+      "rewards/rejected": -4.721249103546143,
+      "step": 2650
+    },
+    {
+      "epoch": 2.16,
+      "grad_norm": 16.142436981201172,
+      "learning_rate": 2.2078952094652705e-06,
+      "logits/chosen": 1.0400832891464233,
+      "logits/rejected": 1.1018368005752563,
+      "logps/chosen": -24.179841995239258,
+      "logps/rejected": -50.50990295410156,
+      "loss": 1.5025,
+      "nll_loss": 0.14027239382266998,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -2.4179842472076416,
+      "rewards/margins": 2.6330063343048096,
+      "rewards/rejected": -5.050990104675293,
+      "step": 2700
+    },
+    {
+      "epoch": 2.2,
+      "grad_norm": 19.593612670898438,
+      "learning_rate": 2.017941585351591e-06,
+      "logits/chosen": 1.0729751586914062,
+      "logits/rejected": 1.1096735000610352,
+      "logps/chosen": -24.066631317138672,
+      "logps/rejected": -47.574344635009766,
+      "loss": 1.5804,
+      "nll_loss": 0.14690618216991425,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -2.406662940979004,
+      "rewards/margins": 2.3507721424102783,
+      "rewards/rejected": -4.757434844970703,
+      "step": 2750
+    },
+    {
+      "epoch": 2.24,
+      "grad_norm": 14.078289985656738,
+      "learning_rate": 1.8344464357280722e-06,
+      "logits/chosen": 1.0587913990020752,
+      "logits/rejected": 1.0995149612426758,
+      "logps/chosen": -23.153541564941406,
+      "logps/rejected": -47.50178527832031,
+      "loss": 1.4804,
+      "nll_loss": 0.13591693341732025,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -2.315354347229004,
+      "rewards/margins": 2.4348244667053223,
+      "rewards/rejected": -4.750178813934326,
+      "step": 2800
+    },
+    {
+      "epoch": 2.2800000000000002,
+      "grad_norm": 15.281038284301758,
+      "learning_rate": 1.6578071702286396e-06,
+      "logits/chosen": 1.0582990646362305,
+      "logits/rejected": 1.0928741693496704,
+      "logps/chosen": -23.392786026000977,
+      "logps/rejected": -47.97736358642578,
+      "loss": 1.5417,
+      "nll_loss": 0.1428249180316925,
+      "rewards/accuracies": 0.9175000190734863,
+      "rewards/chosen": -2.33927845954895,
+      "rewards/margins": 2.4584577083587646,
+      "rewards/rejected": -4.797736167907715,
+      "step": 2850
+    },
+    {
+      "epoch": 2.32,
+      "grad_norm": 15.694136619567871,
+      "learning_rate": 1.4884063501688539e-06,
+      "logits/chosen": 1.064022183418274,
+      "logits/rejected": 1.1182899475097656,
+      "logps/chosen": -23.681140899658203,
+      "logps/rejected": -48.91339874267578,
+      "loss": 1.4978,
+      "nll_loss": 0.1389184594154358,
+      "rewards/accuracies": 0.9287499785423279,
+      "rewards/chosen": -2.3681137561798096,
+      "rewards/margins": 2.523225784301758,
+      "rewards/rejected": -4.8913397789001465,
+      "step": 2900
+    },
+    {
+      "epoch": 2.36,
+      "grad_norm": 19.51872444152832,
+      "learning_rate": 1.3266108600032928e-06,
+      "logits/chosen": 1.0682127475738525,
+      "logits/rejected": 1.1111141443252563,
+      "logps/chosen": -24.015024185180664,
+      "logps/rejected": -47.481693267822266,
+      "loss": 1.5333,
+      "nll_loss": 0.14073985815048218,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -2.4015023708343506,
+      "rewards/margins": 2.3466668128967285,
+      "rewards/rejected": -4.748169422149658,
+      "step": 2950
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 21.052757263183594,
+      "learning_rate": 1.1727711127355118e-06,
+      "logits/chosen": 1.054885745048523,
+      "logits/rejected": 1.0945353507995605,
+      "logps/chosen": -22.908018112182617,
+      "logps/rejected": -46.528038024902344,
+      "loss": 1.4819,
+      "nll_loss": 0.13584649562835693,
+      "rewards/accuracies": 0.9100000262260437,
+      "rewards/chosen": -2.290801525115967,
+      "rewards/margins": 2.362002372741699,
+      "rewards/rejected": -4.652803897857666,
+      "step": 3000
+    },
+    {
+      "epoch": 2.44,
+      "grad_norm": 16.752124786376953,
+      "learning_rate": 1.0272202910015083e-06,
+      "logits/chosen": 1.0628950595855713,
+      "logits/rejected": 1.1132755279541016,
+      "logps/chosen": -23.901992797851562,
+      "logps/rejected": -48.75297927856445,
+      "loss": 1.5343,
+      "nll_loss": 0.14184221625328064,
+      "rewards/accuracies": 0.9175000190734863,
+      "rewards/chosen": -2.390199661254883,
+      "rewards/margins": 2.485098361968994,
+      "rewards/rejected": -4.875298023223877,
+      "step": 3050
+    },
+    {
+      "epoch": 2.48,
+      "grad_norm": 20.135562896728516,
+      "learning_rate": 8.902736254703347e-07,
+      "logits/chosen": 1.0642120838165283,
+      "logits/rejected": 1.0948975086212158,
+      "logps/chosen": -22.418310165405273,
+      "logps/rejected": -44.414913177490234,
+      "loss": 1.4982,
+      "nll_loss": 0.13570523262023926,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -2.241831064224243,
+      "rewards/margins": 2.199659824371338,
+      "rewards/rejected": -4.441490650177002,
+      "step": 3100
+    },
+    {
+      "epoch": 2.52,
+      "grad_norm": 17.3437557220459,
+      "learning_rate": 7.622277121246513e-07,
+      "logits/chosen": 1.0698988437652588,
+      "logits/rejected": 1.1142799854278564,
+      "logps/chosen": -23.52094078063965,
+      "logps/rejected": -47.37723159790039,
+      "loss": 1.5275,
+      "nll_loss": 0.13999156653881073,
+      "rewards/accuracies": 0.9137499928474426,
+      "rewards/chosen": -2.3520936965942383,
+      "rewards/margins": 2.385629415512085,
+      "rewards/rejected": -4.737722873687744,
+      "step": 3150
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 19.78534698486328,
+      "learning_rate": 6.433598698998766e-07,
+      "logits/chosen": 1.065538763999939,
+      "logits/rejected": 1.118047833442688,
+      "logps/chosen": -23.858076095581055,
+      "logps/rejected": -46.57390594482422,
+      "loss": 1.5673,
+      "nll_loss": 0.1432953178882599,
+      "rewards/accuracies": 0.9024999737739563,
+      "rewards/chosen": -2.385807752609253,
+      "rewards/margins": 2.271583318710327,
+      "rewards/rejected": -4.657390594482422,
+      "step": 3200
+    },
+    {
+      "epoch": 2.6,
+      "grad_norm": 20.778789520263672,
+      "learning_rate": 5.339275400731331e-07,
+      "logits/chosen": 1.064239263534546,
+      "logits/rejected": 1.1029696464538574,
+      "logps/chosen": -23.444625854492188,
+      "logps/rejected": -45.59990692138672,
+      "loss": 1.5517,
+      "nll_loss": 0.14206433296203613,
+      "rewards/accuracies": 0.9150000214576721,
+      "rewards/chosen": -2.3444626331329346,
+      "rewards/margins": 2.2155280113220215,
+      "rewards/rejected": -4.559990882873535,
+      "step": 3250
+    },
+    {
+      "epoch": 2.64,
+      "grad_norm": 13.808253288269043,
+      "learning_rate": 4.3416772870275295e-07,
+      "logits/chosen": 1.0500396490097046,
+      "logits/rejected": 1.0961949825286865,
+      "logps/chosen": -23.120912551879883,
+      "logps/rejected": -47.897361755371094,
+      "loss": 1.479,
+      "nll_loss": 0.13585034012794495,
+      "rewards/accuracies": 0.9275000095367432,
+      "rewards/chosen": -2.31209135055542,
+      "rewards/margins": 2.4776451587677,
+      "rewards/rejected": -4.789736270904541,
+      "step": 3300
+    },
+    {
+      "epoch": 2.68,
+      "grad_norm": 13.967582702636719,
+      "learning_rate": 3.442964933259474e-07,
+      "logits/chosen": 1.0531511306762695,
+      "logits/rejected": 1.1031429767608643,
+      "logps/chosen": -23.176462173461914,
+      "logps/rejected": -48.2183723449707,
+      "loss": 1.4983,
+      "nll_loss": 0.138520285487175,
+      "rewards/accuracies": 0.9137499928474426,
+      "rewards/chosen": -2.3176462650299072,
+      "rewards/margins": 2.5041911602020264,
+      "rewards/rejected": -4.821837425231934,
+      "step": 3350
+    },
+    {
+      "epoch": 2.7199999999999998,
+      "grad_norm": 18.23953628540039,
+      "learning_rate": 2.6450847502627883e-07,
+      "logits/chosen": 1.038626790046692,
+      "logits/rejected": 1.0861948728561401,
+      "logps/chosen": -23.72449493408203,
+      "logps/rejected": -49.453460693359375,
+      "loss": 1.4878,
+      "nll_loss": 0.13710632920265198,
+      "rewards/accuracies": 0.9175000190734863,
+      "rewards/chosen": -2.3724493980407715,
+      "rewards/margins": 2.572896957397461,
+      "rewards/rejected": -4.945346832275391,
+      "step": 3400
+    },
+    {
+      "epoch": 2.76,
+      "grad_norm": 19.383655548095703,
+      "learning_rate": 1.9497647688442478e-07,
+      "logits/chosen": 1.0585306882858276,
+      "logits/rejected": 1.1055423021316528,
+      "logps/chosen": -22.849088668823242,
+      "logps/rejected": -46.4496955871582,
+      "loss": 1.5048,
+      "nll_loss": 0.13851623237133026,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -2.2849090099334717,
+      "rewards/margins": 2.360060214996338,
+      "rewards/rejected": -4.6449689865112305,
+      "step": 3450
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 18.49037742614746,
+      "learning_rate": 1.358510897251808e-07,
+      "logits/chosen": 1.0519126653671265,
+      "logits/rejected": 1.0990923643112183,
+      "logps/chosen": -22.153770446777344,
+      "logps/rejected": -47.41790008544922,
+      "loss": 1.4731,
+      "nll_loss": 0.13626331090927124,
+      "rewards/accuracies": 0.9212499856948853,
+      "rewards/chosen": -2.21537709236145,
+      "rewards/margins": 2.5264134407043457,
+      "rewards/rejected": -4.741790294647217,
+      "step": 3500
+    },
+    {
+      "epoch": 2.84,
+      "grad_norm": 16.536218643188477,
+      "learning_rate": 8.726036597126619e-08,
+      "logits/chosen": 1.0618226528167725,
+      "logits/rejected": 1.1148923635482788,
+      "logps/chosen": -22.412979125976562,
+      "logps/rejected": -45.536277770996094,
+      "loss": 1.4864,
+      "nll_loss": 0.1364334225654602,
+      "rewards/accuracies": 0.9162499904632568,
+      "rewards/chosen": -2.241297960281372,
+      "rewards/margins": 2.3123297691345215,
+      "rewards/rejected": -4.5536274909973145,
+      "step": 3550
+    },
+    {
+      "epoch": 2.88,
+      "grad_norm": 20.392322540283203,
+      "learning_rate": 4.93095423102935e-08,
+      "logits/chosen": 1.0614595413208008,
+      "logits/rejected": 1.1021995544433594,
+      "logps/chosen": -24.838594436645508,
+      "logps/rejected": -49.431583404541016,
+      "loss": 1.5749,
+      "nll_loss": 0.14667731523513794,
+      "rewards/accuracies": 0.9049999713897705,
+      "rewards/chosen": -2.4838597774505615,
+      "rewards/margins": 2.459298610687256,
+      "rewards/rejected": -4.943158149719238,
+      "step": 3600
+    },
+    {
+      "epoch": 2.92,
+      "grad_norm": 17.081296920776367,
+      "learning_rate": 2.2080811775535006e-08,
+      "logits/chosen": 1.0633502006530762,
+      "logits/rejected": 1.1077544689178467,
+      "logps/chosen": -24.01654052734375,
+      "logps/rejected": -49.075321197509766,
+      "loss": 1.5384,
+      "nll_loss": 0.1433144360780716,
+      "rewards/accuracies": 0.9212499856948853,
+      "rewards/chosen": -2.401654005050659,
+      "rewards/margins": 2.505878210067749,
+      "rewards/rejected": -4.907532215118408,
+      "step": 3650
+    },
+    {
+      "epoch": 2.96,
+      "grad_norm": 19.85245132446289,
+      "learning_rate": 5.633145734114665e-09,
+      "logits/chosen": 1.0758558511734009,
+      "logits/rejected": 1.114052653312683,
+      "logps/chosen": -23.172121047973633,
+      "logps/rejected": -46.996238708496094,
+      "loss": 1.5239,
+      "nll_loss": 0.14142683148384094,
+      "rewards/accuracies": 0.9262499809265137,
+      "rewards/chosen": -2.3172123432159424,
+      "rewards/margins": 2.3824121952056885,
+      "rewards/rejected": -4.699624538421631,
+      "step": 3700
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 19.575937271118164,
+      "learning_rate": 2.1661681620654963e-12,
+      "logits/chosen": 1.0582020282745361,
+      "logits/rejected": 1.1024978160858154,
+      "logps/chosen": -22.676530838012695,
+      "logps/rejected": -48.214569091796875,
+      "loss": 1.4608,
+      "nll_loss": 0.13524094223976135,
+      "rewards/accuracies": 0.9287499785423279,
+      "rewards/chosen": -2.267652988433838,
+      "rewards/margins": 2.5538036823272705,
+      "rewards/rejected": -4.821456432342529,
+      "step": 3750
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 3750,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9aa02136503d05933309aa9f771cfd1d85312a3212ad3e32671f02280841e7f4
+size 6225