Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-1066/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-2132/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-3198/tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,62 @@
+---
+base_model: meta-llama/Llama-3.1-8B-Instruct
+library_name: peft
+model_name: llama3_cot_finetuned
+tags:
+- base_model:adapter:meta-llama/Llama-3.1-8B-Instruct
+- lora
+- sft
+- transformers
+- trl
+licence: license
+pipeline_tag: text-generation
+---
+
+# Model Card for llama3_cot_finetuned
+
+This model is a fine-tuned version of [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/gsr2149-columbia-university/huggingface/runs/diyvruw0) 
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- PEFT 0.18.0
+- TRL: 0.26.1
+- Transformers: 4.57.3
+- Pytorch: 2.9.0+cu126
+- Datasets: 4.0.0
+- Tokenizers: 0.22.1
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

adapter_config.json

@@ -0,0 +1,41 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.1-8B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 24,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.0",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b54df820ee065f28e73fabdacb2f36346384e74c2d4d07dc646ffcea458bb34
+size 27280280

checkpoint-1066/README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.1-8B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.1-8B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.0

checkpoint-1066/adapter_config.json

@@ -0,0 +1,41 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.1-8B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 24,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.0",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-1066/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:939db3bc4ece8b0661cc6e662740d7b75613903f196f73893e489f803469ffaf
+size 27280280

checkpoint-1066/chat_template.jinja

@@ -0,0 +1,109 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- set date_string = "26 Jul 2024" %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message + builtin tools #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if builtin_tools is defined or tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{%- if builtin_tools is defined %}
+    {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- "<|python_tag|>" + tool_call.name + ".call(" }}
+            {%- for arg_name, arg_val in tool_call.arguments | items %}
+                {{- arg_name + '="' + arg_val + '"' }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- endif %}
+                {%- endfor %}
+            {{- ")" }}
+        {%- else  %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- '{"name": "' + tool_call.name + '", ' }}
+            {{- '"parameters": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- "}" }}
+        {%- endif %}
+        {%- if builtin_tools is defined %}
+            {#- This means we're in ipython mode #}
+            {{- "<|eom_id|>" }}
+        {%- else %}
+            {{- "<|eot_id|>" }}
+        {%- endif %}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

checkpoint-1066/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4b2da333c6c4855cb495276d14df5e48cf17100e5bd1cebde54edcc73b9503a
+size 27843851

checkpoint-1066/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57a925e8810ae926276a372728592c0bc0defa4e14f0235e76c5be8b66d29cc1
+size 14645

checkpoint-1066/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d3f8ce642f5027ab2da19fb0cc23dba3718691d985c50f8bc0731463a1a52b3
+size 1465

checkpoint-1066/special_tokens_map.json

@@ -0,0 +1,16 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-1066/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

checkpoint-1066/tokenizer_config.json

@@ -0,0 +1,2062 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

checkpoint-1066/trainer_state.json

@@ -0,0 +1,1094 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1066,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 0.7799350878223776,
+      "epoch": 0.009382513340761156,
+      "grad_norm": 0.251953125,
+      "learning_rate": 0.00019943714821763604,
+      "loss": 0.8229,
+      "mean_token_accuracy": 0.8214783387258648,
+      "num_tokens": 162309.0,
+      "step": 10
+    },
+    {
+      "entropy": 0.6856271875090897,
+      "epoch": 0.01876502668152231,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 0.00019881175734834272,
+      "loss": 0.6706,
+      "mean_token_accuracy": 0.8432094575837255,
+      "num_tokens": 325111.0,
+      "step": 20
+    },
+    {
+      "entropy": 0.5147169048897922,
+      "epoch": 0.02814754002228347,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 0.00019818636647904943,
+      "loss": 0.5152,
+      "mean_token_accuracy": 0.8709522735327482,
+      "num_tokens": 488159.0,
+      "step": 30
+    },
+    {
+      "entropy": 0.48951989263296125,
+      "epoch": 0.03753005336304462,
+      "grad_norm": 0.1552734375,
+      "learning_rate": 0.0001975609756097561,
+      "loss": 0.4875,
+      "mean_token_accuracy": 0.8770332466810942,
+      "num_tokens": 650647.0,
+      "step": 40
+    },
+    {
+      "entropy": 0.4960091439075768,
+      "epoch": 0.04691256670380578,
+      "grad_norm": 0.16796875,
+      "learning_rate": 0.0001969355847404628,
+      "loss": 0.492,
+      "mean_token_accuracy": 0.8768141727894545,
+      "num_tokens": 813419.0,
+      "step": 50
+    },
+    {
+      "entropy": 0.4708784816786647,
+      "epoch": 0.05629508004456694,
+      "grad_norm": 0.197265625,
+      "learning_rate": 0.0001963101938711695,
+      "loss": 0.4648,
+      "mean_token_accuracy": 0.8837848342955112,
+      "num_tokens": 976458.0,
+      "step": 60
+    },
+    {
+      "entropy": 0.546927246497944,
+      "epoch": 0.0656775933853281,
+      "grad_norm": 0.1669921875,
+      "learning_rate": 0.00019568480300187618,
+      "loss": 0.5415,
+      "mean_token_accuracy": 0.8723422776907682,
+      "num_tokens": 1138794.0,
+      "step": 70
+    },
+    {
+      "entropy": 0.47402509227395057,
+      "epoch": 0.07506010672608925,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 0.00019505941213258287,
+      "loss": 0.4702,
+      "mean_token_accuracy": 0.885954425111413,
+      "num_tokens": 1301483.0,
+      "step": 80
+    },
+    {
+      "entropy": 0.4972494073677808,
+      "epoch": 0.08444262006685041,
+      "grad_norm": 0.1708984375,
+      "learning_rate": 0.00019443402126328955,
+      "loss": 0.4901,
+      "mean_token_accuracy": 0.8839348161593079,
+      "num_tokens": 1464240.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.49195765373297035,
+      "epoch": 0.09382513340761156,
+      "grad_norm": 0.1669921875,
+      "learning_rate": 0.00019380863039399627,
+      "loss": 0.4848,
+      "mean_token_accuracy": 0.886236279271543,
+      "num_tokens": 1627457.0,
+      "step": 100
+    },
+    {
+      "entropy": 0.5126075576525182,
+      "epoch": 0.10320764674837273,
+      "grad_norm": 0.1787109375,
+      "learning_rate": 0.00019318323952470295,
+      "loss": 0.5104,
+      "mean_token_accuracy": 0.8793701384216547,
+      "num_tokens": 1790547.0,
+      "step": 110
+    },
+    {
+      "entropy": 0.4117678565904498,
+      "epoch": 0.11259016008913388,
+      "grad_norm": 0.1650390625,
+      "learning_rate": 0.00019255784865540964,
+      "loss": 0.4114,
+      "mean_token_accuracy": 0.8950984058901668,
+      "num_tokens": 1953536.0,
+      "step": 120
+    },
+    {
+      "entropy": 0.4656925258226693,
+      "epoch": 0.12197267342989504,
+      "grad_norm": 0.1875,
+      "learning_rate": 0.00019193245778611632,
+      "loss": 0.4634,
+      "mean_token_accuracy": 0.8897427920252085,
+      "num_tokens": 2115476.0,
+      "step": 130
+    },
+    {
+      "entropy": 0.43195099574513734,
+      "epoch": 0.1313551867706562,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.00019130706691682304,
+      "loss": 0.4246,
+      "mean_token_accuracy": 0.8946435421705246,
+      "num_tokens": 2277936.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.4823188486509025,
+      "epoch": 0.14073770011141734,
+      "grad_norm": 0.20703125,
+      "learning_rate": 0.00019068167604752972,
+      "loss": 0.4796,
+      "mean_token_accuracy": 0.8873342543840408,
+      "num_tokens": 2440643.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.40974695349577817,
+      "epoch": 0.1501202134521785,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 0.0001900562851782364,
+      "loss": 0.4,
+      "mean_token_accuracy": 0.8974028028547764,
+      "num_tokens": 2603401.0,
+      "step": 160
+    },
+    {
+      "entropy": 0.41737876399420204,
+      "epoch": 0.15950272679293967,
+      "grad_norm": 0.177734375,
+      "learning_rate": 0.0001894308943089431,
+      "loss": 0.4136,
+      "mean_token_accuracy": 0.8985418293625116,
+      "num_tokens": 2765836.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.4317509519169107,
+      "epoch": 0.16888524013370082,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.0001888055034396498,
+      "loss": 0.4276,
+      "mean_token_accuracy": 0.8956017823889851,
+      "num_tokens": 2928637.0,
+      "step": 180
+    },
+    {
+      "entropy": 0.39300057776272296,
+      "epoch": 0.17826775347446197,
+      "grad_norm": 0.177734375,
+      "learning_rate": 0.00018818011257035647,
+      "loss": 0.3853,
+      "mean_token_accuracy": 0.903920691087842,
+      "num_tokens": 3090979.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.43899501564446836,
+      "epoch": 0.18765026681522312,
+      "grad_norm": 0.16796875,
+      "learning_rate": 0.00018755472170106316,
+      "loss": 0.4302,
+      "mean_token_accuracy": 0.894548149779439,
+      "num_tokens": 3253538.0,
+      "step": 200
+    },
+    {
+      "entropy": 0.3904935538303107,
+      "epoch": 0.19703278015598427,
+      "grad_norm": 0.1708984375,
+      "learning_rate": 0.00018692933083176987,
+      "loss": 0.3829,
+      "mean_token_accuracy": 0.9031265545636415,
+      "num_tokens": 3415948.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.39792181542143223,
+      "epoch": 0.20641529349674545,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 0.00018630393996247656,
+      "loss": 0.3862,
+      "mean_token_accuracy": 0.9033814109861851,
+      "num_tokens": 3579298.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.38712316513992845,
+      "epoch": 0.2157978068375066,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.00018567854909318324,
+      "loss": 0.3871,
+      "mean_token_accuracy": 0.9008511133491993,
+      "num_tokens": 3742007.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.3974394467426464,
+      "epoch": 0.22518032017826775,
+      "grad_norm": 0.1640625,
+      "learning_rate": 0.00018505315822388993,
+      "loss": 0.39,
+      "mean_token_accuracy": 0.9035492207854986,
+      "num_tokens": 3904988.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.42329446180956437,
+      "epoch": 0.2345628335190289,
+      "grad_norm": 0.19140625,
+      "learning_rate": 0.00018442776735459664,
+      "loss": 0.4136,
+      "mean_token_accuracy": 0.9000132424756885,
+      "num_tokens": 4067806.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.400798928248696,
+      "epoch": 0.24394534685979008,
+      "grad_norm": 0.1953125,
+      "learning_rate": 0.00018380237648530333,
+      "loss": 0.3889,
+      "mean_token_accuracy": 0.8984681818634271,
+      "num_tokens": 4229588.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.3440579119604081,
+      "epoch": 0.2533278602005512,
+      "grad_norm": 0.1748046875,
+      "learning_rate": 0.00018317698561601001,
+      "loss": 0.3355,
+      "mean_token_accuracy": 0.9099012993276119,
+      "num_tokens": 4391775.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.4088153497781605,
+      "epoch": 0.2627103735413124,
+      "grad_norm": 0.2060546875,
+      "learning_rate": 0.0001825515947467167,
+      "loss": 0.3988,
+      "mean_token_accuracy": 0.9000191796571017,
+      "num_tokens": 4554006.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.49478547936305406,
+      "epoch": 0.27209288688207356,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.0001819262038774234,
+      "loss": 0.4851,
+      "mean_token_accuracy": 0.8896255612373352,
+      "num_tokens": 4716601.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.3874137854669243,
+      "epoch": 0.2814754002228347,
+      "grad_norm": 0.1708984375,
+      "learning_rate": 0.0001813008130081301,
+      "loss": 0.3811,
+      "mean_token_accuracy": 0.9031782140955329,
+      "num_tokens": 4879465.0,
+      "step": 300
+    },
+    {
+      "entropy": 0.39499278874136506,
+      "epoch": 0.29085791356359586,
+      "grad_norm": 0.1796875,
+      "learning_rate": 0.00018067542213883679,
+      "loss": 0.3859,
+      "mean_token_accuracy": 0.9037530291825533,
+      "num_tokens": 5041857.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.3148498781956732,
+      "epoch": 0.300240426904357,
+      "grad_norm": 0.146484375,
+      "learning_rate": 0.00018005003126954347,
+      "loss": 0.3105,
+      "mean_token_accuracy": 0.9153206121176481,
+      "num_tokens": 5204416.0,
+      "step": 320
+    },
+    {
+      "entropy": 0.45533317684894425,
+      "epoch": 0.30962294024511816,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.00017942464040025016,
+      "loss": 0.4477,
+      "mean_token_accuracy": 0.8917344063520432,
+      "num_tokens": 5366029.0,
+      "step": 330
+    },
+    {
+      "entropy": 0.4512431137613021,
+      "epoch": 0.31900545358587934,
+      "grad_norm": 0.1787109375,
+      "learning_rate": 0.00017879924953095687,
+      "loss": 0.4435,
+      "mean_token_accuracy": 0.8979733150452376,
+      "num_tokens": 5528662.0,
+      "step": 340
+    },
+    {
+      "entropy": 0.43219919862458483,
+      "epoch": 0.32838796692664046,
+      "grad_norm": 0.1875,
+      "learning_rate": 0.00017817385866166353,
+      "loss": 0.4255,
+      "mean_token_accuracy": 0.8939095275476575,
+      "num_tokens": 5691088.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.39246358472155407,
+      "epoch": 0.33777048026740164,
+      "grad_norm": 0.2060546875,
+      "learning_rate": 0.00017754846779237024,
+      "loss": 0.3792,
+      "mean_token_accuracy": 0.9027158614248038,
+      "num_tokens": 5853586.0,
+      "step": 360
+    },
+    {
+      "entropy": 0.38228728668764234,
+      "epoch": 0.34715299360816276,
+      "grad_norm": 0.158203125,
+      "learning_rate": 0.00017692307692307693,
+      "loss": 0.3814,
+      "mean_token_accuracy": 0.9076376244425773,
+      "num_tokens": 6016273.0,
+      "step": 370
+    },
+    {
+      "entropy": 0.34766899450914934,
+      "epoch": 0.35653550694892394,
+      "grad_norm": 0.16796875,
+      "learning_rate": 0.00017629768605378362,
+      "loss": 0.339,
+      "mean_token_accuracy": 0.9132196143269539,
+      "num_tokens": 6179608.0,
+      "step": 380
+    },
+    {
+      "entropy": 0.419576386269182,
+      "epoch": 0.3659180202896851,
+      "grad_norm": 0.150390625,
+      "learning_rate": 0.0001756722951844903,
+      "loss": 0.4081,
+      "mean_token_accuracy": 0.902265278249979,
+      "num_tokens": 6342688.0,
+      "step": 390
+    },
+    {
+      "entropy": 0.34522315053036434,
+      "epoch": 0.37530053363044624,
+      "grad_norm": 0.16796875,
+      "learning_rate": 0.00017504690431519702,
+      "loss": 0.3398,
+      "mean_token_accuracy": 0.9103329949080944,
+      "num_tokens": 6505076.0,
+      "step": 400
+    },
+    {
+      "entropy": 0.3350997743313201,
+      "epoch": 0.3846830469712074,
+      "grad_norm": 0.1611328125,
+      "learning_rate": 0.0001744215134459037,
+      "loss": 0.3287,
+      "mean_token_accuracy": 0.9135664030909538,
+      "num_tokens": 6667428.0,
+      "step": 410
+    },
+    {
+      "entropy": 0.35369435520842674,
+      "epoch": 0.39406556031196854,
+      "grad_norm": 0.16796875,
+      "learning_rate": 0.0001737961225766104,
+      "loss": 0.3445,
+      "mean_token_accuracy": 0.9136763598769904,
+      "num_tokens": 6829952.0,
+      "step": 420
+    },
+    {
+      "entropy": 0.3438279568566941,
+      "epoch": 0.4034480736527297,
+      "grad_norm": 0.2109375,
+      "learning_rate": 0.00017317073170731708,
+      "loss": 0.3347,
+      "mean_token_accuracy": 0.915488549694419,
+      "num_tokens": 6992752.0,
+      "step": 430
+    },
+    {
+      "entropy": 0.36355263493023815,
+      "epoch": 0.4128305869934909,
+      "grad_norm": 0.134765625,
+      "learning_rate": 0.0001725453408380238,
+      "loss": 0.3514,
+      "mean_token_accuracy": 0.9081276219338179,
+      "num_tokens": 7155395.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.4320855587720871,
+      "epoch": 0.422213100334252,
+      "grad_norm": 0.1787109375,
+      "learning_rate": 0.00017191994996873047,
+      "loss": 0.4275,
+      "mean_token_accuracy": 0.8996547561138868,
+      "num_tokens": 7317871.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.35265264644986016,
+      "epoch": 0.4315956136750132,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 0.00017129455909943713,
+      "loss": 0.346,
+      "mean_token_accuracy": 0.911245071515441,
+      "num_tokens": 7480566.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.38030760866822677,
+      "epoch": 0.4409781270157743,
+      "grad_norm": 0.17578125,
+      "learning_rate": 0.00017066916823014385,
+      "loss": 0.3737,
+      "mean_token_accuracy": 0.9052736137062312,
+      "num_tokens": 7642808.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.37449472132138906,
+      "epoch": 0.4503606403565355,
+      "grad_norm": 0.1923828125,
+      "learning_rate": 0.00017004377736085053,
+      "loss": 0.3668,
+      "mean_token_accuracy": 0.9072099477052689,
+      "num_tokens": 7806054.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.38983453994151207,
+      "epoch": 0.4597431536972967,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 0.00016941838649155725,
+      "loss": 0.3804,
+      "mean_token_accuracy": 0.9052108563482761,
+      "num_tokens": 7968208.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.3876548252766952,
+      "epoch": 0.4691256670380578,
+      "grad_norm": 0.1953125,
+      "learning_rate": 0.0001687929956222639,
+      "loss": 0.3825,
+      "mean_token_accuracy": 0.9071293246001005,
+      "num_tokens": 8130444.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.35602071137400343,
+      "epoch": 0.478508180378819,
+      "grad_norm": 0.1748046875,
+      "learning_rate": 0.00016816760475297062,
+      "loss": 0.3484,
+      "mean_token_accuracy": 0.9137960843741894,
+      "num_tokens": 8292618.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.4106741435127333,
+      "epoch": 0.48789069371958016,
+      "grad_norm": 0.181640625,
+      "learning_rate": 0.0001675422138836773,
+      "loss": 0.3968,
+      "mean_token_accuracy": 0.9058029960840941,
+      "num_tokens": 8455547.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.35365958177717405,
+      "epoch": 0.4972732070603413,
+      "grad_norm": 0.171875,
+      "learning_rate": 0.000166916823014384,
+      "loss": 0.349,
+      "mean_token_accuracy": 0.9107609067112208,
+      "num_tokens": 8617389.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.3634562706924044,
+      "epoch": 0.5066557204011024,
+      "grad_norm": 0.1494140625,
+      "learning_rate": 0.00016629143214509068,
+      "loss": 0.3559,
+      "mean_token_accuracy": 0.9100410873070359,
+      "num_tokens": 8779770.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.32272221232997256,
+      "epoch": 0.5160382337418636,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 0.0001656660412757974,
+      "loss": 0.3192,
+      "mean_token_accuracy": 0.9185531191527844,
+      "num_tokens": 8942167.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.3944925616728142,
+      "epoch": 0.5254207470826248,
+      "grad_norm": 0.189453125,
+      "learning_rate": 0.00016504065040650408,
+      "loss": 0.3822,
+      "mean_token_accuracy": 0.9070056907832622,
+      "num_tokens": 9104106.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.34754261119524016,
+      "epoch": 0.5348032604233859,
+      "grad_norm": 0.1923828125,
+      "learning_rate": 0.00016441525953721076,
+      "loss": 0.341,
+      "mean_token_accuracy": 0.9133739612996579,
+      "num_tokens": 9266326.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.36407561659580095,
+      "epoch": 0.5441857737641471,
+      "grad_norm": 0.146484375,
+      "learning_rate": 0.00016378986866791745,
+      "loss": 0.355,
+      "mean_token_accuracy": 0.9110283955931664,
+      "num_tokens": 9428809.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.36490103751420977,
+      "epoch": 0.5535682871049082,
+      "grad_norm": 0.14453125,
+      "learning_rate": 0.00016316447779862414,
+      "loss": 0.3606,
+      "mean_token_accuracy": 0.9129387583583594,
+      "num_tokens": 9591549.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.4403200134751387,
+      "epoch": 0.5629508004456694,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.00016253908692933085,
+      "loss": 0.4352,
+      "mean_token_accuracy": 0.9009242281317711,
+      "num_tokens": 9754490.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.3376986944698729,
+      "epoch": 0.5723333137864305,
+      "grad_norm": 0.2099609375,
+      "learning_rate": 0.00016191369606003754,
+      "loss": 0.3271,
+      "mean_token_accuracy": 0.9162716951221228,
+      "num_tokens": 9917345.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.30342430413002147,
+      "epoch": 0.5817158271271917,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 0.00016128830519074422,
+      "loss": 0.2956,
+      "mean_token_accuracy": 0.9222736719995737,
+      "num_tokens": 10079523.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.39200085022021086,
+      "epoch": 0.5910983404679528,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 0.0001606629143214509,
+      "loss": 0.3825,
+      "mean_token_accuracy": 0.9086053863167762,
+      "num_tokens": 10241691.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.3571344430092722,
+      "epoch": 0.600480853808714,
+      "grad_norm": 0.17578125,
+      "learning_rate": 0.00016003752345215762,
+      "loss": 0.3541,
+      "mean_token_accuracy": 0.9139539418742061,
+      "num_tokens": 10404189.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.34833492621546613,
+      "epoch": 0.6098633671494752,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 0.00015941213258286428,
+      "loss": 0.3376,
+      "mean_token_accuracy": 0.9150689322501421,
+      "num_tokens": 10566544.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.3005343694472685,
+      "epoch": 0.6192458804902363,
+      "grad_norm": 0.158203125,
+      "learning_rate": 0.000158786741713571,
+      "loss": 0.2928,
+      "mean_token_accuracy": 0.9205218695104123,
+      "num_tokens": 10728072.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.28468642103252934,
+      "epoch": 0.6286283938309974,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 0.00015816135084427768,
+      "loss": 0.2813,
+      "mean_token_accuracy": 0.9226693190634251,
+      "num_tokens": 10890089.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.3512667278526351,
+      "epoch": 0.6380109071717587,
+      "grad_norm": 0.15625,
+      "learning_rate": 0.0001575359599749844,
+      "loss": 0.3421,
+      "mean_token_accuracy": 0.9135142650455237,
+      "num_tokens": 11052295.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.38745955675840377,
+      "epoch": 0.6473934205125198,
+      "grad_norm": 0.16015625,
+      "learning_rate": 0.00015691056910569105,
+      "loss": 0.3774,
+      "mean_token_accuracy": 0.9076902076601983,
+      "num_tokens": 11214249.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.3123638934805058,
+      "epoch": 0.6567759338532809,
+      "grad_norm": 0.1669921875,
+      "learning_rate": 0.00015628517823639777,
+      "loss": 0.3099,
+      "mean_token_accuracy": 0.9233202930539847,
+      "num_tokens": 11377548.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.36946490925038233,
+      "epoch": 0.6661584471940422,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 0.00015565978736710445,
+      "loss": 0.3584,
+      "mean_token_accuracy": 0.9100467972457409,
+      "num_tokens": 11539730.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.44331123513402415,
+      "epoch": 0.6755409605348033,
+      "grad_norm": 0.185546875,
+      "learning_rate": 0.00015503439649781114,
+      "loss": 0.4357,
+      "mean_token_accuracy": 0.9018381554633379,
+      "num_tokens": 11702656.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.35576051749521864,
+      "epoch": 0.6849234738755644,
+      "grad_norm": 0.1396484375,
+      "learning_rate": 0.00015440900562851783,
+      "loss": 0.3529,
+      "mean_token_accuracy": 0.9145626701414585,
+      "num_tokens": 11866221.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.3530123558419291,
+      "epoch": 0.6943059872163255,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 0.0001537836147592245,
+      "loss": 0.3413,
+      "mean_token_accuracy": 0.9174593914300203,
+      "num_tokens": 12028438.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.3903426473028958,
+      "epoch": 0.7036885005570868,
+      "grad_norm": 0.1953125,
+      "learning_rate": 0.00015315822388993123,
+      "loss": 0.385,
+      "mean_token_accuracy": 0.9088658891618252,
+      "num_tokens": 12190615.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.3577064266020898,
+      "epoch": 0.7130710138978479,
+      "grad_norm": 0.154296875,
+      "learning_rate": 0.0001525328330206379,
+      "loss": 0.3477,
+      "mean_token_accuracy": 0.9163919541984796,
+      "num_tokens": 12353059.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.29131998592056335,
+      "epoch": 0.722453527238609,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 0.0001519074421513446,
+      "loss": 0.2891,
+      "mean_token_accuracy": 0.9242078192532063,
+      "num_tokens": 12515436.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.3846006996463984,
+      "epoch": 0.7318360405793702,
+      "grad_norm": 0.193359375,
+      "learning_rate": 0.00015128205128205128,
+      "loss": 0.3775,
+      "mean_token_accuracy": 0.9100886223837733,
+      "num_tokens": 12677546.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.34870610454818235,
+      "epoch": 0.7412185539201314,
+      "grad_norm": 0.1728515625,
+      "learning_rate": 0.000150656660412758,
+      "loss": 0.3401,
+      "mean_token_accuracy": 0.9157310061156749,
+      "num_tokens": 12840366.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.33407560451305474,
+      "epoch": 0.7506010672608925,
+      "grad_norm": 0.169921875,
+      "learning_rate": 0.00015003126954346466,
+      "loss": 0.3285,
+      "mean_token_accuracy": 0.9150595877319574,
+      "num_tokens": 13002637.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.33455972147057766,
+      "epoch": 0.7599835806016537,
+      "grad_norm": 0.173828125,
+      "learning_rate": 0.00014940587867417137,
+      "loss": 0.3268,
+      "mean_token_accuracy": 0.9206027369946241,
+      "num_tokens": 13165555.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.3760374941455666,
+      "epoch": 0.7693660939424148,
+      "grad_norm": 0.212890625,
+      "learning_rate": 0.00014878048780487806,
+      "loss": 0.3689,
+      "mean_token_accuracy": 0.9126812819391489,
+      "num_tokens": 13328334.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.4276343748439103,
+      "epoch": 0.778748607283176,
+      "grad_norm": 0.16015625,
+      "learning_rate": 0.00014815509693558477,
+      "loss": 0.4228,
+      "mean_token_accuracy": 0.9042647287249566,
+      "num_tokens": 13490265.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.35269592416007073,
+      "epoch": 0.7881311206239371,
+      "grad_norm": 0.1416015625,
+      "learning_rate": 0.00014752970606629143,
+      "loss": 0.3436,
+      "mean_token_accuracy": 0.9150325283408165,
+      "num_tokens": 13652700.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.3484025130281225,
+      "epoch": 0.7975136339646983,
+      "grad_norm": 0.1650390625,
+      "learning_rate": 0.00014690431519699812,
+      "loss": 0.3433,
+      "mean_token_accuracy": 0.9187351956963539,
+      "num_tokens": 13814997.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.34064410558785313,
+      "epoch": 0.8068961473054594,
+      "grad_norm": 0.169921875,
+      "learning_rate": 0.00014627892432770483,
+      "loss": 0.3319,
+      "mean_token_accuracy": 0.917700307443738,
+      "num_tokens": 13977550.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.3676424361299723,
+      "epoch": 0.8162786606462206,
+      "grad_norm": 0.1474609375,
+      "learning_rate": 0.00014565353345841151,
+      "loss": 0.3611,
+      "mean_token_accuracy": 0.9167136520147323,
+      "num_tokens": 14140493.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.3216154734662268,
+      "epoch": 0.8256611739869818,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 0.0001450281425891182,
+      "loss": 0.3133,
+      "mean_token_accuracy": 0.920833321288228,
+      "num_tokens": 14302638.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.3731617649958935,
+      "epoch": 0.8350436873277429,
+      "grad_norm": 0.1640625,
+      "learning_rate": 0.0001444027517198249,
+      "loss": 0.3708,
+      "mean_token_accuracy": 0.9144421305507422,
+      "num_tokens": 14465632.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.3741735242540017,
+      "epoch": 0.844426200668504,
+      "grad_norm": 0.1474609375,
+      "learning_rate": 0.0001437773608505316,
+      "loss": 0.3628,
+      "mean_token_accuracy": 0.9130110245198011,
+      "num_tokens": 14627986.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.363976987201022,
+      "epoch": 0.8538087140092653,
+      "grad_norm": 0.1611328125,
+      "learning_rate": 0.0001431519699812383,
+      "loss": 0.3585,
+      "mean_token_accuracy": 0.9120528548955917,
+      "num_tokens": 14790551.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.3448503586114384,
+      "epoch": 0.8631912273500264,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 0.00014252657911194497,
+      "loss": 0.3421,
+      "mean_token_accuracy": 0.9182739958167077,
+      "num_tokens": 14952661.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.27787273108260707,
+      "epoch": 0.8725737406907875,
+      "grad_norm": 0.150390625,
+      "learning_rate": 0.00014190118824265166,
+      "loss": 0.2687,
+      "mean_token_accuracy": 0.92897635884583,
+      "num_tokens": 15115022.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.36200930876657367,
+      "epoch": 0.8819562540315486,
+      "grad_norm": 0.150390625,
+      "learning_rate": 0.00014127579737335837,
+      "loss": 0.3497,
+      "mean_token_accuracy": 0.9132607497274876,
+      "num_tokens": 15277207.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.2834262179443613,
+      "epoch": 0.8913387673723099,
+      "grad_norm": 0.1494140625,
+      "learning_rate": 0.00014065040650406506,
+      "loss": 0.2814,
+      "mean_token_accuracy": 0.9268792189657689,
+      "num_tokens": 15440092.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.3499863417411689,
+      "epoch": 0.900721280713071,
+      "grad_norm": 0.1484375,
+      "learning_rate": 0.00014002501563477175,
+      "loss": 0.3417,
+      "mean_token_accuracy": 0.9159077113494277,
+      "num_tokens": 15603417.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.2800866418518126,
+      "epoch": 0.9101037940538321,
+      "grad_norm": 0.1826171875,
+      "learning_rate": 0.00013939962476547843,
+      "loss": 0.27,
+      "mean_token_accuracy": 0.9285330750048161,
+      "num_tokens": 15765520.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.3038740568445064,
+      "epoch": 0.9194863073945934,
+      "grad_norm": 0.1669921875,
+      "learning_rate": 0.00013877423389618512,
+      "loss": 0.2992,
+      "mean_token_accuracy": 0.9227856550365686,
+      "num_tokens": 15928370.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.39133549834950826,
+      "epoch": 0.9288688207353545,
+      "grad_norm": 0.19921875,
+      "learning_rate": 0.0001381488430268918,
+      "loss": 0.3845,
+      "mean_token_accuracy": 0.9107819177210331,
+      "num_tokens": 16090841.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.368834131793119,
+      "epoch": 0.9382513340761156,
+      "grad_norm": 0.16796875,
+      "learning_rate": 0.0001375234521575985,
+      "loss": 0.3592,
+      "mean_token_accuracy": 0.9155494935810566,
+      "num_tokens": 16253510.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.3432939130580053,
+      "epoch": 0.9476338474168768,
+      "grad_norm": 0.181640625,
+      "learning_rate": 0.0001368980612883052,
+      "loss": 0.3434,
+      "mean_token_accuracy": 0.9178479440510273,
+      "num_tokens": 16416142.0,
+      "step": 1010
+    },
+    {
+      "entropy": 0.3263048173219431,
+      "epoch": 0.957016360757638,
+      "grad_norm": 0.158203125,
+      "learning_rate": 0.0001362726704190119,
+      "loss": 0.3157,
+      "mean_token_accuracy": 0.919898977316916,
+      "num_tokens": 16578145.0,
+      "step": 1020
+    },
+    {
+      "entropy": 0.30497189780580813,
+      "epoch": 0.9663988740983991,
+      "grad_norm": 0.162109375,
+      "learning_rate": 0.00013564727954971858,
+      "loss": 0.2985,
+      "mean_token_accuracy": 0.922874777764082,
+      "num_tokens": 16741103.0,
+      "step": 1030
+    },
+    {
+      "entropy": 0.2940454826748464,
+      "epoch": 0.9757813874391603,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 0.00013502188868042526,
+      "loss": 0.2874,
+      "mean_token_accuracy": 0.9239259999245405,
+      "num_tokens": 16903947.0,
+      "step": 1040
+    },
+    {
+      "entropy": 0.367372212279588,
+      "epoch": 0.9851639007799214,
+      "grad_norm": 0.1748046875,
+      "learning_rate": 0.00013439649781113198,
+      "loss": 0.3601,
+      "mean_token_accuracy": 0.9152833472937345,
+      "num_tokens": 17065800.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.34914421315770594,
+      "epoch": 0.9945464141206826,
+      "grad_norm": 0.1826171875,
+      "learning_rate": 0.00013377110694183866,
+      "loss": 0.3425,
+      "mean_token_accuracy": 0.9175648283213377,
+      "num_tokens": 17228736.0,
+      "step": 1060
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 3198,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7.814835842576794e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1066/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db83ac20cd1ea46a30b9db82ee3c07152c60e62ceb73c1d01e3725eeebe949ed
+size 6289

checkpoint-2132/README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.1-8B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.1-8B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.0

checkpoint-2132/adapter_config.json

@@ -0,0 +1,41 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.1-8B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 24,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.0",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-2132/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:763724c4fdc269f123767099353d736454e2a50a10fd23abdb439d247f4c2670
+size 27280280

checkpoint-2132/chat_template.jinja

@@ -0,0 +1,109 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- set date_string = "26 Jul 2024" %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message + builtin tools #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if builtin_tools is defined or tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{%- if builtin_tools is defined %}
+    {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- "<|python_tag|>" + tool_call.name + ".call(" }}
+            {%- for arg_name, arg_val in tool_call.arguments | items %}
+                {{- arg_name + '="' + arg_val + '"' }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- endif %}
+                {%- endfor %}
+            {{- ")" }}
+        {%- else  %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- '{"name": "' + tool_call.name + '", ' }}
+            {{- '"parameters": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- "}" }}
+        {%- endif %}
+        {%- if builtin_tools is defined %}
+            {#- This means we're in ipython mode #}
+            {{- "<|eom_id|>" }}
+        {%- else %}
+            {{- "<|eot_id|>" }}
+        {%- endif %}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

checkpoint-2132/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48f195f6b6079460747c78513c74755ca071ac5895619aa98b2c0f74adefa102
+size 27843851

checkpoint-2132/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f812ed3d34392df14ad31ce3caefce0af4e7edbb6c3ac2e45e11c68314a7872
+size 14645

checkpoint-2132/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3a5930b268616eb65112dc31be9eb4769cd9ebb68b8a649be3f808253344d22
+size 1465

checkpoint-2132/special_tokens_map.json

@@ -0,0 +1,16 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-2132/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

checkpoint-2132/tokenizer_config.json

@@ -0,0 +1,2062 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

checkpoint-2132/trainer_state.json

@@ -0,0 +1,2164 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 2132,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 0.7799350878223776,
+      "epoch": 0.009382513340761156,
+      "grad_norm": 0.251953125,
+      "learning_rate": 0.00019943714821763604,
+      "loss": 0.8229,
+      "mean_token_accuracy": 0.8214783387258648,
+      "num_tokens": 162309.0,
+      "step": 10
+    },
+    {
+      "entropy": 0.6856271875090897,
+      "epoch": 0.01876502668152231,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 0.00019881175734834272,
+      "loss": 0.6706,
+      "mean_token_accuracy": 0.8432094575837255,
+      "num_tokens": 325111.0,
+      "step": 20
+    },
+    {
+      "entropy": 0.5147169048897922,
+      "epoch": 0.02814754002228347,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 0.00019818636647904943,
+      "loss": 0.5152,
+      "mean_token_accuracy": 0.8709522735327482,
+      "num_tokens": 488159.0,
+      "step": 30
+    },
+    {
+      "entropy": 0.48951989263296125,
+      "epoch": 0.03753005336304462,
+      "grad_norm": 0.1552734375,
+      "learning_rate": 0.0001975609756097561,
+      "loss": 0.4875,
+      "mean_token_accuracy": 0.8770332466810942,
+      "num_tokens": 650647.0,
+      "step": 40
+    },
+    {
+      "entropy": 0.4960091439075768,
+      "epoch": 0.04691256670380578,
+      "grad_norm": 0.16796875,
+      "learning_rate": 0.0001969355847404628,
+      "loss": 0.492,
+      "mean_token_accuracy": 0.8768141727894545,
+      "num_tokens": 813419.0,
+      "step": 50
+    },
+    {
+      "entropy": 0.4708784816786647,
+      "epoch": 0.05629508004456694,
+      "grad_norm": 0.197265625,
+      "learning_rate": 0.0001963101938711695,
+      "loss": 0.4648,
+      "mean_token_accuracy": 0.8837848342955112,
+      "num_tokens": 976458.0,
+      "step": 60
+    },
+    {
+      "entropy": 0.546927246497944,
+      "epoch": 0.0656775933853281,
+      "grad_norm": 0.1669921875,
+      "learning_rate": 0.00019568480300187618,
+      "loss": 0.5415,
+      "mean_token_accuracy": 0.8723422776907682,
+      "num_tokens": 1138794.0,
+      "step": 70
+    },
+    {
+      "entropy": 0.47402509227395057,
+      "epoch": 0.07506010672608925,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 0.00019505941213258287,
+      "loss": 0.4702,
+      "mean_token_accuracy": 0.885954425111413,
+      "num_tokens": 1301483.0,
+      "step": 80
+    },
+    {
+      "entropy": 0.4972494073677808,
+      "epoch": 0.08444262006685041,
+      "grad_norm": 0.1708984375,
+      "learning_rate": 0.00019443402126328955,
+      "loss": 0.4901,
+      "mean_token_accuracy": 0.8839348161593079,
+      "num_tokens": 1464240.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.49195765373297035,
+      "epoch": 0.09382513340761156,
+      "grad_norm": 0.1669921875,
+      "learning_rate": 0.00019380863039399627,
+      "loss": 0.4848,
+      "mean_token_accuracy": 0.886236279271543,
+      "num_tokens": 1627457.0,
+      "step": 100
+    },
+    {
+      "entropy": 0.5126075576525182,
+      "epoch": 0.10320764674837273,
+      "grad_norm": 0.1787109375,
+      "learning_rate": 0.00019318323952470295,
+      "loss": 0.5104,
+      "mean_token_accuracy": 0.8793701384216547,
+      "num_tokens": 1790547.0,
+      "step": 110
+    },
+    {
+      "entropy": 0.4117678565904498,
+      "epoch": 0.11259016008913388,
+      "grad_norm": 0.1650390625,
+      "learning_rate": 0.00019255784865540964,
+      "loss": 0.4114,
+      "mean_token_accuracy": 0.8950984058901668,
+      "num_tokens": 1953536.0,
+      "step": 120
+    },
+    {
+      "entropy": 0.4656925258226693,
+      "epoch": 0.12197267342989504,
+      "grad_norm": 0.1875,
+      "learning_rate": 0.00019193245778611632,
+      "loss": 0.4634,
+      "mean_token_accuracy": 0.8897427920252085,
+      "num_tokens": 2115476.0,
+      "step": 130
+    },
+    {
+      "entropy": 0.43195099574513734,
+      "epoch": 0.1313551867706562,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.00019130706691682304,
+      "loss": 0.4246,
+      "mean_token_accuracy": 0.8946435421705246,
+      "num_tokens": 2277936.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.4823188486509025,
+      "epoch": 0.14073770011141734,
+      "grad_norm": 0.20703125,
+      "learning_rate": 0.00019068167604752972,
+      "loss": 0.4796,
+      "mean_token_accuracy": 0.8873342543840408,
+      "num_tokens": 2440643.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.40974695349577817,
+      "epoch": 0.1501202134521785,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 0.0001900562851782364,
+      "loss": 0.4,
+      "mean_token_accuracy": 0.8974028028547764,
+      "num_tokens": 2603401.0,
+      "step": 160
+    },
+    {
+      "entropy": 0.41737876399420204,
+      "epoch": 0.15950272679293967,
+      "grad_norm": 0.177734375,
+      "learning_rate": 0.0001894308943089431,
+      "loss": 0.4136,
+      "mean_token_accuracy": 0.8985418293625116,
+      "num_tokens": 2765836.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.4317509519169107,
+      "epoch": 0.16888524013370082,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.0001888055034396498,
+      "loss": 0.4276,
+      "mean_token_accuracy": 0.8956017823889851,
+      "num_tokens": 2928637.0,
+      "step": 180
+    },
+    {
+      "entropy": 0.39300057776272296,
+      "epoch": 0.17826775347446197,
+      "grad_norm": 0.177734375,
+      "learning_rate": 0.00018818011257035647,
+      "loss": 0.3853,
+      "mean_token_accuracy": 0.903920691087842,
+      "num_tokens": 3090979.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.43899501564446836,
+      "epoch": 0.18765026681522312,
+      "grad_norm": 0.16796875,
+      "learning_rate": 0.00018755472170106316,
+      "loss": 0.4302,
+      "mean_token_accuracy": 0.894548149779439,
+      "num_tokens": 3253538.0,
+      "step": 200
+    },
+    {
+      "entropy": 0.3904935538303107,
+      "epoch": 0.19703278015598427,
+      "grad_norm": 0.1708984375,
+      "learning_rate": 0.00018692933083176987,
+      "loss": 0.3829,
+      "mean_token_accuracy": 0.9031265545636415,
+      "num_tokens": 3415948.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.39792181542143223,
+      "epoch": 0.20641529349674545,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 0.00018630393996247656,
+      "loss": 0.3862,
+      "mean_token_accuracy": 0.9033814109861851,
+      "num_tokens": 3579298.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.38712316513992845,
+      "epoch": 0.2157978068375066,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.00018567854909318324,
+      "loss": 0.3871,
+      "mean_token_accuracy": 0.9008511133491993,
+      "num_tokens": 3742007.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.3974394467426464,
+      "epoch": 0.22518032017826775,
+      "grad_norm": 0.1640625,
+      "learning_rate": 0.00018505315822388993,
+      "loss": 0.39,
+      "mean_token_accuracy": 0.9035492207854986,
+      "num_tokens": 3904988.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.42329446180956437,
+      "epoch": 0.2345628335190289,
+      "grad_norm": 0.19140625,
+      "learning_rate": 0.00018442776735459664,
+      "loss": 0.4136,
+      "mean_token_accuracy": 0.9000132424756885,
+      "num_tokens": 4067806.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.400798928248696,
+      "epoch": 0.24394534685979008,
+      "grad_norm": 0.1953125,
+      "learning_rate": 0.00018380237648530333,
+      "loss": 0.3889,
+      "mean_token_accuracy": 0.8984681818634271,
+      "num_tokens": 4229588.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.3440579119604081,
+      "epoch": 0.2533278602005512,
+      "grad_norm": 0.1748046875,
+      "learning_rate": 0.00018317698561601001,
+      "loss": 0.3355,
+      "mean_token_accuracy": 0.9099012993276119,
+      "num_tokens": 4391775.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.4088153497781605,
+      "epoch": 0.2627103735413124,
+      "grad_norm": 0.2060546875,
+      "learning_rate": 0.0001825515947467167,
+      "loss": 0.3988,
+      "mean_token_accuracy": 0.9000191796571017,
+      "num_tokens": 4554006.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.49478547936305406,
+      "epoch": 0.27209288688207356,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.0001819262038774234,
+      "loss": 0.4851,
+      "mean_token_accuracy": 0.8896255612373352,
+      "num_tokens": 4716601.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.3874137854669243,
+      "epoch": 0.2814754002228347,
+      "grad_norm": 0.1708984375,
+      "learning_rate": 0.0001813008130081301,
+      "loss": 0.3811,
+      "mean_token_accuracy": 0.9031782140955329,
+      "num_tokens": 4879465.0,
+      "step": 300
+    },
+    {
+      "entropy": 0.39499278874136506,
+      "epoch": 0.29085791356359586,
+      "grad_norm": 0.1796875,
+      "learning_rate": 0.00018067542213883679,
+      "loss": 0.3859,
+      "mean_token_accuracy": 0.9037530291825533,
+      "num_tokens": 5041857.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.3148498781956732,
+      "epoch": 0.300240426904357,
+      "grad_norm": 0.146484375,
+      "learning_rate": 0.00018005003126954347,
+      "loss": 0.3105,
+      "mean_token_accuracy": 0.9153206121176481,
+      "num_tokens": 5204416.0,
+      "step": 320
+    },
+    {
+      "entropy": 0.45533317684894425,
+      "epoch": 0.30962294024511816,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.00017942464040025016,
+      "loss": 0.4477,
+      "mean_token_accuracy": 0.8917344063520432,
+      "num_tokens": 5366029.0,
+      "step": 330
+    },
+    {
+      "entropy": 0.4512431137613021,
+      "epoch": 0.31900545358587934,
+      "grad_norm": 0.1787109375,
+      "learning_rate": 0.00017879924953095687,
+      "loss": 0.4435,
+      "mean_token_accuracy": 0.8979733150452376,
+      "num_tokens": 5528662.0,
+      "step": 340
+    },
+    {
+      "entropy": 0.43219919862458483,
+      "epoch": 0.32838796692664046,
+      "grad_norm": 0.1875,
+      "learning_rate": 0.00017817385866166353,
+      "loss": 0.4255,
+      "mean_token_accuracy": 0.8939095275476575,
+      "num_tokens": 5691088.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.39246358472155407,
+      "epoch": 0.33777048026740164,
+      "grad_norm": 0.2060546875,
+      "learning_rate": 0.00017754846779237024,
+      "loss": 0.3792,
+      "mean_token_accuracy": 0.9027158614248038,
+      "num_tokens": 5853586.0,
+      "step": 360
+    },
+    {
+      "entropy": 0.38228728668764234,
+      "epoch": 0.34715299360816276,
+      "grad_norm": 0.158203125,
+      "learning_rate": 0.00017692307692307693,
+      "loss": 0.3814,
+      "mean_token_accuracy": 0.9076376244425773,
+      "num_tokens": 6016273.0,
+      "step": 370
+    },
+    {
+      "entropy": 0.34766899450914934,
+      "epoch": 0.35653550694892394,
+      "grad_norm": 0.16796875,
+      "learning_rate": 0.00017629768605378362,
+      "loss": 0.339,
+      "mean_token_accuracy": 0.9132196143269539,
+      "num_tokens": 6179608.0,
+      "step": 380
+    },
+    {
+      "entropy": 0.419576386269182,
+      "epoch": 0.3659180202896851,
+      "grad_norm": 0.150390625,
+      "learning_rate": 0.0001756722951844903,
+      "loss": 0.4081,
+      "mean_token_accuracy": 0.902265278249979,
+      "num_tokens": 6342688.0,
+      "step": 390
+    },
+    {
+      "entropy": 0.34522315053036434,
+      "epoch": 0.37530053363044624,
+      "grad_norm": 0.16796875,
+      "learning_rate": 0.00017504690431519702,
+      "loss": 0.3398,
+      "mean_token_accuracy": 0.9103329949080944,
+      "num_tokens": 6505076.0,
+      "step": 400
+    },
+    {
+      "entropy": 0.3350997743313201,
+      "epoch": 0.3846830469712074,
+      "grad_norm": 0.1611328125,
+      "learning_rate": 0.0001744215134459037,
+      "loss": 0.3287,
+      "mean_token_accuracy": 0.9135664030909538,
+      "num_tokens": 6667428.0,
+      "step": 410
+    },
+    {
+      "entropy": 0.35369435520842674,
+      "epoch": 0.39406556031196854,
+      "grad_norm": 0.16796875,
+      "learning_rate": 0.0001737961225766104,
+      "loss": 0.3445,
+      "mean_token_accuracy": 0.9136763598769904,
+      "num_tokens": 6829952.0,
+      "step": 420
+    },
+    {
+      "entropy": 0.3438279568566941,
+      "epoch": 0.4034480736527297,
+      "grad_norm": 0.2109375,
+      "learning_rate": 0.00017317073170731708,
+      "loss": 0.3347,
+      "mean_token_accuracy": 0.915488549694419,
+      "num_tokens": 6992752.0,
+      "step": 430
+    },
+    {
+      "entropy": 0.36355263493023815,
+      "epoch": 0.4128305869934909,
+      "grad_norm": 0.134765625,
+      "learning_rate": 0.0001725453408380238,
+      "loss": 0.3514,
+      "mean_token_accuracy": 0.9081276219338179,
+      "num_tokens": 7155395.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.4320855587720871,
+      "epoch": 0.422213100334252,
+      "grad_norm": 0.1787109375,
+      "learning_rate": 0.00017191994996873047,
+      "loss": 0.4275,
+      "mean_token_accuracy": 0.8996547561138868,
+      "num_tokens": 7317871.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.35265264644986016,
+      "epoch": 0.4315956136750132,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 0.00017129455909943713,
+      "loss": 0.346,
+      "mean_token_accuracy": 0.911245071515441,
+      "num_tokens": 7480566.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.38030760866822677,
+      "epoch": 0.4409781270157743,
+      "grad_norm": 0.17578125,
+      "learning_rate": 0.00017066916823014385,
+      "loss": 0.3737,
+      "mean_token_accuracy": 0.9052736137062312,
+      "num_tokens": 7642808.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.37449472132138906,
+      "epoch": 0.4503606403565355,
+      "grad_norm": 0.1923828125,
+      "learning_rate": 0.00017004377736085053,
+      "loss": 0.3668,
+      "mean_token_accuracy": 0.9072099477052689,
+      "num_tokens": 7806054.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.38983453994151207,
+      "epoch": 0.4597431536972967,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 0.00016941838649155725,
+      "loss": 0.3804,
+      "mean_token_accuracy": 0.9052108563482761,
+      "num_tokens": 7968208.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.3876548252766952,
+      "epoch": 0.4691256670380578,
+      "grad_norm": 0.1953125,
+      "learning_rate": 0.0001687929956222639,
+      "loss": 0.3825,
+      "mean_token_accuracy": 0.9071293246001005,
+      "num_tokens": 8130444.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.35602071137400343,
+      "epoch": 0.478508180378819,
+      "grad_norm": 0.1748046875,
+      "learning_rate": 0.00016816760475297062,
+      "loss": 0.3484,
+      "mean_token_accuracy": 0.9137960843741894,
+      "num_tokens": 8292618.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.4106741435127333,
+      "epoch": 0.48789069371958016,
+      "grad_norm": 0.181640625,
+      "learning_rate": 0.0001675422138836773,
+      "loss": 0.3968,
+      "mean_token_accuracy": 0.9058029960840941,
+      "num_tokens": 8455547.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.35365958177717405,
+      "epoch": 0.4972732070603413,
+      "grad_norm": 0.171875,
+      "learning_rate": 0.000166916823014384,
+      "loss": 0.349,
+      "mean_token_accuracy": 0.9107609067112208,
+      "num_tokens": 8617389.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.3634562706924044,
+      "epoch": 0.5066557204011024,
+      "grad_norm": 0.1494140625,
+      "learning_rate": 0.00016629143214509068,
+      "loss": 0.3559,
+      "mean_token_accuracy": 0.9100410873070359,
+      "num_tokens": 8779770.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.32272221232997256,
+      "epoch": 0.5160382337418636,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 0.0001656660412757974,
+      "loss": 0.3192,
+      "mean_token_accuracy": 0.9185531191527844,
+      "num_tokens": 8942167.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.3944925616728142,
+      "epoch": 0.5254207470826248,
+      "grad_norm": 0.189453125,
+      "learning_rate": 0.00016504065040650408,
+      "loss": 0.3822,
+      "mean_token_accuracy": 0.9070056907832622,
+      "num_tokens": 9104106.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.34754261119524016,
+      "epoch": 0.5348032604233859,
+      "grad_norm": 0.1923828125,
+      "learning_rate": 0.00016441525953721076,
+      "loss": 0.341,
+      "mean_token_accuracy": 0.9133739612996579,
+      "num_tokens": 9266326.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.36407561659580095,
+      "epoch": 0.5441857737641471,
+      "grad_norm": 0.146484375,
+      "learning_rate": 0.00016378986866791745,
+      "loss": 0.355,
+      "mean_token_accuracy": 0.9110283955931664,
+      "num_tokens": 9428809.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.36490103751420977,
+      "epoch": 0.5535682871049082,
+      "grad_norm": 0.14453125,
+      "learning_rate": 0.00016316447779862414,
+      "loss": 0.3606,
+      "mean_token_accuracy": 0.9129387583583594,
+      "num_tokens": 9591549.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.4403200134751387,
+      "epoch": 0.5629508004456694,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.00016253908692933085,
+      "loss": 0.4352,
+      "mean_token_accuracy": 0.9009242281317711,
+      "num_tokens": 9754490.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.3376986944698729,
+      "epoch": 0.5723333137864305,
+      "grad_norm": 0.2099609375,
+      "learning_rate": 0.00016191369606003754,
+      "loss": 0.3271,
+      "mean_token_accuracy": 0.9162716951221228,
+      "num_tokens": 9917345.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.30342430413002147,
+      "epoch": 0.5817158271271917,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 0.00016128830519074422,
+      "loss": 0.2956,
+      "mean_token_accuracy": 0.9222736719995737,
+      "num_tokens": 10079523.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.39200085022021086,
+      "epoch": 0.5910983404679528,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 0.0001606629143214509,
+      "loss": 0.3825,
+      "mean_token_accuracy": 0.9086053863167762,
+      "num_tokens": 10241691.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.3571344430092722,
+      "epoch": 0.600480853808714,
+      "grad_norm": 0.17578125,
+      "learning_rate": 0.00016003752345215762,
+      "loss": 0.3541,
+      "mean_token_accuracy": 0.9139539418742061,
+      "num_tokens": 10404189.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.34833492621546613,
+      "epoch": 0.6098633671494752,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 0.00015941213258286428,
+      "loss": 0.3376,
+      "mean_token_accuracy": 0.9150689322501421,
+      "num_tokens": 10566544.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.3005343694472685,
+      "epoch": 0.6192458804902363,
+      "grad_norm": 0.158203125,
+      "learning_rate": 0.000158786741713571,
+      "loss": 0.2928,
+      "mean_token_accuracy": 0.9205218695104123,
+      "num_tokens": 10728072.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.28468642103252934,
+      "epoch": 0.6286283938309974,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 0.00015816135084427768,
+      "loss": 0.2813,
+      "mean_token_accuracy": 0.9226693190634251,
+      "num_tokens": 10890089.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.3512667278526351,
+      "epoch": 0.6380109071717587,
+      "grad_norm": 0.15625,
+      "learning_rate": 0.0001575359599749844,
+      "loss": 0.3421,
+      "mean_token_accuracy": 0.9135142650455237,
+      "num_tokens": 11052295.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.38745955675840377,
+      "epoch": 0.6473934205125198,
+      "grad_norm": 0.16015625,
+      "learning_rate": 0.00015691056910569105,
+      "loss": 0.3774,
+      "mean_token_accuracy": 0.9076902076601983,
+      "num_tokens": 11214249.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.3123638934805058,
+      "epoch": 0.6567759338532809,
+      "grad_norm": 0.1669921875,
+      "learning_rate": 0.00015628517823639777,
+      "loss": 0.3099,
+      "mean_token_accuracy": 0.9233202930539847,
+      "num_tokens": 11377548.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.36946490925038233,
+      "epoch": 0.6661584471940422,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 0.00015565978736710445,
+      "loss": 0.3584,
+      "mean_token_accuracy": 0.9100467972457409,
+      "num_tokens": 11539730.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.44331123513402415,
+      "epoch": 0.6755409605348033,
+      "grad_norm": 0.185546875,
+      "learning_rate": 0.00015503439649781114,
+      "loss": 0.4357,
+      "mean_token_accuracy": 0.9018381554633379,
+      "num_tokens": 11702656.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.35576051749521864,
+      "epoch": 0.6849234738755644,
+      "grad_norm": 0.1396484375,
+      "learning_rate": 0.00015440900562851783,
+      "loss": 0.3529,
+      "mean_token_accuracy": 0.9145626701414585,
+      "num_tokens": 11866221.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.3530123558419291,
+      "epoch": 0.6943059872163255,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 0.0001537836147592245,
+      "loss": 0.3413,
+      "mean_token_accuracy": 0.9174593914300203,
+      "num_tokens": 12028438.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.3903426473028958,
+      "epoch": 0.7036885005570868,
+      "grad_norm": 0.1953125,
+      "learning_rate": 0.00015315822388993123,
+      "loss": 0.385,
+      "mean_token_accuracy": 0.9088658891618252,
+      "num_tokens": 12190615.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.3577064266020898,
+      "epoch": 0.7130710138978479,
+      "grad_norm": 0.154296875,
+      "learning_rate": 0.0001525328330206379,
+      "loss": 0.3477,
+      "mean_token_accuracy": 0.9163919541984796,
+      "num_tokens": 12353059.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.29131998592056335,
+      "epoch": 0.722453527238609,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 0.0001519074421513446,
+      "loss": 0.2891,
+      "mean_token_accuracy": 0.9242078192532063,
+      "num_tokens": 12515436.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.3846006996463984,
+      "epoch": 0.7318360405793702,
+      "grad_norm": 0.193359375,
+      "learning_rate": 0.00015128205128205128,
+      "loss": 0.3775,
+      "mean_token_accuracy": 0.9100886223837733,
+      "num_tokens": 12677546.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.34870610454818235,
+      "epoch": 0.7412185539201314,
+      "grad_norm": 0.1728515625,
+      "learning_rate": 0.000150656660412758,
+      "loss": 0.3401,
+      "mean_token_accuracy": 0.9157310061156749,
+      "num_tokens": 12840366.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.33407560451305474,
+      "epoch": 0.7506010672608925,
+      "grad_norm": 0.169921875,
+      "learning_rate": 0.00015003126954346466,
+      "loss": 0.3285,
+      "mean_token_accuracy": 0.9150595877319574,
+      "num_tokens": 13002637.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.33455972147057766,
+      "epoch": 0.7599835806016537,
+      "grad_norm": 0.173828125,
+      "learning_rate": 0.00014940587867417137,
+      "loss": 0.3268,
+      "mean_token_accuracy": 0.9206027369946241,
+      "num_tokens": 13165555.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.3760374941455666,
+      "epoch": 0.7693660939424148,
+      "grad_norm": 0.212890625,
+      "learning_rate": 0.00014878048780487806,
+      "loss": 0.3689,
+      "mean_token_accuracy": 0.9126812819391489,
+      "num_tokens": 13328334.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.4276343748439103,
+      "epoch": 0.778748607283176,
+      "grad_norm": 0.16015625,
+      "learning_rate": 0.00014815509693558477,
+      "loss": 0.4228,
+      "mean_token_accuracy": 0.9042647287249566,
+      "num_tokens": 13490265.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.35269592416007073,
+      "epoch": 0.7881311206239371,
+      "grad_norm": 0.1416015625,
+      "learning_rate": 0.00014752970606629143,
+      "loss": 0.3436,
+      "mean_token_accuracy": 0.9150325283408165,
+      "num_tokens": 13652700.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.3484025130281225,
+      "epoch": 0.7975136339646983,
+      "grad_norm": 0.1650390625,
+      "learning_rate": 0.00014690431519699812,
+      "loss": 0.3433,
+      "mean_token_accuracy": 0.9187351956963539,
+      "num_tokens": 13814997.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.34064410558785313,
+      "epoch": 0.8068961473054594,
+      "grad_norm": 0.169921875,
+      "learning_rate": 0.00014627892432770483,
+      "loss": 0.3319,
+      "mean_token_accuracy": 0.917700307443738,
+      "num_tokens": 13977550.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.3676424361299723,
+      "epoch": 0.8162786606462206,
+      "grad_norm": 0.1474609375,
+      "learning_rate": 0.00014565353345841151,
+      "loss": 0.3611,
+      "mean_token_accuracy": 0.9167136520147323,
+      "num_tokens": 14140493.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.3216154734662268,
+      "epoch": 0.8256611739869818,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 0.0001450281425891182,
+      "loss": 0.3133,
+      "mean_token_accuracy": 0.920833321288228,
+      "num_tokens": 14302638.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.3731617649958935,
+      "epoch": 0.8350436873277429,
+      "grad_norm": 0.1640625,
+      "learning_rate": 0.0001444027517198249,
+      "loss": 0.3708,
+      "mean_token_accuracy": 0.9144421305507422,
+      "num_tokens": 14465632.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.3741735242540017,
+      "epoch": 0.844426200668504,
+      "grad_norm": 0.1474609375,
+      "learning_rate": 0.0001437773608505316,
+      "loss": 0.3628,
+      "mean_token_accuracy": 0.9130110245198011,
+      "num_tokens": 14627986.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.363976987201022,
+      "epoch": 0.8538087140092653,
+      "grad_norm": 0.1611328125,
+      "learning_rate": 0.0001431519699812383,
+      "loss": 0.3585,
+      "mean_token_accuracy": 0.9120528548955917,
+      "num_tokens": 14790551.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.3448503586114384,
+      "epoch": 0.8631912273500264,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 0.00014252657911194497,
+      "loss": 0.3421,
+      "mean_token_accuracy": 0.9182739958167077,
+      "num_tokens": 14952661.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.27787273108260707,
+      "epoch": 0.8725737406907875,
+      "grad_norm": 0.150390625,
+      "learning_rate": 0.00014190118824265166,
+      "loss": 0.2687,
+      "mean_token_accuracy": 0.92897635884583,
+      "num_tokens": 15115022.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.36200930876657367,
+      "epoch": 0.8819562540315486,
+      "grad_norm": 0.150390625,
+      "learning_rate": 0.00014127579737335837,
+      "loss": 0.3497,
+      "mean_token_accuracy": 0.9132607497274876,
+      "num_tokens": 15277207.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.2834262179443613,
+      "epoch": 0.8913387673723099,
+      "grad_norm": 0.1494140625,
+      "learning_rate": 0.00014065040650406506,
+      "loss": 0.2814,
+      "mean_token_accuracy": 0.9268792189657689,
+      "num_tokens": 15440092.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.3499863417411689,
+      "epoch": 0.900721280713071,
+      "grad_norm": 0.1484375,
+      "learning_rate": 0.00014002501563477175,
+      "loss": 0.3417,
+      "mean_token_accuracy": 0.9159077113494277,
+      "num_tokens": 15603417.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.2800866418518126,
+      "epoch": 0.9101037940538321,
+      "grad_norm": 0.1826171875,
+      "learning_rate": 0.00013939962476547843,
+      "loss": 0.27,
+      "mean_token_accuracy": 0.9285330750048161,
+      "num_tokens": 15765520.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.3038740568445064,
+      "epoch": 0.9194863073945934,
+      "grad_norm": 0.1669921875,
+      "learning_rate": 0.00013877423389618512,
+      "loss": 0.2992,
+      "mean_token_accuracy": 0.9227856550365686,
+      "num_tokens": 15928370.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.39133549834950826,
+      "epoch": 0.9288688207353545,
+      "grad_norm": 0.19921875,
+      "learning_rate": 0.0001381488430268918,
+      "loss": 0.3845,
+      "mean_token_accuracy": 0.9107819177210331,
+      "num_tokens": 16090841.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.368834131793119,
+      "epoch": 0.9382513340761156,
+      "grad_norm": 0.16796875,
+      "learning_rate": 0.0001375234521575985,
+      "loss": 0.3592,
+      "mean_token_accuracy": 0.9155494935810566,
+      "num_tokens": 16253510.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.3432939130580053,
+      "epoch": 0.9476338474168768,
+      "grad_norm": 0.181640625,
+      "learning_rate": 0.0001368980612883052,
+      "loss": 0.3434,
+      "mean_token_accuracy": 0.9178479440510273,
+      "num_tokens": 16416142.0,
+      "step": 1010
+    },
+    {
+      "entropy": 0.3263048173219431,
+      "epoch": 0.957016360757638,
+      "grad_norm": 0.158203125,
+      "learning_rate": 0.0001362726704190119,
+      "loss": 0.3157,
+      "mean_token_accuracy": 0.919898977316916,
+      "num_tokens": 16578145.0,
+      "step": 1020
+    },
+    {
+      "entropy": 0.30497189780580813,
+      "epoch": 0.9663988740983991,
+      "grad_norm": 0.162109375,
+      "learning_rate": 0.00013564727954971858,
+      "loss": 0.2985,
+      "mean_token_accuracy": 0.922874777764082,
+      "num_tokens": 16741103.0,
+      "step": 1030
+    },
+    {
+      "entropy": 0.2940454826748464,
+      "epoch": 0.9757813874391603,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 0.00013502188868042526,
+      "loss": 0.2874,
+      "mean_token_accuracy": 0.9239259999245405,
+      "num_tokens": 16903947.0,
+      "step": 1040
+    },
+    {
+      "entropy": 0.367372212279588,
+      "epoch": 0.9851639007799214,
+      "grad_norm": 0.1748046875,
+      "learning_rate": 0.00013439649781113198,
+      "loss": 0.3601,
+      "mean_token_accuracy": 0.9152833472937345,
+      "num_tokens": 17065800.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.34914421315770594,
+      "epoch": 0.9945464141206826,
+      "grad_norm": 0.1826171875,
+      "learning_rate": 0.00013377110694183866,
+      "loss": 0.3425,
+      "mean_token_accuracy": 0.9175648283213377,
+      "num_tokens": 17228736.0,
+      "step": 1060
+    },
+    {
+      "entropy": 0.2900184833320083,
+      "epoch": 1.0037530053363044,
+      "grad_norm": 0.13671875,
+      "learning_rate": 0.00013314571607254535,
+      "loss": 0.2782,
+      "mean_token_accuracy": 0.9317144804699405,
+      "num_tokens": 17388924.0,
+      "step": 1070
+    },
+    {
+      "entropy": 0.2829561754653696,
+      "epoch": 1.0131355186770656,
+      "grad_norm": 0.1259765625,
+      "learning_rate": 0.00013252032520325203,
+      "loss": 0.2765,
+      "mean_token_accuracy": 0.9315391689538955,
+      "num_tokens": 17550836.0,
+      "step": 1080
+    },
+    {
+      "entropy": 0.3159750218328554,
+      "epoch": 1.0225180320178269,
+      "grad_norm": 0.154296875,
+      "learning_rate": 0.00013189493433395872,
+      "loss": 0.3097,
+      "mean_token_accuracy": 0.924969544634223,
+      "num_tokens": 17713453.0,
+      "step": 1090
+    },
+    {
+      "entropy": 0.3149459302192554,
+      "epoch": 1.0319005453585879,
+      "grad_norm": 0.181640625,
+      "learning_rate": 0.00013126954346466543,
+      "loss": 0.307,
+      "mean_token_accuracy": 0.9239892680197954,
+      "num_tokens": 17875306.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.3264659658656456,
+      "epoch": 1.0412830586993491,
+      "grad_norm": 0.140625,
+      "learning_rate": 0.0001306441525953721,
+      "loss": 0.3177,
+      "mean_token_accuracy": 0.9219260595738887,
+      "num_tokens": 18037829.0,
+      "step": 1110
+    },
+    {
+      "entropy": 0.3119678142364137,
+      "epoch": 1.0506655720401104,
+      "grad_norm": 0.1376953125,
+      "learning_rate": 0.0001300187617260788,
+      "loss": 0.3047,
+      "mean_token_accuracy": 0.9250960987061262,
+      "num_tokens": 18201027.0,
+      "step": 1120
+    },
+    {
+      "entropy": 0.3200692283746321,
+      "epoch": 1.0600480853808714,
+      "grad_norm": 0.115234375,
+      "learning_rate": 0.0001293933708567855,
+      "loss": 0.3129,
+      "mean_token_accuracy": 0.9225547250360251,
+      "num_tokens": 18364107.0,
+      "step": 1130
+    },
+    {
+      "entropy": 0.2799913567607291,
+      "epoch": 1.0694305987216326,
+      "grad_norm": 0.1865234375,
+      "learning_rate": 0.0001287679799874922,
+      "loss": 0.2668,
+      "mean_token_accuracy": 0.9307141672819853,
+      "num_tokens": 18526662.0,
+      "step": 1140
+    },
+    {
+      "entropy": 0.3025812109815888,
+      "epoch": 1.0788131120623938,
+      "grad_norm": 0.1728515625,
+      "learning_rate": 0.00012814258911819887,
+      "loss": 0.3015,
+      "mean_token_accuracy": 0.9235180430114269,
+      "num_tokens": 18689065.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.2885304512223229,
+      "epoch": 1.0881956254031548,
+      "grad_norm": 0.263671875,
+      "learning_rate": 0.00012751719824890558,
+      "loss": 0.2786,
+      "mean_token_accuracy": 0.9279053185135127,
+      "num_tokens": 18851798.0,
+      "step": 1160
+    },
+    {
+      "entropy": 0.33722220184863544,
+      "epoch": 1.097578138743916,
+      "grad_norm": 0.1962890625,
+      "learning_rate": 0.00012689180737961227,
+      "loss": 0.3301,
+      "mean_token_accuracy": 0.9198755659162998,
+      "num_tokens": 19013951.0,
+      "step": 1170
+    },
+    {
+      "entropy": 0.35752617044490764,
+      "epoch": 1.106960652084677,
+      "grad_norm": 0.1826171875,
+      "learning_rate": 0.00012626641651031895,
+      "loss": 0.3525,
+      "mean_token_accuracy": 0.9136955734342337,
+      "num_tokens": 19176391.0,
+      "step": 1180
+    },
+    {
+      "entropy": 0.3285738020436838,
+      "epoch": 1.1163431654254383,
+      "grad_norm": 0.169921875,
+      "learning_rate": 0.00012564102564102564,
+      "loss": 0.3174,
+      "mean_token_accuracy": 0.923049738816917,
+      "num_tokens": 19338315.0,
+      "step": 1190
+    },
+    {
+      "entropy": 0.33181785854394547,
+      "epoch": 1.1257256787661996,
+      "grad_norm": 0.154296875,
+      "learning_rate": 0.00012501563477173235,
+      "loss": 0.3252,
+      "mean_token_accuracy": 0.9195261619985103,
+      "num_tokens": 19501276.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.33468560994951985,
+      "epoch": 1.1351081921069606,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.00012439024390243904,
+      "loss": 0.3297,
+      "mean_token_accuracy": 0.9180046692490578,
+      "num_tokens": 19663199.0,
+      "step": 1210
+    },
+    {
+      "entropy": 0.35496956037823113,
+      "epoch": 1.1444907054477218,
+      "grad_norm": 0.14453125,
+      "learning_rate": 0.00012376485303314572,
+      "loss": 0.3452,
+      "mean_token_accuracy": 0.9190296337008477,
+      "num_tokens": 19825833.0,
+      "step": 1220
+    },
+    {
+      "entropy": 0.3228163967374712,
+      "epoch": 1.153873218788483,
+      "grad_norm": 0.173828125,
+      "learning_rate": 0.0001231394621638524,
+      "loss": 0.3184,
+      "mean_token_accuracy": 0.9187171328812838,
+      "num_tokens": 19988129.0,
+      "step": 1230
+    },
+    {
+      "entropy": 0.36049368806998244,
+      "epoch": 1.163255732129244,
+      "grad_norm": 0.177734375,
+      "learning_rate": 0.0001225140712945591,
+      "loss": 0.3561,
+      "mean_token_accuracy": 0.9135892525315285,
+      "num_tokens": 20149704.0,
+      "step": 1240
+    },
+    {
+      "entropy": 0.29329942048643715,
+      "epoch": 1.1726382454700053,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 0.0001218886804252658,
+      "loss": 0.2808,
+      "mean_token_accuracy": 0.9278272874653339,
+      "num_tokens": 20312458.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.3245240814285353,
+      "epoch": 1.1820207588107665,
+      "grad_norm": 0.1328125,
+      "learning_rate": 0.00012126328955597248,
+      "loss": 0.3206,
+      "mean_token_accuracy": 0.9241520136594772,
+      "num_tokens": 20474875.0,
+      "step": 1260
+    },
+    {
+      "entropy": 0.31488763078814375,
+      "epoch": 1.1914032721515275,
+      "grad_norm": 0.134765625,
+      "learning_rate": 0.00012063789868667918,
+      "loss": 0.3092,
+      "mean_token_accuracy": 0.9249691208824515,
+      "num_tokens": 20636775.0,
+      "step": 1270
+    },
+    {
+      "entropy": 0.32809503517928534,
+      "epoch": 1.2007857854922888,
+      "grad_norm": 0.1748046875,
+      "learning_rate": 0.00012001250781738587,
+      "loss": 0.3175,
+      "mean_token_accuracy": 0.9244366209954024,
+      "num_tokens": 20799311.0,
+      "step": 1280
+    },
+    {
+      "entropy": 0.3452982397924643,
+      "epoch": 1.21016829883305,
+      "grad_norm": 0.1337890625,
+      "learning_rate": 0.00011938711694809257,
+      "loss": 0.3417,
+      "mean_token_accuracy": 0.9209324043244124,
+      "num_tokens": 20961671.0,
+      "step": 1290
+    },
+    {
+      "entropy": 0.33462724700802937,
+      "epoch": 1.219550812173811,
+      "grad_norm": 0.1728515625,
+      "learning_rate": 0.00011876172607879925,
+      "loss": 0.3274,
+      "mean_token_accuracy": 0.9196479767560959,
+      "num_tokens": 21124643.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.32381630715099163,
+      "epoch": 1.2289333255145722,
+      "grad_norm": 0.1376953125,
+      "learning_rate": 0.00011813633520950595,
+      "loss": 0.3161,
+      "mean_token_accuracy": 0.9226799987256527,
+      "num_tokens": 21286738.0,
+      "step": 1310
+    },
+    {
+      "entropy": 0.3411265324510168,
+      "epoch": 1.2383158388553333,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 0.00011751094434021264,
+      "loss": 0.3347,
+      "mean_token_accuracy": 0.9211051788181066,
+      "num_tokens": 21448827.0,
+      "step": 1320
+    },
+    {
+      "entropy": 0.32648798704030924,
+      "epoch": 1.2476983521960945,
+      "grad_norm": 0.1669921875,
+      "learning_rate": 0.00011688555347091934,
+      "loss": 0.3159,
+      "mean_token_accuracy": 0.922034990042448,
+      "num_tokens": 21610617.0,
+      "step": 1330
+    },
+    {
+      "entropy": 0.3225275243108626,
+      "epoch": 1.2570808655368557,
+      "grad_norm": 0.1337890625,
+      "learning_rate": 0.00011626016260162603,
+      "loss": 0.3217,
+      "mean_token_accuracy": 0.924454814940691,
+      "num_tokens": 21773343.0,
+      "step": 1340
+    },
+    {
+      "entropy": 0.31472907413262874,
+      "epoch": 1.2664633788776167,
+      "grad_norm": 0.158203125,
+      "learning_rate": 0.0001156347717323327,
+      "loss": 0.3056,
+      "mean_token_accuracy": 0.9251250382512808,
+      "num_tokens": 21936161.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.34041504978667947,
+      "epoch": 1.275845892218378,
+      "grad_norm": 0.15625,
+      "learning_rate": 0.00011500938086303941,
+      "loss": 0.3337,
+      "mean_token_accuracy": 0.9195377014577388,
+      "num_tokens": 22099004.0,
+      "step": 1360
+    },
+    {
+      "entropy": 0.32660515860188755,
+      "epoch": 1.2852284055591392,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 0.00011438398999374609,
+      "loss": 0.3177,
+      "mean_token_accuracy": 0.9214879527688027,
+      "num_tokens": 22261465.0,
+      "step": 1370
+    },
+    {
+      "entropy": 0.27577626157435586,
+      "epoch": 1.2946109188999002,
+      "grad_norm": 0.171875,
+      "learning_rate": 0.00011375859912445279,
+      "loss": 0.2718,
+      "mean_token_accuracy": 0.9287902843207121,
+      "num_tokens": 22424003.0,
+      "step": 1380
+    },
+    {
+      "entropy": 0.3049365603888873,
+      "epoch": 1.3039934322406614,
+      "grad_norm": 0.150390625,
+      "learning_rate": 0.00011313320825515947,
+      "loss": 0.296,
+      "mean_token_accuracy": 0.9254100024700165,
+      "num_tokens": 22586414.0,
+      "step": 1390
+    },
+    {
+      "entropy": 0.3391640084271785,
+      "epoch": 1.3133759455814227,
+      "grad_norm": 0.158203125,
+      "learning_rate": 0.00011250781738586617,
+      "loss": 0.3316,
+      "mean_token_accuracy": 0.9204599536955357,
+      "num_tokens": 22748801.0,
+      "step": 1400
+    },
+    {
+      "entropy": 0.29096004691673444,
+      "epoch": 1.3227584589221837,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 0.00011188242651657286,
+      "loss": 0.2829,
+      "mean_token_accuracy": 0.926516629010439,
+      "num_tokens": 22911078.0,
+      "step": 1410
+    },
+    {
+      "entropy": 0.30442387533839793,
+      "epoch": 1.332140972262945,
+      "grad_norm": 0.1435546875,
+      "learning_rate": 0.00011125703564727956,
+      "loss": 0.2997,
+      "mean_token_accuracy": 0.928035743534565,
+      "num_tokens": 23073932.0,
+      "step": 1420
+    },
+    {
+      "entropy": 0.2672559514408931,
+      "epoch": 1.3415234856037062,
+      "grad_norm": 0.1669921875,
+      "learning_rate": 0.00011063164477798624,
+      "loss": 0.2567,
+      "mean_token_accuracy": 0.934470995515585,
+      "num_tokens": 23234861.0,
+      "step": 1430
+    },
+    {
+      "entropy": 0.28968692439375443,
+      "epoch": 1.3509059989444672,
+      "grad_norm": 0.189453125,
+      "learning_rate": 0.00011000625390869294,
+      "loss": 0.2833,
+      "mean_token_accuracy": 0.9286899168044329,
+      "num_tokens": 23397575.0,
+      "step": 1440
+    },
+    {
+      "entropy": 0.3179404502676334,
+      "epoch": 1.3602885122852284,
+      "grad_norm": 0.154296875,
+      "learning_rate": 0.00010938086303939963,
+      "loss": 0.3106,
+      "mean_token_accuracy": 0.9237559959292412,
+      "num_tokens": 23560621.0,
+      "step": 1450
+    },
+    {
+      "entropy": 0.3825915275025181,
+      "epoch": 1.3696710256259896,
+      "grad_norm": 0.1796875,
+      "learning_rate": 0.00010875547217010633,
+      "loss": 0.3812,
+      "mean_token_accuracy": 0.9096528366208076,
+      "num_tokens": 23723326.0,
+      "step": 1460
+    },
+    {
+      "entropy": 0.352222545677796,
+      "epoch": 1.3790535389667506,
+      "grad_norm": 0.1611328125,
+      "learning_rate": 0.00010813008130081302,
+      "loss": 0.3445,
+      "mean_token_accuracy": 0.9214517448097468,
+      "num_tokens": 23885973.0,
+      "step": 1470
+    },
+    {
+      "entropy": 0.3833993759646546,
+      "epoch": 1.3884360523075119,
+      "grad_norm": 0.1591796875,
+      "learning_rate": 0.00010750469043151969,
+      "loss": 0.3772,
+      "mean_token_accuracy": 0.9148985600098968,
+      "num_tokens": 24049379.0,
+      "step": 1480
+    },
+    {
+      "entropy": 0.3407844117609784,
+      "epoch": 1.3978185656482731,
+      "grad_norm": 0.15625,
+      "learning_rate": 0.0001068792995622264,
+      "loss": 0.3332,
+      "mean_token_accuracy": 0.9178143370896578,
+      "num_tokens": 24211771.0,
+      "step": 1490
+    },
+    {
+      "entropy": 0.3129306449787691,
+      "epoch": 1.4072010789890341,
+      "grad_norm": 0.166015625,
+      "learning_rate": 0.00010625390869293307,
+      "loss": 0.3105,
+      "mean_token_accuracy": 0.9243231456726789,
+      "num_tokens": 24374782.0,
+      "step": 1500
+    },
+    {
+      "entropy": 0.2853265376936179,
+      "epoch": 1.4165835923297954,
+      "grad_norm": 0.14453125,
+      "learning_rate": 0.00010562851782363979,
+      "loss": 0.272,
+      "mean_token_accuracy": 0.9318001717329025,
+      "num_tokens": 24537798.0,
+      "step": 1510
+    },
+    {
+      "entropy": 0.32060901625081895,
+      "epoch": 1.4259661056705566,
+      "grad_norm": 0.169921875,
+      "learning_rate": 0.00010500312695434646,
+      "loss": 0.3189,
+      "mean_token_accuracy": 0.923735149204731,
+      "num_tokens": 24700222.0,
+      "step": 1520
+    },
+    {
+      "entropy": 0.28523959064041265,
+      "epoch": 1.4353486190113176,
+      "grad_norm": 0.138671875,
+      "learning_rate": 0.00010437773608505317,
+      "loss": 0.2765,
+      "mean_token_accuracy": 0.9289753165096044,
+      "num_tokens": 24862698.0,
+      "step": 1530
+    },
+    {
+      "entropy": 0.28084768254193476,
+      "epoch": 1.4447311323520788,
+      "grad_norm": 0.1435546875,
+      "learning_rate": 0.00010375234521575985,
+      "loss": 0.2735,
+      "mean_token_accuracy": 0.9312996946275234,
+      "num_tokens": 25025816.0,
+      "step": 1540
+    },
+    {
+      "entropy": 0.3017741791787557,
+      "epoch": 1.45411364569284,
+      "grad_norm": 0.15625,
+      "learning_rate": 0.00010312695434646656,
+      "loss": 0.2951,
+      "mean_token_accuracy": 0.9261103741824627,
+      "num_tokens": 25189369.0,
+      "step": 1550
+    },
+    {
+      "entropy": 0.3212110122665763,
+      "epoch": 1.463496159033601,
+      "grad_norm": 0.181640625,
+      "learning_rate": 0.00010250156347717323,
+      "loss": 0.3141,
+      "mean_token_accuracy": 0.9228064101189375,
+      "num_tokens": 25351206.0,
+      "step": 1560
+    },
+    {
+      "entropy": 0.3111924729484599,
+      "epoch": 1.4728786723743623,
+      "grad_norm": 0.1591796875,
+      "learning_rate": 0.00010187617260787993,
+      "loss": 0.3073,
+      "mean_token_accuracy": 0.9231358967721462,
+      "num_tokens": 25514005.0,
+      "step": 1570
+    },
+    {
+      "entropy": 0.35193296511424704,
+      "epoch": 1.4822611857151236,
+      "grad_norm": 0.1748046875,
+      "learning_rate": 0.00010125078173858662,
+      "loss": 0.3426,
+      "mean_token_accuracy": 0.9204717528074979,
+      "num_tokens": 25676907.0,
+      "step": 1580
+    },
+    {
+      "entropy": 0.33642219280591235,
+      "epoch": 1.4916436990558846,
+      "grad_norm": 0.1611328125,
+      "learning_rate": 0.00010062539086929332,
+      "loss": 0.3297,
+      "mean_token_accuracy": 0.9194286223500967,
+      "num_tokens": 25839935.0,
+      "step": 1590
+    },
+    {
+      "entropy": 0.26263106021797283,
+      "epoch": 1.5010262123966458,
+      "grad_norm": 0.142578125,
+      "learning_rate": 0.0001,
+      "loss": 0.2573,
+      "mean_token_accuracy": 0.935544315353036,
+      "num_tokens": 26003218.0,
+      "step": 1600
+    },
+    {
+      "entropy": 0.3122893640131224,
+      "epoch": 1.510408725737407,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 9.937460913070669e-05,
+      "loss": 0.3049,
+      "mean_token_accuracy": 0.9271042123436928,
+      "num_tokens": 26165724.0,
+      "step": 1610
+    },
+    {
+      "entropy": 0.29348382144235075,
+      "epoch": 1.519791239078168,
+      "grad_norm": 0.1640625,
+      "learning_rate": 9.874921826141339e-05,
+      "loss": 0.2887,
+      "mean_token_accuracy": 0.9292786654084921,
+      "num_tokens": 26328312.0,
+      "step": 1620
+    },
+    {
+      "entropy": 0.34205422176164574,
+      "epoch": 1.529173752418929,
+      "grad_norm": 0.1484375,
+      "learning_rate": 9.812382739212008e-05,
+      "loss": 0.3353,
+      "mean_token_accuracy": 0.9192357573658227,
+      "num_tokens": 26490508.0,
+      "step": 1630
+    },
+    {
+      "entropy": 0.3207799144263845,
+      "epoch": 1.5385562657596905,
+      "grad_norm": 0.1591796875,
+      "learning_rate": 9.749843652282678e-05,
+      "loss": 0.3142,
+      "mean_token_accuracy": 0.9239265900105238,
+      "num_tokens": 26652241.0,
+      "step": 1640
+    },
+    {
+      "entropy": 0.3212417629547417,
+      "epoch": 1.5479387791004515,
+      "grad_norm": 0.12890625,
+      "learning_rate": 9.687304565353346e-05,
+      "loss": 0.3105,
+      "mean_token_accuracy": 0.9247593119740486,
+      "num_tokens": 26815313.0,
+      "step": 1650
+    },
+    {
+      "entropy": 0.2918511720839888,
+      "epoch": 1.5573212924412125,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 9.624765478424016e-05,
+      "loss": 0.2874,
+      "mean_token_accuracy": 0.9265463493764401,
+      "num_tokens": 26978291.0,
+      "step": 1660
+    },
+    {
+      "entropy": 0.3887786120234523,
+      "epoch": 1.566703805781974,
+      "grad_norm": 0.177734375,
+      "learning_rate": 9.562226391494684e-05,
+      "loss": 0.3832,
+      "mean_token_accuracy": 0.9116566512733698,
+      "num_tokens": 27140951.0,
+      "step": 1670
+    },
+    {
+      "entropy": 0.3554312888765708,
+      "epoch": 1.576086319122735,
+      "grad_norm": 0.2080078125,
+      "learning_rate": 9.499687304565354e-05,
+      "loss": 0.3498,
+      "mean_token_accuracy": 0.9205850351601839,
+      "num_tokens": 27303423.0,
+      "step": 1680
+    },
+    {
+      "entropy": 0.3036960868688766,
+      "epoch": 1.585468832463496,
+      "grad_norm": 0.1455078125,
+      "learning_rate": 9.437148217636022e-05,
+      "loss": 0.2982,
+      "mean_token_accuracy": 0.9223078574985266,
+      "num_tokens": 27465935.0,
+      "step": 1690
+    },
+    {
+      "entropy": 0.34383984084706753,
+      "epoch": 1.5948513458042575,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 9.374609130706692e-05,
+      "loss": 0.3352,
+      "mean_token_accuracy": 0.9215784965083003,
+      "num_tokens": 27628448.0,
+      "step": 1700
+    },
+    {
+      "entropy": 0.3477696883957833,
+      "epoch": 1.6042338591450185,
+      "grad_norm": 0.15234375,
+      "learning_rate": 9.312070043777361e-05,
+      "loss": 0.345,
+      "mean_token_accuracy": 0.9204984463751316,
+      "num_tokens": 27790993.0,
+      "step": 1710
+    },
+    {
+      "entropy": 0.4038438654737547,
+      "epoch": 1.6136163724857795,
+      "grad_norm": 0.1455078125,
+      "learning_rate": 9.249530956848031e-05,
+      "loss": 0.3942,
+      "mean_token_accuracy": 0.9117133766412735,
+      "num_tokens": 27953367.0,
+      "step": 1720
+    },
+    {
+      "entropy": 0.35322881376487203,
+      "epoch": 1.622998885826541,
+      "grad_norm": 0.2255859375,
+      "learning_rate": 9.1869918699187e-05,
+      "loss": 0.3508,
+      "mean_token_accuracy": 0.9202003927901388,
+      "num_tokens": 28115525.0,
+      "step": 1730
+    },
+    {
+      "entropy": 0.3633168841304723,
+      "epoch": 1.632381399167302,
+      "grad_norm": 0.1376953125,
+      "learning_rate": 9.12445278298937e-05,
+      "loss": 0.3537,
+      "mean_token_accuracy": 0.9178501278162002,
+      "num_tokens": 28278195.0,
+      "step": 1740
+    },
+    {
+      "entropy": 0.29921169562148864,
+      "epoch": 1.641763912508063,
+      "grad_norm": 0.169921875,
+      "learning_rate": 9.061913696060038e-05,
+      "loss": 0.2928,
+      "mean_token_accuracy": 0.929632369056344,
+      "num_tokens": 28441365.0,
+      "step": 1750
+    },
+    {
+      "entropy": 0.343074598867679,
+      "epoch": 1.6511464258488242,
+      "grad_norm": 0.1826171875,
+      "learning_rate": 8.999374609130708e-05,
+      "loss": 0.3418,
+      "mean_token_accuracy": 0.9203568341210484,
+      "num_tokens": 28604025.0,
+      "step": 1760
+    },
+    {
+      "entropy": 0.3354411284439266,
+      "epoch": 1.6605289391895854,
+      "grad_norm": 0.19921875,
+      "learning_rate": 8.936835522201377e-05,
+      "loss": 0.3247,
+      "mean_token_accuracy": 0.9247291401028633,
+      "num_tokens": 28766627.0,
+      "step": 1770
+    },
+    {
+      "entropy": 0.29062830022885466,
+      "epoch": 1.6699114525303465,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 8.874296435272045e-05,
+      "loss": 0.2871,
+      "mean_token_accuracy": 0.9279629901051522,
+      "num_tokens": 28928424.0,
+      "step": 1780
+    },
+    {
+      "entropy": 0.3109960203932133,
+      "epoch": 1.6792939658711077,
+      "grad_norm": 0.158203125,
+      "learning_rate": 8.811757348342715e-05,
+      "loss": 0.3046,
+      "mean_token_accuracy": 0.925516726821661,
+      "num_tokens": 29090881.0,
+      "step": 1790
+    },
+    {
+      "entropy": 0.29157741767703554,
+      "epoch": 1.688676479211869,
+      "grad_norm": 0.1474609375,
+      "learning_rate": 8.749218261413384e-05,
+      "loss": 0.2872,
+      "mean_token_accuracy": 0.9311040572822094,
+      "num_tokens": 29253614.0,
+      "step": 1800
+    },
+    {
+      "entropy": 0.2842859643569682,
+      "epoch": 1.69805899255263,
+      "grad_norm": 0.19921875,
+      "learning_rate": 8.686679174484052e-05,
+      "loss": 0.2787,
+      "mean_token_accuracy": 0.930610903352499,
+      "num_tokens": 29415480.0,
+      "step": 1810
+    },
+    {
+      "entropy": 0.2779356167244259,
+      "epoch": 1.7074415058933912,
+      "grad_norm": 0.1865234375,
+      "learning_rate": 8.624140087554722e-05,
+      "loss": 0.2712,
+      "mean_token_accuracy": 0.9308344077318906,
+      "num_tokens": 29578036.0,
+      "step": 1820
+    },
+    {
+      "entropy": 0.318134157784516,
+      "epoch": 1.7168240192341524,
+      "grad_norm": 0.130859375,
+      "learning_rate": 8.561601000625391e-05,
+      "loss": 0.311,
+      "mean_token_accuracy": 0.9251023806631565,
+      "num_tokens": 29740251.0,
+      "step": 1830
+    },
+    {
+      "entropy": 0.30244565989705735,
+      "epoch": 1.7262065325749134,
+      "grad_norm": 0.2451171875,
+      "learning_rate": 8.49906191369606e-05,
+      "loss": 0.2985,
+      "mean_token_accuracy": 0.925722798705101,
+      "num_tokens": 29902140.0,
+      "step": 1840
+    },
+    {
+      "entropy": 0.32010520926560276,
+      "epoch": 1.7355890459156746,
+      "grad_norm": 0.125,
+      "learning_rate": 8.43652282676673e-05,
+      "loss": 0.3077,
+      "mean_token_accuracy": 0.9267256014049053,
+      "num_tokens": 30064277.0,
+      "step": 1850
+    },
+    {
+      "entropy": 0.2397772744880058,
+      "epoch": 1.7449715592564359,
+      "grad_norm": 0.123046875,
+      "learning_rate": 8.373983739837398e-05,
+      "loss": 0.2367,
+      "mean_token_accuracy": 0.9358306158334017,
+      "num_tokens": 30227035.0,
+      "step": 1860
+    },
+    {
+      "entropy": 0.2846862503385637,
+      "epoch": 1.754354072597197,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 8.311444652908068e-05,
+      "loss": 0.2791,
+      "mean_token_accuracy": 0.9296355988830328,
+      "num_tokens": 30389480.0,
+      "step": 1870
+    },
+    {
+      "entropy": 0.3066696587542538,
+      "epoch": 1.7637365859379581,
+      "grad_norm": 0.2021484375,
+      "learning_rate": 8.248905565978737e-05,
+      "loss": 0.3006,
+      "mean_token_accuracy": 0.9251831691712141,
+      "num_tokens": 30552091.0,
+      "step": 1880
+    },
+    {
+      "entropy": 0.31332365355920044,
+      "epoch": 1.7731190992787194,
+      "grad_norm": 0.1552734375,
+      "learning_rate": 8.186366479049407e-05,
+      "loss": 0.3065,
+      "mean_token_accuracy": 0.9259384579956531,
+      "num_tokens": 30714856.0,
+      "step": 1890
+    },
+    {
+      "entropy": 0.26562274275929665,
+      "epoch": 1.7825016126194804,
+      "grad_norm": 0.1201171875,
+      "learning_rate": 8.123827392120076e-05,
+      "loss": 0.2575,
+      "mean_token_accuracy": 0.9345466572791338,
+      "num_tokens": 30877977.0,
+      "step": 1900
+    },
+    {
+      "entropy": 0.26213877720292655,
+      "epoch": 1.7918841259602416,
+      "grad_norm": 0.1396484375,
+      "learning_rate": 8.061288305190746e-05,
+      "loss": 0.2569,
+      "mean_token_accuracy": 0.9343690615147352,
+      "num_tokens": 31040248.0,
+      "step": 1910
+    },
+    {
+      "entropy": 0.31369023660663514,
+      "epoch": 1.8012666393010028,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 7.998749218261413e-05,
+      "loss": 0.3079,
+      "mean_token_accuracy": 0.9239290647208691,
+      "num_tokens": 31202363.0,
+      "step": 1920
+    },
+    {
+      "entropy": 0.2883313969883602,
+      "epoch": 1.8106491526417638,
+      "grad_norm": 0.162109375,
+      "learning_rate": 7.936210131332083e-05,
+      "loss": 0.2846,
+      "mean_token_accuracy": 0.9300382232293487,
+      "num_tokens": 31365512.0,
+      "step": 1930
+    },
+    {
+      "entropy": 0.2506348692520987,
+      "epoch": 1.820031665982525,
+      "grad_norm": 0.171875,
+      "learning_rate": 7.873671044402751e-05,
+      "loss": 0.2448,
+      "mean_token_accuracy": 0.9374226000159979,
+      "num_tokens": 31527448.0,
+      "step": 1940
+    },
+    {
+      "entropy": 0.37234054349246437,
+      "epoch": 1.8294141793232863,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 7.811131957473421e-05,
+      "loss": 0.3625,
+      "mean_token_accuracy": 0.9161295138299466,
+      "num_tokens": 31690266.0,
+      "step": 1950
+    },
+    {
+      "entropy": 0.30422380041563885,
+      "epoch": 1.8387966926640473,
+      "grad_norm": 0.193359375,
+      "learning_rate": 7.74859287054409e-05,
+      "loss": 0.3022,
+      "mean_token_accuracy": 0.9249883979558945,
+      "num_tokens": 31852062.0,
+      "step": 1960
+    },
+    {
+      "entropy": 0.28159310154733247,
+      "epoch": 1.8481792060048086,
+      "grad_norm": 0.12890625,
+      "learning_rate": 7.68605378361476e-05,
+      "loss": 0.2734,
+      "mean_token_accuracy": 0.9310276363044977,
+      "num_tokens": 32015366.0,
+      "step": 1970
+    },
+    {
+      "entropy": 0.34929422264685855,
+      "epoch": 1.8575617193455698,
+      "grad_norm": 0.134765625,
+      "learning_rate": 7.623514696685429e-05,
+      "loss": 0.3437,
+      "mean_token_accuracy": 0.9215225942432881,
+      "num_tokens": 32178520.0,
+      "step": 1980
+    },
+    {
+      "entropy": 0.3168531317904126,
+      "epoch": 1.8669442326863308,
+      "grad_norm": 0.1787109375,
+      "learning_rate": 7.560975609756099e-05,
+      "loss": 0.311,
+      "mean_token_accuracy": 0.9265505827963352,
+      "num_tokens": 32341136.0,
+      "step": 1990
+    },
+    {
+      "entropy": 0.34503675387823024,
+      "epoch": 1.876326746027092,
+      "grad_norm": 0.1337890625,
+      "learning_rate": 7.498436522826767e-05,
+      "loss": 0.3371,
+      "mean_token_accuracy": 0.9186861624941229,
+      "num_tokens": 32503824.0,
+      "step": 2000
+    },
+    {
+      "entropy": 0.3137181800149847,
+      "epoch": 1.8857092593678533,
+      "grad_norm": 0.12890625,
+      "learning_rate": 7.435897435897436e-05,
+      "loss": 0.3072,
+      "mean_token_accuracy": 0.9252266064286232,
+      "num_tokens": 32665961.0,
+      "step": 2010
+    },
+    {
+      "entropy": 0.3118691684619989,
+      "epoch": 1.8950917727086143,
+      "grad_norm": 0.1259765625,
+      "learning_rate": 7.373358348968106e-05,
+      "loss": 0.3092,
+      "mean_token_accuracy": 0.9266283692792058,
+      "num_tokens": 32828930.0,
+      "step": 2020
+    },
+    {
+      "entropy": 0.36025177109404466,
+      "epoch": 1.9044742860493755,
+      "grad_norm": 0.1337890625,
+      "learning_rate": 7.310819262038774e-05,
+      "loss": 0.3483,
+      "mean_token_accuracy": 0.919862699508667,
+      "num_tokens": 32991199.0,
+      "step": 2030
+    },
+    {
+      "entropy": 0.2960929194523487,
+      "epoch": 1.9138567993901368,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 7.248280175109444e-05,
+      "loss": 0.2911,
+      "mean_token_accuracy": 0.9277313150465488,
+      "num_tokens": 33153591.0,
+      "step": 2040
+    },
+    {
+      "entropy": 0.3087802886031568,
+      "epoch": 1.9232393127308978,
+      "grad_norm": 0.1318359375,
+      "learning_rate": 7.185741088180113e-05,
+      "loss": 0.3034,
+      "mean_token_accuracy": 0.9258704790845513,
+      "num_tokens": 33316611.0,
+      "step": 2050
+    },
+    {
+      "entropy": 0.3371582037769258,
+      "epoch": 1.932621826071659,
+      "grad_norm": 0.1484375,
+      "learning_rate": 7.123202001250782e-05,
+      "loss": 0.3246,
+      "mean_token_accuracy": 0.9226027946919203,
+      "num_tokens": 33479599.0,
+      "step": 2060
+    },
+    {
+      "entropy": 0.3058539542311337,
+      "epoch": 1.9420043394124202,
+      "grad_norm": 0.19140625,
+      "learning_rate": 7.06066291432145e-05,
+      "loss": 0.3052,
+      "mean_token_accuracy": 0.926361970603466,
+      "num_tokens": 33642025.0,
+      "step": 2070
+    },
+    {
+      "entropy": 0.30661272021825425,
+      "epoch": 1.9513868527531812,
+      "grad_norm": 0.142578125,
+      "learning_rate": 6.99812382739212e-05,
+      "loss": 0.3019,
+      "mean_token_accuracy": 0.9264740001410245,
+      "num_tokens": 33804960.0,
+      "step": 2080
+    },
+    {
+      "entropy": 0.3367635572212748,
+      "epoch": 1.9607693660939423,
+      "grad_norm": 0.140625,
+      "learning_rate": 6.935584740462789e-05,
+      "loss": 0.3255,
+      "mean_token_accuracy": 0.9226295977830887,
+      "num_tokens": 33967617.0,
+      "step": 2090
+    },
+    {
+      "entropy": 0.30951609945041125,
+      "epoch": 1.9701518794347037,
+      "grad_norm": 0.1455078125,
+      "learning_rate": 6.873045653533459e-05,
+      "loss": 0.3099,
+      "mean_token_accuracy": 0.9269402369856834,
+      "num_tokens": 34130354.0,
+      "step": 2100
+    },
+    {
+      "entropy": 0.27725380276096984,
+      "epoch": 1.9795343927754647,
+      "grad_norm": 0.138671875,
+      "learning_rate": 6.810506566604128e-05,
+      "loss": 0.2718,
+      "mean_token_accuracy": 0.9319463197141886,
+      "num_tokens": 34292449.0,
+      "step": 2110
+    },
+    {
+      "entropy": 0.3474087182315998,
+      "epoch": 1.9889169061162257,
+      "grad_norm": 0.158203125,
+      "learning_rate": 6.747967479674798e-05,
+      "loss": 0.3388,
+      "mean_token_accuracy": 0.918656725436449,
+      "num_tokens": 34454991.0,
+      "step": 2120
+    },
+    {
+      "entropy": 0.3628831981972326,
+      "epoch": 1.9982994194569872,
+      "grad_norm": 0.1220703125,
+      "learning_rate": 6.685428392745466e-05,
+      "loss": 0.3612,
+      "mean_token_accuracy": 0.9168768994510174,
+      "num_tokens": 34617319.0,
+      "step": 2130
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 3198,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.5629671685153587e+18,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-2132/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db83ac20cd1ea46a30b9db82ee3c07152c60e62ceb73c1d01e3725eeebe949ed
+size 6289

checkpoint-3198/README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.1-8B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.1-8B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.0

checkpoint-3198/adapter_config.json

@@ -0,0 +1,41 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.1-8B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 24,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.0",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-3198/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b54df820ee065f28e73fabdacb2f36346384e74c2d4d07dc646ffcea458bb34
+size 27280280

checkpoint-3198/chat_template.jinja

@@ -0,0 +1,109 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- set date_string = "26 Jul 2024" %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message + builtin tools #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if builtin_tools is defined or tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{%- if builtin_tools is defined %}
+    {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- "<|python_tag|>" + tool_call.name + ".call(" }}
+            {%- for arg_name, arg_val in tool_call.arguments | items %}
+                {{- arg_name + '="' + arg_val + '"' }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- endif %}
+                {%- endfor %}
+            {{- ")" }}
+        {%- else  %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- '{"name": "' + tool_call.name + '", ' }}
+            {{- '"parameters": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- "}" }}
+        {%- endif %}
+        {%- if builtin_tools is defined %}
+            {#- This means we're in ipython mode #}
+            {{- "<|eom_id|>" }}
+        {%- else %}
+            {{- "<|eot_id|>" }}
+        {%- endif %}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

checkpoint-3198/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d05a7b7fe3eb272321eac6c556d9740f72b1cf4175e604e2eab986ac48f3209
+size 27843851

checkpoint-3198/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b39b2b652316a4c251bf2db9a2ea5b6349c912b5ec7083e58d3608b50dafeffe
+size 14645

checkpoint-3198/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:598d66ec6c532715d17084f9d482bc685671827aa24d0b943b6aa04cee8d2c54
+size 1465

checkpoint-3198/special_tokens_map.json

@@ -0,0 +1,16 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-3198/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

checkpoint-3198/tokenizer_config.json

@@ -0,0 +1,2062 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

checkpoint-3198/trainer_state.json

@@ -0,0 +1,3224 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 3198,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 0.7799350878223776,
+      "epoch": 0.009382513340761156,
+      "grad_norm": 0.251953125,
+      "learning_rate": 0.00019943714821763604,
+      "loss": 0.8229,
+      "mean_token_accuracy": 0.8214783387258648,
+      "num_tokens": 162309.0,
+      "step": 10
+    },
+    {
+      "entropy": 0.6856271875090897,
+      "epoch": 0.01876502668152231,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 0.00019881175734834272,
+      "loss": 0.6706,
+      "mean_token_accuracy": 0.8432094575837255,
+      "num_tokens": 325111.0,
+      "step": 20
+    },
+    {
+      "entropy": 0.5147169048897922,
+      "epoch": 0.02814754002228347,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 0.00019818636647904943,
+      "loss": 0.5152,
+      "mean_token_accuracy": 0.8709522735327482,
+      "num_tokens": 488159.0,
+      "step": 30
+    },
+    {
+      "entropy": 0.48951989263296125,
+      "epoch": 0.03753005336304462,
+      "grad_norm": 0.1552734375,
+      "learning_rate": 0.0001975609756097561,
+      "loss": 0.4875,
+      "mean_token_accuracy": 0.8770332466810942,
+      "num_tokens": 650647.0,
+      "step": 40
+    },
+    {
+      "entropy": 0.4960091439075768,
+      "epoch": 0.04691256670380578,
+      "grad_norm": 0.16796875,
+      "learning_rate": 0.0001969355847404628,
+      "loss": 0.492,
+      "mean_token_accuracy": 0.8768141727894545,
+      "num_tokens": 813419.0,
+      "step": 50
+    },
+    {
+      "entropy": 0.4708784816786647,
+      "epoch": 0.05629508004456694,
+      "grad_norm": 0.197265625,
+      "learning_rate": 0.0001963101938711695,
+      "loss": 0.4648,
+      "mean_token_accuracy": 0.8837848342955112,
+      "num_tokens": 976458.0,
+      "step": 60
+    },
+    {
+      "entropy": 0.546927246497944,
+      "epoch": 0.0656775933853281,
+      "grad_norm": 0.1669921875,
+      "learning_rate": 0.00019568480300187618,
+      "loss": 0.5415,
+      "mean_token_accuracy": 0.8723422776907682,
+      "num_tokens": 1138794.0,
+      "step": 70
+    },
+    {
+      "entropy": 0.47402509227395057,
+      "epoch": 0.07506010672608925,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 0.00019505941213258287,
+      "loss": 0.4702,
+      "mean_token_accuracy": 0.885954425111413,
+      "num_tokens": 1301483.0,
+      "step": 80
+    },
+    {
+      "entropy": 0.4972494073677808,
+      "epoch": 0.08444262006685041,
+      "grad_norm": 0.1708984375,
+      "learning_rate": 0.00019443402126328955,
+      "loss": 0.4901,
+      "mean_token_accuracy": 0.8839348161593079,
+      "num_tokens": 1464240.0,
+      "step": 90
+    },
+    {
+      "entropy": 0.49195765373297035,
+      "epoch": 0.09382513340761156,
+      "grad_norm": 0.1669921875,
+      "learning_rate": 0.00019380863039399627,
+      "loss": 0.4848,
+      "mean_token_accuracy": 0.886236279271543,
+      "num_tokens": 1627457.0,
+      "step": 100
+    },
+    {
+      "entropy": 0.5126075576525182,
+      "epoch": 0.10320764674837273,
+      "grad_norm": 0.1787109375,
+      "learning_rate": 0.00019318323952470295,
+      "loss": 0.5104,
+      "mean_token_accuracy": 0.8793701384216547,
+      "num_tokens": 1790547.0,
+      "step": 110
+    },
+    {
+      "entropy": 0.4117678565904498,
+      "epoch": 0.11259016008913388,
+      "grad_norm": 0.1650390625,
+      "learning_rate": 0.00019255784865540964,
+      "loss": 0.4114,
+      "mean_token_accuracy": 0.8950984058901668,
+      "num_tokens": 1953536.0,
+      "step": 120
+    },
+    {
+      "entropy": 0.4656925258226693,
+      "epoch": 0.12197267342989504,
+      "grad_norm": 0.1875,
+      "learning_rate": 0.00019193245778611632,
+      "loss": 0.4634,
+      "mean_token_accuracy": 0.8897427920252085,
+      "num_tokens": 2115476.0,
+      "step": 130
+    },
+    {
+      "entropy": 0.43195099574513734,
+      "epoch": 0.1313551867706562,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.00019130706691682304,
+      "loss": 0.4246,
+      "mean_token_accuracy": 0.8946435421705246,
+      "num_tokens": 2277936.0,
+      "step": 140
+    },
+    {
+      "entropy": 0.4823188486509025,
+      "epoch": 0.14073770011141734,
+      "grad_norm": 0.20703125,
+      "learning_rate": 0.00019068167604752972,
+      "loss": 0.4796,
+      "mean_token_accuracy": 0.8873342543840408,
+      "num_tokens": 2440643.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.40974695349577817,
+      "epoch": 0.1501202134521785,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 0.0001900562851782364,
+      "loss": 0.4,
+      "mean_token_accuracy": 0.8974028028547764,
+      "num_tokens": 2603401.0,
+      "step": 160
+    },
+    {
+      "entropy": 0.41737876399420204,
+      "epoch": 0.15950272679293967,
+      "grad_norm": 0.177734375,
+      "learning_rate": 0.0001894308943089431,
+      "loss": 0.4136,
+      "mean_token_accuracy": 0.8985418293625116,
+      "num_tokens": 2765836.0,
+      "step": 170
+    },
+    {
+      "entropy": 0.4317509519169107,
+      "epoch": 0.16888524013370082,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.0001888055034396498,
+      "loss": 0.4276,
+      "mean_token_accuracy": 0.8956017823889851,
+      "num_tokens": 2928637.0,
+      "step": 180
+    },
+    {
+      "entropy": 0.39300057776272296,
+      "epoch": 0.17826775347446197,
+      "grad_norm": 0.177734375,
+      "learning_rate": 0.00018818011257035647,
+      "loss": 0.3853,
+      "mean_token_accuracy": 0.903920691087842,
+      "num_tokens": 3090979.0,
+      "step": 190
+    },
+    {
+      "entropy": 0.43899501564446836,
+      "epoch": 0.18765026681522312,
+      "grad_norm": 0.16796875,
+      "learning_rate": 0.00018755472170106316,
+      "loss": 0.4302,
+      "mean_token_accuracy": 0.894548149779439,
+      "num_tokens": 3253538.0,
+      "step": 200
+    },
+    {
+      "entropy": 0.3904935538303107,
+      "epoch": 0.19703278015598427,
+      "grad_norm": 0.1708984375,
+      "learning_rate": 0.00018692933083176987,
+      "loss": 0.3829,
+      "mean_token_accuracy": 0.9031265545636415,
+      "num_tokens": 3415948.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.39792181542143223,
+      "epoch": 0.20641529349674545,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 0.00018630393996247656,
+      "loss": 0.3862,
+      "mean_token_accuracy": 0.9033814109861851,
+      "num_tokens": 3579298.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.38712316513992845,
+      "epoch": 0.2157978068375066,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.00018567854909318324,
+      "loss": 0.3871,
+      "mean_token_accuracy": 0.9008511133491993,
+      "num_tokens": 3742007.0,
+      "step": 230
+    },
+    {
+      "entropy": 0.3974394467426464,
+      "epoch": 0.22518032017826775,
+      "grad_norm": 0.1640625,
+      "learning_rate": 0.00018505315822388993,
+      "loss": 0.39,
+      "mean_token_accuracy": 0.9035492207854986,
+      "num_tokens": 3904988.0,
+      "step": 240
+    },
+    {
+      "entropy": 0.42329446180956437,
+      "epoch": 0.2345628335190289,
+      "grad_norm": 0.19140625,
+      "learning_rate": 0.00018442776735459664,
+      "loss": 0.4136,
+      "mean_token_accuracy": 0.9000132424756885,
+      "num_tokens": 4067806.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.400798928248696,
+      "epoch": 0.24394534685979008,
+      "grad_norm": 0.1953125,
+      "learning_rate": 0.00018380237648530333,
+      "loss": 0.3889,
+      "mean_token_accuracy": 0.8984681818634271,
+      "num_tokens": 4229588.0,
+      "step": 260
+    },
+    {
+      "entropy": 0.3440579119604081,
+      "epoch": 0.2533278602005512,
+      "grad_norm": 0.1748046875,
+      "learning_rate": 0.00018317698561601001,
+      "loss": 0.3355,
+      "mean_token_accuracy": 0.9099012993276119,
+      "num_tokens": 4391775.0,
+      "step": 270
+    },
+    {
+      "entropy": 0.4088153497781605,
+      "epoch": 0.2627103735413124,
+      "grad_norm": 0.2060546875,
+      "learning_rate": 0.0001825515947467167,
+      "loss": 0.3988,
+      "mean_token_accuracy": 0.9000191796571017,
+      "num_tokens": 4554006.0,
+      "step": 280
+    },
+    {
+      "entropy": 0.49478547936305406,
+      "epoch": 0.27209288688207356,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.0001819262038774234,
+      "loss": 0.4851,
+      "mean_token_accuracy": 0.8896255612373352,
+      "num_tokens": 4716601.0,
+      "step": 290
+    },
+    {
+      "entropy": 0.3874137854669243,
+      "epoch": 0.2814754002228347,
+      "grad_norm": 0.1708984375,
+      "learning_rate": 0.0001813008130081301,
+      "loss": 0.3811,
+      "mean_token_accuracy": 0.9031782140955329,
+      "num_tokens": 4879465.0,
+      "step": 300
+    },
+    {
+      "entropy": 0.39499278874136506,
+      "epoch": 0.29085791356359586,
+      "grad_norm": 0.1796875,
+      "learning_rate": 0.00018067542213883679,
+      "loss": 0.3859,
+      "mean_token_accuracy": 0.9037530291825533,
+      "num_tokens": 5041857.0,
+      "step": 310
+    },
+    {
+      "entropy": 0.3148498781956732,
+      "epoch": 0.300240426904357,
+      "grad_norm": 0.146484375,
+      "learning_rate": 0.00018005003126954347,
+      "loss": 0.3105,
+      "mean_token_accuracy": 0.9153206121176481,
+      "num_tokens": 5204416.0,
+      "step": 320
+    },
+    {
+      "entropy": 0.45533317684894425,
+      "epoch": 0.30962294024511816,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.00017942464040025016,
+      "loss": 0.4477,
+      "mean_token_accuracy": 0.8917344063520432,
+      "num_tokens": 5366029.0,
+      "step": 330
+    },
+    {
+      "entropy": 0.4512431137613021,
+      "epoch": 0.31900545358587934,
+      "grad_norm": 0.1787109375,
+      "learning_rate": 0.00017879924953095687,
+      "loss": 0.4435,
+      "mean_token_accuracy": 0.8979733150452376,
+      "num_tokens": 5528662.0,
+      "step": 340
+    },
+    {
+      "entropy": 0.43219919862458483,
+      "epoch": 0.32838796692664046,
+      "grad_norm": 0.1875,
+      "learning_rate": 0.00017817385866166353,
+      "loss": 0.4255,
+      "mean_token_accuracy": 0.8939095275476575,
+      "num_tokens": 5691088.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.39246358472155407,
+      "epoch": 0.33777048026740164,
+      "grad_norm": 0.2060546875,
+      "learning_rate": 0.00017754846779237024,
+      "loss": 0.3792,
+      "mean_token_accuracy": 0.9027158614248038,
+      "num_tokens": 5853586.0,
+      "step": 360
+    },
+    {
+      "entropy": 0.38228728668764234,
+      "epoch": 0.34715299360816276,
+      "grad_norm": 0.158203125,
+      "learning_rate": 0.00017692307692307693,
+      "loss": 0.3814,
+      "mean_token_accuracy": 0.9076376244425773,
+      "num_tokens": 6016273.0,
+      "step": 370
+    },
+    {
+      "entropy": 0.34766899450914934,
+      "epoch": 0.35653550694892394,
+      "grad_norm": 0.16796875,
+      "learning_rate": 0.00017629768605378362,
+      "loss": 0.339,
+      "mean_token_accuracy": 0.9132196143269539,
+      "num_tokens": 6179608.0,
+      "step": 380
+    },
+    {
+      "entropy": 0.419576386269182,
+      "epoch": 0.3659180202896851,
+      "grad_norm": 0.150390625,
+      "learning_rate": 0.0001756722951844903,
+      "loss": 0.4081,
+      "mean_token_accuracy": 0.902265278249979,
+      "num_tokens": 6342688.0,
+      "step": 390
+    },
+    {
+      "entropy": 0.34522315053036434,
+      "epoch": 0.37530053363044624,
+      "grad_norm": 0.16796875,
+      "learning_rate": 0.00017504690431519702,
+      "loss": 0.3398,
+      "mean_token_accuracy": 0.9103329949080944,
+      "num_tokens": 6505076.0,
+      "step": 400
+    },
+    {
+      "entropy": 0.3350997743313201,
+      "epoch": 0.3846830469712074,
+      "grad_norm": 0.1611328125,
+      "learning_rate": 0.0001744215134459037,
+      "loss": 0.3287,
+      "mean_token_accuracy": 0.9135664030909538,
+      "num_tokens": 6667428.0,
+      "step": 410
+    },
+    {
+      "entropy": 0.35369435520842674,
+      "epoch": 0.39406556031196854,
+      "grad_norm": 0.16796875,
+      "learning_rate": 0.0001737961225766104,
+      "loss": 0.3445,
+      "mean_token_accuracy": 0.9136763598769904,
+      "num_tokens": 6829952.0,
+      "step": 420
+    },
+    {
+      "entropy": 0.3438279568566941,
+      "epoch": 0.4034480736527297,
+      "grad_norm": 0.2109375,
+      "learning_rate": 0.00017317073170731708,
+      "loss": 0.3347,
+      "mean_token_accuracy": 0.915488549694419,
+      "num_tokens": 6992752.0,
+      "step": 430
+    },
+    {
+      "entropy": 0.36355263493023815,
+      "epoch": 0.4128305869934909,
+      "grad_norm": 0.134765625,
+      "learning_rate": 0.0001725453408380238,
+      "loss": 0.3514,
+      "mean_token_accuracy": 0.9081276219338179,
+      "num_tokens": 7155395.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.4320855587720871,
+      "epoch": 0.422213100334252,
+      "grad_norm": 0.1787109375,
+      "learning_rate": 0.00017191994996873047,
+      "loss": 0.4275,
+      "mean_token_accuracy": 0.8996547561138868,
+      "num_tokens": 7317871.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.35265264644986016,
+      "epoch": 0.4315956136750132,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 0.00017129455909943713,
+      "loss": 0.346,
+      "mean_token_accuracy": 0.911245071515441,
+      "num_tokens": 7480566.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.38030760866822677,
+      "epoch": 0.4409781270157743,
+      "grad_norm": 0.17578125,
+      "learning_rate": 0.00017066916823014385,
+      "loss": 0.3737,
+      "mean_token_accuracy": 0.9052736137062312,
+      "num_tokens": 7642808.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.37449472132138906,
+      "epoch": 0.4503606403565355,
+      "grad_norm": 0.1923828125,
+      "learning_rate": 0.00017004377736085053,
+      "loss": 0.3668,
+      "mean_token_accuracy": 0.9072099477052689,
+      "num_tokens": 7806054.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.38983453994151207,
+      "epoch": 0.4597431536972967,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 0.00016941838649155725,
+      "loss": 0.3804,
+      "mean_token_accuracy": 0.9052108563482761,
+      "num_tokens": 7968208.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.3876548252766952,
+      "epoch": 0.4691256670380578,
+      "grad_norm": 0.1953125,
+      "learning_rate": 0.0001687929956222639,
+      "loss": 0.3825,
+      "mean_token_accuracy": 0.9071293246001005,
+      "num_tokens": 8130444.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.35602071137400343,
+      "epoch": 0.478508180378819,
+      "grad_norm": 0.1748046875,
+      "learning_rate": 0.00016816760475297062,
+      "loss": 0.3484,
+      "mean_token_accuracy": 0.9137960843741894,
+      "num_tokens": 8292618.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.4106741435127333,
+      "epoch": 0.48789069371958016,
+      "grad_norm": 0.181640625,
+      "learning_rate": 0.0001675422138836773,
+      "loss": 0.3968,
+      "mean_token_accuracy": 0.9058029960840941,
+      "num_tokens": 8455547.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.35365958177717405,
+      "epoch": 0.4972732070603413,
+      "grad_norm": 0.171875,
+      "learning_rate": 0.000166916823014384,
+      "loss": 0.349,
+      "mean_token_accuracy": 0.9107609067112208,
+      "num_tokens": 8617389.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.3634562706924044,
+      "epoch": 0.5066557204011024,
+      "grad_norm": 0.1494140625,
+      "learning_rate": 0.00016629143214509068,
+      "loss": 0.3559,
+      "mean_token_accuracy": 0.9100410873070359,
+      "num_tokens": 8779770.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.32272221232997256,
+      "epoch": 0.5160382337418636,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 0.0001656660412757974,
+      "loss": 0.3192,
+      "mean_token_accuracy": 0.9185531191527844,
+      "num_tokens": 8942167.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.3944925616728142,
+      "epoch": 0.5254207470826248,
+      "grad_norm": 0.189453125,
+      "learning_rate": 0.00016504065040650408,
+      "loss": 0.3822,
+      "mean_token_accuracy": 0.9070056907832622,
+      "num_tokens": 9104106.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.34754261119524016,
+      "epoch": 0.5348032604233859,
+      "grad_norm": 0.1923828125,
+      "learning_rate": 0.00016441525953721076,
+      "loss": 0.341,
+      "mean_token_accuracy": 0.9133739612996579,
+      "num_tokens": 9266326.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.36407561659580095,
+      "epoch": 0.5441857737641471,
+      "grad_norm": 0.146484375,
+      "learning_rate": 0.00016378986866791745,
+      "loss": 0.355,
+      "mean_token_accuracy": 0.9110283955931664,
+      "num_tokens": 9428809.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.36490103751420977,
+      "epoch": 0.5535682871049082,
+      "grad_norm": 0.14453125,
+      "learning_rate": 0.00016316447779862414,
+      "loss": 0.3606,
+      "mean_token_accuracy": 0.9129387583583594,
+      "num_tokens": 9591549.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.4403200134751387,
+      "epoch": 0.5629508004456694,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.00016253908692933085,
+      "loss": 0.4352,
+      "mean_token_accuracy": 0.9009242281317711,
+      "num_tokens": 9754490.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.3376986944698729,
+      "epoch": 0.5723333137864305,
+      "grad_norm": 0.2099609375,
+      "learning_rate": 0.00016191369606003754,
+      "loss": 0.3271,
+      "mean_token_accuracy": 0.9162716951221228,
+      "num_tokens": 9917345.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.30342430413002147,
+      "epoch": 0.5817158271271917,
+      "grad_norm": 0.2041015625,
+      "learning_rate": 0.00016128830519074422,
+      "loss": 0.2956,
+      "mean_token_accuracy": 0.9222736719995737,
+      "num_tokens": 10079523.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.39200085022021086,
+      "epoch": 0.5910983404679528,
+      "grad_norm": 0.1806640625,
+      "learning_rate": 0.0001606629143214509,
+      "loss": 0.3825,
+      "mean_token_accuracy": 0.9086053863167762,
+      "num_tokens": 10241691.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.3571344430092722,
+      "epoch": 0.600480853808714,
+      "grad_norm": 0.17578125,
+      "learning_rate": 0.00016003752345215762,
+      "loss": 0.3541,
+      "mean_token_accuracy": 0.9139539418742061,
+      "num_tokens": 10404189.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.34833492621546613,
+      "epoch": 0.6098633671494752,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 0.00015941213258286428,
+      "loss": 0.3376,
+      "mean_token_accuracy": 0.9150689322501421,
+      "num_tokens": 10566544.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.3005343694472685,
+      "epoch": 0.6192458804902363,
+      "grad_norm": 0.158203125,
+      "learning_rate": 0.000158786741713571,
+      "loss": 0.2928,
+      "mean_token_accuracy": 0.9205218695104123,
+      "num_tokens": 10728072.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.28468642103252934,
+      "epoch": 0.6286283938309974,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 0.00015816135084427768,
+      "loss": 0.2813,
+      "mean_token_accuracy": 0.9226693190634251,
+      "num_tokens": 10890089.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.3512667278526351,
+      "epoch": 0.6380109071717587,
+      "grad_norm": 0.15625,
+      "learning_rate": 0.0001575359599749844,
+      "loss": 0.3421,
+      "mean_token_accuracy": 0.9135142650455237,
+      "num_tokens": 11052295.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.38745955675840377,
+      "epoch": 0.6473934205125198,
+      "grad_norm": 0.16015625,
+      "learning_rate": 0.00015691056910569105,
+      "loss": 0.3774,
+      "mean_token_accuracy": 0.9076902076601983,
+      "num_tokens": 11214249.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.3123638934805058,
+      "epoch": 0.6567759338532809,
+      "grad_norm": 0.1669921875,
+      "learning_rate": 0.00015628517823639777,
+      "loss": 0.3099,
+      "mean_token_accuracy": 0.9233202930539847,
+      "num_tokens": 11377548.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.36946490925038233,
+      "epoch": 0.6661584471940422,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 0.00015565978736710445,
+      "loss": 0.3584,
+      "mean_token_accuracy": 0.9100467972457409,
+      "num_tokens": 11539730.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.44331123513402415,
+      "epoch": 0.6755409605348033,
+      "grad_norm": 0.185546875,
+      "learning_rate": 0.00015503439649781114,
+      "loss": 0.4357,
+      "mean_token_accuracy": 0.9018381554633379,
+      "num_tokens": 11702656.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.35576051749521864,
+      "epoch": 0.6849234738755644,
+      "grad_norm": 0.1396484375,
+      "learning_rate": 0.00015440900562851783,
+      "loss": 0.3529,
+      "mean_token_accuracy": 0.9145626701414585,
+      "num_tokens": 11866221.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.3530123558419291,
+      "epoch": 0.6943059872163255,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 0.0001537836147592245,
+      "loss": 0.3413,
+      "mean_token_accuracy": 0.9174593914300203,
+      "num_tokens": 12028438.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.3903426473028958,
+      "epoch": 0.7036885005570868,
+      "grad_norm": 0.1953125,
+      "learning_rate": 0.00015315822388993123,
+      "loss": 0.385,
+      "mean_token_accuracy": 0.9088658891618252,
+      "num_tokens": 12190615.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.3577064266020898,
+      "epoch": 0.7130710138978479,
+      "grad_norm": 0.154296875,
+      "learning_rate": 0.0001525328330206379,
+      "loss": 0.3477,
+      "mean_token_accuracy": 0.9163919541984796,
+      "num_tokens": 12353059.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.29131998592056335,
+      "epoch": 0.722453527238609,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 0.0001519074421513446,
+      "loss": 0.2891,
+      "mean_token_accuracy": 0.9242078192532063,
+      "num_tokens": 12515436.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.3846006996463984,
+      "epoch": 0.7318360405793702,
+      "grad_norm": 0.193359375,
+      "learning_rate": 0.00015128205128205128,
+      "loss": 0.3775,
+      "mean_token_accuracy": 0.9100886223837733,
+      "num_tokens": 12677546.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.34870610454818235,
+      "epoch": 0.7412185539201314,
+      "grad_norm": 0.1728515625,
+      "learning_rate": 0.000150656660412758,
+      "loss": 0.3401,
+      "mean_token_accuracy": 0.9157310061156749,
+      "num_tokens": 12840366.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.33407560451305474,
+      "epoch": 0.7506010672608925,
+      "grad_norm": 0.169921875,
+      "learning_rate": 0.00015003126954346466,
+      "loss": 0.3285,
+      "mean_token_accuracy": 0.9150595877319574,
+      "num_tokens": 13002637.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.33455972147057766,
+      "epoch": 0.7599835806016537,
+      "grad_norm": 0.173828125,
+      "learning_rate": 0.00014940587867417137,
+      "loss": 0.3268,
+      "mean_token_accuracy": 0.9206027369946241,
+      "num_tokens": 13165555.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.3760374941455666,
+      "epoch": 0.7693660939424148,
+      "grad_norm": 0.212890625,
+      "learning_rate": 0.00014878048780487806,
+      "loss": 0.3689,
+      "mean_token_accuracy": 0.9126812819391489,
+      "num_tokens": 13328334.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.4276343748439103,
+      "epoch": 0.778748607283176,
+      "grad_norm": 0.16015625,
+      "learning_rate": 0.00014815509693558477,
+      "loss": 0.4228,
+      "mean_token_accuracy": 0.9042647287249566,
+      "num_tokens": 13490265.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.35269592416007073,
+      "epoch": 0.7881311206239371,
+      "grad_norm": 0.1416015625,
+      "learning_rate": 0.00014752970606629143,
+      "loss": 0.3436,
+      "mean_token_accuracy": 0.9150325283408165,
+      "num_tokens": 13652700.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.3484025130281225,
+      "epoch": 0.7975136339646983,
+      "grad_norm": 0.1650390625,
+      "learning_rate": 0.00014690431519699812,
+      "loss": 0.3433,
+      "mean_token_accuracy": 0.9187351956963539,
+      "num_tokens": 13814997.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.34064410558785313,
+      "epoch": 0.8068961473054594,
+      "grad_norm": 0.169921875,
+      "learning_rate": 0.00014627892432770483,
+      "loss": 0.3319,
+      "mean_token_accuracy": 0.917700307443738,
+      "num_tokens": 13977550.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.3676424361299723,
+      "epoch": 0.8162786606462206,
+      "grad_norm": 0.1474609375,
+      "learning_rate": 0.00014565353345841151,
+      "loss": 0.3611,
+      "mean_token_accuracy": 0.9167136520147323,
+      "num_tokens": 14140493.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.3216154734662268,
+      "epoch": 0.8256611739869818,
+      "grad_norm": 0.1767578125,
+      "learning_rate": 0.0001450281425891182,
+      "loss": 0.3133,
+      "mean_token_accuracy": 0.920833321288228,
+      "num_tokens": 14302638.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.3731617649958935,
+      "epoch": 0.8350436873277429,
+      "grad_norm": 0.1640625,
+      "learning_rate": 0.0001444027517198249,
+      "loss": 0.3708,
+      "mean_token_accuracy": 0.9144421305507422,
+      "num_tokens": 14465632.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.3741735242540017,
+      "epoch": 0.844426200668504,
+      "grad_norm": 0.1474609375,
+      "learning_rate": 0.0001437773608505316,
+      "loss": 0.3628,
+      "mean_token_accuracy": 0.9130110245198011,
+      "num_tokens": 14627986.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.363976987201022,
+      "epoch": 0.8538087140092653,
+      "grad_norm": 0.1611328125,
+      "learning_rate": 0.0001431519699812383,
+      "loss": 0.3585,
+      "mean_token_accuracy": 0.9120528548955917,
+      "num_tokens": 14790551.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.3448503586114384,
+      "epoch": 0.8631912273500264,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 0.00014252657911194497,
+      "loss": 0.3421,
+      "mean_token_accuracy": 0.9182739958167077,
+      "num_tokens": 14952661.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.27787273108260707,
+      "epoch": 0.8725737406907875,
+      "grad_norm": 0.150390625,
+      "learning_rate": 0.00014190118824265166,
+      "loss": 0.2687,
+      "mean_token_accuracy": 0.92897635884583,
+      "num_tokens": 15115022.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.36200930876657367,
+      "epoch": 0.8819562540315486,
+      "grad_norm": 0.150390625,
+      "learning_rate": 0.00014127579737335837,
+      "loss": 0.3497,
+      "mean_token_accuracy": 0.9132607497274876,
+      "num_tokens": 15277207.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.2834262179443613,
+      "epoch": 0.8913387673723099,
+      "grad_norm": 0.1494140625,
+      "learning_rate": 0.00014065040650406506,
+      "loss": 0.2814,
+      "mean_token_accuracy": 0.9268792189657689,
+      "num_tokens": 15440092.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.3499863417411689,
+      "epoch": 0.900721280713071,
+      "grad_norm": 0.1484375,
+      "learning_rate": 0.00014002501563477175,
+      "loss": 0.3417,
+      "mean_token_accuracy": 0.9159077113494277,
+      "num_tokens": 15603417.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.2800866418518126,
+      "epoch": 0.9101037940538321,
+      "grad_norm": 0.1826171875,
+      "learning_rate": 0.00013939962476547843,
+      "loss": 0.27,
+      "mean_token_accuracy": 0.9285330750048161,
+      "num_tokens": 15765520.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.3038740568445064,
+      "epoch": 0.9194863073945934,
+      "grad_norm": 0.1669921875,
+      "learning_rate": 0.00013877423389618512,
+      "loss": 0.2992,
+      "mean_token_accuracy": 0.9227856550365686,
+      "num_tokens": 15928370.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.39133549834950826,
+      "epoch": 0.9288688207353545,
+      "grad_norm": 0.19921875,
+      "learning_rate": 0.0001381488430268918,
+      "loss": 0.3845,
+      "mean_token_accuracy": 0.9107819177210331,
+      "num_tokens": 16090841.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.368834131793119,
+      "epoch": 0.9382513340761156,
+      "grad_norm": 0.16796875,
+      "learning_rate": 0.0001375234521575985,
+      "loss": 0.3592,
+      "mean_token_accuracy": 0.9155494935810566,
+      "num_tokens": 16253510.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.3432939130580053,
+      "epoch": 0.9476338474168768,
+      "grad_norm": 0.181640625,
+      "learning_rate": 0.0001368980612883052,
+      "loss": 0.3434,
+      "mean_token_accuracy": 0.9178479440510273,
+      "num_tokens": 16416142.0,
+      "step": 1010
+    },
+    {
+      "entropy": 0.3263048173219431,
+      "epoch": 0.957016360757638,
+      "grad_norm": 0.158203125,
+      "learning_rate": 0.0001362726704190119,
+      "loss": 0.3157,
+      "mean_token_accuracy": 0.919898977316916,
+      "num_tokens": 16578145.0,
+      "step": 1020
+    },
+    {
+      "entropy": 0.30497189780580813,
+      "epoch": 0.9663988740983991,
+      "grad_norm": 0.162109375,
+      "learning_rate": 0.00013564727954971858,
+      "loss": 0.2985,
+      "mean_token_accuracy": 0.922874777764082,
+      "num_tokens": 16741103.0,
+      "step": 1030
+    },
+    {
+      "entropy": 0.2940454826748464,
+      "epoch": 0.9757813874391603,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 0.00013502188868042526,
+      "loss": 0.2874,
+      "mean_token_accuracy": 0.9239259999245405,
+      "num_tokens": 16903947.0,
+      "step": 1040
+    },
+    {
+      "entropy": 0.367372212279588,
+      "epoch": 0.9851639007799214,
+      "grad_norm": 0.1748046875,
+      "learning_rate": 0.00013439649781113198,
+      "loss": 0.3601,
+      "mean_token_accuracy": 0.9152833472937345,
+      "num_tokens": 17065800.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.34914421315770594,
+      "epoch": 0.9945464141206826,
+      "grad_norm": 0.1826171875,
+      "learning_rate": 0.00013377110694183866,
+      "loss": 0.3425,
+      "mean_token_accuracy": 0.9175648283213377,
+      "num_tokens": 17228736.0,
+      "step": 1060
+    },
+    {
+      "entropy": 0.2900184833320083,
+      "epoch": 1.0037530053363044,
+      "grad_norm": 0.13671875,
+      "learning_rate": 0.00013314571607254535,
+      "loss": 0.2782,
+      "mean_token_accuracy": 0.9317144804699405,
+      "num_tokens": 17388924.0,
+      "step": 1070
+    },
+    {
+      "entropy": 0.2829561754653696,
+      "epoch": 1.0131355186770656,
+      "grad_norm": 0.1259765625,
+      "learning_rate": 0.00013252032520325203,
+      "loss": 0.2765,
+      "mean_token_accuracy": 0.9315391689538955,
+      "num_tokens": 17550836.0,
+      "step": 1080
+    },
+    {
+      "entropy": 0.3159750218328554,
+      "epoch": 1.0225180320178269,
+      "grad_norm": 0.154296875,
+      "learning_rate": 0.00013189493433395872,
+      "loss": 0.3097,
+      "mean_token_accuracy": 0.924969544634223,
+      "num_tokens": 17713453.0,
+      "step": 1090
+    },
+    {
+      "entropy": 0.3149459302192554,
+      "epoch": 1.0319005453585879,
+      "grad_norm": 0.181640625,
+      "learning_rate": 0.00013126954346466543,
+      "loss": 0.307,
+      "mean_token_accuracy": 0.9239892680197954,
+      "num_tokens": 17875306.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.3264659658656456,
+      "epoch": 1.0412830586993491,
+      "grad_norm": 0.140625,
+      "learning_rate": 0.0001306441525953721,
+      "loss": 0.3177,
+      "mean_token_accuracy": 0.9219260595738887,
+      "num_tokens": 18037829.0,
+      "step": 1110
+    },
+    {
+      "entropy": 0.3119678142364137,
+      "epoch": 1.0506655720401104,
+      "grad_norm": 0.1376953125,
+      "learning_rate": 0.0001300187617260788,
+      "loss": 0.3047,
+      "mean_token_accuracy": 0.9250960987061262,
+      "num_tokens": 18201027.0,
+      "step": 1120
+    },
+    {
+      "entropy": 0.3200692283746321,
+      "epoch": 1.0600480853808714,
+      "grad_norm": 0.115234375,
+      "learning_rate": 0.0001293933708567855,
+      "loss": 0.3129,
+      "mean_token_accuracy": 0.9225547250360251,
+      "num_tokens": 18364107.0,
+      "step": 1130
+    },
+    {
+      "entropy": 0.2799913567607291,
+      "epoch": 1.0694305987216326,
+      "grad_norm": 0.1865234375,
+      "learning_rate": 0.0001287679799874922,
+      "loss": 0.2668,
+      "mean_token_accuracy": 0.9307141672819853,
+      "num_tokens": 18526662.0,
+      "step": 1140
+    },
+    {
+      "entropy": 0.3025812109815888,
+      "epoch": 1.0788131120623938,
+      "grad_norm": 0.1728515625,
+      "learning_rate": 0.00012814258911819887,
+      "loss": 0.3015,
+      "mean_token_accuracy": 0.9235180430114269,
+      "num_tokens": 18689065.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.2885304512223229,
+      "epoch": 1.0881956254031548,
+      "grad_norm": 0.263671875,
+      "learning_rate": 0.00012751719824890558,
+      "loss": 0.2786,
+      "mean_token_accuracy": 0.9279053185135127,
+      "num_tokens": 18851798.0,
+      "step": 1160
+    },
+    {
+      "entropy": 0.33722220184863544,
+      "epoch": 1.097578138743916,
+      "grad_norm": 0.1962890625,
+      "learning_rate": 0.00012689180737961227,
+      "loss": 0.3301,
+      "mean_token_accuracy": 0.9198755659162998,
+      "num_tokens": 19013951.0,
+      "step": 1170
+    },
+    {
+      "entropy": 0.35752617044490764,
+      "epoch": 1.106960652084677,
+      "grad_norm": 0.1826171875,
+      "learning_rate": 0.00012626641651031895,
+      "loss": 0.3525,
+      "mean_token_accuracy": 0.9136955734342337,
+      "num_tokens": 19176391.0,
+      "step": 1180
+    },
+    {
+      "entropy": 0.3285738020436838,
+      "epoch": 1.1163431654254383,
+      "grad_norm": 0.169921875,
+      "learning_rate": 0.00012564102564102564,
+      "loss": 0.3174,
+      "mean_token_accuracy": 0.923049738816917,
+      "num_tokens": 19338315.0,
+      "step": 1190
+    },
+    {
+      "entropy": 0.33181785854394547,
+      "epoch": 1.1257256787661996,
+      "grad_norm": 0.154296875,
+      "learning_rate": 0.00012501563477173235,
+      "loss": 0.3252,
+      "mean_token_accuracy": 0.9195261619985103,
+      "num_tokens": 19501276.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.33468560994951985,
+      "epoch": 1.1351081921069606,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 0.00012439024390243904,
+      "loss": 0.3297,
+      "mean_token_accuracy": 0.9180046692490578,
+      "num_tokens": 19663199.0,
+      "step": 1210
+    },
+    {
+      "entropy": 0.35496956037823113,
+      "epoch": 1.1444907054477218,
+      "grad_norm": 0.14453125,
+      "learning_rate": 0.00012376485303314572,
+      "loss": 0.3452,
+      "mean_token_accuracy": 0.9190296337008477,
+      "num_tokens": 19825833.0,
+      "step": 1220
+    },
+    {
+      "entropy": 0.3228163967374712,
+      "epoch": 1.153873218788483,
+      "grad_norm": 0.173828125,
+      "learning_rate": 0.0001231394621638524,
+      "loss": 0.3184,
+      "mean_token_accuracy": 0.9187171328812838,
+      "num_tokens": 19988129.0,
+      "step": 1230
+    },
+    {
+      "entropy": 0.36049368806998244,
+      "epoch": 1.163255732129244,
+      "grad_norm": 0.177734375,
+      "learning_rate": 0.0001225140712945591,
+      "loss": 0.3561,
+      "mean_token_accuracy": 0.9135892525315285,
+      "num_tokens": 20149704.0,
+      "step": 1240
+    },
+    {
+      "entropy": 0.29329942048643715,
+      "epoch": 1.1726382454700053,
+      "grad_norm": 0.1982421875,
+      "learning_rate": 0.0001218886804252658,
+      "loss": 0.2808,
+      "mean_token_accuracy": 0.9278272874653339,
+      "num_tokens": 20312458.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.3245240814285353,
+      "epoch": 1.1820207588107665,
+      "grad_norm": 0.1328125,
+      "learning_rate": 0.00012126328955597248,
+      "loss": 0.3206,
+      "mean_token_accuracy": 0.9241520136594772,
+      "num_tokens": 20474875.0,
+      "step": 1260
+    },
+    {
+      "entropy": 0.31488763078814375,
+      "epoch": 1.1914032721515275,
+      "grad_norm": 0.134765625,
+      "learning_rate": 0.00012063789868667918,
+      "loss": 0.3092,
+      "mean_token_accuracy": 0.9249691208824515,
+      "num_tokens": 20636775.0,
+      "step": 1270
+    },
+    {
+      "entropy": 0.32809503517928534,
+      "epoch": 1.2007857854922888,
+      "grad_norm": 0.1748046875,
+      "learning_rate": 0.00012001250781738587,
+      "loss": 0.3175,
+      "mean_token_accuracy": 0.9244366209954024,
+      "num_tokens": 20799311.0,
+      "step": 1280
+    },
+    {
+      "entropy": 0.3452982397924643,
+      "epoch": 1.21016829883305,
+      "grad_norm": 0.1337890625,
+      "learning_rate": 0.00011938711694809257,
+      "loss": 0.3417,
+      "mean_token_accuracy": 0.9209324043244124,
+      "num_tokens": 20961671.0,
+      "step": 1290
+    },
+    {
+      "entropy": 0.33462724700802937,
+      "epoch": 1.219550812173811,
+      "grad_norm": 0.1728515625,
+      "learning_rate": 0.00011876172607879925,
+      "loss": 0.3274,
+      "mean_token_accuracy": 0.9196479767560959,
+      "num_tokens": 21124643.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.32381630715099163,
+      "epoch": 1.2289333255145722,
+      "grad_norm": 0.1376953125,
+      "learning_rate": 0.00011813633520950595,
+      "loss": 0.3161,
+      "mean_token_accuracy": 0.9226799987256527,
+      "num_tokens": 21286738.0,
+      "step": 1310
+    },
+    {
+      "entropy": 0.3411265324510168,
+      "epoch": 1.2383158388553333,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 0.00011751094434021264,
+      "loss": 0.3347,
+      "mean_token_accuracy": 0.9211051788181066,
+      "num_tokens": 21448827.0,
+      "step": 1320
+    },
+    {
+      "entropy": 0.32648798704030924,
+      "epoch": 1.2476983521960945,
+      "grad_norm": 0.1669921875,
+      "learning_rate": 0.00011688555347091934,
+      "loss": 0.3159,
+      "mean_token_accuracy": 0.922034990042448,
+      "num_tokens": 21610617.0,
+      "step": 1330
+    },
+    {
+      "entropy": 0.3225275243108626,
+      "epoch": 1.2570808655368557,
+      "grad_norm": 0.1337890625,
+      "learning_rate": 0.00011626016260162603,
+      "loss": 0.3217,
+      "mean_token_accuracy": 0.924454814940691,
+      "num_tokens": 21773343.0,
+      "step": 1340
+    },
+    {
+      "entropy": 0.31472907413262874,
+      "epoch": 1.2664633788776167,
+      "grad_norm": 0.158203125,
+      "learning_rate": 0.0001156347717323327,
+      "loss": 0.3056,
+      "mean_token_accuracy": 0.9251250382512808,
+      "num_tokens": 21936161.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.34041504978667947,
+      "epoch": 1.275845892218378,
+      "grad_norm": 0.15625,
+      "learning_rate": 0.00011500938086303941,
+      "loss": 0.3337,
+      "mean_token_accuracy": 0.9195377014577388,
+      "num_tokens": 22099004.0,
+      "step": 1360
+    },
+    {
+      "entropy": 0.32660515860188755,
+      "epoch": 1.2852284055591392,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 0.00011438398999374609,
+      "loss": 0.3177,
+      "mean_token_accuracy": 0.9214879527688027,
+      "num_tokens": 22261465.0,
+      "step": 1370
+    },
+    {
+      "entropy": 0.27577626157435586,
+      "epoch": 1.2946109188999002,
+      "grad_norm": 0.171875,
+      "learning_rate": 0.00011375859912445279,
+      "loss": 0.2718,
+      "mean_token_accuracy": 0.9287902843207121,
+      "num_tokens": 22424003.0,
+      "step": 1380
+    },
+    {
+      "entropy": 0.3049365603888873,
+      "epoch": 1.3039934322406614,
+      "grad_norm": 0.150390625,
+      "learning_rate": 0.00011313320825515947,
+      "loss": 0.296,
+      "mean_token_accuracy": 0.9254100024700165,
+      "num_tokens": 22586414.0,
+      "step": 1390
+    },
+    {
+      "entropy": 0.3391640084271785,
+      "epoch": 1.3133759455814227,
+      "grad_norm": 0.158203125,
+      "learning_rate": 0.00011250781738586617,
+      "loss": 0.3316,
+      "mean_token_accuracy": 0.9204599536955357,
+      "num_tokens": 22748801.0,
+      "step": 1400
+    },
+    {
+      "entropy": 0.29096004691673444,
+      "epoch": 1.3227584589221837,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 0.00011188242651657286,
+      "loss": 0.2829,
+      "mean_token_accuracy": 0.926516629010439,
+      "num_tokens": 22911078.0,
+      "step": 1410
+    },
+    {
+      "entropy": 0.30442387533839793,
+      "epoch": 1.332140972262945,
+      "grad_norm": 0.1435546875,
+      "learning_rate": 0.00011125703564727956,
+      "loss": 0.2997,
+      "mean_token_accuracy": 0.928035743534565,
+      "num_tokens": 23073932.0,
+      "step": 1420
+    },
+    {
+      "entropy": 0.2672559514408931,
+      "epoch": 1.3415234856037062,
+      "grad_norm": 0.1669921875,
+      "learning_rate": 0.00011063164477798624,
+      "loss": 0.2567,
+      "mean_token_accuracy": 0.934470995515585,
+      "num_tokens": 23234861.0,
+      "step": 1430
+    },
+    {
+      "entropy": 0.28968692439375443,
+      "epoch": 1.3509059989444672,
+      "grad_norm": 0.189453125,
+      "learning_rate": 0.00011000625390869294,
+      "loss": 0.2833,
+      "mean_token_accuracy": 0.9286899168044329,
+      "num_tokens": 23397575.0,
+      "step": 1440
+    },
+    {
+      "entropy": 0.3179404502676334,
+      "epoch": 1.3602885122852284,
+      "grad_norm": 0.154296875,
+      "learning_rate": 0.00010938086303939963,
+      "loss": 0.3106,
+      "mean_token_accuracy": 0.9237559959292412,
+      "num_tokens": 23560621.0,
+      "step": 1450
+    },
+    {
+      "entropy": 0.3825915275025181,
+      "epoch": 1.3696710256259896,
+      "grad_norm": 0.1796875,
+      "learning_rate": 0.00010875547217010633,
+      "loss": 0.3812,
+      "mean_token_accuracy": 0.9096528366208076,
+      "num_tokens": 23723326.0,
+      "step": 1460
+    },
+    {
+      "entropy": 0.352222545677796,
+      "epoch": 1.3790535389667506,
+      "grad_norm": 0.1611328125,
+      "learning_rate": 0.00010813008130081302,
+      "loss": 0.3445,
+      "mean_token_accuracy": 0.9214517448097468,
+      "num_tokens": 23885973.0,
+      "step": 1470
+    },
+    {
+      "entropy": 0.3833993759646546,
+      "epoch": 1.3884360523075119,
+      "grad_norm": 0.1591796875,
+      "learning_rate": 0.00010750469043151969,
+      "loss": 0.3772,
+      "mean_token_accuracy": 0.9148985600098968,
+      "num_tokens": 24049379.0,
+      "step": 1480
+    },
+    {
+      "entropy": 0.3407844117609784,
+      "epoch": 1.3978185656482731,
+      "grad_norm": 0.15625,
+      "learning_rate": 0.0001068792995622264,
+      "loss": 0.3332,
+      "mean_token_accuracy": 0.9178143370896578,
+      "num_tokens": 24211771.0,
+      "step": 1490
+    },
+    {
+      "entropy": 0.3129306449787691,
+      "epoch": 1.4072010789890341,
+      "grad_norm": 0.166015625,
+      "learning_rate": 0.00010625390869293307,
+      "loss": 0.3105,
+      "mean_token_accuracy": 0.9243231456726789,
+      "num_tokens": 24374782.0,
+      "step": 1500
+    },
+    {
+      "entropy": 0.2853265376936179,
+      "epoch": 1.4165835923297954,
+      "grad_norm": 0.14453125,
+      "learning_rate": 0.00010562851782363979,
+      "loss": 0.272,
+      "mean_token_accuracy": 0.9318001717329025,
+      "num_tokens": 24537798.0,
+      "step": 1510
+    },
+    {
+      "entropy": 0.32060901625081895,
+      "epoch": 1.4259661056705566,
+      "grad_norm": 0.169921875,
+      "learning_rate": 0.00010500312695434646,
+      "loss": 0.3189,
+      "mean_token_accuracy": 0.923735149204731,
+      "num_tokens": 24700222.0,
+      "step": 1520
+    },
+    {
+      "entropy": 0.28523959064041265,
+      "epoch": 1.4353486190113176,
+      "grad_norm": 0.138671875,
+      "learning_rate": 0.00010437773608505317,
+      "loss": 0.2765,
+      "mean_token_accuracy": 0.9289753165096044,
+      "num_tokens": 24862698.0,
+      "step": 1530
+    },
+    {
+      "entropy": 0.28084768254193476,
+      "epoch": 1.4447311323520788,
+      "grad_norm": 0.1435546875,
+      "learning_rate": 0.00010375234521575985,
+      "loss": 0.2735,
+      "mean_token_accuracy": 0.9312996946275234,
+      "num_tokens": 25025816.0,
+      "step": 1540
+    },
+    {
+      "entropy": 0.3017741791787557,
+      "epoch": 1.45411364569284,
+      "grad_norm": 0.15625,
+      "learning_rate": 0.00010312695434646656,
+      "loss": 0.2951,
+      "mean_token_accuracy": 0.9261103741824627,
+      "num_tokens": 25189369.0,
+      "step": 1550
+    },
+    {
+      "entropy": 0.3212110122665763,
+      "epoch": 1.463496159033601,
+      "grad_norm": 0.181640625,
+      "learning_rate": 0.00010250156347717323,
+      "loss": 0.3141,
+      "mean_token_accuracy": 0.9228064101189375,
+      "num_tokens": 25351206.0,
+      "step": 1560
+    },
+    {
+      "entropy": 0.3111924729484599,
+      "epoch": 1.4728786723743623,
+      "grad_norm": 0.1591796875,
+      "learning_rate": 0.00010187617260787993,
+      "loss": 0.3073,
+      "mean_token_accuracy": 0.9231358967721462,
+      "num_tokens": 25514005.0,
+      "step": 1570
+    },
+    {
+      "entropy": 0.35193296511424704,
+      "epoch": 1.4822611857151236,
+      "grad_norm": 0.1748046875,
+      "learning_rate": 0.00010125078173858662,
+      "loss": 0.3426,
+      "mean_token_accuracy": 0.9204717528074979,
+      "num_tokens": 25676907.0,
+      "step": 1580
+    },
+    {
+      "entropy": 0.33642219280591235,
+      "epoch": 1.4916436990558846,
+      "grad_norm": 0.1611328125,
+      "learning_rate": 0.00010062539086929332,
+      "loss": 0.3297,
+      "mean_token_accuracy": 0.9194286223500967,
+      "num_tokens": 25839935.0,
+      "step": 1590
+    },
+    {
+      "entropy": 0.26263106021797283,
+      "epoch": 1.5010262123966458,
+      "grad_norm": 0.142578125,
+      "learning_rate": 0.0001,
+      "loss": 0.2573,
+      "mean_token_accuracy": 0.935544315353036,
+      "num_tokens": 26003218.0,
+      "step": 1600
+    },
+    {
+      "entropy": 0.3122893640131224,
+      "epoch": 1.510408725737407,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 9.937460913070669e-05,
+      "loss": 0.3049,
+      "mean_token_accuracy": 0.9271042123436928,
+      "num_tokens": 26165724.0,
+      "step": 1610
+    },
+    {
+      "entropy": 0.29348382144235075,
+      "epoch": 1.519791239078168,
+      "grad_norm": 0.1640625,
+      "learning_rate": 9.874921826141339e-05,
+      "loss": 0.2887,
+      "mean_token_accuracy": 0.9292786654084921,
+      "num_tokens": 26328312.0,
+      "step": 1620
+    },
+    {
+      "entropy": 0.34205422176164574,
+      "epoch": 1.529173752418929,
+      "grad_norm": 0.1484375,
+      "learning_rate": 9.812382739212008e-05,
+      "loss": 0.3353,
+      "mean_token_accuracy": 0.9192357573658227,
+      "num_tokens": 26490508.0,
+      "step": 1630
+    },
+    {
+      "entropy": 0.3207799144263845,
+      "epoch": 1.5385562657596905,
+      "grad_norm": 0.1591796875,
+      "learning_rate": 9.749843652282678e-05,
+      "loss": 0.3142,
+      "mean_token_accuracy": 0.9239265900105238,
+      "num_tokens": 26652241.0,
+      "step": 1640
+    },
+    {
+      "entropy": 0.3212417629547417,
+      "epoch": 1.5479387791004515,
+      "grad_norm": 0.12890625,
+      "learning_rate": 9.687304565353346e-05,
+      "loss": 0.3105,
+      "mean_token_accuracy": 0.9247593119740486,
+      "num_tokens": 26815313.0,
+      "step": 1650
+    },
+    {
+      "entropy": 0.2918511720839888,
+      "epoch": 1.5573212924412125,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 9.624765478424016e-05,
+      "loss": 0.2874,
+      "mean_token_accuracy": 0.9265463493764401,
+      "num_tokens": 26978291.0,
+      "step": 1660
+    },
+    {
+      "entropy": 0.3887786120234523,
+      "epoch": 1.566703805781974,
+      "grad_norm": 0.177734375,
+      "learning_rate": 9.562226391494684e-05,
+      "loss": 0.3832,
+      "mean_token_accuracy": 0.9116566512733698,
+      "num_tokens": 27140951.0,
+      "step": 1670
+    },
+    {
+      "entropy": 0.3554312888765708,
+      "epoch": 1.576086319122735,
+      "grad_norm": 0.2080078125,
+      "learning_rate": 9.499687304565354e-05,
+      "loss": 0.3498,
+      "mean_token_accuracy": 0.9205850351601839,
+      "num_tokens": 27303423.0,
+      "step": 1680
+    },
+    {
+      "entropy": 0.3036960868688766,
+      "epoch": 1.585468832463496,
+      "grad_norm": 0.1455078125,
+      "learning_rate": 9.437148217636022e-05,
+      "loss": 0.2982,
+      "mean_token_accuracy": 0.9223078574985266,
+      "num_tokens": 27465935.0,
+      "step": 1690
+    },
+    {
+      "entropy": 0.34383984084706753,
+      "epoch": 1.5948513458042575,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 9.374609130706692e-05,
+      "loss": 0.3352,
+      "mean_token_accuracy": 0.9215784965083003,
+      "num_tokens": 27628448.0,
+      "step": 1700
+    },
+    {
+      "entropy": 0.3477696883957833,
+      "epoch": 1.6042338591450185,
+      "grad_norm": 0.15234375,
+      "learning_rate": 9.312070043777361e-05,
+      "loss": 0.345,
+      "mean_token_accuracy": 0.9204984463751316,
+      "num_tokens": 27790993.0,
+      "step": 1710
+    },
+    {
+      "entropy": 0.4038438654737547,
+      "epoch": 1.6136163724857795,
+      "grad_norm": 0.1455078125,
+      "learning_rate": 9.249530956848031e-05,
+      "loss": 0.3942,
+      "mean_token_accuracy": 0.9117133766412735,
+      "num_tokens": 27953367.0,
+      "step": 1720
+    },
+    {
+      "entropy": 0.35322881376487203,
+      "epoch": 1.622998885826541,
+      "grad_norm": 0.2255859375,
+      "learning_rate": 9.1869918699187e-05,
+      "loss": 0.3508,
+      "mean_token_accuracy": 0.9202003927901388,
+      "num_tokens": 28115525.0,
+      "step": 1730
+    },
+    {
+      "entropy": 0.3633168841304723,
+      "epoch": 1.632381399167302,
+      "grad_norm": 0.1376953125,
+      "learning_rate": 9.12445278298937e-05,
+      "loss": 0.3537,
+      "mean_token_accuracy": 0.9178501278162002,
+      "num_tokens": 28278195.0,
+      "step": 1740
+    },
+    {
+      "entropy": 0.29921169562148864,
+      "epoch": 1.641763912508063,
+      "grad_norm": 0.169921875,
+      "learning_rate": 9.061913696060038e-05,
+      "loss": 0.2928,
+      "mean_token_accuracy": 0.929632369056344,
+      "num_tokens": 28441365.0,
+      "step": 1750
+    },
+    {
+      "entropy": 0.343074598867679,
+      "epoch": 1.6511464258488242,
+      "grad_norm": 0.1826171875,
+      "learning_rate": 8.999374609130708e-05,
+      "loss": 0.3418,
+      "mean_token_accuracy": 0.9203568341210484,
+      "num_tokens": 28604025.0,
+      "step": 1760
+    },
+    {
+      "entropy": 0.3354411284439266,
+      "epoch": 1.6605289391895854,
+      "grad_norm": 0.19921875,
+      "learning_rate": 8.936835522201377e-05,
+      "loss": 0.3247,
+      "mean_token_accuracy": 0.9247291401028633,
+      "num_tokens": 28766627.0,
+      "step": 1770
+    },
+    {
+      "entropy": 0.29062830022885466,
+      "epoch": 1.6699114525303465,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 8.874296435272045e-05,
+      "loss": 0.2871,
+      "mean_token_accuracy": 0.9279629901051522,
+      "num_tokens": 28928424.0,
+      "step": 1780
+    },
+    {
+      "entropy": 0.3109960203932133,
+      "epoch": 1.6792939658711077,
+      "grad_norm": 0.158203125,
+      "learning_rate": 8.811757348342715e-05,
+      "loss": 0.3046,
+      "mean_token_accuracy": 0.925516726821661,
+      "num_tokens": 29090881.0,
+      "step": 1790
+    },
+    {
+      "entropy": 0.29157741767703554,
+      "epoch": 1.688676479211869,
+      "grad_norm": 0.1474609375,
+      "learning_rate": 8.749218261413384e-05,
+      "loss": 0.2872,
+      "mean_token_accuracy": 0.9311040572822094,
+      "num_tokens": 29253614.0,
+      "step": 1800
+    },
+    {
+      "entropy": 0.2842859643569682,
+      "epoch": 1.69805899255263,
+      "grad_norm": 0.19921875,
+      "learning_rate": 8.686679174484052e-05,
+      "loss": 0.2787,
+      "mean_token_accuracy": 0.930610903352499,
+      "num_tokens": 29415480.0,
+      "step": 1810
+    },
+    {
+      "entropy": 0.2779356167244259,
+      "epoch": 1.7074415058933912,
+      "grad_norm": 0.1865234375,
+      "learning_rate": 8.624140087554722e-05,
+      "loss": 0.2712,
+      "mean_token_accuracy": 0.9308344077318906,
+      "num_tokens": 29578036.0,
+      "step": 1820
+    },
+    {
+      "entropy": 0.318134157784516,
+      "epoch": 1.7168240192341524,
+      "grad_norm": 0.130859375,
+      "learning_rate": 8.561601000625391e-05,
+      "loss": 0.311,
+      "mean_token_accuracy": 0.9251023806631565,
+      "num_tokens": 29740251.0,
+      "step": 1830
+    },
+    {
+      "entropy": 0.30244565989705735,
+      "epoch": 1.7262065325749134,
+      "grad_norm": 0.2451171875,
+      "learning_rate": 8.49906191369606e-05,
+      "loss": 0.2985,
+      "mean_token_accuracy": 0.925722798705101,
+      "num_tokens": 29902140.0,
+      "step": 1840
+    },
+    {
+      "entropy": 0.32010520926560276,
+      "epoch": 1.7355890459156746,
+      "grad_norm": 0.125,
+      "learning_rate": 8.43652282676673e-05,
+      "loss": 0.3077,
+      "mean_token_accuracy": 0.9267256014049053,
+      "num_tokens": 30064277.0,
+      "step": 1850
+    },
+    {
+      "entropy": 0.2397772744880058,
+      "epoch": 1.7449715592564359,
+      "grad_norm": 0.123046875,
+      "learning_rate": 8.373983739837398e-05,
+      "loss": 0.2367,
+      "mean_token_accuracy": 0.9358306158334017,
+      "num_tokens": 30227035.0,
+      "step": 1860
+    },
+    {
+      "entropy": 0.2846862503385637,
+      "epoch": 1.754354072597197,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 8.311444652908068e-05,
+      "loss": 0.2791,
+      "mean_token_accuracy": 0.9296355988830328,
+      "num_tokens": 30389480.0,
+      "step": 1870
+    },
+    {
+      "entropy": 0.3066696587542538,
+      "epoch": 1.7637365859379581,
+      "grad_norm": 0.2021484375,
+      "learning_rate": 8.248905565978737e-05,
+      "loss": 0.3006,
+      "mean_token_accuracy": 0.9251831691712141,
+      "num_tokens": 30552091.0,
+      "step": 1880
+    },
+    {
+      "entropy": 0.31332365355920044,
+      "epoch": 1.7731190992787194,
+      "grad_norm": 0.1552734375,
+      "learning_rate": 8.186366479049407e-05,
+      "loss": 0.3065,
+      "mean_token_accuracy": 0.9259384579956531,
+      "num_tokens": 30714856.0,
+      "step": 1890
+    },
+    {
+      "entropy": 0.26562274275929665,
+      "epoch": 1.7825016126194804,
+      "grad_norm": 0.1201171875,
+      "learning_rate": 8.123827392120076e-05,
+      "loss": 0.2575,
+      "mean_token_accuracy": 0.9345466572791338,
+      "num_tokens": 30877977.0,
+      "step": 1900
+    },
+    {
+      "entropy": 0.26213877720292655,
+      "epoch": 1.7918841259602416,
+      "grad_norm": 0.1396484375,
+      "learning_rate": 8.061288305190746e-05,
+      "loss": 0.2569,
+      "mean_token_accuracy": 0.9343690615147352,
+      "num_tokens": 31040248.0,
+      "step": 1910
+    },
+    {
+      "entropy": 0.31369023660663514,
+      "epoch": 1.8012666393010028,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 7.998749218261413e-05,
+      "loss": 0.3079,
+      "mean_token_accuracy": 0.9239290647208691,
+      "num_tokens": 31202363.0,
+      "step": 1920
+    },
+    {
+      "entropy": 0.2883313969883602,
+      "epoch": 1.8106491526417638,
+      "grad_norm": 0.162109375,
+      "learning_rate": 7.936210131332083e-05,
+      "loss": 0.2846,
+      "mean_token_accuracy": 0.9300382232293487,
+      "num_tokens": 31365512.0,
+      "step": 1930
+    },
+    {
+      "entropy": 0.2506348692520987,
+      "epoch": 1.820031665982525,
+      "grad_norm": 0.171875,
+      "learning_rate": 7.873671044402751e-05,
+      "loss": 0.2448,
+      "mean_token_accuracy": 0.9374226000159979,
+      "num_tokens": 31527448.0,
+      "step": 1940
+    },
+    {
+      "entropy": 0.37234054349246437,
+      "epoch": 1.8294141793232863,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 7.811131957473421e-05,
+      "loss": 0.3625,
+      "mean_token_accuracy": 0.9161295138299466,
+      "num_tokens": 31690266.0,
+      "step": 1950
+    },
+    {
+      "entropy": 0.30422380041563885,
+      "epoch": 1.8387966926640473,
+      "grad_norm": 0.193359375,
+      "learning_rate": 7.74859287054409e-05,
+      "loss": 0.3022,
+      "mean_token_accuracy": 0.9249883979558945,
+      "num_tokens": 31852062.0,
+      "step": 1960
+    },
+    {
+      "entropy": 0.28159310154733247,
+      "epoch": 1.8481792060048086,
+      "grad_norm": 0.12890625,
+      "learning_rate": 7.68605378361476e-05,
+      "loss": 0.2734,
+      "mean_token_accuracy": 0.9310276363044977,
+      "num_tokens": 32015366.0,
+      "step": 1970
+    },
+    {
+      "entropy": 0.34929422264685855,
+      "epoch": 1.8575617193455698,
+      "grad_norm": 0.134765625,
+      "learning_rate": 7.623514696685429e-05,
+      "loss": 0.3437,
+      "mean_token_accuracy": 0.9215225942432881,
+      "num_tokens": 32178520.0,
+      "step": 1980
+    },
+    {
+      "entropy": 0.3168531317904126,
+      "epoch": 1.8669442326863308,
+      "grad_norm": 0.1787109375,
+      "learning_rate": 7.560975609756099e-05,
+      "loss": 0.311,
+      "mean_token_accuracy": 0.9265505827963352,
+      "num_tokens": 32341136.0,
+      "step": 1990
+    },
+    {
+      "entropy": 0.34503675387823024,
+      "epoch": 1.876326746027092,
+      "grad_norm": 0.1337890625,
+      "learning_rate": 7.498436522826767e-05,
+      "loss": 0.3371,
+      "mean_token_accuracy": 0.9186861624941229,
+      "num_tokens": 32503824.0,
+      "step": 2000
+    },
+    {
+      "entropy": 0.3137181800149847,
+      "epoch": 1.8857092593678533,
+      "grad_norm": 0.12890625,
+      "learning_rate": 7.435897435897436e-05,
+      "loss": 0.3072,
+      "mean_token_accuracy": 0.9252266064286232,
+      "num_tokens": 32665961.0,
+      "step": 2010
+    },
+    {
+      "entropy": 0.3118691684619989,
+      "epoch": 1.8950917727086143,
+      "grad_norm": 0.1259765625,
+      "learning_rate": 7.373358348968106e-05,
+      "loss": 0.3092,
+      "mean_token_accuracy": 0.9266283692792058,
+      "num_tokens": 32828930.0,
+      "step": 2020
+    },
+    {
+      "entropy": 0.36025177109404466,
+      "epoch": 1.9044742860493755,
+      "grad_norm": 0.1337890625,
+      "learning_rate": 7.310819262038774e-05,
+      "loss": 0.3483,
+      "mean_token_accuracy": 0.919862699508667,
+      "num_tokens": 32991199.0,
+      "step": 2030
+    },
+    {
+      "entropy": 0.2960929194523487,
+      "epoch": 1.9138567993901368,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 7.248280175109444e-05,
+      "loss": 0.2911,
+      "mean_token_accuracy": 0.9277313150465488,
+      "num_tokens": 33153591.0,
+      "step": 2040
+    },
+    {
+      "entropy": 0.3087802886031568,
+      "epoch": 1.9232393127308978,
+      "grad_norm": 0.1318359375,
+      "learning_rate": 7.185741088180113e-05,
+      "loss": 0.3034,
+      "mean_token_accuracy": 0.9258704790845513,
+      "num_tokens": 33316611.0,
+      "step": 2050
+    },
+    {
+      "entropy": 0.3371582037769258,
+      "epoch": 1.932621826071659,
+      "grad_norm": 0.1484375,
+      "learning_rate": 7.123202001250782e-05,
+      "loss": 0.3246,
+      "mean_token_accuracy": 0.9226027946919203,
+      "num_tokens": 33479599.0,
+      "step": 2060
+    },
+    {
+      "entropy": 0.3058539542311337,
+      "epoch": 1.9420043394124202,
+      "grad_norm": 0.19140625,
+      "learning_rate": 7.06066291432145e-05,
+      "loss": 0.3052,
+      "mean_token_accuracy": 0.926361970603466,
+      "num_tokens": 33642025.0,
+      "step": 2070
+    },
+    {
+      "entropy": 0.30661272021825425,
+      "epoch": 1.9513868527531812,
+      "grad_norm": 0.142578125,
+      "learning_rate": 6.99812382739212e-05,
+      "loss": 0.3019,
+      "mean_token_accuracy": 0.9264740001410245,
+      "num_tokens": 33804960.0,
+      "step": 2080
+    },
+    {
+      "entropy": 0.3367635572212748,
+      "epoch": 1.9607693660939423,
+      "grad_norm": 0.140625,
+      "learning_rate": 6.935584740462789e-05,
+      "loss": 0.3255,
+      "mean_token_accuracy": 0.9226295977830887,
+      "num_tokens": 33967617.0,
+      "step": 2090
+    },
+    {
+      "entropy": 0.30951609945041125,
+      "epoch": 1.9701518794347037,
+      "grad_norm": 0.1455078125,
+      "learning_rate": 6.873045653533459e-05,
+      "loss": 0.3099,
+      "mean_token_accuracy": 0.9269402369856834,
+      "num_tokens": 34130354.0,
+      "step": 2100
+    },
+    {
+      "entropy": 0.27725380276096984,
+      "epoch": 1.9795343927754647,
+      "grad_norm": 0.138671875,
+      "learning_rate": 6.810506566604128e-05,
+      "loss": 0.2718,
+      "mean_token_accuracy": 0.9319463197141886,
+      "num_tokens": 34292449.0,
+      "step": 2110
+    },
+    {
+      "entropy": 0.3474087182315998,
+      "epoch": 1.9889169061162257,
+      "grad_norm": 0.158203125,
+      "learning_rate": 6.747967479674798e-05,
+      "loss": 0.3388,
+      "mean_token_accuracy": 0.918656725436449,
+      "num_tokens": 34454991.0,
+      "step": 2120
+    },
+    {
+      "entropy": 0.3628831981972326,
+      "epoch": 1.9982994194569872,
+      "grad_norm": 0.1220703125,
+      "learning_rate": 6.685428392745466e-05,
+      "loss": 0.3612,
+      "mean_token_accuracy": 0.9168768994510174,
+      "num_tokens": 34617319.0,
+      "step": 2130
+    },
+    {
+      "entropy": 0.30754789969866064,
+      "epoch": 2.007506010672609,
+      "grad_norm": 0.1416015625,
+      "learning_rate": 6.622889305816136e-05,
+      "loss": 0.2987,
+      "mean_token_accuracy": 0.9273600897211938,
+      "num_tokens": 34776727.0,
+      "step": 2140
+    },
+    {
+      "entropy": 0.2743127902853303,
+      "epoch": 2.0168885240133703,
+      "grad_norm": 0.138671875,
+      "learning_rate": 6.560350218886805e-05,
+      "loss": 0.2625,
+      "mean_token_accuracy": 0.9338100612163543,
+      "num_tokens": 34939582.0,
+      "step": 2150
+    },
+    {
+      "entropy": 0.23109424414578825,
+      "epoch": 2.0262710373541313,
+      "grad_norm": 0.1259765625,
+      "learning_rate": 6.497811131957475e-05,
+      "loss": 0.2269,
+      "mean_token_accuracy": 0.9402029298245906,
+      "num_tokens": 35102107.0,
+      "step": 2160
+    },
+    {
+      "entropy": 0.32085173048544674,
+      "epoch": 2.0356535506948923,
+      "grad_norm": 0.17578125,
+      "learning_rate": 6.435272045028143e-05,
+      "loss": 0.3157,
+      "mean_token_accuracy": 0.9279470354318619,
+      "num_tokens": 35264637.0,
+      "step": 2170
+    },
+    {
+      "entropy": 0.2932477945287246,
+      "epoch": 2.0450360640356537,
+      "grad_norm": 0.130859375,
+      "learning_rate": 6.372732958098812e-05,
+      "loss": 0.2874,
+      "mean_token_accuracy": 0.9301197096705437,
+      "num_tokens": 35427805.0,
+      "step": 2180
+    },
+    {
+      "entropy": 0.31676896529388615,
+      "epoch": 2.0544185773764148,
+      "grad_norm": 0.166015625,
+      "learning_rate": 6.31019387116948e-05,
+      "loss": 0.3111,
+      "mean_token_accuracy": 0.923145342990756,
+      "num_tokens": 35590042.0,
+      "step": 2190
+    },
+    {
+      "entropy": 0.2916707712749485,
+      "epoch": 2.0638010907171758,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 6.24765478424015e-05,
+      "loss": 0.283,
+      "mean_token_accuracy": 0.929805812984705,
+      "num_tokens": 35752824.0,
+      "step": 2200
+    },
+    {
+      "entropy": 0.2904738457233179,
+      "epoch": 2.0731836040579372,
+      "grad_norm": 0.12353515625,
+      "learning_rate": 6.185115697310819e-05,
+      "loss": 0.2835,
+      "mean_token_accuracy": 0.929260141402483,
+      "num_tokens": 35915457.0,
+      "step": 2210
+    },
+    {
+      "entropy": 0.34058672086102887,
+      "epoch": 2.0825661173986982,
+      "grad_norm": 0.177734375,
+      "learning_rate": 6.122576610381489e-05,
+      "loss": 0.3331,
+      "mean_token_accuracy": 0.921688269637525,
+      "num_tokens": 36077094.0,
+      "step": 2220
+    },
+    {
+      "entropy": 0.3112417246331461,
+      "epoch": 2.0919486307394592,
+      "grad_norm": 0.1455078125,
+      "learning_rate": 6.060037523452158e-05,
+      "loss": 0.3079,
+      "mean_token_accuracy": 0.9285556517541409,
+      "num_tokens": 36239464.0,
+      "step": 2230
+    },
+    {
+      "entropy": 0.34240057118586265,
+      "epoch": 2.1013311440802207,
+      "grad_norm": 0.1357421875,
+      "learning_rate": 5.997498436522827e-05,
+      "loss": 0.3332,
+      "mean_token_accuracy": 0.9241030484437942,
+      "num_tokens": 36402734.0,
+      "step": 2240
+    },
+    {
+      "entropy": 0.3349809938808903,
+      "epoch": 2.1107136574209817,
+      "grad_norm": 0.1416015625,
+      "learning_rate": 5.9349593495934964e-05,
+      "loss": 0.3297,
+      "mean_token_accuracy": 0.9225290399044752,
+      "num_tokens": 36565940.0,
+      "step": 2250
+    },
+    {
+      "entropy": 0.29396202315110714,
+      "epoch": 2.1200961707617427,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 5.872420262664166e-05,
+      "loss": 0.2881,
+      "mean_token_accuracy": 0.9296416111290455,
+      "num_tokens": 36728649.0,
+      "step": 2260
+    },
+    {
+      "entropy": 0.2903045032813679,
+      "epoch": 2.129478684102504,
+      "grad_norm": 0.09765625,
+      "learning_rate": 5.809881175734835e-05,
+      "loss": 0.2833,
+      "mean_token_accuracy": 0.9330372145399451,
+      "num_tokens": 36890880.0,
+      "step": 2270
+    },
+    {
+      "entropy": 0.27020151492906735,
+      "epoch": 2.138861197443265,
+      "grad_norm": 0.134765625,
+      "learning_rate": 5.747342088805504e-05,
+      "loss": 0.2674,
+      "mean_token_accuracy": 0.9339799758046865,
+      "num_tokens": 37053255.0,
+      "step": 2280
+    },
+    {
+      "entropy": 0.31789518537698314,
+      "epoch": 2.148243710784026,
+      "grad_norm": 0.150390625,
+      "learning_rate": 5.6848030018761736e-05,
+      "loss": 0.306,
+      "mean_token_accuracy": 0.9265004731714725,
+      "num_tokens": 37215790.0,
+      "step": 2290
+    },
+    {
+      "entropy": 0.2779642059525941,
+      "epoch": 2.1576262241247877,
+      "grad_norm": 0.1494140625,
+      "learning_rate": 5.622263914946842e-05,
+      "loss": 0.276,
+      "mean_token_accuracy": 0.9299513600766659,
+      "num_tokens": 37377844.0,
+      "step": 2300
+    },
+    {
+      "entropy": 0.2879410418216139,
+      "epoch": 2.1670087374655487,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 5.559724828017511e-05,
+      "loss": 0.2804,
+      "mean_token_accuracy": 0.9302285347133875,
+      "num_tokens": 37540641.0,
+      "step": 2310
+    },
+    {
+      "entropy": 0.3198163398075849,
+      "epoch": 2.1763912508063097,
+      "grad_norm": 0.1494140625,
+      "learning_rate": 5.49718574108818e-05,
+      "loss": 0.3094,
+      "mean_token_accuracy": 0.9261773098260164,
+      "num_tokens": 37702995.0,
+      "step": 2320
+    },
+    {
+      "entropy": 0.30028246028232386,
+      "epoch": 2.1857737641470707,
+      "grad_norm": 0.1279296875,
+      "learning_rate": 5.4346466541588495e-05,
+      "loss": 0.2997,
+      "mean_token_accuracy": 0.9303501343354583,
+      "num_tokens": 37866156.0,
+      "step": 2330
+    },
+    {
+      "entropy": 0.30868017480243,
+      "epoch": 2.195156277487832,
+      "grad_norm": 0.1787109375,
+      "learning_rate": 5.372107567229519e-05,
+      "loss": 0.2983,
+      "mean_token_accuracy": 0.9272098153829574,
+      "num_tokens": 38028751.0,
+      "step": 2340
+    },
+    {
+      "entropy": 0.35966923176893034,
+      "epoch": 2.204538790828593,
+      "grad_norm": 0.1591796875,
+      "learning_rate": 5.3095684803001874e-05,
+      "loss": 0.3501,
+      "mean_token_accuracy": 0.9204060444608331,
+      "num_tokens": 38191815.0,
+      "step": 2350
+    },
+    {
+      "entropy": 0.3018774308031425,
+      "epoch": 2.213921304169354,
+      "grad_norm": 0.1318359375,
+      "learning_rate": 5.247029393370857e-05,
+      "loss": 0.2989,
+      "mean_token_accuracy": 0.9272254019975662,
+      "num_tokens": 38354651.0,
+      "step": 2360
+    },
+    {
+      "entropy": 0.2737217928515747,
+      "epoch": 2.2233038175101156,
+      "grad_norm": 0.1279296875,
+      "learning_rate": 5.184490306441526e-05,
+      "loss": 0.2656,
+      "mean_token_accuracy": 0.9318129748106003,
+      "num_tokens": 38517176.0,
+      "step": 2370
+    },
+    {
+      "entropy": 0.3319847055245191,
+      "epoch": 2.2326863308508766,
+      "grad_norm": 0.109375,
+      "learning_rate": 5.121951219512195e-05,
+      "loss": 0.3264,
+      "mean_token_accuracy": 0.9221911679953336,
+      "num_tokens": 38679769.0,
+      "step": 2380
+    },
+    {
+      "entropy": 0.30262379703926856,
+      "epoch": 2.2420688441916377,
+      "grad_norm": 0.1240234375,
+      "learning_rate": 5.0594121325828646e-05,
+      "loss": 0.2941,
+      "mean_token_accuracy": 0.9280214164406061,
+      "num_tokens": 38842724.0,
+      "step": 2390
+    },
+    {
+      "entropy": 0.2472104947839398,
+      "epoch": 2.251451357532399,
+      "grad_norm": 0.10888671875,
+      "learning_rate": 4.996873045653533e-05,
+      "loss": 0.2411,
+      "mean_token_accuracy": 0.9378781199455262,
+      "num_tokens": 39005513.0,
+      "step": 2400
+    },
+    {
+      "entropy": 0.25551886922330597,
+      "epoch": 2.26083387087316,
+      "grad_norm": 0.1279296875,
+      "learning_rate": 4.9343339587242026e-05,
+      "loss": 0.2547,
+      "mean_token_accuracy": 0.9324965570122004,
+      "num_tokens": 39167728.0,
+      "step": 2410
+    },
+    {
+      "entropy": 0.3002851114142686,
+      "epoch": 2.270216384213921,
+      "grad_norm": 0.1142578125,
+      "learning_rate": 4.871794871794872e-05,
+      "loss": 0.2892,
+      "mean_token_accuracy": 0.9301211513578892,
+      "num_tokens": 39330194.0,
+      "step": 2420
+    },
+    {
+      "entropy": 0.301609949831618,
+      "epoch": 2.2795988975546826,
+      "grad_norm": 0.1396484375,
+      "learning_rate": 4.809255784865541e-05,
+      "loss": 0.2937,
+      "mean_token_accuracy": 0.9278855569660663,
+      "num_tokens": 39493237.0,
+      "step": 2430
+    },
+    {
+      "entropy": 0.31216122427140364,
+      "epoch": 2.2889814108954436,
+      "grad_norm": 0.1123046875,
+      "learning_rate": 4.7467166979362105e-05,
+      "loss": 0.3093,
+      "mean_token_accuracy": 0.9252140417695045,
+      "num_tokens": 39655553.0,
+      "step": 2440
+    },
+    {
+      "entropy": 0.23702631431515328,
+      "epoch": 2.2983639242362046,
+      "grad_norm": 0.1435546875,
+      "learning_rate": 4.68417761100688e-05,
+      "loss": 0.2252,
+      "mean_token_accuracy": 0.9393304187804461,
+      "num_tokens": 39817974.0,
+      "step": 2450
+    },
+    {
+      "entropy": 0.30330076812533663,
+      "epoch": 2.307746437576966,
+      "grad_norm": 0.1474609375,
+      "learning_rate": 4.621638524077549e-05,
+      "loss": 0.2995,
+      "mean_token_accuracy": 0.926243145391345,
+      "num_tokens": 39981087.0,
+      "step": 2460
+    },
+    {
+      "entropy": 0.30959747194428927,
+      "epoch": 2.317128950917727,
+      "grad_norm": 0.1376953125,
+      "learning_rate": 4.559099437148218e-05,
+      "loss": 0.305,
+      "mean_token_accuracy": 0.927550395578146,
+      "num_tokens": 40144139.0,
+      "step": 2470
+    },
+    {
+      "entropy": 0.28602821550448426,
+      "epoch": 2.326511464258488,
+      "grad_norm": 0.1220703125,
+      "learning_rate": 4.496560350218887e-05,
+      "loss": 0.2737,
+      "mean_token_accuracy": 0.93137735016644,
+      "num_tokens": 40307179.0,
+      "step": 2480
+    },
+    {
+      "entropy": 0.32213802009937353,
+      "epoch": 2.3358939775992495,
+      "grad_norm": 0.12890625,
+      "learning_rate": 4.434021263289556e-05,
+      "loss": 0.3179,
+      "mean_token_accuracy": 0.9258282888680697,
+      "num_tokens": 40470205.0,
+      "step": 2490
+    },
+    {
+      "entropy": 0.2958646867133211,
+      "epoch": 2.3452764909400106,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 4.3714821763602256e-05,
+      "loss": 0.2905,
+      "mean_token_accuracy": 0.9263608150184155,
+      "num_tokens": 40631651.0,
+      "step": 2500
+    },
+    {
+      "entropy": 0.27999189621768894,
+      "epoch": 2.3546590042807716,
+      "grad_norm": 0.1552734375,
+      "learning_rate": 4.308943089430895e-05,
+      "loss": 0.2729,
+      "mean_token_accuracy": 0.9314939744770527,
+      "num_tokens": 40793827.0,
+      "step": 2510
+    },
+    {
+      "entropy": 0.2795608859625645,
+      "epoch": 2.364041517621533,
+      "grad_norm": 0.10400390625,
+      "learning_rate": 4.246404002501564e-05,
+      "loss": 0.2746,
+      "mean_token_accuracy": 0.9295887473970652,
+      "num_tokens": 40956707.0,
+      "step": 2520
+    },
+    {
+      "entropy": 0.3216266447561793,
+      "epoch": 2.373424030962294,
+      "grad_norm": 0.134765625,
+      "learning_rate": 4.183864915572233e-05,
+      "loss": 0.3164,
+      "mean_token_accuracy": 0.9245435189455747,
+      "num_tokens": 41118930.0,
+      "step": 2530
+    },
+    {
+      "entropy": 0.3690712286042981,
+      "epoch": 2.382806544303055,
+      "grad_norm": 0.14453125,
+      "learning_rate": 4.121325828642902e-05,
+      "loss": 0.3642,
+      "mean_token_accuracy": 0.9181511271744967,
+      "num_tokens": 41282031.0,
+      "step": 2540
+    },
+    {
+      "entropy": 0.3252036487159785,
+      "epoch": 2.3921890576438165,
+      "grad_norm": 0.142578125,
+      "learning_rate": 4.058786741713571e-05,
+      "loss": 0.3189,
+      "mean_token_accuracy": 0.9277937490493059,
+      "num_tokens": 41444176.0,
+      "step": 2550
+    },
+    {
+      "entropy": 0.26126941335387527,
+      "epoch": 2.4015715709845775,
+      "grad_norm": 0.126953125,
+      "learning_rate": 3.99624765478424e-05,
+      "loss": 0.2558,
+      "mean_token_accuracy": 0.9345622129738331,
+      "num_tokens": 41606952.0,
+      "step": 2560
+    },
+    {
+      "entropy": 0.32813997849007137,
+      "epoch": 2.4109540843253385,
+      "grad_norm": 0.1396484375,
+      "learning_rate": 3.9337085678549094e-05,
+      "loss": 0.3231,
+      "mean_token_accuracy": 0.9266963802278042,
+      "num_tokens": 41769441.0,
+      "step": 2570
+    },
+    {
+      "entropy": 0.3256168618972879,
+      "epoch": 2.4203365976661,
+      "grad_norm": 0.1494140625,
+      "learning_rate": 3.871169480925579e-05,
+      "loss": 0.3166,
+      "mean_token_accuracy": 0.922471884265542,
+      "num_tokens": 41932618.0,
+      "step": 2580
+    },
+    {
+      "entropy": 0.30511015201336705,
+      "epoch": 2.429719111006861,
+      "grad_norm": 0.11279296875,
+      "learning_rate": 3.808630393996248e-05,
+      "loss": 0.2983,
+      "mean_token_accuracy": 0.9276284482330084,
+      "num_tokens": 42094647.0,
+      "step": 2590
+    },
+    {
+      "entropy": 0.2836356892774347,
+      "epoch": 2.439101624347622,
+      "grad_norm": 0.1611328125,
+      "learning_rate": 3.7460913070669166e-05,
+      "loss": 0.2756,
+      "mean_token_accuracy": 0.9297160636633635,
+      "num_tokens": 42257314.0,
+      "step": 2600
+    },
+    {
+      "entropy": 0.26650271246326157,
+      "epoch": 2.4484841376883835,
+      "grad_norm": 0.150390625,
+      "learning_rate": 3.683552220137586e-05,
+      "loss": 0.2597,
+      "mean_token_accuracy": 0.9355631563812494,
+      "num_tokens": 42419215.0,
+      "step": 2610
+    },
+    {
+      "entropy": 0.339943853468867,
+      "epoch": 2.4578666510291445,
+      "grad_norm": 0.142578125,
+      "learning_rate": 3.621013133208255e-05,
+      "loss": 0.3348,
+      "mean_token_accuracy": 0.9231440974399447,
+      "num_tokens": 42581049.0,
+      "step": 2620
+    },
+    {
+      "entropy": 0.2780093678680714,
+      "epoch": 2.4672491643699055,
+      "grad_norm": 0.1474609375,
+      "learning_rate": 3.5584740462789245e-05,
+      "loss": 0.2706,
+      "mean_token_accuracy": 0.9318165834993124,
+      "num_tokens": 42743424.0,
+      "step": 2630
+    },
+    {
+      "entropy": 0.38994275986333377,
+      "epoch": 2.4766316777106665,
+      "grad_norm": 0.1630859375,
+      "learning_rate": 3.495934959349594e-05,
+      "loss": 0.3856,
+      "mean_token_accuracy": 0.914062000438571,
+      "num_tokens": 42905298.0,
+      "step": 2640
+    },
+    {
+      "entropy": 0.25893269666703417,
+      "epoch": 2.486014191051428,
+      "grad_norm": 0.1328125,
+      "learning_rate": 3.433395872420263e-05,
+      "loss": 0.2499,
+      "mean_token_accuracy": 0.9351230334490538,
+      "num_tokens": 43067214.0,
+      "step": 2650
+    },
+    {
+      "entropy": 0.3018484404310584,
+      "epoch": 2.495396704392189,
+      "grad_norm": 0.11962890625,
+      "learning_rate": 3.370856785490932e-05,
+      "loss": 0.2957,
+      "mean_token_accuracy": 0.9283837772905826,
+      "num_tokens": 43229346.0,
+      "step": 2660
+    },
+    {
+      "entropy": 0.2423785132705234,
+      "epoch": 2.50477921773295,
+      "grad_norm": 0.1298828125,
+      "learning_rate": 3.308317698561601e-05,
+      "loss": 0.2388,
+      "mean_token_accuracy": 0.9364284005016088,
+      "num_tokens": 43391495.0,
+      "step": 2670
+    },
+    {
+      "entropy": 0.3937613701389637,
+      "epoch": 2.5141617310737114,
+      "grad_norm": 0.146484375,
+      "learning_rate": 3.2457786116322704e-05,
+      "loss": 0.3887,
+      "mean_token_accuracy": 0.9135016176849604,
+      "num_tokens": 43554181.0,
+      "step": 2680
+    },
+    {
+      "entropy": 0.28887827979051506,
+      "epoch": 2.5235442444144724,
+      "grad_norm": 0.1845703125,
+      "learning_rate": 3.18323952470294e-05,
+      "loss": 0.2816,
+      "mean_token_accuracy": 0.9309012487530708,
+      "num_tokens": 43716755.0,
+      "step": 2690
+    },
+    {
+      "entropy": 0.3326025055255741,
+      "epoch": 2.5329267577552335,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 3.120700437773609e-05,
+      "loss": 0.3246,
+      "mean_token_accuracy": 0.9202332761138677,
+      "num_tokens": 43878960.0,
+      "step": 2700
+    },
+    {
+      "entropy": 0.3055231410660781,
+      "epoch": 2.542309271095995,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 3.058161350844278e-05,
+      "loss": 0.3023,
+      "mean_token_accuracy": 0.9283943830057979,
+      "num_tokens": 44041403.0,
+      "step": 2710
+    },
+    {
+      "entropy": 0.28135751393274405,
+      "epoch": 2.551691784436756,
+      "grad_norm": 0.1396484375,
+      "learning_rate": 2.9956222639149466e-05,
+      "loss": 0.2764,
+      "mean_token_accuracy": 0.9312219850718975,
+      "num_tokens": 44204489.0,
+      "step": 2720
+    },
+    {
+      "entropy": 0.2601934680482373,
+      "epoch": 2.561074297777517,
+      "grad_norm": 0.2060546875,
+      "learning_rate": 2.933083176985616e-05,
+      "loss": 0.2534,
+      "mean_token_accuracy": 0.9365647614002228,
+      "num_tokens": 44367457.0,
+      "step": 2730
+    },
+    {
+      "entropy": 0.2961357018793933,
+      "epoch": 2.5704568111182784,
+      "grad_norm": 0.1318359375,
+      "learning_rate": 2.8705440900562852e-05,
+      "loss": 0.2925,
+      "mean_token_accuracy": 0.9294706670567393,
+      "num_tokens": 44531018.0,
+      "step": 2740
+    },
+    {
+      "entropy": 0.3402582473878283,
+      "epoch": 2.5798393244590394,
+      "grad_norm": 0.142578125,
+      "learning_rate": 2.8080050031269545e-05,
+      "loss": 0.3349,
+      "mean_token_accuracy": 0.9241237251088024,
+      "num_tokens": 44693846.0,
+      "step": 2750
+    },
+    {
+      "entropy": 0.2918738658190705,
+      "epoch": 2.5892218377998004,
+      "grad_norm": 0.1455078125,
+      "learning_rate": 2.7454659161976238e-05,
+      "loss": 0.2832,
+      "mean_token_accuracy": 0.9334099506959319,
+      "num_tokens": 44856457.0,
+      "step": 2760
+    },
+    {
+      "entropy": 0.3172753399936482,
+      "epoch": 2.598604351140562,
+      "grad_norm": 0.1552734375,
+      "learning_rate": 2.682926829268293e-05,
+      "loss": 0.3109,
+      "mean_token_accuracy": 0.9242209114134312,
+      "num_tokens": 45018390.0,
+      "step": 2770
+    },
+    {
+      "entropy": 0.29830688254442067,
+      "epoch": 2.607986864481323,
+      "grad_norm": 0.142578125,
+      "learning_rate": 2.620387742338962e-05,
+      "loss": 0.2932,
+      "mean_token_accuracy": 0.9281566303223372,
+      "num_tokens": 45180693.0,
+      "step": 2780
+    },
+    {
+      "entropy": 0.32780712051899175,
+      "epoch": 2.617369377822084,
+      "grad_norm": 0.1328125,
+      "learning_rate": 2.557848655409631e-05,
+      "loss": 0.3174,
+      "mean_token_accuracy": 0.926155343092978,
+      "num_tokens": 45343376.0,
+      "step": 2790
+    },
+    {
+      "entropy": 0.2601445548760239,
+      "epoch": 2.6267518911628454,
+      "grad_norm": 0.146484375,
+      "learning_rate": 2.4953095684803003e-05,
+      "loss": 0.2511,
+      "mean_token_accuracy": 0.9344733364880085,
+      "num_tokens": 45505376.0,
+      "step": 2800
+    },
+    {
+      "entropy": 0.329776706371922,
+      "epoch": 2.6361344045036064,
+      "grad_norm": 0.14453125,
+      "learning_rate": 2.4327704815509693e-05,
+      "loss": 0.3268,
+      "mean_token_accuracy": 0.9248848807066679,
+      "num_tokens": 45667838.0,
+      "step": 2810
+    },
+    {
+      "entropy": 0.2883093195501715,
+      "epoch": 2.6455169178443674,
+      "grad_norm": 0.1181640625,
+      "learning_rate": 2.3702313946216386e-05,
+      "loss": 0.2863,
+      "mean_token_accuracy": 0.930842773616314,
+      "num_tokens": 45830525.0,
+      "step": 2820
+    },
+    {
+      "entropy": 0.24201850754907356,
+      "epoch": 2.654899431185129,
+      "grad_norm": 0.146484375,
+      "learning_rate": 2.307692307692308e-05,
+      "loss": 0.2331,
+      "mean_token_accuracy": 0.9388571180403232,
+      "num_tokens": 45993227.0,
+      "step": 2830
+    },
+    {
+      "entropy": 0.32533247509854846,
+      "epoch": 2.66428194452589,
+      "grad_norm": 0.150390625,
+      "learning_rate": 2.245153220762977e-05,
+      "loss": 0.3168,
+      "mean_token_accuracy": 0.9255710192024708,
+      "num_tokens": 46155827.0,
+      "step": 2840
+    },
+    {
+      "entropy": 0.3312520031700842,
+      "epoch": 2.673664457866651,
+      "grad_norm": 0.1181640625,
+      "learning_rate": 2.1826141338336462e-05,
+      "loss": 0.3231,
+      "mean_token_accuracy": 0.9246046051383019,
+      "num_tokens": 46318131.0,
+      "step": 2850
+    },
+    {
+      "entropy": 0.3453969239722937,
+      "epoch": 2.6830469712074123,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 2.1200750469043155e-05,
+      "loss": 0.3423,
+      "mean_token_accuracy": 0.9209890190511942,
+      "num_tokens": 46480591.0,
+      "step": 2860
+    },
+    {
+      "entropy": 0.29795768808107825,
+      "epoch": 2.6924294845481733,
+      "grad_norm": 0.1748046875,
+      "learning_rate": 2.0575359599749844e-05,
+      "loss": 0.2897,
+      "mean_token_accuracy": 0.9284091215580702,
+      "num_tokens": 46642465.0,
+      "step": 2870
+    },
+    {
+      "entropy": 0.31598600526340304,
+      "epoch": 2.7018119978889343,
+      "grad_norm": 0.1494140625,
+      "learning_rate": 1.9949968730456537e-05,
+      "loss": 0.3058,
+      "mean_token_accuracy": 0.9263520397245883,
+      "num_tokens": 46805289.0,
+      "step": 2880
+    },
+    {
+      "entropy": 0.2473068662569858,
+      "epoch": 2.711194511229696,
+      "grad_norm": 0.1083984375,
+      "learning_rate": 1.932457786116323e-05,
+      "loss": 0.2443,
+      "mean_token_accuracy": 0.9364898830652237,
+      "num_tokens": 46968250.0,
+      "step": 2890
+    },
+    {
+      "entropy": 0.32329227214795536,
+      "epoch": 2.720577024570457,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 1.869918699186992e-05,
+      "loss": 0.3206,
+      "mean_token_accuracy": 0.925135574862361,
+      "num_tokens": 47130591.0,
+      "step": 2900
+    },
+    {
+      "entropy": 0.2909718307200819,
+      "epoch": 2.729959537911218,
+      "grad_norm": 0.140625,
+      "learning_rate": 1.807379612257661e-05,
+      "loss": 0.2792,
+      "mean_token_accuracy": 0.9289011463522912,
+      "num_tokens": 47293338.0,
+      "step": 2910
+    },
+    {
+      "entropy": 0.35400011245510543,
+      "epoch": 2.7393420512519793,
+      "grad_norm": 0.11376953125,
+      "learning_rate": 1.7448405253283303e-05,
+      "loss": 0.3465,
+      "mean_token_accuracy": 0.9220137037336826,
+      "num_tokens": 47456230.0,
+      "step": 2920
+    },
+    {
+      "entropy": 0.2967787377536297,
+      "epoch": 2.7487245645927403,
+      "grad_norm": 0.150390625,
+      "learning_rate": 1.6823014383989993e-05,
+      "loss": 0.2901,
+      "mean_token_accuracy": 0.9279664635658265,
+      "num_tokens": 47618021.0,
+      "step": 2930
+    },
+    {
+      "entropy": 0.2816445629810914,
+      "epoch": 2.7581070779335013,
+      "grad_norm": 0.1474609375,
+      "learning_rate": 1.6197623514696686e-05,
+      "loss": 0.2777,
+      "mean_token_accuracy": 0.9283030040562152,
+      "num_tokens": 47781112.0,
+      "step": 2940
+    },
+    {
+      "entropy": 0.2784802548063453,
+      "epoch": 2.7674895912742628,
+      "grad_norm": 0.11328125,
+      "learning_rate": 1.557223264540338e-05,
+      "loss": 0.2742,
+      "mean_token_accuracy": 0.9345203250646591,
+      "num_tokens": 47943069.0,
+      "step": 2950
+    },
+    {
+      "entropy": 0.2651161406189203,
+      "epoch": 2.7768721046150238,
+      "grad_norm": 0.1259765625,
+      "learning_rate": 1.4946841776110068e-05,
+      "loss": 0.2572,
+      "mean_token_accuracy": 0.9342164538800717,
+      "num_tokens": 48105640.0,
+      "step": 2960
+    },
+    {
+      "entropy": 0.3173203022684902,
+      "epoch": 2.7862546179557848,
+      "grad_norm": 0.1611328125,
+      "learning_rate": 1.4321450906816761e-05,
+      "loss": 0.3152,
+      "mean_token_accuracy": 0.9248944539576769,
+      "num_tokens": 48268433.0,
+      "step": 2970
+    },
+    {
+      "entropy": 0.35732815401279366,
+      "epoch": 2.7956371312965462,
+      "grad_norm": 0.154296875,
+      "learning_rate": 1.3696060037523453e-05,
+      "loss": 0.3507,
+      "mean_token_accuracy": 0.9167288988828659,
+      "num_tokens": 48430662.0,
+      "step": 2980
+    },
+    {
+      "entropy": 0.33472506381222045,
+      "epoch": 2.8050196446373072,
+      "grad_norm": 0.15625,
+      "learning_rate": 1.3070669168230146e-05,
+      "loss": 0.3277,
+      "mean_token_accuracy": 0.9212579555809498,
+      "num_tokens": 48593324.0,
+      "step": 2990
+    },
+    {
+      "entropy": 0.2915991071087774,
+      "epoch": 2.8144021579780683,
+      "grad_norm": 0.11279296875,
+      "learning_rate": 1.2445278298936835e-05,
+      "loss": 0.2848,
+      "mean_token_accuracy": 0.9322345092892647,
+      "num_tokens": 48755803.0,
+      "step": 3000
+    },
+    {
+      "entropy": 0.29993935831007545,
+      "epoch": 2.8237846713188297,
+      "grad_norm": 0.146484375,
+      "learning_rate": 1.1819887429643528e-05,
+      "loss": 0.2917,
+      "mean_token_accuracy": 0.9273430518805981,
+      "num_tokens": 48918179.0,
+      "step": 3010
+    },
+    {
+      "entropy": 0.2874399486405309,
+      "epoch": 2.8331671846595907,
+      "grad_norm": 0.16015625,
+      "learning_rate": 1.119449656035022e-05,
+      "loss": 0.2776,
+      "mean_token_accuracy": 0.931094466149807,
+      "num_tokens": 49080584.0,
+      "step": 3020
+    },
+    {
+      "entropy": 0.3395154855912551,
+      "epoch": 2.8425496980003517,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 1.0569105691056911e-05,
+      "loss": 0.3299,
+      "mean_token_accuracy": 0.924785427749157,
+      "num_tokens": 49243601.0,
+      "step": 3030
+    },
+    {
+      "entropy": 0.3259556777135003,
+      "epoch": 2.851932211341113,
+      "grad_norm": 0.1337890625,
+      "learning_rate": 9.943714821763602e-06,
+      "loss": 0.3207,
+      "mean_token_accuracy": 0.9251650847494602,
+      "num_tokens": 49405613.0,
+      "step": 3040
+    },
+    {
+      "entropy": 0.2658869383740239,
+      "epoch": 2.861314724681874,
+      "grad_norm": 0.1455078125,
+      "learning_rate": 9.318323952470294e-06,
+      "loss": 0.2584,
+      "mean_token_accuracy": 0.9330834452062845,
+      "num_tokens": 49568051.0,
+      "step": 3050
+    },
+    {
+      "entropy": 0.2719281981640961,
+      "epoch": 2.870697238022635,
+      "grad_norm": 0.16015625,
+      "learning_rate": 8.692933083176987e-06,
+      "loss": 0.2644,
+      "mean_token_accuracy": 0.9362422559410334,
+      "num_tokens": 49730469.0,
+      "step": 3060
+    },
+    {
+      "entropy": 0.3221181625209283,
+      "epoch": 2.8800797513633967,
+      "grad_norm": 0.146484375,
+      "learning_rate": 8.067542213883678e-06,
+      "loss": 0.3129,
+      "mean_token_accuracy": 0.9273162662982941,
+      "num_tokens": 49893439.0,
+      "step": 3070
+    },
+    {
+      "entropy": 0.26640188965247946,
+      "epoch": 2.8894622647041577,
+      "grad_norm": 0.1611328125,
+      "learning_rate": 7.442151344590369e-06,
+      "loss": 0.2571,
+      "mean_token_accuracy": 0.9348009482026101,
+      "num_tokens": 50055539.0,
+      "step": 3080
+    },
+    {
+      "entropy": 0.3002611313131638,
+      "epoch": 2.8988447780449187,
+      "grad_norm": 0.1083984375,
+      "learning_rate": 6.816760475297062e-06,
+      "loss": 0.2954,
+      "mean_token_accuracy": 0.9285491608083248,
+      "num_tokens": 50218430.0,
+      "step": 3090
+    },
+    {
+      "entropy": 0.36435635392554105,
+      "epoch": 2.90822729138568,
+      "grad_norm": 0.1396484375,
+      "learning_rate": 6.191369606003752e-06,
+      "loss": 0.3591,
+      "mean_token_accuracy": 0.9154221881181002,
+      "num_tokens": 50380974.0,
+      "step": 3100
+    },
+    {
+      "entropy": 0.27004491952247917,
+      "epoch": 2.917609804726441,
+      "grad_norm": 0.11572265625,
+      "learning_rate": 5.565978736710444e-06,
+      "loss": 0.2659,
+      "mean_token_accuracy": 0.9321655161678791,
+      "num_tokens": 50543527.0,
+      "step": 3110
+    },
+    {
+      "entropy": 0.2986813228810206,
+      "epoch": 2.926992318067202,
+      "grad_norm": 0.1572265625,
+      "learning_rate": 4.940587867417136e-06,
+      "loss": 0.296,
+      "mean_token_accuracy": 0.928958417288959,
+      "num_tokens": 50706274.0,
+      "step": 3120
+    },
+    {
+      "entropy": 0.3143228885368444,
+      "epoch": 2.9363748314079636,
+      "grad_norm": 0.1455078125,
+      "learning_rate": 4.315196998123828e-06,
+      "loss": 0.3108,
+      "mean_token_accuracy": 0.9247220842167735,
+      "num_tokens": 50869052.0,
+      "step": 3130
+    },
+    {
+      "entropy": 0.293576886493247,
+      "epoch": 2.9457573447487246,
+      "grad_norm": 0.1455078125,
+      "learning_rate": 3.6898061288305188e-06,
+      "loss": 0.2885,
+      "mean_token_accuracy": 0.9293860323727131,
+      "num_tokens": 51031710.0,
+      "step": 3140
+    },
+    {
+      "entropy": 0.2768217631499283,
+      "epoch": 2.9551398580894856,
+      "grad_norm": 0.138671875,
+      "learning_rate": 3.064415259537211e-06,
+      "loss": 0.2731,
+      "mean_token_accuracy": 0.9313604889437557,
+      "num_tokens": 51194073.0,
+      "step": 3150
+    },
+    {
+      "entropy": 0.29631736621959137,
+      "epoch": 2.964522371430247,
+      "grad_norm": 0.169921875,
+      "learning_rate": 2.4390243902439027e-06,
+      "loss": 0.2903,
+      "mean_token_accuracy": 0.9294600717723369,
+      "num_tokens": 51356082.0,
+      "step": 3160
+    },
+    {
+      "entropy": 0.28854622174985706,
+      "epoch": 2.973904884771008,
+      "grad_norm": 0.1337890625,
+      "learning_rate": 1.813633520950594e-06,
+      "loss": 0.2842,
+      "mean_token_accuracy": 0.9323527112603187,
+      "num_tokens": 51517707.0,
+      "step": 3170
+    },
+    {
+      "entropy": 0.2872266868769657,
+      "epoch": 2.983287398111769,
+      "grad_norm": 0.1337890625,
+      "learning_rate": 1.1882426516572859e-06,
+      "loss": 0.2816,
+      "mean_token_accuracy": 0.9318976398557425,
+      "num_tokens": 51680325.0,
+      "step": 3180
+    },
+    {
+      "entropy": 0.254745813657064,
+      "epoch": 2.9926699114525306,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 5.628517823639775e-07,
+      "loss": 0.2483,
+      "mean_token_accuracy": 0.9362917259335518,
+      "num_tokens": 51842805.0,
+      "step": 3190
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 3198,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.344450752773038e+18,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-3198/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db83ac20cd1ea46a30b9db82ee3c07152c60e62ceb73c1d01e3725eeebe949ed
+size 6289

runs/Dec16_19-13-33_c97b5462718c/events.out.tfevents.1765912482.c97b5462718c.242.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0aad25b8f3ede9c8765ced339a85a4c648e0c58b06986bdc4830f5d4bc69c316
+size 128580