Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.1-8B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.1-8B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.17.1

adapter_config.json

@@ -0,0 +1,38 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.1-8B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:459bcdaaf80546a85a4f59f821c0f534e87bc18d784267f02f90a98908d855af
+size 151020944

chat_template.jinja

@@ -0,0 +1,156 @@
+
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- set date_string = "26 Jul 2024" %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message + builtin tools #}
+{{- "<|start_header_id|>system<|end_header_id|>
+
+" }}
+{%- if builtin_tools is defined or tools is not none %}
+    {{- "Environment: ipython
+" }}
+{%- endif %}
+{%- if builtin_tools is defined %}
+    {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "
+
+"}}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023
+" }}
+{{- "Today Date: " + date_string + "
+
+" }}
+{{- "You are an expert in evaluating political news and events.
+"}}
+{{- "The user has provided the news story below.
+"}}
+{{- "Produce an event data record similar to GDELT or ICEWS that represents the main event described in the article.
+"}}
+{{- "Give the output in json format {'actors':[{'name':<actor name>,'code':<actor classification codes>,'role':<actor semantic roles>},], 'date':<date of event>, 'city':<city in which event occurred>, 'country':<country in which event occurred>, 'event_class':<the event class name>, 'event_code':<the event class code>}}
+"}}
+{{- "Just give JSON and not a single word more.
+"}}
+{{- "Use context clues to make assumptions when necessary.
+"}}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.
+
+" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "
+
+" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>
+
+' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.
+
+" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.
+
+" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "
+
+" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'assistant' or message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>
+
+'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif message.role == 'assistant' %}
+    {% generation %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>
+
+'}}
+        {%- if add_generation_prompt %}
+            {{- message['content'] | trim + '<|eot_id|>' }}
+        {%- endif %}
+    {% endgeneration %}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>
+
+' -}}
+            {{- "<|python_tag|>" + tool_call.name + ".call(" }}
+            {%- for arg_name, arg_val in tool_call.arguments | items %}
+                {{- arg_name + '="' + arg_val + '"' }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- endif %}
+                {%- endfor %}
+            {{- ")" }}
+        {%- else  %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>
+
+' -}}
+            {{- '{"name": "' + tool_call.name + '", ' }}
+            {{- '"parameters": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- "}" }}
+        {%- endif %}
+        {%- if builtin_tools is defined %}
+            {#- This means we're in ipython mode #}
+            {{- "<|eom_id|>" }}
+        {%- else %}
+            {{- "<|eot_id|>" }}
+        {%- endif %}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>
+
+" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba6804a116cfad11fd6ccaa06f77f0ef1f7dcec4fe87f794d0febf23295ca328
+size 302107915

rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d52b17a69f0da05793c63883fc3490c88f89d0dd5e17c438b4e134118378e74
+size 15429

rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5719eb5bc5ae9f3232c0351a8f0a4cb4038244e74942fb0462e2a0afe7985f5e
+size 15429

rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b8e7514b14a72c0cec89ed6c672d91b583947b914fb2d1bdf9c876ecd4e73de
+size 15429

rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2011fa43320bcfbab4e2b7d478756e0d5c379a7baef8dbd241ae09e6c4a0ffef
+size 15429

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef1506d52e3a95af1f30dd8738cf398b30e77abea357c24f406cde12f2cd8a73
+size 1465

special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|eot_id|>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

trainer_state.json

@@ -0,0 +1,1214 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 2960,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.6544005966186524,
+      "epoch": 0.008445945945945946,
+      "grad_norm": 1.3735458850860596,
+      "learning_rate": 4.0449438202247185e-05,
+      "loss": 1.0469,
+      "mean_token_accuracy": 0.8033061313629151,
+      "num_tokens": 907427.0,
+      "step": 25
+    },
+    {
+      "entropy": 1.6733982801437377,
+      "epoch": 0.016891891891891893,
+      "grad_norm": 0.6318637728691101,
+      "learning_rate": 8.258426966292134e-05,
+      "loss": 0.405,
+      "mean_token_accuracy": 0.9122987079620362,
+      "num_tokens": 1819974.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.6036081886291504,
+      "epoch": 0.02533783783783784,
+      "grad_norm": 0.6783556342124939,
+      "learning_rate": 0.0001247191011235955,
+      "loss": 0.2662,
+      "mean_token_accuracy": 0.9371256637573242,
+      "num_tokens": 2733501.0,
+      "step": 75
+    },
+    {
+      "entropy": 1.4985664987564087,
+      "epoch": 0.033783783783783786,
+      "grad_norm": 0.6065422296524048,
+      "learning_rate": 0.00014999550985573648,
+      "loss": 0.1934,
+      "mean_token_accuracy": 0.9516211485862732,
+      "num_tokens": 3644957.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.4797721529006957,
+      "epoch": 0.04222972972972973,
+      "grad_norm": 0.7198048830032349,
+      "learning_rate": 0.00014994500190699606,
+      "loss": 0.1433,
+      "mean_token_accuracy": 0.9621564221382141,
+      "num_tokens": 4550854.0,
+      "step": 125
+    },
+    {
+      "entropy": 1.4920766258239746,
+      "epoch": 0.05067567567567568,
+      "grad_norm": 0.5299217104911804,
+      "learning_rate": 0.00014983841125121148,
+      "loss": 0.118,
+      "mean_token_accuracy": 0.9688123917579651,
+      "num_tokens": 5457417.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.5022115802764893,
+      "epoch": 0.05912162162162162,
+      "grad_norm": 0.4025590419769287,
+      "learning_rate": 0.00014967581765210774,
+      "loss": 0.1057,
+      "mean_token_accuracy": 0.9718510413169861,
+      "num_tokens": 6368975.0,
+      "step": 175
+    },
+    {
+      "entropy": 1.6094658851623536,
+      "epoch": 0.06756756756756757,
+      "grad_norm": 0.478301465511322,
+      "learning_rate": 0.00014945734278143014,
+      "loss": 0.0969,
+      "mean_token_accuracy": 0.974509973526001,
+      "num_tokens": 7279965.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.698849024772644,
+      "epoch": 0.07601351351351351,
+      "grad_norm": 0.443584680557251,
+      "learning_rate": 0.0001491831501278952,
+      "loss": 0.0883,
+      "mean_token_accuracy": 0.9762599396705628,
+      "num_tokens": 8188065.0,
+      "step": 225
+    },
+    {
+      "entropy": 1.6420823621749878,
+      "epoch": 0.08445945945945946,
+      "grad_norm": 0.3269982933998108,
+      "learning_rate": 0.00014885344487484893,
+      "loss": 0.0821,
+      "mean_token_accuracy": 0.977983922958374,
+      "num_tokens": 9094149.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.5950000524520873,
+      "epoch": 0.0929054054054054,
+      "grad_norm": 0.3920482099056244,
+      "learning_rate": 0.0001484684737467246,
+      "loss": 0.0817,
+      "mean_token_accuracy": 0.9783960843086242,
+      "num_tokens": 10006726.0,
+      "step": 275
+    },
+    {
+      "entropy": 1.4868795156478882,
+      "epoch": 0.10135135135135136,
+      "grad_norm": 0.3517569303512573,
+      "learning_rate": 0.00014802852482441415,
+      "loss": 0.077,
+      "mean_token_accuracy": 0.979105396270752,
+      "num_tokens": 10925844.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.4533446311950684,
+      "epoch": 0.1097972972972973,
+      "grad_norm": 0.38413992524147034,
+      "learning_rate": 0.00014753392732969221,
+      "loss": 0.0741,
+      "mean_token_accuracy": 0.9798829126358032,
+      "num_tokens": 11837660.0,
+      "step": 325
+    },
+    {
+      "entropy": 1.2654137516021728,
+      "epoch": 0.11824324324324324,
+      "grad_norm": 0.34504997730255127,
+      "learning_rate": 0.0001469850513788533,
+      "loss": 0.0735,
+      "mean_token_accuracy": 0.9800626587867737,
+      "num_tokens": 12746057.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.3292001867294312,
+      "epoch": 0.1266891891891892,
+      "grad_norm": 0.33613255620002747,
+      "learning_rate": 0.0001463823077057472,
+      "loss": 0.0719,
+      "mean_token_accuracy": 0.9804170989990234,
+      "num_tokens": 13650462.0,
+      "step": 375
+    },
+    {
+      "entropy": 1.3403068208694457,
+      "epoch": 0.13513513513513514,
+      "grad_norm": 0.31635916233062744,
+      "learning_rate": 0.00014572614735441927,
+      "loss": 0.0661,
+      "mean_token_accuracy": 0.9817720007896423,
+      "num_tokens": 14564250.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.272653160095215,
+      "epoch": 0.14358108108108109,
+      "grad_norm": 0.3426429033279419,
+      "learning_rate": 0.00014501706134158613,
+      "loss": 0.0645,
+      "mean_token_accuracy": 0.9820195341110229,
+      "num_tokens": 15478716.0,
+      "step": 425
+    },
+    {
+      "entropy": 1.2841063594818116,
+      "epoch": 0.15202702702702703,
+      "grad_norm": 0.2935279607772827,
+      "learning_rate": 0.00014425558028919864,
+      "loss": 0.0598,
+      "mean_token_accuracy": 0.9836338186264038,
+      "num_tokens": 16393588.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.3311733865737916,
+      "epoch": 0.16047297297297297,
+      "grad_norm": 0.3083665072917938,
+      "learning_rate": 0.00014344227402736823,
+      "loss": 0.0617,
+      "mean_token_accuracy": 0.9829438376426697,
+      "num_tokens": 17305794.0,
+      "step": 475
+    },
+    {
+      "entropy": 1.2743083667755126,
+      "epoch": 0.16891891891891891,
+      "grad_norm": 0.265240341424942,
+      "learning_rate": 0.00014257775116795232,
+      "loss": 0.0639,
+      "mean_token_accuracy": 0.9826490211486817,
+      "num_tokens": 18208894.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.3156098794937134,
+      "epoch": 0.17736486486486486,
+      "grad_norm": 0.28603339195251465,
+      "learning_rate": 0.00014166265864911924,
+      "loss": 0.061,
+      "mean_token_accuracy": 0.9837357306480408,
+      "num_tokens": 19114063.0,
+      "step": 525
+    },
+    {
+      "entropy": 1.1023945260047912,
+      "epoch": 0.1858108108108108,
+      "grad_norm": 0.28375494480133057,
+      "learning_rate": 0.00014069768125123255,
+      "loss": 0.06,
+      "mean_token_accuracy": 0.9836070895195007,
+      "num_tokens": 20028518.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.1739844036102296,
+      "epoch": 0.19425675675675674,
+      "grad_norm": 0.2670102119445801,
+      "learning_rate": 0.0001396835410844174,
+      "loss": 0.058,
+      "mean_token_accuracy": 0.9842915415763855,
+      "num_tokens": 20941124.0,
+      "step": 575
+    },
+    {
+      "entropy": 1.1551535773277282,
+      "epoch": 0.20270270270270271,
+      "grad_norm": 0.25598305463790894,
+      "learning_rate": 0.0001386209970481926,
+      "loss": 0.0546,
+      "mean_token_accuracy": 0.9847545146942138,
+      "num_tokens": 21862872.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.159937448501587,
+      "epoch": 0.21114864864864866,
+      "grad_norm": 0.2896850109100342,
+      "learning_rate": 0.00013751084426357195,
+      "loss": 0.0542,
+      "mean_token_accuracy": 0.9850636339187622,
+      "num_tokens": 22773515.0,
+      "step": 625
+    },
+    {
+      "entropy": 1.2576635265350342,
+      "epoch": 0.2195945945945946,
+      "grad_norm": 0.25722506642341614,
+      "learning_rate": 0.00013635391347806118,
+      "loss": 0.0563,
+      "mean_token_accuracy": 0.9844811201095581,
+      "num_tokens": 23678425.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.3517053985595704,
+      "epoch": 0.22804054054054054,
+      "grad_norm": 0.2554025650024414,
+      "learning_rate": 0.00013515107044399414,
+      "loss": 0.0526,
+      "mean_token_accuracy": 0.9854999375343323,
+      "num_tokens": 24591066.0,
+      "step": 675
+    },
+    {
+      "entropy": 1.333014817237854,
+      "epoch": 0.23648648648648649,
+      "grad_norm": 0.2464788407087326,
+      "learning_rate": 0.00013390321527067476,
+      "loss": 0.0511,
+      "mean_token_accuracy": 0.9857523179054261,
+      "num_tokens": 25499037.0,
+      "step": 700
+    },
+    {
+      "entropy": 1.4412504625320435,
+      "epoch": 0.24493243243243243,
+      "grad_norm": 0.2875170409679413,
+      "learning_rate": 0.0001326112817508089,
+      "loss": 0.0522,
+      "mean_token_accuracy": 0.9856375217437744,
+      "num_tokens": 26408782.0,
+      "step": 725
+    },
+    {
+      "entropy": 1.448633713722229,
+      "epoch": 0.2533783783783784,
+      "grad_norm": 0.22021101415157318,
+      "learning_rate": 0.00013127623666173017,
+      "loss": 0.0491,
+      "mean_token_accuracy": 0.9864472603797912,
+      "num_tokens": 27325252.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.9574100494384765,
+      "epoch": 0.26182432432432434,
+      "grad_norm": 0.272896409034729,
+      "learning_rate": 0.00012989907904194307,
+      "loss": 0.0471,
+      "mean_token_accuracy": 0.9870141172409057,
+      "num_tokens": 28233870.0,
+      "step": 775
+    },
+    {
+      "entropy": 2.094488110542297,
+      "epoch": 0.2702702702702703,
+      "grad_norm": 0.2583068907260895,
+      "learning_rate": 0.00012848083944352415,
+      "loss": 0.0489,
+      "mean_token_accuracy": 0.986718726158142,
+      "num_tokens": 29146336.0,
+      "step": 800
+    },
+    {
+      "entropy": 1.959323434829712,
+      "epoch": 0.27871621621621623,
+      "grad_norm": 0.3295232951641083,
+      "learning_rate": 0.0001270225791609413,
+      "loss": 0.0489,
+      "mean_token_accuracy": 0.9866770005226135,
+      "num_tokens": 30046527.0,
+      "step": 825
+    },
+    {
+      "entropy": 1.9984667921066284,
+      "epoch": 0.28716216216216217,
+      "grad_norm": 0.3672245442867279,
+      "learning_rate": 0.00012552538943686763,
+      "loss": 0.047,
+      "mean_token_accuracy": 0.9869811415672303,
+      "num_tokens": 30952079.0,
+      "step": 850
+    },
+    {
+      "entropy": 2.488990459442139,
+      "epoch": 0.2956081081081081,
+      "grad_norm": 0.24156689643859863,
+      "learning_rate": 0.00012399039064558498,
+      "loss": 0.0522,
+      "mean_token_accuracy": 0.9861561584472657,
+      "num_tokens": 31865272.0,
+      "step": 875
+    },
+    {
+      "entropy": 2.9546525478363037,
+      "epoch": 0.30405405405405406,
+      "grad_norm": 0.28694045543670654,
+      "learning_rate": 0.00012241873145458728,
+      "loss": 0.0452,
+      "mean_token_accuracy": 0.987418737411499,
+      "num_tokens": 32782599.0,
+      "step": 900
+    },
+    {
+      "entropy": 3.1015477085113528,
+      "epoch": 0.3125,
+      "grad_norm": 0.3419845402240753,
+      "learning_rate": 0.00012081158796501186,
+      "loss": 0.0445,
+      "mean_token_accuracy": 0.987585904598236,
+      "num_tokens": 33691106.0,
+      "step": 925
+    },
+    {
+      "entropy": 3.269675178527832,
+      "epoch": 0.32094594594594594,
+      "grad_norm": 0.2529977262020111,
+      "learning_rate": 0.00011917016283154167,
+      "loss": 0.0445,
+      "mean_token_accuracy": 0.9878652453422546,
+      "num_tokens": 34601435.0,
+      "step": 950
+    },
+    {
+      "entropy": 3.800573854446411,
+      "epoch": 0.3293918918918919,
+      "grad_norm": 0.27715784311294556,
+      "learning_rate": 0.00011749568436243676,
+      "loss": 0.0435,
+      "mean_token_accuracy": 0.9883425831794739,
+      "num_tokens": 35510489.0,
+      "step": 975
+    },
+    {
+      "entropy": 4.043784437179565,
+      "epoch": 0.33783783783783783,
+      "grad_norm": 0.5143760442733765,
+      "learning_rate": 0.00011578940560036907,
+      "loss": 0.0413,
+      "mean_token_accuracy": 0.9888231515884399,
+      "num_tokens": 36429237.0,
+      "step": 1000
+    },
+    {
+      "entropy": 2.057482042312622,
+      "epoch": 0.34628378378378377,
+      "grad_norm": 0.2672846019268036,
+      "learning_rate": 0.00011405260338474767,
+      "loss": 0.0446,
+      "mean_token_accuracy": 0.9876713585853577,
+      "num_tokens": 37339576.0,
+      "step": 1025
+    },
+    {
+      "entropy": 2.6275680255889893,
+      "epoch": 0.3547297297297297,
+      "grad_norm": 0.24197888374328613,
+      "learning_rate": 0.00011228657739623669,
+      "loss": 0.0437,
+      "mean_token_accuracy": 0.9879350781440734,
+      "num_tokens": 38247985.0,
+      "step": 1050
+    },
+    {
+      "entropy": 3.461882781982422,
+      "epoch": 0.36317567567567566,
+      "grad_norm": 0.2765347361564636,
+      "learning_rate": 0.00011049264918418065,
+      "loss": 0.0426,
+      "mean_token_accuracy": 0.9884021711349488,
+      "num_tokens": 39157574.0,
+      "step": 1075
+    },
+    {
+      "entropy": 4.308964958190918,
+      "epoch": 0.3716216216216216,
+      "grad_norm": 0.20039749145507812,
+      "learning_rate": 0.00010867216117766509,
+      "loss": 0.0399,
+      "mean_token_accuracy": 0.9890323376655579,
+      "num_tokens": 40067400.0,
+      "step": 1100
+    },
+    {
+      "entropy": 4.554050693511963,
+      "epoch": 0.38006756756756754,
+      "grad_norm": 0.35294127464294434,
+      "learning_rate": 0.00010682647568095248,
+      "loss": 0.0411,
+      "mean_token_accuracy": 0.9890136790275573,
+      "num_tokens": 40982186.0,
+      "step": 1125
+    },
+    {
+      "entropy": 4.567013301849365,
+      "epoch": 0.3885135135135135,
+      "grad_norm": 0.3076898455619812,
+      "learning_rate": 0.0001049569738540451,
+      "loss": 0.0386,
+      "mean_token_accuracy": 0.9893850111961364,
+      "num_tokens": 41885395.0,
+      "step": 1150
+    },
+    {
+      "entropy": 4.846361865997315,
+      "epoch": 0.3969594594594595,
+      "grad_norm": 0.23663102090358734,
+      "learning_rate": 0.00010306505467913805,
+      "loss": 0.0413,
+      "mean_token_accuracy": 0.9891546440124511,
+      "num_tokens": 42795848.0,
+      "step": 1175
+    },
+    {
+      "entropy": 5.00790548324585,
+      "epoch": 0.40540540540540543,
+      "grad_norm": 0.2782009243965149,
+      "learning_rate": 0.00010115213391373531,
+      "loss": 0.0391,
+      "mean_token_accuracy": 0.989229564666748,
+      "num_tokens": 43712886.0,
+      "step": 1200
+    },
+    {
+      "entropy": 5.715256175994873,
+      "epoch": 0.41385135135135137,
+      "grad_norm": 0.2141530066728592,
+      "learning_rate": 9.921964303121276e-05,
+      "loss": 0.0401,
+      "mean_token_accuracy": 0.9891216611862182,
+      "num_tokens": 44623546.0,
+      "step": 1225
+    },
+    {
+      "entropy": 4.9959554100036625,
+      "epoch": 0.4222972972972973,
+      "grad_norm": 0.22451816499233246,
+      "learning_rate": 9.726902814962062e-05,
+      "loss": 0.0411,
+      "mean_token_accuracy": 0.9891446828842163,
+      "num_tokens": 45537294.0,
+      "step": 1250
+    },
+    {
+      "entropy": 5.725631275177002,
+      "epoch": 0.43074324324324326,
+      "grad_norm": 0.18682850897312164,
+      "learning_rate": 9.530174894952696e-05,
+      "loss": 0.0385,
+      "mean_token_accuracy": 0.9893482565879822,
+      "num_tokens": 46450644.0,
+      "step": 1275
+    },
+    {
+      "entropy": 5.857726764678955,
+      "epoch": 0.4391891891891892,
+      "grad_norm": 0.3280983567237854,
+      "learning_rate": 9.331927758171233e-05,
+      "loss": 0.0404,
+      "mean_token_accuracy": 0.9892752313613892,
+      "num_tokens": 47363194.0,
+      "step": 1300
+    },
+    {
+      "entropy": 5.824376182556152,
+      "epoch": 0.44763513513513514,
+      "grad_norm": 0.337335467338562,
+      "learning_rate": 9.132309756553246e-05,
+      "loss": 0.0379,
+      "mean_token_accuracy": 0.9895437502861023,
+      "num_tokens": 48272493.0,
+      "step": 1325
+    },
+    {
+      "entropy": 5.87083625793457,
+      "epoch": 0.4560810810810811,
+      "grad_norm": 0.24668747186660767,
+      "learning_rate": 8.931470267877376e-05,
+      "loss": 0.0365,
+      "mean_token_accuracy": 0.989769902229309,
+      "num_tokens": 49182533.0,
+      "step": 1350
+    },
+    {
+      "entropy": 5.90132287979126,
+      "epoch": 0.46452702702702703,
+      "grad_norm": 0.2317751944065094,
+      "learning_rate": 8.729559583983243e-05,
+      "loss": 0.0395,
+      "mean_token_accuracy": 0.9894163608551025,
+      "num_tokens": 50087647.0,
+      "step": 1375
+    },
+    {
+      "entropy": 5.919566116333008,
+      "epoch": 0.47297297297297297,
+      "grad_norm": 0.22726328670978546,
+      "learning_rate": 8.526728798305301e-05,
+      "loss": 0.0374,
+      "mean_token_accuracy": 0.989595787525177,
+      "num_tokens": 51000947.0,
+      "step": 1400
+    },
+    {
+      "entropy": 6.049031200408936,
+      "epoch": 0.4814189189189189,
+      "grad_norm": 0.34049370884895325,
+      "learning_rate": 8.323129692806908e-05,
+      "loss": 0.0372,
+      "mean_token_accuracy": 0.9898285675048828,
+      "num_tokens": 51908120.0,
+      "step": 1425
+    },
+    {
+      "entropy": 2.902804012298584,
+      "epoch": 0.48986486486486486,
+      "grad_norm": 0.19970372319221497,
+      "learning_rate": 8.118914624399095e-05,
+      "loss": 0.0369,
+      "mean_token_accuracy": 0.9899203157424927,
+      "num_tokens": 52813161.0,
+      "step": 1450
+    },
+    {
+      "entropy": 3.857316732406616,
+      "epoch": 0.4983108108108108,
+      "grad_norm": 0.24421635270118713,
+      "learning_rate": 7.914236410929138e-05,
+      "loss": 0.0376,
+      "mean_token_accuracy": 0.989795663356781,
+      "num_tokens": 53716192.0,
+      "step": 1475
+    },
+    {
+      "entropy": 4.954489555358887,
+      "epoch": 0.5067567567567568,
+      "grad_norm": 0.23250356316566467,
+      "learning_rate": 7.709248216824183e-05,
+      "loss": 0.0342,
+      "mean_token_accuracy": 0.9905585527420044,
+      "num_tokens": 54625350.0,
+      "step": 1500
+    },
+    {
+      "entropy": 4.959287452697754,
+      "epoch": 0.5152027027027027,
+      "grad_norm": 0.17365384101867676,
+      "learning_rate": 7.504103438475512e-05,
+      "loss": 0.0351,
+      "mean_token_accuracy": 0.9902826499938965,
+      "num_tokens": 55535591.0,
+      "step": 1525
+    },
+    {
+      "entropy": 5.005143127441406,
+      "epoch": 0.5236486486486487,
+      "grad_norm": 0.17304693162441254,
+      "learning_rate": 7.298955589449254e-05,
+      "loss": 0.0345,
+      "mean_token_accuracy": 0.9904901576042175,
+      "num_tokens": 56445704.0,
+      "step": 1550
+    },
+    {
+      "entropy": 5.455631046295166,
+      "epoch": 0.5320945945945946,
+      "grad_norm": 0.2527100443840027,
+      "learning_rate": 7.093958185609368e-05,
+      "loss": 0.0351,
+      "mean_token_accuracy": 0.990511257648468,
+      "num_tokens": 57360195.0,
+      "step": 1575
+    },
+    {
+      "entropy": 5.283494300842285,
+      "epoch": 0.5405405405405406,
+      "grad_norm": 0.1954471617937088,
+      "learning_rate": 6.889264630238958e-05,
+      "loss": 0.0334,
+      "mean_token_accuracy": 0.9907509636878967,
+      "num_tokens": 58264821.0,
+      "step": 1600
+    },
+    {
+      "entropy": 5.037727890014648,
+      "epoch": 0.5489864864864865,
+      "grad_norm": 0.22263263165950775,
+      "learning_rate": 6.685028099245787e-05,
+      "loss": 0.0345,
+      "mean_token_accuracy": 0.9907543969154358,
+      "num_tokens": 59170912.0,
+      "step": 1625
+    },
+    {
+      "entropy": 5.550898399353027,
+      "epoch": 0.5574324324324325,
+      "grad_norm": 0.21726706624031067,
+      "learning_rate": 6.481401426537975e-05,
+      "loss": 0.0366,
+      "mean_token_accuracy": 0.9905391645431518,
+      "num_tokens": 60080420.0,
+      "step": 1650
+    },
+    {
+      "entropy": 6.31848970413208,
+      "epoch": 0.5658783783783784,
+      "grad_norm": 0.2131570726633072,
+      "learning_rate": 6.278536989655604e-05,
+      "loss": 0.0344,
+      "mean_token_accuracy": 0.9905202126502991,
+      "num_tokens": 60991740.0,
+      "step": 1675
+    },
+    {
+      "entropy": 6.03132999420166,
+      "epoch": 0.5743243243243243,
+      "grad_norm": 0.4251382648944855,
+      "learning_rate": 6.076586595743827e-05,
+      "loss": 0.0358,
+      "mean_token_accuracy": 0.9904899668693542,
+      "num_tokens": 61902212.0,
+      "step": 1700
+    },
+    {
+      "entropy": 5.921144943237305,
+      "epoch": 0.5827702702702703,
+      "grad_norm": 0.2580125629901886,
+      "learning_rate": 5.8757013679528376e-05,
+      "loss": 0.035,
+      "mean_token_accuracy": 0.9907992458343506,
+      "num_tokens": 62807039.0,
+      "step": 1725
+    },
+    {
+      "entropy": 5.778346786499023,
+      "epoch": 0.5912162162162162,
+      "grad_norm": 0.38355374336242676,
+      "learning_rate": 5.676031632349657e-05,
+      "loss": 0.0338,
+      "mean_token_accuracy": 0.9909575366973877,
+      "num_tokens": 63719784.0,
+      "step": 1750
+    },
+    {
+      "entropy": 6.197827320098877,
+      "epoch": 0.5996621621621622,
+      "grad_norm": 0.20510102808475494,
+      "learning_rate": 5.477726805426417e-05,
+      "loss": 0.0319,
+      "mean_token_accuracy": 0.9913099408149719,
+      "num_tokens": 64627651.0,
+      "step": 1775
+    },
+    {
+      "entropy": 6.37231632232666,
+      "epoch": 0.6081081081081081,
+      "grad_norm": 0.3134596347808838,
+      "learning_rate": 5.280935282289267e-05,
+      "loss": 0.0359,
+      "mean_token_accuracy": 0.9905921173095703,
+      "num_tokens": 65542454.0,
+      "step": 1800
+    },
+    {
+      "entropy": 5.984546794891357,
+      "epoch": 0.6165540540540541,
+      "grad_norm": 0.22857634723186493,
+      "learning_rate": 5.085804325611637e-05,
+      "loss": 0.0344,
+      "mean_token_accuracy": 0.9908464002609253,
+      "num_tokens": 66451331.0,
+      "step": 1825
+    },
+    {
+      "entropy": 5.947293300628662,
+      "epoch": 0.625,
+      "grad_norm": 0.20945040881633759,
+      "learning_rate": 4.8924799554348946e-05,
+      "loss": 0.033,
+      "mean_token_accuracy": 0.9910026931762695,
+      "num_tokens": 67368771.0,
+      "step": 1850
+    },
+    {
+      "entropy": 6.113097095489502,
+      "epoch": 0.6334459459459459,
+      "grad_norm": 0.21207907795906067,
+      "learning_rate": 4.701106839898888e-05,
+      "loss": 0.0312,
+      "mean_token_accuracy": 0.9914006876945496,
+      "num_tokens": 68283078.0,
+      "step": 1875
+    },
+    {
+      "entropy": 6.131697692871094,
+      "epoch": 0.6418918918918919,
+      "grad_norm": 0.2148510068655014,
+      "learning_rate": 4.511828186984159e-05,
+      "loss": 0.0318,
+      "mean_token_accuracy": 0.9913706135749817,
+      "num_tokens": 69194564.0,
+      "step": 1900
+    },
+    {
+      "entropy": 6.441943588256836,
+      "epoch": 0.6503378378378378,
+      "grad_norm": 1.0836849212646484,
+      "learning_rate": 4.324785637346806e-05,
+      "loss": 0.0339,
+      "mean_token_accuracy": 0.991126937866211,
+      "num_tokens": 70105433.0,
+      "step": 1925
+    },
+    {
+      "entropy": 5.870137577056885,
+      "epoch": 0.6587837837837838,
+      "grad_norm": 0.22901445627212524,
+      "learning_rate": 4.1401191583261976e-05,
+      "loss": 0.0328,
+      "mean_token_accuracy": 0.9911576080322265,
+      "num_tokens": 71012300.0,
+      "step": 1950
+    },
+    {
+      "entropy": 5.879305953979492,
+      "epoch": 0.6672297297297297,
+      "grad_norm": 0.1556059569120407,
+      "learning_rate": 3.9579669392048805e-05,
+      "loss": 0.0317,
+      "mean_token_accuracy": 0.9913742136955261,
+      "num_tokens": 71926567.0,
+      "step": 1975
+    },
+    {
+      "entropy": 5.809606380462647,
+      "epoch": 0.6756756756756757,
+      "grad_norm": 0.19562895596027374,
+      "learning_rate": 3.778465287799013e-05,
+      "loss": 0.0314,
+      "mean_token_accuracy": 0.9914613127708435,
+      "num_tokens": 72837494.0,
+      "step": 2000
+    },
+    {
+      "entropy": 5.884324359893799,
+      "epoch": 0.6841216216216216,
+      "grad_norm": 0.2343241572380066,
+      "learning_rate": 3.6017485284567764e-05,
+      "loss": 0.0314,
+      "mean_token_accuracy": 0.9914584946632385,
+      "num_tokens": 73744984.0,
+      "step": 2025
+    },
+    {
+      "entropy": 5.99851276397705,
+      "epoch": 0.6925675675675675,
+      "grad_norm": 0.21915408968925476,
+      "learning_rate": 3.427948901540981e-05,
+      "loss": 0.0297,
+      "mean_token_accuracy": 0.9918388199806213,
+      "num_tokens": 74649523.0,
+      "step": 2050
+    },
+    {
+      "entropy": 6.2692474937438964,
+      "epoch": 0.7010135135135135,
+      "grad_norm": 0.18452119827270508,
+      "learning_rate": 3.257196464471247e-05,
+      "loss": 0.033,
+      "mean_token_accuracy": 0.991136999130249,
+      "num_tokens": 75560392.0,
+      "step": 2075
+    },
+    {
+      "entropy": 6.153740520477295,
+      "epoch": 0.7094594594594594,
+      "grad_norm": 0.5608646273612976,
+      "learning_rate": 3.0896189943996347e-05,
+      "loss": 0.0308,
+      "mean_token_accuracy": 0.9917735600471497,
+      "num_tokens": 76474807.0,
+      "step": 2100
+    },
+    {
+      "entropy": 4.738718566894531,
+      "epoch": 0.7179054054054054,
+      "grad_norm": 0.18798308074474335,
+      "learning_rate": 2.9253418925927042e-05,
+      "loss": 0.0299,
+      "mean_token_accuracy": 0.991875262260437,
+      "num_tokens": 77388576.0,
+      "step": 2125
+    },
+    {
+      "entropy": 5.610414485931397,
+      "epoch": 0.7263513513513513,
+      "grad_norm": 0.1727709174156189,
+      "learning_rate": 2.7644880905914516e-05,
+      "loss": 0.034,
+      "mean_token_accuracy": 0.9910853981971741,
+      "num_tokens": 78295311.0,
+      "step": 2150
+    },
+    {
+      "entropy": 6.087554264068603,
+      "epoch": 0.7347972972972973,
+      "grad_norm": 0.2110079526901245,
+      "learning_rate": 2.6071779582194204e-05,
+      "loss": 0.0286,
+      "mean_token_accuracy": 0.9919546890258789,
+      "num_tokens": 79207082.0,
+      "step": 2175
+    },
+    {
+      "entropy": 6.270893592834472,
+      "epoch": 0.7432432432432432,
+      "grad_norm": 0.2463434487581253,
+      "learning_rate": 2.4535292135077557e-05,
+      "loss": 0.0324,
+      "mean_token_accuracy": 0.9914210176467896,
+      "num_tokens": 80112279.0,
+      "step": 2200
+    },
+    {
+      "entropy": 5.859546127319336,
+      "epoch": 0.7516891891891891,
+      "grad_norm": 0.2690972089767456,
+      "learning_rate": 2.303656834604675e-05,
+      "loss": 0.0297,
+      "mean_token_accuracy": 0.9921945381164551,
+      "num_tokens": 81025674.0,
+      "step": 2225
+    },
+    {
+      "entropy": 5.928163032531739,
+      "epoch": 0.7601351351351351,
+      "grad_norm": 0.3094039261341095,
+      "learning_rate": 2.1576729737352222e-05,
+      "loss": 0.0324,
+      "mean_token_accuracy": 0.9912599349021911,
+      "num_tokens": 81936249.0,
+      "step": 2250
+    },
+    {
+      "entropy": 6.425680274963379,
+      "epoch": 0.768581081081081,
+      "grad_norm": 0.19043578207492828,
+      "learning_rate": 2.0156868732757316e-05,
+      "loss": 0.0311,
+      "mean_token_accuracy": 0.9918688058853149,
+      "num_tokens": 82842722.0,
+      "step": 2275
+    },
+    {
+      "entropy": 6.4502605438232425,
+      "epoch": 0.777027027027027,
+      "grad_norm": 0.2400105744600296,
+      "learning_rate": 1.8778047840057574e-05,
+      "loss": 0.0294,
+      "mean_token_accuracy": 0.9918547129631042,
+      "num_tokens": 83754198.0,
+      "step": 2300
+    },
+    {
+      "entropy": 6.494881954193115,
+      "epoch": 0.785472972972973,
+      "grad_norm": 0.23887406289577484,
+      "learning_rate": 1.744129885598691e-05,
+      "loss": 0.0302,
+      "mean_token_accuracy": 0.99188218832016,
+      "num_tokens": 84659431.0,
+      "step": 2325
+    },
+    {
+      "entropy": 6.008076190948486,
+      "epoch": 0.793918918918919,
+      "grad_norm": 0.18670912086963654,
+      "learning_rate": 1.614762209410536e-05,
+      "loss": 0.0321,
+      "mean_token_accuracy": 0.9914862275123596,
+      "num_tokens": 85567791.0,
+      "step": 2350
+    },
+    {
+      "entropy": 6.224152889251709,
+      "epoch": 0.8023648648648649,
+      "grad_norm": 0.22409085929393768,
+      "learning_rate": 1.4897985636246282e-05,
+      "loss": 0.0298,
+      "mean_token_accuracy": 0.9919859480857849,
+      "num_tokens": 86480574.0,
+      "step": 2375
+    },
+    {
+      "entropy": 6.168068618774414,
+      "epoch": 0.8108108108108109,
+      "grad_norm": 0.25881311297416687,
+      "learning_rate": 1.3693324608083114e-05,
+      "loss": 0.0327,
+      "mean_token_accuracy": 0.9915460610389709,
+      "num_tokens": 87387849.0,
+      "step": 2400
+    },
+    {
+      "entropy": 6.423477592468262,
+      "epoch": 0.8192567567567568,
+      "grad_norm": 0.3354607820510864,
+      "learning_rate": 1.2534540479357805e-05,
+      "loss": 0.0299,
+      "mean_token_accuracy": 0.9918716263771057,
+      "num_tokens": 88300702.0,
+      "step": 2425
+    },
+    {
+      "entropy": 6.327016792297363,
+      "epoch": 0.8277027027027027,
+      "grad_norm": 0.2097429782152176,
+      "learning_rate": 1.1422500389294636e-05,
+      "loss": 0.0321,
+      "mean_token_accuracy": 0.9915898013114929,
+      "num_tokens": 89203547.0,
+      "step": 2450
+    },
+    {
+      "entropy": 6.420764675140381,
+      "epoch": 0.8361486486486487,
+      "grad_norm": 0.18369197845458984,
+      "learning_rate": 1.0358036497704212e-05,
+      "loss": 0.0327,
+      "mean_token_accuracy": 0.9911897611618042,
+      "num_tokens": 90112731.0,
+      "step": 2475
+    },
+    {
+      "entropy": 6.386304302215576,
+      "epoch": 0.8445945945945946,
+      "grad_norm": 0.23658068478107452,
+      "learning_rate": 9.341945362263148e-06,
+      "loss": 0.0292,
+      "mean_token_accuracy": 0.992039077281952,
+      "num_tokens": 91021384.0,
+      "step": 2500
+    },
+    {
+      "entropy": 6.52750301361084,
+      "epoch": 0.8530405405405406,
+      "grad_norm": 0.18866685032844543,
+      "learning_rate": 8.374987342435422e-06,
+      "loss": 0.0307,
+      "mean_token_accuracy": 0.9917124676704406,
+      "num_tokens": 91928983.0,
+      "step": 2525
+    },
+    {
+      "entropy": 6.484247798919678,
+      "epoch": 0.8614864864864865,
+      "grad_norm": 0.24947895109653473,
+      "learning_rate": 7.457886030481624e-06,
+      "loss": 0.0292,
+      "mean_token_accuracy": 0.9923151659965516,
+      "num_tokens": 92839727.0,
+      "step": 2550
+    },
+    {
+      "entropy": 6.5589457130432125,
+      "epoch": 0.8699324324324325,
+      "grad_norm": 0.17680883407592773,
+      "learning_rate": 6.591327709981706e-06,
+      "loss": 0.0306,
+      "mean_token_accuracy": 0.9916476058959961,
+      "num_tokens": 93748359.0,
+      "step": 2575
+    },
+    {
+      "entropy": 6.7248397636413575,
+      "epoch": 0.8783783783783784,
+      "grad_norm": 0.2523375451564789,
+      "learning_rate": 5.775960842276523e-06,
+      "loss": 0.0299,
+      "mean_token_accuracy": 0.9919104433059692,
+      "num_tokens": 94659666.0,
+      "step": 2600
+    },
+    {
+      "entropy": 6.729737434387207,
+      "epoch": 0.8868243243243243,
+      "grad_norm": 0.2729496359825134,
+      "learning_rate": 5.0123955812123304e-06,
+      "loss": 0.0287,
+      "mean_token_accuracy": 0.9919739437103271,
+      "num_tokens": 95570628.0,
+      "step": 2625
+    },
+    {
+      "entropy": 6.524206485748291,
+      "epoch": 0.8952702702702703,
+      "grad_norm": 0.23744052648544312,
+      "learning_rate": 4.301203316551743e-06,
+      "loss": 0.0278,
+      "mean_token_accuracy": 0.9923106765747071,
+      "num_tokens": 96483593.0,
+      "step": 2650
+    },
+    {
+      "entropy": 6.161054248809815,
+      "epoch": 0.9037162162162162,
+      "grad_norm": 0.22951319813728333,
+      "learning_rate": 3.6429162463922272e-06,
+      "loss": 0.0312,
+      "mean_token_accuracy": 0.9918225383758545,
+      "num_tokens": 97394540.0,
+      "step": 2675
+    },
+    {
+      "entropy": 6.499143791198731,
+      "epoch": 0.9121621621621622,
+      "grad_norm": 0.26686224341392517,
+      "learning_rate": 3.0380269789127053e-06,
+      "loss": 0.0313,
+      "mean_token_accuracy": 0.9920451307296753,
+      "num_tokens": 98306196.0,
+      "step": 2700
+    },
+    {
+      "entropy": 6.535239028930664,
+      "epoch": 0.9206081081081081,
+      "grad_norm": 0.2154003530740738,
+      "learning_rate": 2.486988163745787e-06,
+      "loss": 0.0332,
+      "mean_token_accuracy": 0.9912288475036621,
+      "num_tokens": 99216806.0,
+      "step": 2725
+    },
+    {
+      "entropy": 6.6383766746521,
+      "epoch": 0.9290540540540541,
+      "grad_norm": 0.23152807354927063,
+      "learning_rate": 1.990212153251955e-06,
+      "loss": 0.03,
+      "mean_token_accuracy": 0.9921779370307923,
+      "num_tokens": 100123787.0,
+      "step": 2750
+    },
+    {
+      "entropy": 6.710671405792237,
+      "epoch": 0.9375,
+      "grad_norm": 0.2653197944164276,
+      "learning_rate": 1.548070693948711e-06,
+      "loss": 0.029,
+      "mean_token_accuracy": 0.9920233678817749,
+      "num_tokens": 101042182.0,
+      "step": 2775
+    },
+    {
+      "entropy": 6.694545707702637,
+      "epoch": 0.9459459459459459,
+      "grad_norm": 0.27205508947372437,
+      "learning_rate": 1.160894648325919e-06,
+      "loss": 0.0311,
+      "mean_token_accuracy": 0.991715407371521,
+      "num_tokens": 101950342.0,
+      "step": 2800
+    },
+    {
+      "entropy": 6.713838577270508,
+      "epoch": 0.9543918918918919,
+      "grad_norm": 0.22778886556625366,
+      "learning_rate": 8.289737472553986e-07,
+      "loss": 0.0311,
+      "mean_token_accuracy": 0.9918222999572754,
+      "num_tokens": 102853666.0,
+      "step": 2825
+    },
+    {
+      "entropy": 6.7134253120422365,
+      "epoch": 0.9628378378378378,
+      "grad_norm": 0.20736610889434814,
+      "learning_rate": 5.525563731800548e-07,
+      "loss": 0.0323,
+      "mean_token_accuracy": 0.9916398048400878,
+      "num_tokens": 103761774.0,
+      "step": 2850
+    },
+    {
+      "entropy": 6.728495311737061,
+      "epoch": 0.9712837837837838,
+      "grad_norm": 0.21934311091899872,
+      "learning_rate": 3.318493742447664e-07,
+      "loss": 0.0279,
+      "mean_token_accuracy": 0.992308053970337,
+      "num_tokens": 104670941.0,
+      "step": 2875
+    },
+    {
+      "entropy": 6.710691089630127,
+      "epoch": 0.9797297297297297,
+      "grad_norm": 0.2065826803445816,
+      "learning_rate": 1.670179095081381e-07,
+      "loss": 0.0307,
+      "mean_token_accuracy": 0.9915909337997436,
+      "num_tokens": 105582221.0,
+      "step": 2900
+    },
+    {
+      "entropy": 6.70396333694458,
+      "epoch": 0.9881756756756757,
+      "grad_norm": 0.2730462849140167,
+      "learning_rate": 5.818532535105391e-08,
+      "loss": 0.0309,
+      "mean_token_accuracy": 0.991929578781128,
+      "num_tokens": 106482818.0,
+      "step": 2925
+    },
+    {
+      "entropy": 6.706233367919922,
+      "epoch": 0.9966216216216216,
+      "grad_norm": 0.22958749532699585,
+      "learning_rate": 5.433063174253849e-09,
+      "loss": 0.0304,
+      "mean_token_accuracy": 0.9921355462074279,
+      "num_tokens": 107383906.0,
+      "step": 2950
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 2960,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 0,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.923592891424571e+18,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dbcc795ca9422eea8b17b3437218ac369c019c5f6af7f4748e77f52f39f01e3
+size 6225