Upload folder using huggingface_hub

README.md

@@ -0,0 +1,80 @@
+---
+language:
+- en
+license: apache-2.0
+base_model: microsoft/WizardLM-2-8x22B
+tags:
+- exl2
+---
+
+# WizardLM-2-8x22B - EXL2 3.0bpw
+
+This is a 3.0bpw EXL2 quant of [microsoft/WizardLM-2-8x22B](https://huggingface.co/microsoft/WizardLM-2-8x22B)
+
+Details about the model can be found at the above model page.
+
+## EXL2 Version
+
+These quants were made with exllamav2 version 0.0.18. Quants made on this version of EXL2 may not work on older versions of the exllamav2 library.
+
+If you have problems loading these models, please update Text Generation WebUI to the latest version.
+
+
+
+## Quant Details
+
+This is the script used for quantization.
+
+```bash
+#!/bin/bash
+
+# Activate the conda environment
+source ~/miniconda3/etc/profile.d/conda.sh
+conda activate exllamav2
+
+# Set the model name and bit size
+MODEL_NAME="WizardLM-2-8x22B"
+
+# Define variables
+MODEL_DIR="/mnt/storage/models/$MODEL_NAME"
+OUTPUT_DIR="exl2_$MODEL_NAME"
+MEASUREMENT_FILE="measurements/$MODEL_NAME.json"
+
+# Create the measurement file if needed
+if [ ! -f "$MEASUREMENT_FILE" ]; then
+    echo "Creating $MEASUREMENT_FILE"
+    # Create directories
+    if [ -d "$OUTPUT_DIR" ]; then
+        rm -r "$OUTPUT_DIR"
+    fi
+    mkdir "$OUTPUT_DIR"
+
+    python convert.py -i $MODEL_DIR -o $OUTPUT_DIR -nr -om $MEASUREMENT_FILE
+fi
+
+# Choose one of the below. Either create a single quant for testing or a batch of them.
+# BIT_PRECISIONS=(2.25)
+BIT_PRECISIONS=(5.0 4.5 4.0 3.5 3.0 2.75 2.5 2.25)
+
+for BIT_PRECISION in "${BIT_PRECISIONS[@]}"
+do
+    CONVERTED_FOLDER="models/${MODEL_NAME}_exl2_${BIT_PRECISION}bpw"
+
+    # If it doesn't already exist, make the quant
+    if [ ! -d "$CONVERTED_FOLDER" ]; then
+
+        echo "Creating $CONVERTED_FOLDER"
+
+        # Create directories
+        if [ -d "$OUTPUT_DIR" ]; then
+            rm -r "$OUTPUT_DIR"
+        fi
+        mkdir "$OUTPUT_DIR"
+        mkdir "$CONVERTED_FOLDER"
+        
+        # Run conversion commands  
+        python convert.py -i $MODEL_DIR -o $OUTPUT_DIR -nr -m $MEASUREMENT_FILE -b $BIT_PRECISION -cf $CONVERTED_FOLDER
+
+    fi
+done
+```

config.json

@@ -0,0 +1,42 @@
+{
+    "_name_or_path": "",
+    "architectures": [
+        "MixtralForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "bos_token_id": 1,
+    "eos_token_id": 2,
+    "hidden_act": "silu",
+    "hidden_size": 6144,
+    "initializer_range": 0.02,
+    "intermediate_size": 16384,
+    "max_position_embeddings": 65536,
+    "model_type": "mixtral",
+    "num_attention_heads": 48,
+    "num_experts_per_tok": 2,
+    "num_hidden_layers": 56,
+    "num_key_value_heads": 8,
+    "num_local_experts": 8,
+    "output_router_logits": false,
+    "rms_norm_eps": 1e-05,
+    "rope_theta": 1000000,
+    "router_aux_loss_coef": 0.001,
+    "router_jitter_noise": 0.0,
+    "sliding_window": null,
+    "tie_word_embeddings": false,
+    "torch_dtype": "bfloat16",
+    "transformers_version": "4.36.2",
+    "use_cache": false,
+    "vocab_size": 32000,
+    "quantization_config": {
+        "quant_method": "exl2",
+        "version": "0.0.18",
+        "bits": 3.0,
+        "head_bits": 6,
+        "calibration": {
+            "rows": 100,
+            "length": 2048,
+            "dataset": "(default)"
+        }
+    }
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.36.2"
+}

output-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:092493caa2b8aac0f4a7ecaa2e0eda713658ba17795fc6a5821fce26767a1fe5
+size 8585439496

output-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:883d5d5c58575987a2d19ce87ab6f526a249ef96da4c02d5605b86d45daacaaa
+size 8590129464

output-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdb4b8f37c2fe13020cf9dbd7c6349857bf1494e63742465eceaef50293f8a93
+size 8574778736

output-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a23dded0243cef3f32679f8152ec01f86f0b429a2a78b75667e4d2155edb2d52
+size 8588906504

output-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85ec964d8b585a793591c6639d953a7537f215f258f7495b744b68f9905123d8
+size 8590070936

output-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:095dc462722d6eb8879b955060c94e9a3549704868095fdd5e3037a3bba44753
+size 8548224288

output-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9ed94bfe9123b12ddda27f48a4893f8141ae2829e7a19211dc41cac81fd8445
+size 1663890936

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<unk>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<unk>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}