|
--- |
|
license: other |
|
license_name: tongyi-qianwen |
|
license_link: >- |
|
https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat/blob/main/LICENSE |
|
language: |
|
- en |
|
pipeline_tag: text-generation |
|
base_model: Qwen/CodeQwen1.5-7B-Chat |
|
tags: |
|
- exl2 |
|
- chat |
|
--- |
|
|
|
# CodeQwen1.5-7B-Chat - EXL2 6.0bpw |
|
|
|
This is a 6.0bpw EXL2 quant of [Qwen/CodeQwen1.5-7B-Chat](https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat) |
|
|
|
Details about the model can be found at the above model page. |
|
|
|
## EXL2 Version |
|
|
|
These quants were made with exllamav2 version 0.0.18. Quants made on this version of EXL2 may not work on older versions of the exllamav2 library. |
|
|
|
If you have problems loading these models, please update Text Generation WebUI to the latest version. |
|
|
|
## Perplexity Scoring |
|
|
|
Below are the perplexity scores for the EXL2 models. A lower score is better. |
|
|
|
| Quant Level | Perplexity Score | |
|
|-------------|------------------| |
|
| 8.0 | 13.6136 | |
|
| 7.0 | 13.6220 | |
|
| 6.0 | 13.6524 | |
|
| 5.0 | 13.7689 | |
|
| 4.0 | 13.9466 | |
|
| 3.5 | 14.2961 | |
|
| 3.0 | 16.8038 | |
|
| 2.75 | 16.9662 | |
|
| 2.5 | 17.4515 | |
|
|
|
### Perplexity Script |
|
|
|
This was the script used for perplexity testing. |
|
|
|
```bash |
|
#!/bin/bash |
|
|
|
source ~/miniconda3/etc/profile.d/conda.sh |
|
conda activate exllamav2 |
|
|
|
# Set the model name and bit size |
|
MODEL_NAME="CodeQwen1.5-7B-Chat" |
|
BIT_PRECISIONS=(8.0 7.0 6.0 5.0 4.0 3.5 3.0 2.75 2.5) |
|
|
|
# Print the markdown table header |
|
echo "| Quant Level | Perplexity Score |" |
|
echo "|-------------|------------------|" |
|
|
|
for BIT_PRECISION in "${BIT_PRECISIONS[@]}" |
|
do |
|
MODEL_DIR="models/${MODEL_NAME}_exl2_${BIT_PRECISION}bpw" |
|
if [ -d "$MODEL_DIR" ]; then |
|
output=$(python test_inference.py -m "$MODEL_DIR" -gs 17,24 -ed data/wikitext/wikitext-2-v1.parquet) |
|
score=$(echo "$output" | grep -oP 'Evaluation perplexity: \K[\d.]+') |
|
echo "| $BIT_PRECISION | $score |" |
|
fi |
|
done |
|
``` |
|
|
|
|
|
|
|
## Quant Details |
|
|
|
This is the script used for quantization. |
|
|
|
```bash |
|
#!/bin/bash |
|
|
|
# Activate the conda environment |
|
source ~/miniconda3/etc/profile.d/conda.sh |
|
conda activate exllamav2 |
|
|
|
# Set the model name and bit size |
|
MODEL_NAME="CodeQwen1.5-7B-Chat" |
|
|
|
# Define variables |
|
MODEL_DIR="models/$MODEL_NAME" |
|
OUTPUT_DIR="exl2_$MODEL_NAME" |
|
MEASUREMENT_FILE="measurements/$MODEL_NAME.json" |
|
|
|
# Create the measurement file if needed |
|
if [ ! -f "$MEASUREMENT_FILE" ]; then |
|
echo "Creating $MEASUREMENT_FILE" |
|
# Create directories |
|
if [ -d "$OUTPUT_DIR" ]; then |
|
rm -r "$OUTPUT_DIR" |
|
fi |
|
mkdir "$OUTPUT_DIR" |
|
|
|
python convert.py -i $MODEL_DIR -o $OUTPUT_DIR -nr -om $MEASUREMENT_FILE |
|
fi |
|
|
|
# Choose one of the below. Either create a single quant for testing or a batch of them. |
|
# BIT_PRECISIONS=(2.25) |
|
BIT_PRECISIONS=(8.0 7.0 6.0 5.0 4.0 3.5 3.0 2.75 2.5) |
|
|
|
for BIT_PRECISION in "${BIT_PRECISIONS[@]}" |
|
do |
|
CONVERTED_FOLDER="models/${MODEL_NAME}_exl2_${BIT_PRECISION}bpw" |
|
|
|
# If it doesn't already exist, make the quant |
|
if [ ! -d "$CONVERTED_FOLDER" ]; then |
|
|
|
echo "Creating $CONVERTED_FOLDER" |
|
|
|
# Create directories |
|
if [ -d "$OUTPUT_DIR" ]; then |
|
rm -r "$OUTPUT_DIR" |
|
fi |
|
mkdir "$OUTPUT_DIR" |
|
mkdir "$CONVERTED_FOLDER" |
|
|
|
# Run conversion commands |
|
python convert.py -i $MODEL_DIR -o $OUTPUT_DIR -nr -m $MEASUREMENT_FILE -b $BIT_PRECISION -cf $CONVERTED_FOLDER |
|
|
|
fi |
|
done |
|
``` |
|
|