--- base_model: - Qwen/Qwen2.5-3B-Instruct base_model_relation: quantized tags: - ctranslate2 - Qwen2.5 - chat --- Conversion of https://huggingface.co/Qwen/Qwen2.5-3B-Instruct into the ```ctranslate2``` format using ```int8``` quantization. NOTE #1: This requires a version of ```ctranslate2``` GREATER THAN 4.5.0. NOTE #2: The sample scripts below require ```pip``` installing the necessary ```CUDA``` and ```CUDNN``` libraries. If you rely on a systemwide installation instead, adjust your code accordingly. Requirements: - torch 2.4.0+cu124 - nvidia-cublas-cu12 12.4.2.65 - nvidia-cuda-nvrtc-cu12 12.4.99 - nvidia-cuda-runtime-cu12 12.4.99 - nvidia-cudnn-cu12 9.1.0.70 - numpy==1.26.4 (YOU MUST DOWNGRADE FROM THE NUMPY VERSION THAT CTRANSLATE2 INSTALLS BY DEFAULT) - All other traditional dependencies like ```transformers```, ```accelerate```, etc.
Sample Script #1 (non-streaming): ```Python import sys import os os.environ['KMP_DUPLICATE_LIB_OK']='TRUE' from pathlib import Path def set_cuda_paths(): venv_base = Path(sys.executable).parent.parent nvidia_base_path = venv_base / 'Lib' / 'site-packages' / 'nvidia' cuda_path = nvidia_base_path / 'cuda_runtime' / 'bin' cublas_path = nvidia_base_path / 'cublas' / 'bin' cudnn_path = nvidia_base_path / 'cudnn' / 'bin' nvrtc_path = nvidia_base_path / 'cuda_nvrtc' / 'bin' paths_to_add = [ str(cuda_path), str(cublas_path), str(cudnn_path), str(nvrtc_path), ] env_vars = ['CUDA_PATH', 'CUDA_PATH_V12_4', 'PATH'] for env_var in env_vars: current_value = os.environ.get(env_var, '') new_value = os.pathsep.join(paths_to_add + [current_value] if current_value else paths_to_add) os.environ[env_var] = new_value set_cuda_paths() import ctranslate2 import gc import torch from transformers import AutoTokenizer import pynvml from constants import user_message, system_message pynvml.nvmlInit() handle = pynvml.nvmlDeviceGetHandleByIndex(0) model_dir = r"[INSERT PATH TO FOLDER CONTAINING THE MODEL FILES HERE]" def build_prompt(): prompt = f"""<|im_start|>system {system_message}<|im_end|> <|im_start|>user {user_message}<|im_end|> <|im_start|>assistant """ return prompt def main(): model_name = os.path.basename(model_dir) beam_size_value = 1 intra_threads = max(os.cpu_count() - 4, 4) generator = ctranslate2.Generator( model_dir, device="cuda", compute_type="int8", intra_threads=intra_threads ) tokenizer = AutoTokenizer.from_pretrained(model_dir, add_prefix_space=None) prompt = build_prompt() tokens = tokenizer.convert_ids_to_tokens(tokenizer.encode(prompt)) results_batch = generator.generate_batch( [tokens], include_prompt_in_result=False, max_batch_size=4096, batch_type="tokens", beam_size=beam_size_value, num_hypotheses=1, max_length=512, sampling_temperature=0.0, ) output = tokenizer.decode(results_batch[0].sequences_ids[0]) print("\nGenerated response:\n") print(output) del generator del tokenizer torch.cuda.empty_cache() gc.collect() if __name__ == "__main__": main() ```
Sample Script #2 (streaming) ```Python import sys import os os.environ['KMP_DUPLICATE_LIB_OK']='TRUE' from pathlib import Path def set_cuda_paths(): venv_base = Path(sys.executable).parent.parent nvidia_base_path = venv_base / 'Lib' / 'site-packages' / 'nvidia' cuda_path = nvidia_base_path / 'cuda_runtime' / 'bin' cublas_path = nvidia_base_path / 'cublas' / 'bin' cudnn_path = nvidia_base_path / 'cudnn' / 'bin' nvrtc_path = nvidia_base_path / 'cuda_nvrtc' / 'bin' paths_to_add = [ str(cuda_path), str(cublas_path), str(cudnn_path), str(nvrtc_path), ] env_vars = ['CUDA_PATH', 'CUDA_PATH_V12_4', 'PATH'] for env_var in env_vars: current_value = os.environ.get(env_var, '') new_value = os.pathsep.join(paths_to_add + [current_value] if current_value else paths_to_add) os.environ[env_var] = new_value set_cuda_paths() import ctranslate2 import gc import torch from transformers import AutoTokenizer import pynvml from constants import user_message, system_message pynvml.nvmlInit() handle = pynvml.nvmlDeviceGetHandleByIndex(0) model_dir = r"[PATH TO FOLDER CONTAINING THE MODEL FILES]" def build_prompt(): prompt = f"""<|im_start|>system {system_message}<|im_end|> <|im_start|>user {user_message}<|im_end|> <|im_start|>assistant """ return prompt def main(): generator = ctranslate2.Generator( model_dir, device="cuda", compute_type="int8", ) tokenizer = AutoTokenizer.from_pretrained(model_dir) prompt = build_prompt() tokens = tokenizer.convert_ids_to_tokens(tokenizer.encode(prompt)) # Initialize token iterator token_iterator = generator.generate_tokens( [tokens], max_length=512, sampling_temperature=0.0 ) decoded_output = "" tokens_buffer = [] try: for token_result in token_iterator: token_id = token_result.token_id token = tokenizer.convert_ids_to_tokens(token_id) if token_id == tokenizer.eos_token_id: break is_new_word = token.startswith("Ġ") if is_new_word and tokens_buffer: word = tokenizer.decode(tokens_buffer) print(word, end='', flush=True) decoded_output += word tokens_buffer = [] tokens_buffer.append(token_id) if tokens_buffer: word = tokenizer.decode(tokens_buffer) print(word, end='', flush=True) decoded_output += word except KeyboardInterrupt: print("\nGeneration interrupted") del generator del tokenizer torch.cuda.empty_cache() gc.collect() if __name__ == "__main__": main() ```