Delete QwenTT

QwenTT/README.md

@@ -1,59 +0,0 @@
----
-license: other
-library_name: peft
-tags:
-- llama-factory
-- lora
-- generated_from_trainer
-base_model: Qwen/Qwen1.5-0.5B
-model-index:
-- name: QwenTT
-  results: []
----
-
-<!-- This model card has been generated automatically according to the information the Trainer had access to. You
-should probably proofread and complete it, then remove this comment. -->
-
-# QwenTT
-
-This model is a fine-tuned version of [Qwen/Qwen1.5-0.5B](https://huggingface.co/Qwen/Qwen1.5-0.5B) on the identity and the wikipedia_en datasets.
-
-## Model description
-
-More information needed
-
-## Intended uses & limitations
-
-More information needed
-
-## Training and evaluation data
-
-More information needed
-
-## Training procedure
-
-### Training hyperparameters
-
-The following hyperparameters were used during training:
-- learning_rate: 5e-05
-- train_batch_size: 2
-- eval_batch_size: 8
-- seed: 42
-- gradient_accumulation_steps: 8
-- total_train_batch_size: 16
-- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
-- lr_scheduler_type: cosine
-- num_epochs: 3.0
-- mixed_precision_training: Native AMP
-
-### Training results
-
-
-
-### Framework versions
-
-- PEFT 0.11.1
-- Transformers 4.40.2
-- Pytorch 2.2.1+cu121
-- Datasets 2.19.1
-- Tokenizers 0.19.1

QwenTT/added_tokens.json

@@ -1,5 +0,0 @@
-{
-  "<|endoftext|>": 151643,
-  "<|im_end|>": 151645,
-  "<|im_start|>": 151644
-}

QwenTT/all_results.json

@@ -1,8 +0,0 @@
-{
-    "epoch": 2.608695652173913,
-    "total_flos": 27871774801920.0,
-    "train_loss": 3.2534534454345705,
-    "train_runtime": 85.4079,
-    "train_samples_per_second": 3.196,
-    "train_steps_per_second": 0.176
-}

QwenTT/config.json

@@ -1,29 +0,0 @@
-{
-  "alpha_pattern": {},
-  "auto_mapping": null,
-  "base_model_name_or_path": "Qwen/Qwen1.5-0.5B",
-  "bias": "none",
-  "fan_in_fan_out": false,
-  "inference_mode": true,
-  "init_lora_weights": true,
-  "layer_replication": null,
-  "layers_pattern": null,
-  "layers_to_transform": null,
-  "loftq_config": {},
-  "lora_alpha": 16,
-  "lora_dropout": 0,
-  "megatron_config": null,
-  "megatron_core": "megatron.core",
-  "modules_to_save": null,
-  "peft_type": "LORA",
-  "r": 8,
-  "rank_pattern": {},
-  "revision": null,
-  "target_modules": [
-    "v_proj",
-    "q_proj"
-  ],
-  "task_type": "CAUSAL_LM",
-  "use_dora": false,
-  "use_rslora": false
-}

QwenTT/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:31f5d9f1b65b5a0006ffc61f8ef6bd7ccffd1a64c896a8454f8101ec8c9c78e7
-size 3158328

QwenTT/running_log.txt

@@ -1,161 +0,0 @@
-05/17/2024 21:21:12 - INFO - transformers.tokenization_utils_base - loading file vocab.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen1.5-0.5B/snapshots/8f445e3628f3500ee69f24e1303c9f10f5342a39/vocab.json
-
-05/17/2024 21:21:12 - INFO - transformers.tokenization_utils_base - loading file merges.txt from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen1.5-0.5B/snapshots/8f445e3628f3500ee69f24e1303c9f10f5342a39/merges.txt
-
-05/17/2024 21:21:12 - INFO - transformers.tokenization_utils_base - loading file tokenizer.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen1.5-0.5B/snapshots/8f445e3628f3500ee69f24e1303c9f10f5342a39/tokenizer.json
-
-05/17/2024 21:21:12 - INFO - transformers.tokenization_utils_base - loading file added_tokens.json from cache at None
-
-05/17/2024 21:21:12 - INFO - transformers.tokenization_utils_base - loading file special_tokens_map.json from cache at None
-
-05/17/2024 21:21:12 - INFO - transformers.tokenization_utils_base - loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen1.5-0.5B/snapshots/8f445e3628f3500ee69f24e1303c9f10f5342a39/tokenizer_config.json
-
-05/17/2024 21:21:12 - WARNING - transformers.tokenization_utils_base - Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
-
-05/17/2024 21:21:12 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>
-
-05/17/2024 21:21:12 - INFO - llamafactory.data.loader - Loading dataset identity.json...
-
-05/17/2024 21:21:13 - INFO - llamafactory.data.loader - Loading dataset olm/olm-wikipedia-20221220...
-
-05/17/2024 21:22:39 - INFO - transformers.configuration_utils - loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen1.5-0.5B/snapshots/8f445e3628f3500ee69f24e1303c9f10f5342a39/config.json
-
-05/17/2024 21:22:39 - INFO - transformers.configuration_utils - Model config Qwen2Config {
-  "_name_or_path": "Qwen/Qwen1.5-0.5B",
-  "architectures": [
-    "Qwen2ForCausalLM"
-  ],
-  "attention_dropout": 0.0,
-  "bos_token_id": 151643,
-  "eos_token_id": 151643,
-  "hidden_act": "silu",
-  "hidden_size": 1024,
-  "initializer_range": 0.02,
-  "intermediate_size": 2816,
-  "max_position_embeddings": 32768,
-  "max_window_layers": 21,
-  "model_type": "qwen2",
-  "num_attention_heads": 16,
-  "num_hidden_layers": 24,
-  "num_key_value_heads": 16,
-  "rms_norm_eps": 1e-06,
-  "rope_theta": 1000000.0,
-  "sliding_window": 32768,
-  "tie_word_embeddings": true,
-  "torch_dtype": "bfloat16",
-  "transformers_version": "4.40.2",
-  "use_cache": true,
-  "use_sliding_window": false,
-  "vocab_size": 151936
-}
-
-
-05/17/2024 21:22:39 - INFO - llamafactory.model.utils.quantization - Quantizing model to 8 bit.
-
-05/17/2024 21:22:45 - INFO - transformers.modeling_utils - loading weights file model.safetensors from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen1.5-0.5B/snapshots/8f445e3628f3500ee69f24e1303c9f10f5342a39/model.safetensors
-
-05/17/2024 21:22:45 - INFO - transformers.modeling_utils - Instantiating Qwen2ForCausalLM model under default dtype torch.float16.
-
-05/17/2024 21:22:45 - INFO - transformers.generation.configuration_utils - Generate config GenerationConfig {
-  "bos_token_id": 151643,
-  "eos_token_id": 151643
-}
-
-
-05/17/2024 21:22:49 - INFO - transformers.modeling_utils - All model checkpoint weights were used when initializing Qwen2ForCausalLM.
-
-
-05/17/2024 21:22:49 - INFO - transformers.modeling_utils - All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen1.5-0.5B.
-If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.
-
-05/17/2024 21:22:49 - INFO - transformers.generation.configuration_utils - loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen1.5-0.5B/snapshots/8f445e3628f3500ee69f24e1303c9f10f5342a39/generation_config.json
-
-05/17/2024 21:22:49 - INFO - transformers.generation.configuration_utils - Generate config GenerationConfig {
-  "bos_token_id": 151643,
-  "eos_token_id": 151643,
-  "max_new_tokens": 2048
-}
-
-
-05/17/2024 21:22:49 - INFO - llamafactory.model.utils.checkpointing - Gradient checkpointing enabled.
-
-05/17/2024 21:22:49 - INFO - llamafactory.model.utils.attention - Using torch SDPA for faster training and inference.
-
-05/17/2024 21:22:49 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
-
-05/17/2024 21:22:49 - INFO - llamafactory.model.adapter - Fine-tuning method: LoRA
-
-05/17/2024 21:22:50 - INFO - llamafactory.model.loader - trainable params: 786432 || all params: 464774144 || trainable%: 0.1692
-
-05/17/2024 21:22:50 - INFO - transformers.trainer - Using auto half precision backend
-
-05/17/2024 21:22:50 - INFO - transformers.trainer - ***** Running training *****
-
-05/17/2024 21:22:50 - INFO - transformers.trainer -   Num examples = 91
-
-05/17/2024 21:22:50 - INFO - transformers.trainer -   Num Epochs = 3
-
-05/17/2024 21:22:50 - INFO - transformers.trainer -   Instantaneous batch size per device = 2
-
-05/17/2024 21:22:50 - INFO - transformers.trainer -   Total train batch size (w. parallel, distributed & accumulation) = 16
-
-05/17/2024 21:22:50 - INFO - transformers.trainer -   Gradient Accumulation steps = 8
-
-05/17/2024 21:22:50 - INFO - transformers.trainer -   Total optimization steps = 15
-
-05/17/2024 21:22:50 - INFO - transformers.trainer -   Number of trainable parameters = 786,432
-
-05/17/2024 21:23:19 - INFO - llamafactory.extras.callbacks - {'loss': 3.2997, 'learning_rate': 3.7500e-05, 'epoch': 0.87}
-
-05/17/2024 21:23:47 - INFO - llamafactory.extras.callbacks - {'loss': 3.2782, 'learning_rate': 1.2500e-05, 'epoch': 1.74}
-
-05/17/2024 21:24:15 - INFO - llamafactory.extras.callbacks - {'loss': 3.1825, 'learning_rate': 0.0000e+00, 'epoch': 2.61}
-
-05/17/2024 21:24:15 - INFO - transformers.trainer - 
-
-Training completed. Do not forget to share your model on huggingface.co/models =)
-
-
-
-05/17/2024 21:24:15 - INFO - transformers.trainer - Saving model checkpoint to saves/Qwen1.5-0.5B/lora/QwenTT
-
-05/17/2024 21:24:16 - INFO - transformers.configuration_utils - loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen1.5-0.5B/snapshots/8f445e3628f3500ee69f24e1303c9f10f5342a39/config.json
-
-05/17/2024 21:24:16 - INFO - transformers.configuration_utils - Model config Qwen2Config {
-  "architectures": [
-    "Qwen2ForCausalLM"
-  ],
-  "attention_dropout": 0.0,
-  "bos_token_id": 151643,
-  "eos_token_id": 151643,
-  "hidden_act": "silu",
-  "hidden_size": 1024,
-  "initializer_range": 0.02,
-  "intermediate_size": 2816,
-  "max_position_embeddings": 32768,
-  "max_window_layers": 21,
-  "model_type": "qwen2",
-  "num_attention_heads": 16,
-  "num_hidden_layers": 24,
-  "num_key_value_heads": 16,
-  "rms_norm_eps": 1e-06,
-  "rope_theta": 1000000.0,
-  "sliding_window": 32768,
-  "tie_word_embeddings": true,
-  "torch_dtype": "bfloat16",
-  "transformers_version": "4.40.2",
-  "use_cache": true,
-  "use_sliding_window": false,
-  "vocab_size": 151936
-}
-
-
-05/17/2024 21:24:16 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in saves/Qwen1.5-0.5B/lora/QwenTT/tokenizer_config.json
-
-05/17/2024 21:24:16 - INFO - transformers.tokenization_utils_base - Special tokens file saved in saves/Qwen1.5-0.5B/lora/QwenTT/special_tokens_map.json
-
-05/17/2024 21:24:16 - WARNING - llamafactory.extras.ploting - No metric eval_loss to plot.
-
-05/17/2024 21:24:16 - INFO - transformers.modelcard - Dropping the following result as it does not have all the necessary fields:
-{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
-

QwenTT/special_tokens_map.json

@@ -1,20 +0,0 @@
-{
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>"
-  ],
-  "eos_token": {
-    "content": "<|im_end|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
-}

QwenTT/tokenizer_config.json

@@ -1,44 +0,0 @@
-{
-  "add_prefix_space": false,
-  "added_tokens_decoder": {
-    "151643": {
-      "content": "<|endoftext|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151644": {
-      "content": "<|im_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151645": {
-      "content": "<|im_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>"
-  ],
-  "bos_token": null,
-  "chat_template": "{% set system_message = 'You are a helpful assistant.' %}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ '<|im_start|>system\\n' + system_message + '<|im_end|>\\n' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user\\n' + content + '<|im_end|>\\n<|im_start|>assistant\\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|>' + '\\n' }}{% endif %}{% endfor %}",
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "<|im_end|>",
-  "errors": "replace",
-  "model_max_length": 32768,
-  "pad_token": "<|endoftext|>",
-  "padding_side": "right",
-  "split_special_tokens": false,
-  "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
-}

QwenTT/train_results.json

@@ -1,8 +0,0 @@
-{
-    "epoch": 2.608695652173913,
-    "total_flos": 27871774801920.0,
-    "train_loss": 3.2534534454345705,
-    "train_runtime": 85.4079,
-    "train_samples_per_second": 3.196,
-    "train_steps_per_second": 0.176
-}

QwenTT/trainer_config.yaml

@@ -1,31 +0,0 @@
-cutoff_len: 1024
-dataset: identity,wikipedia_en
-dataset_dir: data
-do_train: true
-finetuning_type: lora
-flash_attn: auto
-fp16: true
-gradient_accumulation_steps: 8
-learning_rate: 5.0e-05
-logging_steps: 5
-lora_alpha: 16
-lora_dropout: 0
-lora_rank: 8
-lora_target: q_proj,v_proj
-lr_scheduler_type: cosine
-max_grad_norm: 1.0
-max_samples: 100000
-model_name_or_path: Qwen/Qwen1.5-0.5B
-num_train_epochs: 3.0
-optim: adamw_torch
-output_dir: saves/Qwen1.5-0.5B/lora/QwenTT
-packing: false
-per_device_train_batch_size: 2
-plot_loss: true
-preprocessing_num_workers: 16
-quantization_bit: 8
-report_to: none
-save_steps: 100
-stage: sft
-template: qwen
-warmup_steps: 0

QwenTT/trainer_log.jsonl

@@ -1,4 +0,0 @@
-{"current_steps": 5, "total_steps": 15, "loss": 3.2997, "learning_rate": 3.7500000000000003e-05, "epoch": 0.8695652173913043, "percentage": 33.33, "elapsed_time": "0:00:29", "remaining_time": "0:00:58"}
-{"current_steps": 10, "total_steps": 15, "loss": 3.2782, "learning_rate": 1.2500000000000006e-05, "epoch": 1.7391304347826086, "percentage": 66.67, "elapsed_time": "0:00:56", "remaining_time": "0:00:28"}
-{"current_steps": 15, "total_steps": 15, "loss": 3.1825, "learning_rate": 0.0, "epoch": 2.608695652173913, "percentage": 100.0, "elapsed_time": "0:01:25", "remaining_time": "0:00:00"}
-{"current_steps": 15, "total_steps": 15, "epoch": 2.608695652173913, "percentage": 100.0, "elapsed_time": "0:01:25", "remaining_time": "0:00:00"}

QwenTT/trainer_state.json

@@ -1,51 +0,0 @@
-{
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 2.608695652173913,
-  "eval_steps": 500,
-  "global_step": 15,
-  "is_hyper_param_search": false,
-  "is_local_process_zero": true,
-  "is_world_process_zero": true,
-  "log_history": [
-    {
-      "epoch": 0.8695652173913043,
-      "grad_norm": 1.7076869010925293,
-      "learning_rate": 3.7500000000000003e-05,
-      "loss": 3.2997,
-      "step": 5
-    },
-    {
-      "epoch": 1.7391304347826086,
-      "grad_norm": 1.5296180248260498,
-      "learning_rate": 1.2500000000000006e-05,
-      "loss": 3.2782,
-      "step": 10
-    },
-    {
-      "epoch": 2.608695652173913,
-      "grad_norm": 1.4705257415771484,
-      "learning_rate": 0.0,
-      "loss": 3.1825,
-      "step": 15
-    },
-    {
-      "epoch": 2.608695652173913,
-      "step": 15,
-      "total_flos": 27871774801920.0,
-      "train_loss": 3.2534534454345705,
-      "train_runtime": 85.4079,
-      "train_samples_per_second": 3.196,
-      "train_steps_per_second": 0.176
-    }
-  ],
-  "logging_steps": 5,
-  "max_steps": 15,
-  "num_input_tokens_seen": 0,
-  "num_train_epochs": 3,
-  "save_steps": 100,
-  "total_flos": 27871774801920.0,
-  "train_batch_size": 2,
-  "trial_name": null,
-  "trial_params": null
-}

QwenTT/training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:60718a321f9cf36940833e02a0917fbb713ef04b94919bba3b1ad6020bc6b6ac
-size 5176