|
warning: not compiled with GPU offload support, --n-gpu-layers option will be ignored |
|
warning: see main README.md for information on enabling GPU BLAS support |
|
main: build = 8 (589f21d) |
|
main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu |
|
main: seed = 1717069408 |
|
llama_model_loader: loaded meta data with 24 key-value pairs and 1875 tensors from Yuan2-M32-hf-IMat-GGUF/Yuan2-M32-hf.gguf (version GGUF V3 (latest)) |
|
llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. |
|
llama_model_loader: - kv 0: general.architecture str = yuan2_moe |
|
llama_model_loader: - kv 1: general.name str = yuan2_moe |
|
llama_model_loader: - kv 2: yuan2_moe.context_length u32 = 4096 |
|
llama_model_loader: - kv 3: yuan2_moe.embedding_length u32 = 2048 |
|
llama_model_loader: - kv 4: yuan2_moe.block_count u32 = 24 |
|
llama_model_loader: - kv 5: yuan2_moe.feed_forward_length u32 = 8192 |
|
llama_model_loader: - kv 6: yuan2_moe.rope.dimension_count u32 = 128 |
|
llama_model_loader: - kv 7: yuan2_moe.attention.head_count u32 = 16 |
|
llama_model_loader: - kv 8: yuan2_moe.attention.head_count_kv u32 = 16 |
|
llama_model_loader: - kv 9: yuan2_moe.expert_count u32 = 32 |
|
llama_model_loader: - kv 10: yuan2_moe.expert_used_count u32 = 2 |
|
llama_model_loader: - kv 11: yuan2_moe.attention.layer_norm_rms_epsilon f32 = 0.000001 |
|
llama_model_loader: - kv 12: general.file_type u32 = 1 |
|
llama_model_loader: - kv 13: tokenizer.ggml.model str = llama |
|
llama_model_loader: - kv 14: tokenizer.ggml.tokens arr[str,135040] = ["<unk>", "<s>", "</s>", "<0x00>", "<... |
|
llama_model_loader: - kv 15: tokenizer.ggml.scores arr[f32,135040] = [0.000000, 0.000000, 0.000000, 0.0000... |
|
llama_model_loader: - kv 16: tokenizer.ggml.token_type arr[i32,135040] = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... |
|
llama_model_loader: - kv 17: tokenizer.ggml.bos_token_id u32 = 77185 |
|
llama_model_loader: - kv 18: tokenizer.ggml.eos_token_id u32 = 77185 |
|
llama_model_loader: - kv 19: tokenizer.ggml.seperator_token_id u32 = 77185 |
|
llama_model_loader: - kv 20: tokenizer.ggml.padding_token_id u32 = 77185 |
|
llama_model_loader: - kv 21: tokenizer.ggml.mask_token_id u32 = 77185 |
|
llama_model_loader: - kv 22: tokenizer.ggml.add_bos_token bool = false |
|
llama_model_loader: - kv 23: tokenizer.ggml.add_eos_token bool = false |
|
llama_model_loader: - type f32: 121 tensors |
|
llama_model_loader: - type f16: 1754 tensors |
|
llm_load_vocab: mismatch in special tokens definition ( 2729/135040 vs 346/135040 ). |
|
llm_load_print_meta: format = GGUF V3 (latest) |
|
llm_load_print_meta: arch = yuan2_moe |
|
llm_load_print_meta: vocab type = SPM |
|
llm_load_print_meta: n_vocab = 135040 |
|
llm_load_print_meta: n_merges = 0 |
|
llm_load_print_meta: n_ctx_train = 4096 |
|
llm_load_print_meta: n_embd = 2048 |
|
llm_load_print_meta: n_head = 16 |
|
llm_load_print_meta: n_head_kv = 16 |
|
llm_load_print_meta: n_layer = 24 |
|
llm_load_print_meta: n_rot = 128 |
|
llm_load_print_meta: n_embd_head_k = 256 |
|
llm_load_print_meta: n_embd_head_v = 256 |
|
llm_load_print_meta: n_gqa = 1 |
|
llm_load_print_meta: n_embd_k_gqa = 4096 |
|
llm_load_print_meta: n_embd_v_gqa = 4096 |
|
llm_load_print_meta: f_norm_eps = 0.0e+00 |
|
llm_load_print_meta: f_norm_rms_eps = 1.0e-06 |
|
llm_load_print_meta: f_clamp_kqv = 0.0e+00 |
|
llm_load_print_meta: f_max_alibi_bias = 0.0e+00 |
|
llm_load_print_meta: n_ff = 8192 |
|
llm_load_print_meta: n_expert = 32 |
|
llm_load_print_meta: n_expert_used = 2 |
|
llm_load_print_meta: rope scaling = linear |
|
llm_load_print_meta: freq_base_train = 10000.0 |
|
llm_load_print_meta: freq_scale_train = 1 |
|
llm_load_print_meta: n_yarn_orig_ctx = 4096 |
|
llm_load_print_meta: rope_finetuned = unknown |
|
llm_load_print_meta: model type = 40B |
|
llm_load_print_meta: model ftype = F16 |
|
llm_load_print_meta: model params = 40.22 B |
|
llm_load_print_meta: model size = 74.91 GiB (16.00 BPW) |
|
llm_load_print_meta: general.name = yuan2_moe |
|
llm_load_print_meta: BOS token = 77185 '<eod>' |
|
llm_load_print_meta: EOS token = 77185 '<eod>' |
|
llm_load_print_meta: UNK token = 0 '<unk>' |
|
llm_load_print_meta: SEP token = 77185 '<eod>' |
|
llm_load_print_meta: PAD token = 77185 '<eod>' |
|
llm_load_print_meta: LF token = 13 '<0x0A>' |
|
llm_load_tensors: ggml ctx size = 1.06 MiB |
|
llm_load_tensors: CPU buffer size = 76712.85 MiB |
|
.................................................................................................... |
|
llama_new_context_with_model: n_ctx = 512 |
|
llama_new_context_with_model: freq_base = 10000.0 |
|
llama_new_context_with_model: freq_scale = 1 |
|
llama_kv_cache_init: CPU KV buffer size = 192.38 MiB |
|
llama_new_context_with_model: KV self size = 192.00 MiB, K (f16): 96.00 MiB, V (f16): 96.00 MiB |
|
llama_new_context_with_model: CPU input buffer size = 6.01 MiB |
|
llama_new_context_with_model: CPU compute buffer size = 275.88 MiB |
|
llama_new_context_with_model: graph splits (measure): 1 |
|
|
|
system_info: n_threads = 25 / 32 | AVX = 1 | AVX_VNNI = 0 | AVX2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 0 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | MATMUL_INT8 = 0 | |
|
compute_imatrix: tokenizing the input .. |
|
compute_imatrix: tokenization took 145.195 ms |
|
compute_imatrix: computing over 227 chunks with batch_size 512 |
|
compute_imatrix: 6.78 seconds per pass - ETA 25.65 minutes |
|
[1]6.5558,[2]5.0024,[3]5.8688,[4]6.8477, |
|
save_imatrix: stored collected data after 10 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[5]7.1188,[6]7.3938,[7]6.3059,[8]6.7000,[9]6.7433, |
|
save_imatrix: stored collected data after 20 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[10]7.2226,[11]7.5399,[12]7.9695,[13]8.3739,[14]7.8757, |
|
save_imatrix: stored collected data after 30 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[15]8.2691,[16]8.3867,[17]8.6503,[18]8.8671,[19]9.0091, |
|
save_imatrix: stored collected data after 40 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[20]9.1840,[21]9.2799,[22]9.0400,[23]8.6569,[24]8.7209, |
|
save_imatrix: stored collected data after 50 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[25]8.1930,[26]7.7850,[27]7.5167,[28]7.2632,[29]6.9825, |
|
save_imatrix: stored collected data after 60 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[30]7.3951,[31]7.3448,[32]7.5309,[33]7.3902,[34]7.3415, |
|
save_imatrix: stored collected data after 70 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[35]7.0182,[36]6.7552,[37]6.4881,[38]6.4493,[39]6.4314, |
|
save_imatrix: stored collected data after 80 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[40]6.4113,[41]6.4790,[42]6.5106,[43]6.6147,[44]6.7387, |
|
save_imatrix: stored collected data after 90 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[45]6.8810,[46]6.9200,[47]7.0145,[48]7.1681,[49]7.3595, |
|
save_imatrix: stored collected data after 100 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[50]7.5659,[51]7.6763,[52]7.6824,[53]7.6250,[54]7.5348, |
|
save_imatrix: stored collected data after 110 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[55]7.4143,[56]7.5141,[57]7.5952,[58]7.6657,[59]7.7468, |
|
save_imatrix: stored collected data after 120 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[60]7.7793,[61]7.8927,[62]7.9396,[63]8.0284,[64]8.0642, |
|
save_imatrix: stored collected data after 130 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[65]8.1467,[66]8.2391,[67]8.2560,[68]8.2841,[69]8.3739, |
|
save_imatrix: stored collected data after 140 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[70]8.5129,[71]8.6025,[72]8.6186,[73]8.5808,[74]8.5795, |
|
save_imatrix: stored collected data after 150 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[75]8.6234,[76]8.6382,[77]8.6283,[78]8.5947,[79]8.5555, |
|
save_imatrix: stored collected data after 160 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[80]8.5826,[81]8.5578,[82]8.5244,[83]8.5821,[84]8.6244, |
|
save_imatrix: stored collected data after 170 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[85]8.6236,[86]8.6298,[87]8.6283,[88]8.6120,[89]8.5763, |
|
save_imatrix: stored collected data after 180 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[90]8.5623,[91]8.5481,[92]8.5802,[93]8.5909,[94]8.6161, |
|
save_imatrix: stored collected data after 190 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[95]8.6344,[96]8.6459,[97]8.6175,[98]8.5774,[99]8.5987, |
|
save_imatrix: stored collected data after 200 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[100]8.6399,[101]8.6517,[102]8.6304,[103]8.5887,[104]8.5743, |
|
save_imatrix: stored collected data after 210 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[105]8.5806,[106]8.6410,[107]8.6477,[108]8.6413,[109]8.6781, |
|
save_imatrix: stored collected data after 220 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[110]8.7056,[111]8.7298,[112]8.7006,[113]8.7188,[114]8.7299, |
|
save_imatrix: stored collected data after 230 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[115]8.7238,[116]8.7355,[117]8.7230,[118]8.7087,[119]8.6851, |
|
save_imatrix: stored collected data after 240 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[120]8.6678,[121]8.7011,[122]8.7138,[123]8.7270,[124]8.7333, |
|
save_imatrix: stored collected data after 250 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[125]8.7264,[126]8.7703,[127]8.6855,[128]8.6916,[129]8.6864, |
|
save_imatrix: stored collected data after 260 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[130]8.6621,[131]8.6297,[132]8.5725,[133]8.5187,[134]8.4626, |
|
save_imatrix: stored collected data after 270 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[135]8.4089,[136]8.3466,[137]8.2958,[138]8.2468,[139]8.1934, |
|
save_imatrix: stored collected data after 280 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[140]8.1758,[141]8.1481,[142]8.1078,[143]8.0591,[144]8.0292, |
|
save_imatrix: stored collected data after 290 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[145]8.0041,[146]7.9806,[147]7.9489,[148]7.9131,[149]7.8880, |
|
save_imatrix: stored collected data after 300 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[150]7.8550,[151]7.8215,[152]7.8359,[153]7.7834,[154]7.7633, |
|
save_imatrix: stored collected data after 310 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[155]7.8523,[156]7.9496,[157]8.0368,[158]8.1009,[159]8.1471, |
|
save_imatrix: stored collected data after 320 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[160]8.1647,[161]8.1946,[162]8.2248,[163]8.2249,[164]8.2250, |
|
save_imatrix: stored collected data after 330 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[165]8.2689,[166]8.2907,[167]8.3209,[168]8.3421,[169]8.3452, |
|
save_imatrix: stored collected data after 340 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[170]8.3679,[171]8.3964,[172]8.4240,[173]8.4850,[174]8.5304, |
|
save_imatrix: stored collected data after 350 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[175]8.5662,[176]8.5695,[177]8.5835,[178]8.5995,[179]8.6060, |
|
save_imatrix: stored collected data after 360 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[180]8.5969,[181]8.6079,[182]8.6110,[183]8.6246,[184]8.6350, |
|
save_imatrix: stored collected data after 370 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[185]8.6868,[186]8.6822,[187]8.6447,[188]8.6949,[189]8.7372, |
|
save_imatrix: stored collected data after 380 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[190]8.7771,[191]8.8391,[192]8.9060,[193]8.8979,[194]8.8961, |
|
save_imatrix: stored collected data after 390 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[195]8.8519,[196]8.8544,[197]8.8626,[198]8.8997,[199]8.8840, |
|
save_imatrix: stored collected data after 400 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[200]8.8698,[201]8.8631,[202]8.8561,[203]8.8675,[204]8.8596, |
|
save_imatrix: stored collected data after 410 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[205]8.8652,[206]8.8837,[207]8.8930,[208]8.9143,[209]8.9504, |
|
save_imatrix: stored collected data after 420 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[210]8.9338,[211]8.9362,[212]8.9262,[213]8.9121,[214]8.9058, |
|
save_imatrix: stored collected data after 430 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[215]8.8761,[216]8.8852,[217]8.8815,[218]8.8830,[219]8.8819, |
|
save_imatrix: stored collected data after 440 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[220]8.9316,[221]8.9905,[222]9.0298,[223]9.0636,[224]9.1087, |
|
save_imatrix: stored collected data after 450 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
[225]9.0895,[226]9.0782,[227]9.0585, |
|
save_imatrix: stored collected data after 454 chunks in Yuan2-M32-hf-IMat-GGUF/imatrix.dat |
|
|
|
llama_print_timings: load time = 51125.10 ms |
|
llama_print_timings: sample time = 0.00 ms / 1 runs ( 0.00 ms per token, inf tokens per second) |
|
llama_print_timings: prompt eval time = 1527383.56 ms / 116224 tokens ( 13.14 ms per token, 76.09 tokens per second) |
|
llama_print_timings: eval time = 0.00 ms / 1 runs ( 0.00 ms per token, inf tokens per second) |
|
llama_print_timings: total time = 1572971.60 ms / 116225 tokens |
|
|
|
Final estimate: PPL = 9.0585 +/- 0.10484 |
|
|
