Hugging Face's logo

mlc-ai
/
Hermes-2-Pro-Llama-3-8B-q4f32_1-MLC

MLC-LLM
web-llm
Model card Files Community
Charlie Ruan commited on
Commit
dfce914
•
1 Parent(s): e302f96

Fix duplicated token in tokenizer.json

Browse files

See https://huggingface.co/NousResearch/Hermes-2-Pro-Llama-3-8B/discussions/15 for more

Files changed (1) hide show
  1. tokenizer.json +19 -19
tokenizer.json CHANGED
@@ -2426,7 +2426,7 @@
2426
  },
2427
  {
2428
  "id": 128269,
2429
- "content": "<|reserved_special_token_262|>",
2430
  "single_word": false,
2431
  "lstrip": false,
2432
  "rstrip": false,
@@ -2435,7 +2435,7 @@
2435
  },
2436
  {
2437
  "id": 128270,
2438
- "content": "<|reserved_special_token_263|>",
2439
  "single_word": false,
2440
  "lstrip": false,
2441
  "rstrip": false,
@@ -2444,7 +2444,7 @@
2444
  },
2445
  {
2446
  "id": 128271,
2447
- "content": "<|reserved_special_token_264|>",
2448
  "single_word": false,
2449
  "lstrip": false,
2450
  "rstrip": false,
@@ -2453,7 +2453,7 @@
2453
  },
2454
  {
2455
  "id": 128272,
2456
- "content": "<|reserved_special_token_265|>",
2457
  "single_word": false,
2458
  "lstrip": false,
2459
  "rstrip": false,
@@ -2462,7 +2462,7 @@
2462
  },
2463
  {
2464
  "id": 128273,
2465
- "content": "<|reserved_special_token_266|>",
2466
  "single_word": false,
2467
  "lstrip": false,
2468
  "rstrip": false,
@@ -2471,7 +2471,7 @@
2471
  },
2472
  {
2473
  "id": 128274,
2474
- "content": "<|reserved_special_token_267|>",
2475
  "single_word": false,
2476
  "lstrip": false,
2477
  "rstrip": false,
@@ -2480,7 +2480,7 @@
2480
  },
2481
  {
2482
  "id": 128275,
2483
- "content": "<|reserved_special_token_268|>",
2484
  "single_word": false,
2485
  "lstrip": false,
2486
  "rstrip": false,
@@ -2489,7 +2489,7 @@
2489
  },
2490
  {
2491
  "id": 128276,
2492
- "content": "<|reserved_special_token_269|>",
2493
  "single_word": false,
2494
  "lstrip": false,
2495
  "rstrip": false,
@@ -2498,7 +2498,7 @@
2498
  },
2499
  {
2500
  "id": 128277,
2501
- "content": "<|reserved_special_token_270|>",
2502
  "single_word": false,
2503
  "lstrip": false,
2504
  "rstrip": false,
@@ -2507,7 +2507,7 @@
2507
  },
2508
  {
2509
  "id": 128278,
2510
- "content": "<|reserved_special_token_271|>",
2511
  "single_word": false,
2512
  "lstrip": false,
2513
  "rstrip": false,
@@ -2516,7 +2516,7 @@
2516
  },
2517
  {
2518
  "id": 128279,
2519
- "content": "<|reserved_special_token_272|>",
2520
  "single_word": false,
2521
  "lstrip": false,
2522
  "rstrip": false,
@@ -2525,7 +2525,7 @@
2525
  },
2526
  {
2527
  "id": 128280,
2528
- "content": "<|reserved_special_token_273|>",
2529
  "single_word": false,
2530
  "lstrip": false,
2531
  "rstrip": false,
@@ -2534,7 +2534,7 @@
2534
  },
2535
  {
2536
  "id": 128281,
2537
- "content": "<|reserved_special_token_274|>",
2538
  "single_word": false,
2539
  "lstrip": false,
2540
  "rstrip": false,
@@ -2543,7 +2543,7 @@
2543
  },
2544
  {
2545
  "id": 128282,
2546
- "content": "<|reserved_special_token_275|>",
2547
  "single_word": false,
2548
  "lstrip": false,
2549
  "rstrip": false,
@@ -2552,7 +2552,7 @@
2552
  },
2553
  {
2554
  "id": 128283,
2555
- "content": "<|reserved_special_token_276|>",
2556
  "single_word": false,
2557
  "lstrip": false,
2558
  "rstrip": false,
@@ -2561,7 +2561,7 @@
2561
  },
2562
  {
2563
  "id": 128284,
2564
- "content": "<|reserved_special_token_277|>",
2565
  "single_word": false,
2566
  "lstrip": false,
2567
  "rstrip": false,
@@ -2570,7 +2570,7 @@
2570
  },
2571
  {
2572
  "id": 128285,
2573
- "content": "<|reserved_special_token_278|>",
2574
  "single_word": false,
2575
  "lstrip": false,
2576
  "rstrip": false,
@@ -2579,7 +2579,7 @@
2579
  },
2580
  {
2581
  "id": 128286,
2582
- "content": "<|reserved_special_token_279|>",
2583
  "single_word": false,
2584
  "lstrip": false,
2585
  "rstrip": false,
@@ -2588,7 +2588,7 @@
2588
  },
2589
  {
2590
  "id": 128287,
2591
- "content": "<|reserved_special_token_280|>",
2592
  "single_word": false,
2593
  "lstrip": false,
2594
  "rstrip": false,
 
2426
  },
2427
  {
2428
  "id": 128269,
2429
+ "content": "<|reserved_special_token_263|>",
2430
  "single_word": false,
2431
  "lstrip": false,
2432
  "rstrip": false,
 
2435
  },
2436
  {
2437
  "id": 128270,
2438
+ "content": "<|reserved_special_token_264|>",
2439
  "single_word": false,
2440
  "lstrip": false,
2441
  "rstrip": false,
 
2444
  },
2445
  {
2446
  "id": 128271,
2447
+ "content": "<|reserved_special_token_265|>",
2448
  "single_word": false,
2449
  "lstrip": false,
2450
  "rstrip": false,
 
2453
  },
2454
  {
2455
  "id": 128272,
2456
+ "content": "<|reserved_special_token_266|>",
2457
  "single_word": false,
2458
  "lstrip": false,
2459
  "rstrip": false,
 
2462
  },
2463
  {
2464
  "id": 128273,
2465
+ "content": "<|reserved_special_token_267|>",
2466
  "single_word": false,
2467
  "lstrip": false,
2468
  "rstrip": false,
 
2471
  },
2472
  {
2473
  "id": 128274,
2474
+ "content": "<|reserved_special_token_268|>",
2475
  "single_word": false,
2476
  "lstrip": false,
2477
  "rstrip": false,
 
2480
  },
2481
  {
2482
  "id": 128275,
2483
+ "content": "<|reserved_special_token_269|>",
2484
  "single_word": false,
2485
  "lstrip": false,
2486
  "rstrip": false,
 
2489
  },
2490
  {
2491
  "id": 128276,
2492
+ "content": "<|reserved_special_token_270|>",
2493
  "single_word": false,
2494
  "lstrip": false,
2495
  "rstrip": false,
 
2498
  },
2499
  {
2500
  "id": 128277,
2501
+ "content": "<|reserved_special_token_271|>",
2502
  "single_word": false,
2503
  "lstrip": false,
2504
  "rstrip": false,
 
2507
  },
2508
  {
2509
  "id": 128278,
2510
+ "content": "<|reserved_special_token_272|>",
2511
  "single_word": false,
2512
  "lstrip": false,
2513
  "rstrip": false,
 
2516
  },
2517
  {
2518
  "id": 128279,
2519
+ "content": "<|reserved_special_token_273|>",
2520
  "single_word": false,
2521
  "lstrip": false,
2522
  "rstrip": false,
 
2525
  },
2526
  {
2527
  "id": 128280,
2528
+ "content": "<|reserved_special_token_274|>",
2529
  "single_word": false,
2530
  "lstrip": false,
2531
  "rstrip": false,
 
2534
  },
2535
  {
2536
  "id": 128281,
2537
+ "content": "<|reserved_special_token_275|>",
2538
  "single_word": false,
2539
  "lstrip": false,
2540
  "rstrip": false,
 
2543
  },
2544
  {
2545
  "id": 128282,
2546
+ "content": "<|reserved_special_token_276|>",
2547
  "single_word": false,
2548
  "lstrip": false,
2549
  "rstrip": false,
 
2552
  },
2553
  {
2554
  "id": 128283,
2555
+ "content": "<|reserved_special_token_277|>",
2556
  "single_word": false,
2557
  "lstrip": false,
2558
  "rstrip": false,
 
2561
  },
2562
  {
2563
  "id": 128284,
2564
+ "content": "<|reserved_special_token_278|>",
2565
  "single_word": false,
2566
  "lstrip": false,
2567
  "rstrip": false,
 
2570
  },
2571
  {
2572
  "id": 128285,
2573
+ "content": "<|reserved_special_token_279|>",
2574
  "single_word": false,
2575
  "lstrip": false,
2576
  "rstrip": false,
 
2579
  },
2580
  {
2581
  "id": 128286,
2582
+ "content": "<|reserved_special_token_280|>",
2583
  "single_word": false,
2584
  "lstrip": false,
2585
  "rstrip": false,
 
2588
  },
2589
  {
2590
  "id": 128287,
2591
+ "content": "<|reserved_special_token_281|>",
2592
  "single_word": false,
2593
  "lstrip": false,
2594
  "rstrip": false,