onnx-community
/

camembertav2-base

Feature Extraction

Transformers.js

Model card Files Files and versions Community

Xenova HF staff commited on 10 days ago

Commit

f4e786d

•

1 Parent(s): f76b0c4

Upload tokenizer.json

Files changed (1) hide show

tokenizer.json +1 -1

tokenizer.json CHANGED Viewed

@@ -90,7 +90,7 @@
       {
         "type": "Split",
         "pattern": {
-          "Regex": "(?i:\\b[cmnjstdl]|qu|puisqu|lorsqu|quelqu|presqu|quoiqu|jusqu)['´’]|['´’](?i:[sdmt]|ll|ve|re)\\b|\\p{P}{1,4}|[^\\p{L}\\p{N}\\p{Z}]"
         },
         "behavior": "Isolated",
         "invert": false

       {
         "type": "Split",
         "pattern": {
+          "Regex": "(?:\\b[cCmMnNjJsStTdDlL]|[qQ][uU]|[pP][uU][iI][sS][qQ][uU]|[lL][oO][rR][sS][qQ][uU]|[qQ][uU][eE][lL][qQ][uU]|[pP][rR][eE][sS][qQ][uU]|[qQ][uU][oO][iI][qQ][uU]|[jJ][uU][sS][qQ][uU])['´’]|['´’](?:[sSdDmMtT]|[lL][lL]|[vV][eE]|[rR][eE])\\b|\\p{P}{1,4}|[^\\p{L}\\p{N}\\p{Z}]"
         },
         "behavior": "Isolated",
         "invert": false