2024.09.27/language_model/clip_tokenizer_4.16.2/tokenizer_config.json

{
  "errors": "replace",
  "unk_token": {
    "content": "<|endoftext|>",
    "single_word": false,
    "lstrip": false,
    "rstrip": false,
    "normalized": true,
    "special": false,
    "__type": "AddedToken"
  },
  "bos_token": {
    "content": "<|startoftext|>",
    "single_word": false,
    "lstrip": false,
    "rstrip": false,
    "normalized": true,
    "special": false,
    "__type": "AddedToken"
  },
  "eos_token": {
    "content": "<|endoftext|>",
    "single_word": false,
    "lstrip": false,
    "rstrip": false,
    "normalized": true,
    "special": false,
    "__type": "AddedToken"
  },
  "pad_token": "<|endoftext|>",
  "add_prefix_space": false,
  "do_lower_case": true,
  "name_or_path": "openai/clip-vit-base-patch32",
  "model_max_length": 77,
  "special_tokens_map_file": "/home/ncodella/.cache/huggingface/transformers/18a566598f286c9139f88160c99f84eec492a26bd22738fa9cb44d5b7e0a5c76.cce1206abbad28826f000510f22f354e53e66a97f7c23745a7dfe27609cc07f5",
  "tokenizer_file": "/home/ncodella/.cache/huggingface/transformers/7811def0c53be25ba790cb67ac785669b508a8d1cf8c912b8ac046c5f08aee68.20428ea8b6821af2719b760af844a371643ff49f255c73285f6ea448e15597fe",
  "tokenizer_class": "CLIPTokenizer"
}