import tiktoken import json def count_tokens(string: str, encoding_name: str) -> int: """Returns the number of tokens in a text string.""" encoding = tiktoken.encoding_for_model(encoding_name) num_tokens = len(encoding.encode(string)) return num_tokens def load_json(json_file): with open(json_file) as f: return json.load(f)