f89a7d8
1
2
3
4
5
6
7
8
9
10
import tiktoken encoding = tiktoken.encoding_for_model("gpt-3.5-turbo") def truncate_string(s, max_length=1024) -> str: e = encoding.encode(s)[:max_length] s = encoding.decode(e) return s
