rameshmoorthy
commited on
Commit
•
4333093
1
Parent(s):
09a6682
Update backend/query_llm.py
Browse files- backend/query_llm.py +11 -10
backend/query_llm.py
CHANGED
@@ -9,8 +9,9 @@ from typing import Any, Dict, Generator, List
|
|
9 |
from huggingface_hub import InferenceClient
|
10 |
from transformers import AutoTokenizer
|
11 |
|
12 |
-
|
13 |
-
tokenizer = AutoTokenizer.from_pretrained("mistralai/Mixtral-8x7B-Instruct-v0.1")
|
|
|
14 |
temperature = 0.5
|
15 |
top_p = 0.7
|
16 |
repetition_penalty = 1.2
|
@@ -18,16 +19,16 @@ repetition_penalty = 1.2
|
|
18 |
OPENAI_KEY = getenv("OPENAI_API_KEY")
|
19 |
HF_TOKEN = getenv("HUGGING_FACE_HUB_TOKEN")
|
20 |
|
21 |
-
|
22 |
-
|
23 |
-
|
24 |
-
|
25 |
|
26 |
|
27 |
-
hf_client = InferenceClient(
|
28 |
-
|
29 |
-
|
30 |
-
|
31 |
def format_prompt(message: str, api_kind: str):
|
32 |
"""
|
33 |
Formats the given message using a chat template.
|
|
|
9 |
from huggingface_hub import InferenceClient
|
10 |
from transformers import AutoTokenizer
|
11 |
|
12 |
+
tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1")
|
13 |
+
#tokenizer = AutoTokenizer.from_pretrained("mistralai/Mixtral-8x7B-Instruct-v0.1")
|
14 |
+
#tokenizer = AutoTokenizer.from_pretrained("mistralai/Mixtral-8x22B-Instruct-v0.1")
|
15 |
temperature = 0.5
|
16 |
top_p = 0.7
|
17 |
repetition_penalty = 1.2
|
|
|
19 |
OPENAI_KEY = getenv("OPENAI_API_KEY")
|
20 |
HF_TOKEN = getenv("HUGGING_FACE_HUB_TOKEN")
|
21 |
|
22 |
+
hf_client = InferenceClient(
|
23 |
+
"mistralai/Mistral-7B-Instruct-v0.1",
|
24 |
+
token=HF_TOKEN
|
25 |
+
)
|
26 |
|
27 |
|
28 |
+
# hf_client = InferenceClient(
|
29 |
+
# "mistralai/Mixtral-8x7B-Instruct-v0.1",
|
30 |
+
# token=HF_TOKEN
|
31 |
+
# )
|
32 |
def format_prompt(message: str, api_kind: str):
|
33 |
"""
|
34 |
Formats the given message using a chat template.
|