Added benchmarking

main.py

@@ -1,3 +1,14 @@
+import torch
+
+try:
+    import intel_extension_for_pytorch as ipex
+    ipex_enabled = True
+except:
+    ipex_enabled = False
+
+import time
+import numpy as np
+
 from fastapi import FastAPI, UploadFile, File, HTTPException
 from fastapi.staticfiles import StaticFiles
 from fastapi.responses import FileResponse
@@ -8,8 +19,7 @@ from fastapi.responses import JSONResponse
 from io import BytesIO
 import PyPDF2
 from newspaper import Article
-import torch
-from transformers import AutoModelForMultipleChoice, AutoTokenizer
+from transformers import AutoModelForMultipleChoice, AutoTokenizer, AutoModelForQuestionAnswering
 
 qa_pipeline = pipeline("question-answering", model="roaltopo/scan-u-doc_question-answer")
 bool_q_pipeline = pipeline("text-classification",  model="roaltopo/scan-u-doc_bool-question")
@@ -128,6 +138,69 @@ async def answer_question(uuid: str, question_info: QuestionInfo):
 
     return answer
 
+############
+def get_score1(model_checkpoint, question, context, num_times, warmup_rounds, has_xpu):
+    tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
+    model = AutoModelForQuestionAnswering.from_pretrained(model_checkpoint)
+    model.eval()
+
+    if has_xpu:
+        device = 'xpu'
+    else :
+        device = None
+    
+    qa_pipeline = pipeline("question-answering", model=model, tokenizer=tokenizer, torch_dtype=torch.bfloat16, device=device) #, torch_dtype=torch.bfloat16
+    latency_list = []
+    for i in range(num_times):
+        time_start = time.time()
+        answer = qa_pipeline(question=question, context=context)
+        if i >= warmup_rounds:
+            latency_list.append(time.time() - time_start)
+    pipeline_inference_time = np.mean(latency_list)
+    return pipeline_inference_time
+
+def get_score2(model_checkpoint, question, context, num_times, warmup_rounds, has_xpu):
+    tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
+    model = AutoModelForQuestionAnswering.from_pretrained(model_checkpoint)
+    model.eval()
+
+    if has_xpu:
+        device = 'xpu'
+    else :
+        device = None
+
+    if ipex_enabled:
+        #################### code changes ####################  
+        model = ipex.optimize(model, weights_prepack=False)
+        model = torch.compile(model, backend="ipex")
+        ###################################################### 
+    with torch.no_grad():
+        qa_pipeline = pipeline("question-answering", model=model, tokenizer=tokenizer, torch_dtype=torch.bfloat16, device=device) #, torch_dtype=torch.bfloat16
+        latency_list = []
+        for i in range(num_times):
+            time_start = time.time()
+            answer = qa_pipeline(question=question, context=context)
+            if i >= warmup_rounds:
+                latency_list.append(time.time() - time_start)
+        pipeline_inference_time = np.mean(latency_list)
+    return pipeline_inference_time
+
+@app.get("/benchmark")
+async def benchmark(question: str, context: str):
+    num_times = 50
+    warmup_rounds = 20
+
+    model_checkpoint = "roaltopo/scan-u-doc_question-answer"
+
+
+    has_xpu = torch.xpu.device_count()
+    score1 = get_score1(model_checkpoint, question, context, num_times, warmup_rounds, has_xpu)
+    score2 = get_score2(model_checkpoint, question, context, num_times, warmup_rounds, has_xpu)
+
+    return {'has_xpu': has_xpu, 'ipex_enabled': ipex_enabled,'score1': score1, 'score2': score2}
+
+############
+
 
 app.mount("/", StaticFiles(directory="static", html=True), name="static")