Updated with llama 3.1 model , added logs for the debuging

app.py

@@ -42,7 +42,7 @@ h1 {
 """
 
 # Load the tokenizer and model with quantization
-model_id = "meta-llama/Meta-Llama-3-8B-Instruct"
+model_id = "meta-llama/Meta-Llama-3.1-8B-Instruct"
 bnb_config = BitsAndBytesConfig(
     load_in_4bit=True,
     bnb_4bit_use_double_quant=True,
@@ -82,13 +82,76 @@ terminators = [
     tokenizer.convert_tokens_to_ids("<|eot_id|>")
 ]
 
+# SYS_PROMPT = """
+# Extract all relevant keywords and add quantity from the following text and format the result in nested JSON, ignoring personal details and focusing only on the scope of work as shown in the example:
+# Good JSON example: {'lobby': {'frcm': {'replace': {'carpet': 1, 'carpet_pad': 1, 'base': 1, 'window_treatments': 1, 'artwork_and_decorative_accessories': 1, 'portable_lighting': 1, 'upholstered_furniture_and_decorative_pillows': 1, 'millwork': 1} } } }
+# Bad JSON example: {'lobby': { 'frcm': { 'replace': [ 'carpet', 'carpet_pad', 'base', 'window_treatments', 'artwork_and_decorative_accessories', 'portable_lighting', 'upholstered_furniture_and_decorative_pillows', 'millwork'] } } }
+# Make sure to fetch details from the provided text and ignore unnecessary information. The response should be in JSON format only, without any additional comments.
+# """
+
 SYS_PROMPT = """
-Extract all relevant keywords and add quantity from the following text and format the result in nested JSON, ignoring personal details and focusing only on the scope of work as shown in the example:
-Good JSON example: {'lobby': {'frcm': {'replace': {'carpet': 1, 'carpet_pad': 1, 'base': 1, 'window_treatments': 1, 'artwork_and_decorative_accessories': 1, 'portable_lighting': 1, 'upholstered_furniture_and_decorative_pillows': 1, 'millwork': 1} } } }
-Bad JSON example: {'lobby': { 'frcm': { 'replace': [ 'carpet', 'carpet_pad', 'base', 'window_treatments', 'artwork_and_decorative_accessories', 'portable_lighting', 'upholstered_furniture_and_decorative_pillows', 'millwork'] } } }
+Extract all relevant keywords and add quantities from the following text and format the result in nested JSON, ignoring personal details and focusing only on the area and furniture items as shown in the example. Each item should have a count, which will be set to 1 for simplicity. The response should be in JSON format only, without any additional comments.
+
+Good JSON example:{
+  "Lobby Area/Entrance": {
+    "Vinyl wall covering": 1,
+    "Decorative hardwired lighting": 1
+  },
+  "Lobby": {
+    "Carpet, carpet pad, and base": 1,
+    "Window treatments": 1,
+    "Artwork and decorative accessories": 1,
+    "Portable lighting": 1,
+    "Upholstered furniture and decorative pillows": 1,
+    "Millwork": 1
+  }
+}
 Make sure to fetch details from the provided text and ignore unnecessary information. The response should be in JSON format only, without any additional comments.
-"""
 
+Task:
+Convert the provided extracted text into the JSON format described above.
+
+Provided Text:
+
+PROPERTY IMPROVEMENT PLAN
+PREPARED FOR:
+Springfield, IL
+To be relicensed as Hilton Garden Inn
+...
+Patios/The Terrace - Install patio decorative lighting. Install patio furniture. (lounge chairs, chaise, dining tables/chairs)
+...
+Lobby Area - Replace carpet, carpet pad, and base. Replace window treatments. Replace artwork and decorative accessories. Replace portable lighting. (floor lamps, table lamps) Replace upholstered furniture and decorative pillows. Replace millwork. Replace the television(s).
+...
+Registration Area - Replace vinyl wall covering. Replace hard surface floor covering. Replace artwork. Install new signature graphics on the back wall.
+...
+
+Expected Output (JSON format):
+{
+  "Patios/The Terrace": {
+    "Patio decorative lighting": 1,
+    "Lounge chairs": 1,
+    "Chaise": 1,
+    "Dining tables": 1,
+    "Dining chairs": 1,
+    "Patio furniture": 1
+  },
+  "Lobby Area": {
+    "Carpet, carpet pad, and base": 1,
+    "Window treatments": 1,
+    "Artwork and decorative accessories": 1,
+    "Portable lighting (floor lamps, table lamps)": 1,
+    "Upholstered furniture and decorative pillows": 1,
+    "Millwork": 1,
+    "Television(s)": 1
+  },
+  "Registration Area": {
+    "Vinyl wall covering": 1,
+    "Hard surface floor covering": 1,
+    "Artwork (new signature graphics on the back wall)": 1
+  }
+}
+
+"""
 def chunk_text(text, chunk_size=5000):
     """
     Splits the input text into chunks of specified size.
@@ -102,6 +165,7 @@ def chunk_text(text, chunk_size=5000):
     """
     words = text.split()
     chunks = [' '.join(words[i:i + chunk_size]) for i in range(0, len(words), chunk_size)]
+    logger.info(f"Total chunks created: {len(chunks)}")
     return chunks
 
 def combine_responses(responses):
@@ -175,9 +239,12 @@ def chat_llama3_8b(message: str, history: list, temperature: float, max_new_toke
         
         chunks = chunk_text(message)
         responses = []
+        count=0
         for chunk in chunks:
+            logger.info(f"Processing chunk {count+1}/{len(chunks)}")
             response = generate_response_for_chunk(chunk, history, temperature, max_new_tokens)
             responses.append(response)
+            count+=1
         final_output = combine_responses(responses)
         
         end_time = time.time()

requirements.txt

@@ -1,4 +1,4 @@
 accelerate
-transformers
+transformers==4.43.1
 SentencePiece
 bitsandbytes