Create app.py

app.py

@@ -0,0 +1,50 @@
+import transformers
+from transformers import BloomForCausalLM
+from transformers import BloomTokenizerFast
+import torch
+import gradio as gr
+
+# setting device on GPU if available, else CPU
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+print(device)
+
+model_name = "bigscience/bloom-1b"
+
+model = BloomForCausalLM.from_pretrained(model_name,device_map="auto", load_in_8bit=True)
+tokenizer = BloomTokenizerFast.from_pretrained(model_name)
+
+# Define the pipeline for Gradio purpose
+
+def beam_gradio_pipeline(prompt,length=500):
+
+   result_length = length
+
+   inputs = tokenizer(prompt, return_tensors="pt").to('cuda')
+
+   return tokenizer.decode(model.generate(inputs["input_ids"],
+                       max_length=result_length, 
+                       num_beams=2, 
+                       no_repeat_ngram_size=2,
+                       early_stopping=True
+                      )[0])
+
+with gr.Blocks() as demo:
+  gr.Markdown("<h1><center>Andrew Lim Bloom LLM </center></h1>")
+  gr.Markdown("""<h2><center>Generate your story with a sentence or ask a question:<br><br>
+ <img src=https://aeiljuispo.cloudimg.io/v7/https://s3.amazonaws.com/moonup/production/uploads/1634806038075-5df7e9e5da6d0311fd3d53f9.png?w=200&h=200&f=face width=200px></center></h2>""")
+  gr.Markdown("""<center>******</center>""")
+  
+
+  input_text = gr.Textbox(label="Prompt", lines=6)  
+  buton = gr.Button("Submit ")  
+  output_text = gr.Textbox(lines=6, label="The story start with :")
+  buton.click(beam_gradio_pipeline, inputs=[input_text], outputs=output_text)  
+   
+  gr.HTML("""
+    <div style="border-top: 1px solid #303030;">
+      <br>
+      <p>Help me pay for GPU hours so I can publish faster models!</p>
+      <a href="https://www.buymeacoffee.com/" target="_blank"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" style="height: 45px !important;width: 162px !important;" ></a><br><br>
+    </div>
+    """)
+demo.launch(enable_queue=True, debug=True)