more stuff

app.py

@@ -1,17 +1,18 @@
 import gradio as gr
 import torch
 from PIL import Image
+from huggingface_hub import hf_hub_download
 
 demo_imgs = [
-    ["chinchilla_web-1024x683.jpg", "shiba-inu-dog-in-the-snow.jpg"],
-    ["4645808729_2dfc59b6a5_z.jpg", "5944609705_4664531909_z.jpg"],
-    ["COCO_train2014_000000310472.jpg", "COCO_train2014_000000194806.jpg"],
+    ["images/chinchilla_web-1024x683.jpg", "images/shiba-inu-dog-in-the-snow.jpg"],
+    ["images/4645808729_2dfc59b6a5_z.jpg", "images/5944609705_4664531909_z.jpg"],
+    ["images/COCO_train2014_000000310472.jpg", "images/COCO_train2014_000000194806.jpg"],
     [
-        "bcee7a-20190225-a-london-underground-sign.jpg",
-        "istockphoto-622434332-1024x1024.jpg",
+        "images/bcee7a-20190225-a-london-underground-sign.jpg",
+        "images/istockphoto-622434332-1024x1024.jpg",
     ],
-    ["dogs.jpeg", "pandas.jpg"],
-    ["11887_pesto-pasta_Rita-1x1-1-501c953b29074ab193e2b5ad36e64648.jpg", "hummus.jpg"],
+    ["images/dogs.jpeg", "images/pandas.jpg"],
+    ["images/11887_pesto-pasta_Rita-1x1-1-501c953b29074ab193e2b5ad36e64648.jpg", "images/hummus.jpg"],
 ]
 demo_texts = [
     [
@@ -48,11 +49,14 @@ with open("bad_words.txt", "r") as f:
 model, image_processor, tokenizer = create_model_and_transforms(
     clip_vision_encoder_pretrained="openai",
     clip_vision_encoder_path="ViT-L-14",
-    lang_encoder_path="togethercomputer/RedPajama-INCITE-Base-3B-v1",
-    tokenizer_path="togethercomputer/RedPajama-INCITE-Base-3B-v1",
+    lang_encoder_path="togethercomputer/RedPajama-INCITE-Instruct-3B-v1",
+    tokenizer_path="togethercomputer/RedPajama-INCITE-Instruct-3B-v1",
     cross_attn_every_n_layers=2,
 )
 
+checkpoint_path = hf_hub_download("openflamingo/OpenFlamingo-4B-vitl-rpj3b-langinstruct", "checkpoint.pt")
+model.load_state_dict(torch.load(checkpoint_path), strict=False)
+
 model.eval()
 
 

requirements.txt

@@ -1,3 +1,4 @@
 gradio
 torch
 pillow
+huggingface_hub