Upload 2 files

app.py

@@ -81,6 +81,10 @@ def instruct_generate(
         top_k: The number of top most probable tokens to consider in the sampling process.
         temperature: A value controlling the randomness of the sampling process. Higher values result in more random
     """
+    if input in input_value_2_real.keys():
+        input = input_value_2_real[input]
+    if "..." in input:
+        input = input.replace("...", "")
     sample = {"instruction": prompt, "input": input}
     prompt = generate_prompt(sample)
     encoded = tokenizer.encode(prompt, bos=True, eos=False, device=model.device)
@@ -103,10 +107,12 @@ def instruct_generate(
 
 # 配置具体参数
 pretrained_path = hf_hub_download(
-    repo_id="xxw/tapa_model", filename="lit-llama.pth")
+    repo_id="Gary3410/pretrain_lit_llama", filename="lit-llama.pth")
 tokenizer_path = hf_hub_download(
-    repo_id="xxw/tapa_model", filename="tokenizer.model")
-adapter_path = "lit-llama-adapter-finetuned_15k.pth"
+    repo_id="Gary3410/tapa_model", filename="tokenizer.model")
+adapter_path = hf_hub_download(
+    repo_id="Gary3410/tapa_model", filename="lit-llama-adapter-finetuned_15k.pth")
+# adapter_path = "lit-llama-adapter-finetuned_15k.pth"
 example_path = "example.json"
 # 1024如果不够, 调整为512
 max_seq_len = 1024
@@ -117,18 +123,19 @@ tokenizer = Tokenizer(tokenizer_path)
 with open(example_path, 'r') as f:
     content = f.read()
     example_dict = json.loads(content)
-
+input_value_2_real = {}
+for scene_id, scene_dict in example_dict.items():
+    input_value_2_real[scene_dict["input_display"]] = scene_dict["input"]
 
 def create_instruct_demo():
     with gr.Blocks() as instruct_demo:
         with gr.Row():
             with gr.Column():
                 scene_img = gr.Image(label='Scene', type='filepath')
-                object_list = gr.Textbox(
-                    lines=2, label="Input")
-
                 instruction = gr.Textbox(
                     lines=2, label="Instruction")
+                object_list = gr.Textbox(
+                    lines=5, label="Input")
                 max_len = gr.Slider(minimum=1, maximum=512,
                                     value=128, label="Max length")
                 with gr.Accordion(label='Advanced options', open=False):
@@ -140,18 +147,18 @@ def create_instruct_demo():
                 run_botton = gr.Button("Run")
 
             with gr.Column():
-                outputs = gr.Textbox(lines=10, label="Output")
+                outputs = gr.Textbox(lines=20, label="Output")
 
-        inputs = [instruction, object_list, max_len, top_k, temp]
+        inputs = [scene_img, instruction, object_list, max_len, top_k, temp]
 
         # 接下来设定具体的example格式
         examples_img_list = glob.glob("caption_demo/*.png")
         examples = []
         for example_img_one in examples_img_list:
             scene_name = os.path.basename(example_img_one).split(".")[0]
-            example_object_list = example_dict[scene_name]["input"]
+            example_object_list = example_dict[scene_name]["input_display"]
             example_instruction = example_dict[scene_name]["instruction"]
-            example_one = [example_img_one, example_object_list, example_instruction, 512, 0.8, 200]
+            example_one = [example_img_one, example_instruction, example_object_list, 512, 0.8, 200]
             examples.append(example_one)
 
         gr.Examples(

example.json

@@ -1,34 +1,42 @@
 {
   "FloorPlan21": {
-    "input": "[apple,chair,blender,sink,pottery,oven,egg,table,shredder_(for_paper),seashell,bread,doorknob,fork,plastic_bag,knife,radio_receiver,drawer,person,coffee_maker,inhaler,toaster,plate,cornice,knob,pear,dining_table,tomato,bottle,scale_(measuring_instrument),toilet_tissue,cushion,latch,scissors,soap,handle,balloon,clock,lightbulb,matchbox,refrigerator,trash_can,backpack,alarm_clock,vase,tape_(sticky_cloth_or_paper),printer,cover,faucet,gourd,pan_(for_cooking),ball,spatula,microwave_oven,dispenser,nailfile,cabinet,sweet_potato,lamp,microscope,pot,cup,suitcase,bowl,thermostat,fume_hood,hinge,mirror,spoon,box,]",
-    "instruction": "Can you clean the dishes?"
+    "input": "[apple, chair, blender, sink, pottery, oven, egg, table, shredder_(for_paper), seashell, bread, doorknob, fork, plastic_bag, knife, radio_receiver, drawer, person, coffee_maker, inhaler, toaster, plate, cornice, knob, pear, dining_table, tomato, bottle, scale_(measuring_instrument), toilet_tissue, cushion, latch, scissors, soap, handle, balloon, clock, lightbulb, matchbox, refrigerator, trash_can, backpack, alarm_clock, vase, tape_(sticky_cloth_or_paper), printer, cover, faucet, gourd, pan_(for_cooking), ball, spatula, microwave_oven, dispenser, nailfile, cabinet, sweet_potato, lamp, microscope, pot, cup, suitcase, bowl, thermostat, fume_hood, hinge, mirror, spoon, box,]",
+    "instruction": "Can you clean the dishes?",
+    "input_display": "[apple, chair, blender, sink, pottery, oven, egg, table, shredder_(for_paper), seashell, bread, doorknob, fork, plastic_bag, knife, ...]"
   },
   "FloorPlan24": {
-    "input": "[apple,chair,sink,oven,figurine,shredder_(for_paper),potholder,doorknob,truffle_(chocolate),fork,towel,stove,napkin,knife,drawer,hotplate,coffee_maker,avocado,chopping_board,stool,bolt,toaster,bowling_ball,hand_towel,plate,speaker_(stero_equipment),tag,piggy_bank,knob,dining_table,tomato,scale_(measuring_instrument),toaster_oven,pitcher_(vessel_for_liquid),painting,handle,wineglass,clock,automatic_washer,ice_maker,lightbulb,refrigerator,trash_can,tray,dishwasher,armoire,faucet,gourd,pan_(for_cooking),spatula,microwave_oven,mug,dispenser,cabinet,fire_extinguisher,kitchen_sink,television_set,lamp,cup,bowl,thermostat,water_jug,hinge,spoon,]",
-    "instruction": "Please make me an omelette."
+    "input": "[apple, chair, sink, oven, figurine, shredder_(for_paper), potholder, doorknob, truffle_(chocolate), fork, towel, stove, napkin, knife, drawer, hotplate, coffee_maker, avocado, chopping_board, stool, bolt, toaster, bowling_ball, hand_towel, plate, speaker_(stero_equipment), tag, piggy_bank, knob, dining_table, tomato, scale_(measuring_instrument), toaster_oven, pitcher_(vessel_for_liquid), painting, handle, wineglass, clock, automatic_washer, ice_maker, lightbulb, refrigerator, trash_can, tray, dishwasher, armoire, faucet, gourd, pan_(for_cooking), spatula, microwave_oven, mug, dispenser, cabinet, fire_extinguisher, kitchen_sink, television_set, lamp, cup, bowl, thermostat, water_jug, hinge, spoon,]",
+    "instruction": "Please make me an omelette.",
+    "input_display": "[apple, chair, sink, oven, figurine, shredder_(for_paper), potholder, doorknob, truffle_(chocolate), fork, towel, stove, napkin, knife, drawer, ...]"
   },
   "FloorPlan221": {
-    "input": "[chair,sofa,pen,figurine,table,dog,lampshade,doorknob,bed,toy,drawer,person,statue_(sculpture),flowerpot,stool,monitor_(computer_equipment)computer_monitor,desk,pillow,plate,speaker_(stero_equipment),mouse_(computer_equipment),knob,igniter,dining_table,cushion,painting,dragonfly,laptop_computer,remote_control,vase,trash_can,wall_socket,ashtray,coffee_table,card,computer_keyboard,bird,coaster,television_set,lamp,bowl,thermostat,hinge,curtain,box,]",
-    "instruction": "Could you please close the curtains?"
+    "input": "[chair, sofa, pen, figurine, table, dog, lampshade, doorknob, bed, toy, drawer, person, statue_(sculpture), flowerpot, stool, monitor_(computer_equipment)computer_monitor, desk, pillow, plate, speaker_(stero_equipment), mouse_(computer_equipment), knob, igniter, dining_table, cushion, painting, dragonfly, laptop_computer, remote_control, vase, trash_can, wall_socket, ashtray, coffee_table, card, computer_keyboard, bird, coaster, television_set, lamp, bowl, thermostat, hinge, curtain, box,]",
+    "instruction": "Could you please close the curtains?",
+    "input_display": "[chair, sofa, pen, figurine, table, dog, lampshade, doorknob, bed, toy, drawer, person, statue_(sculpture), flowerpot, stool, ...]"
   },
   "FloorPlan224": {
-    "input": "[chair,sofa,figurine,table,crate,necklace,dog,dresser,lampshade,doorknob,horse,frisbee,deer,screwdriver,oil_lamp,drawer,sweater,person,statue_(sculpture),flowerpot,stool,dress,pole,monitor_(computer_equipment)computer_monitor,hat,easel,umbrella,desk,pillow,speaker_(stero_equipment),book,knob,fireplace,ottoman,dining_table,toilet_tissue,cushion,painting,latch,handle,bathtub,laptop_computer,remote_control,clock,lightbulb,candle,vase,trash_can,wall_socket,hose,coffee_table,computer_keyboard,spotlight,bird,cabinet,television_set,lamp,harmonium,cup,thermostat,newspaper,curtain,runner_(carpet),box,]",
-    "instruction": "Can you turn off the light?"
+    "input": "[chair, sofa, figurine, table, crate, necklace, dog, dresser, lampshade, doorknob, horse, frisbee, deer, screwdriver, oil_lamp, drawer, sweater, person, statue_(sculpture), flowerpot, stool, dress, pole, monitor_(computer_equipment)computer_monitor, hat, easel, umbrella, desk, pillow, speaker_(stero_equipment), book, knob, fireplace, ottoman, dining_table, toilet_tissue, cushion, painting, latch, handle, bathtub, laptop_computer, remote_control, clock, lightbulb, candle, vase, trash_can, wall_socket, hose, coffee_table, computer_keyboard, spotlight, bird, cabinet, television_set, lamp, harmonium, cup, thermostat, newspaper, curtain, runner_(carpet), box,]",
+    "instruction": "Can you turn off the light?",
+    "input_display": "[chair, sofa, figurine, table, crate, necklace, dog, dresser, lampshade, doorknob, horse, frisbee, deer, screwdriver, oil_lamp, ...]"
   },
   "FloorPlan321": {
-    "input": "[chair,sofa,figurine,table,quilt,bed,lampshade,doorknob,tissue_paper,headboard,button,pencil,drawer,place_mat,cigar_box,knitting_needle,monitor_(computer_equipment)computer_monitor,desk,pillow,chandelier,book,knob,armchair,ottoman,dining_table,notebook,cushion,painting,vent,laptop_computer,blanket,lightbulb,cellular_telephone,trash_can,alarm_clock,tape_(sticky_cloth_or_paper),faucet,card,computer_keyboard,coaster,nailfile,bicycle,mattress,lamp,car_(automobile),magazine,thermostat,heart,mirror,box,]",
-    "instruction": "Can you please hand me the pencil on the desk?"
+    "input": "[chair, sofa, figurine, table, quilt, bed, lampshade, doorknob, tissue_paper, headboard, button, pencil, drawer, place_mat, cigar_box, knitting_needle, monitor_(computer_equipment)computer_monitor, desk, pillow, chandelier, book, knob, armchair, ottoman, dining_table, notebook, cushion, painting, vent, laptop_computer, blanket, lightbulb, cellular_telephone, trash_can, alarm_clock, tape_(sticky_cloth_or_paper), faucet, card, computer_keyboard, coaster, nailfile, bicycle, mattress, lamp, car_(automobile), magazine, thermostat, heart, mirror, box,]",
+    "instruction": "Can you please hand me the pencil on the desk?",
+    "input_display": "[chair, sofa, figurine, table, quilt, bed, lampshade, doorknob, tissue_paper, headboard, button, pencil, drawer, place_mat, cigar_box, ...]"
   },
   "FloorPlan323": {
-    "input": "[chair,sofa,sink,dresser,lampshade,bed,doorknob,toy,teddy_bear,towel,headboard,drawer,place_mat,monitor_(computer_equipment)computer_monitor,desk,pillow,speaker_(stero_equipment),mouse_(computer_equipment),piggy_bank,book,cornice,dining_table,cushion,painting,cigarette_case,handle,laptop_computer,remote_control,candle,trash_can,wall_socket,armoire,corkboard,computer_keyboard,lamp,television_set,telephone,cup,hatbox,bowl,thermostat,hinge,mirror,runner_(carpet),box,]",
-    "instruction": "Can you pass me the remote control, please?"
+    "input": "[chair, sofa, sink, dresser, lampshade, bed, doorknob, toy, teddy_bear, towel, headboard, drawer, place_mat, monitor_(computer_equipment)computer_monitor, desk, pillow, speaker_(stero_equipment), mouse_(computer_equipment), piggy_bank, book, cornice, dining_table, cushion, painting, cigarette_case, handle, laptop_computer, remote_control, candle, trash_can, wall_socket, armoire, corkboard, computer_keyboard, lamp, television_set, telephone, cup, hatbox, bowl, thermostat, hinge, mirror, runner_(carpet), box,]",
+    "instruction": "Can you pass me the remote control, please?",
+    "input_display": "[chair, sofa, sink, dresser, lampshade, bed, doorknob, toy, teddy_bear, towel, headboard, drawer, place_mat, monitor_(computer_equipment)computer_monitor, desk, ...]"
   },
   "FloorPlan422": {
-    "input": "[knocker_(on_a_door),sink,hook,clothespin,doorknob,tissue_paper,oil_lamp,drawer,cistern,bottle_cap,desk,hand_towel,knob,bottle,dining_table,toilet_tissue,handle,bathtub,towel_rack,bath_mat,candle_holder,bat_(animal),toilet,wooden_spoon,candle,shower_head,refrigerator,trash_can,cover,hair_dryer,armoire,faucet,scrubbing_brush,dispenser,shower_curtain,cabinet,lamp,bath_towel,cup,thermostat,fume_hood,hinge,mirror,paper_towel,broom,box,]",
-    "instruction": "Open the Cabinet and give me the Soap Bottle"
+    "input": "[knocker_(on_a_door), sink, hook, clothespin, doorknob, tissue_paper, oil_lamp, drawer, cistern, bottle_cap, desk, hand_towel, knob, bottle, dining_table, toilet_tissue, handle, bathtub, towel_rack, bath_mat, candle_holder, bat_(animal), toilet, wooden_spoon, candle, shower_head, refrigerator, trash_can, cover, hair_dryer, armoire, faucet, scrubbing_brush, dispenser, shower_curtain, cabinet, lamp, bath_towel, cup, thermostat, fume_hood, hinge, mirror, paper_towel, broom, box,]",
+    "instruction": "Open the Cabinet and give me the Soap Bottle",
+    "input_display": "[knocker_(on_a_door), sink, hook, clothespin, doorknob, tissue_paper, oil_lamp, drawer, cistern, bottle_cap, desk, hand_towel, knob, bottle, dining_table, ...]"
   },
   "FloorPlan424": {
-    "input": "[sink,bucket,doorknob,towel,wine_bucket,cistern,washbasin,pipe,hand_towel,knob,bottle,toilet_tissue,soap,handle,towel_rack,candle_holder,lightbulb,candle,shower_head,crucifix,vase,cover,wall_socket,faucet,scrubbing_brush,dispenser,cabinet,lamp,bath_towel,cup,thermostat,hinge,mirror,toilet,eraser,]",
-    "instruction": "Please clean the sink"
+    "input": "[sink, bucket, doorknob, towel, wine_bucket, cistern, washbasin, pipe, hand_towel, knob, bottle, toilet_tissue, soap, handle, towel_rack, candle_holder, lightbulb, candle, shower_head, crucifix, vase, cover, wall_socket, faucet, scrubbing_brush, dispenser, cabinet, lamp, bath_towel, cup, thermostat, hinge, mirror, toilet, eraser,]",
+    "instruction": "Please clean the sink",
+    "input_display": "[sink, bucket, doorknob, towel, wine_bucket, cistern, washbasin, pipe, hand_towel, knob, bottle, toilet_tissue, soap, handle, towel_rack, ...]"
   }
 }