Update app.py

app.py

@@ -34,6 +34,25 @@ def caption(input_image):
 import openai
 import os
 openai.api_key= os.getenv('openai_appkey') 
+def gpt3_short(question,vqa_answer,caption):
+    vqa_answer,vqa_score=vqa_answer
+    prompt="prompt: This is a picture of Caption: "+caption+". Question: "+question+" VQA model predicts:"+"A: "+vqa_answer[0]+"socre:"+str(vqa_score[0])+\
+           " B: "+vqa_answer[1]+" score:"+str(vqa_score[1])+" C: "+vqa_answer[2]+" score:"+str(vqa_score[2])+\
+            " D: "+vqa_answer[3]+'score:'+str(vqa_score[3])+\
+           ". Choose A if it is not in conflict with the description of the picture and A's score is bigger than 0.8; otherwise choose the B, C or D based on the description."
+    
+    # prompt=caption+"\n"+question+"\n"+vqa_answer+"\n Tell me the right answer."
+    response = openai.Completion.create(
+    engine="text-davinci-003",
+    prompt=prompt,
+    max_tokens=30,
+    n=1,
+    stop=None,
+    temperature=0.7,
+    )
+    answer = response.choices[0].text.strip()
+    
+    return answer
 def gpt3(question,vqa_answer,caption):
     prompt=caption+"\n"+question+"\n"+vqa_answer+"\n Tell me the right answer."
     response = openai.Completion.create(
@@ -51,7 +70,7 @@ def gpt3(question,vqa_answer,caption):
 def vle(input_image,input_text):
     vqa_answers = vqa_pipeline({"image":input_image, "question":input_text}, top_k=4)
     # return [" ".join([str(value) for key,value in vqa.items()] )for vqa in vqa_answers]
-    return [vqa['answer'] for vqa in vqa_answers]
+    return [vqa['answer'] for vqa in vqa_answers],[vqa['score'] for vqa in vqa_answers]
 def inference_chat(input_image,input_text):
     cap=caption(input_image)
     # inputs = processor(images=input_image, text=input_text,return_tensors="pt")
@@ -62,10 +81,10 @@ def inference_chat(input_image,input_text):
     # out=processor.batch_decode(out, skip_special_tokens=True)
 
     out=vle(input_image,input_text)
-    vqa="\n".join(out)
+    vqa="\n".join(out[0])
     gpt3_out=gpt3(input_text,vqa,cap)
-    gpt3_out1=gpt3(input_text,'',cap)
-    return out[0], gpt3_out,gpt3_out1
+    gpt3_out1=gpt3_short(input_text,out,cap)
+    return out[0][0], gpt3_out,gpt3_out1
 title = """<h1 align="center">VQA</h1>"""
 with gr.Blocks(
     css="""
@@ -105,12 +124,12 @@ with gr.Blocks(
             gpt3_output_v1 = gr.Textbox(lines=0, label="VQA+LLM (long answer)")
             
             
-        image_input.change(
-            lambda: ("", [],"","",""),
-            [],
-            [ caption_output, state,caption_output,gpt3_output_v1,caption_output_v1],
-            queue=False,
-        )
+        # image_input.change(
+        #     lambda: ("", [],"","",""),
+        #     [],
+        #     [ caption_output, state,caption_output,gpt3_output_v1,caption_output_v1],
+        #     queue=False,
+        # )
         chat_input.submit(
                     inference_chat,
                     [