eval_scripts/metrics.py

import sys
sys.path.append('.')

import json
import pandas as pd
import csv
from sentence_transformers import SentenceTransformer, util
from minigpt4.common.eval_utils import computeIoU

# Load pre-trained BERT model
model = SentenceTransformer('paraphrase-MiniLM-L6-v2')


# BERT similarity function will be utilized in the two following functions
def compute_bert_similarity(prediction_caption, ground_truth_caption):
    prediction_embedding = model.encode([prediction_caption])
    ground_truth_embedding = model.encode([ground_truth_caption])
    similarity = util.pytorch_cos_sim(prediction_embedding, ground_truth_embedding)[0][0].item()
    return similarity


def MIMIC_BERT_Sim(gt_pth, pred_pth, output_csv):
    # Read the ground truth and prediction JSON files
    with open(gt_pth, 'r') as f:
        ground_truth_data = json.load(f)
    
    with open(pred_pth, 'r') as f:
        prediction_data = json.load(f)
    
    # Create a list to store BERT similarity data
    bert_similarity_data = []
    
    # Initialize variables to calculate the average
    total_similarity = 0
    total_count = 0
    
    # Iterate over each item in the prediction_data list
    for item in prediction_data:
        # Extract the image_id and corresponding prediction caption
        image_id = item["image_id"]
        prediction_caption = item["caption"]
        
        # Search for the matching ground truth caption based on image_id
        ground_truth_caption = None
        for gt_item in ground_truth_data:
            if gt_item["image_id"] == image_id:
                ground_truth_caption = gt_item["caption"]
                break
        
        if ground_truth_caption is not None:
            bert_similarity = compute_bert_similarity(prediction_caption, ground_truth_caption)
            bert_similarity_data.append({"image_id": image_id, "BERT_score": bert_similarity})
            
            total_similarity += bert_similarity
            total_count += 1
    
    average_similarity = total_similarity / total_count if total_count > 0 else 0
    
    df = pd.DataFrame(bert_similarity_data)
    df_sorted = df.sort_values(by="BERT_score", ascending=True)
    df_sorted.to_csv(output_csv, index=False)
    
    return average_similarity

def VQA_BERT_Sim(gt_pth, pred_pth, output_csv):
    # Load ground truth JSON file
    with open(gt_pth, 'r') as file:
        gt_data = json.load(file)

    # Load prediction JSON file
    with open(pred_pth, 'r') as file:
        prediction_data = json.load(file)

    gt_qa_pairs = {(entry['image_name'], entry['question']): entry['answer'] for entry in gt_data}

    def convert_to_dict(data):
        qa_dict = {}
        for image_name, qa_list in data.items():
            for qa in qa_list:
                key = (image_name, qa['question'])
                qa_dict[key] = qa['answer']
        return qa_dict

    pred_qa_dict = convert_to_dict(prediction_data)

    # Compute BERT similarity and create a list of results
    results = []

    for key, gt_answer in gt_qa_pairs.items():
        if key in pred_qa_dict:
            pred_answer = pred_qa_dict[key]
            gt_answer = str(gt_answer)
            pred_answer = str(pred_answer)

            # Compute BERT similarity
            similarity_score = compute_bert_similarity(pred_answer, gt_answer)

            # Append the result to the list
            results.append({
                "img_name": key[0],
                "question": key[1],
                "answer": pred_answer,
                "BERT_score": similarity_score
            })

    average_similarity = sum(entry["BERT_score"] for entry in results) / len(results) if results else 0
    df = pd.DataFrame(results)
    df_sorted = df.sort_values(by="BERT_score", ascending=True)
    df_sorted.to_csv(output_csv, index=False)
    print(f"Average BERT similarity score: {average_similarity}")


#################################
##############IoU################
#################################

def preprocess_bbox(bbox, original_size, image_size):
    x1 = int((bbox[0] / original_size) * image_size)
    y1 = int((bbox[1] / original_size) * image_size)
    x2 = int((bbox[2] / original_size) * image_size)
    y2 = int((bbox[3] / original_size) * image_size)
    return [x1, y1, x2, y2]

def average_iou(gt_pth, pred_pth, original_size, image_size, dataset_name, csv_filename):
    # Load ground truth
    with open(gt_pth, 'r') as file:
        ground_truth = json.load(file)

    # Load predictions
    with open(pred_pth, 'r') as file:
        predictions = json.load(file)

    iou_list = []

    with open(csv_filename, 'w', newline='') as csvfile:
        fieldnames = ['image_name', 'IoU']
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        writer.writeheader()

        for gt_item in ground_truth:
            gt_key = gt_item['key']
            gt_bboxes = gt_item['bbox']
            original_size = gt_item['height']
            gt_processed_bboxes = [preprocess_bbox(bbox, original_size, image_size) for bbox in gt_bboxes]

            for pred_item in predictions:
                pred_key = pred_item['key'].replace(".png", "")

                if gt_key == pred_key:
                    pred_bboxes = pred_item['bbox']
                    try:
                        for gt_bbox in gt_processed_bboxes:
                            for pred_bbox in pred_bboxes:
                                iou = computeIoU(gt_bbox, pred_bbox)
                                iou_list.append(iou)
                                writer.writerow({'image_name': gt_key, 'IoU': iou})
                                print(gt_key)
                                print(iou)
                    except Exception as e:
                        print("gt_bbox: ", gt_bbox)
                        print("gt_bbox: ", pred_bboxes)

    # average_iou = sum(iou_list) / len(iou_list)
    # print(f"Average IoU for dataset {dataset_name}: {average_iou:.4f}")