evaluate.py

# -*- coding: utf-8 -*-
"""Untitled4.ipynb

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/1eV4-E7VDAHekOoakymatKd3grGwUSTAC
"""

import numpy as np
from sklearn.metrics import precision_recall_fscore_support

def evaluate_similar_case_retrieval(model, test_cases):
    results = {
        'precision': [],
        'recall': [],
        'f1': []
    }

    for case in test_cases:
        retrieved = model.retrieve_similar_cases(case)
        metrics = precision_recall_fscore_support(
            case.relevant,
            retrieved,
            average='binary'
        )
        results['precision'].append(metrics[0])
        results['recall'].append(metrics[1])
        results['f1'].append(metrics[2])

    return {k: np.mean(v) for k, v in results.items()}

def evaluate_precedent_recommendation(model, test_cases):
    correct = 0
    total = len(test_cases)

    for case in test_cases:
        recommendation = model.recommend_precedent(case)
        if recommendation == case.correct_precedent:
            correct += 1

    return {'accuracy': correct/total}

def evaluate_legal_judgment(model, test_cases):
    predictions = []
    actuals = []

    for case in test_cases:
        pred = model.predict_judgment(case)
        predictions.append(pred)
        actuals.append(case.actual_judgment)

    return {
        'judgment_accuracy': sum(p == a for p, a in zip(predictions, actuals)) / len(predictions)
    }

def run_full_evaluation():
    test_cases = load_test_cases()

    results = {
        'retrieval_metrics': evaluate_similar_case_retrieval(model, test_cases),
        'precedent_metrics': evaluate_precedent_recommendation(model, test_cases),
        'judgment_metrics': evaluate_legal_judgment(model, test_cases)
    }

    print("Evaluation Results:")
    print(json.dumps(results, indent=2))
    return results