eval_exp.py

import json
import os
from copy import deepcopy
from pathlib import Path

import numpy as np

from chinatravel.data.load_datasets import load_query
from chinatravel.evaluation.commonsense_constraint import (
    evaluate_commonsense_constraints,
)
from chinatravel.evaluation.hard_constraint import evaluate_hard_constraints_v2
from chinatravel.evaluation.schema_constraint import evaluate_schema_constraints
from chinatravel.evaluation.utils import load_json_file
from chinatravel.symbol_verification.concept_func import func_dict
from chinatravel.symbol_verification.dsl import execute_dsl_code

EVALUATOR_REVISION = "LAMDA-NeSy/ChinaTravel@933744f"
SANDBOX_REVISION = "LAMDA-NeSy/ChinaTravel-Sandbox@3cd0d8f"
PROJECT_ROOT = Path(__file__).resolve().parent

DEFAULT_ATTRACTION_PREFERENCE = """
attraction_count = 0
for activity in allactivities(plan):
    if activity_type(activity) == 'attraction':
        attraction_count += 1
result=attraction_count/(4*day_count(plan))
"""

DEFAULT_TRANSPORT_PREFERENCE = """
time_cost = 0
transport_count = 0
for activity in allactivities(plan):
    transports = activity_transports(activity)
    if transports!=[]:
        transport_count += 1
        time_cost += innercity_transport_time(transports)
if transport_count > 0:
    average_time_cost = time_cost / transport_count
    result= (-1/105) * average_time_cost + 8/7
else:
    result=0
"""

DEFAULT_RESTAURANT_PREFERENCE = """
res_count=0
for activity in allactivities(plan):
    if activity_type(activity) in ['breakfast', 'lunch', 'dinner']:
        res_count+=1
res_count=res_count/(day_count(plan))
result=res_count/3
"""

DEFAULT_PREFERENCES = (
    DEFAULT_ATTRACTION_PREFERENCE,
    DEFAULT_TRANSPORT_PREFERENCE,
    DEFAULT_RESTAURANT_PREFERENCE,
)


def load_result(result_dir, query_index):
    json_files = {
        os.path.splitext(filename)[0]: os.path.join(root, filename)
        for root, _, filenames in os.walk(result_dir)
        for filename in filenames
        if filename.endswith(".json")
    }
    plans = {}
    matched_uid = []
    unmatched_uid = []
    for query_id in query_index:
        result_file = json_files.get(query_id)
        if result_file is None:
            plans[query_id] = {}
            unmatched_uid.append(query_id)
            continue
        try:
            plans[query_id] = load_json_file(result_file)
            matched_uid.append(query_id)
        except (OSError, ValueError, TypeError):
            plans[query_id] = {}
            unmatched_uid.append(query_id)
    return plans, matched_uid, unmatched_uid


def _default_preference_scores(query_index, result_data, all_pass_id):
    all_pass_id = set(all_pass_id)
    all_scores = []
    for query_id in query_index:
        if query_id not in all_pass_id:
            all_scores.append(np.zeros(len(DEFAULT_PREFERENCES)))
            continue

        plan = result_data[query_id]
        scores = []
        for constraint in DEFAULT_PREFERENCES:
            variables = deepcopy(func_dict)
            variables["plan"] = plan
            try:
                execute_dsl_code(
                    constraint,
                    variables,
                    allowed_builtins={"set": set},
                )
                value = float(variables.get("result", 0.0))
                scores.append(max(0.0, min(1.0, value)))
            except Exception:  # noqa: BLE001 - invalid preference code scores zero
                scores.append(0.0)
        all_scores.append(np.asarray(scores))

    if not all_scores:
        return np.zeros(len(DEFAULT_PREFERENCES))
    return np.mean(all_scores, axis=0)


def _frame_records(frame):
    return json.loads(frame.to_json(orient="records", force_ascii=False))


def evaluate(args, _result=None):
    query_index, query_data = load_query(args)
    result_data, matched_uid, unmatched_uid = load_result(
        args.result_dir,
        query_index,
    )

    result = {
        "split": args.splits,
        "language": args.lang,
        "evaluator_revision": EVALUATOR_REVISION,
        "sandbox_revision": SANDBOX_REVISION,
        "matched_uid": matched_uid,
        "unmatched_uid": unmatched_uid,
    }

    schema = load_json_file(
        PROJECT_ROOT / "chinatravel/evaluation/output_schema.json"
    )
    yield {"stage": "schema", "progress": 0}
    schema_rate, schema_details, schema_pass_id = evaluate_schema_constraints(
        query_index,
        result_data,
        schema=schema,
    )
    result["DR"] = schema_rate
    yield {"stage": "schema", "progress": 100}

    yield {"stage": "commonsense", "progress": 0}
    (
        epr_macro,
        epr_micro,
        commonsense_details,
        commonsense_pass_id,
    ) = evaluate_commonsense_constraints(
        query_index,
        query_data,
        result_data,
        verbose=False,
        lang=args.lang,
    )
    result["EPR_micro"] = epr_micro
    result["EPR_macro"] = epr_macro
    yield {"stage": "commonsense", "progress": 100}

    yield {"stage": "logic", "progress": 0}
    (
        lpr_macro,
        lpr_micro,
        conditional_lpr_macro,
        conditional_lpr_micro,
        logic_details,
        logic_pass_id,
    ) = evaluate_hard_constraints_v2(
        query_index,
        query_data,
        result_data,
        env_pass_id=commonsense_pass_id,
        verbose=False,
        lang=args.lang,
    )
    result["LPR_micro"] = lpr_micro
    result["LPR_macro"] = lpr_macro
    result["C-LPR"] = conditional_lpr_micro
    result["C-LPR_macro"] = conditional_lpr_macro
    yield {"stage": "logic", "progress": 100}

    all_pass_id = sorted(
        set(schema_pass_id)
        & set(commonsense_pass_id)
        & set(logic_pass_id)
    )
    fpr = len(all_pass_id) / len(query_index) * 100
    dav, att, ddr = _default_preference_scores(
        query_index,
        result_data,
        all_pass_id,
    ) * 100
    overall = (
        0.1 * epr_micro
        + 0.1 * epr_macro
        + 0.25 * conditional_lpr_micro
        + 0.05 * dav
        + 0.05 * att
        + 0.05 * ddr
        + 0.4 * fpr
    )
    result.update(
        {
            "FPR": fpr,
            "DAV": dav,
            "ATT": att,
            "DDR": ddr,
            "overall": overall,
            "all_pass_uid": all_pass_id,
            "details": {
                "schema": _frame_records(schema_details),
                "commonsense": _frame_records(commonsense_details),
                "logic": _frame_records(logic_details),
            },
        }
    )
    yield {"stage": "final", "progress": 100, "result": result}