299 lines
15 KiB
Python
299 lines
15 KiB
Python
from __future__ import annotations
|
||
|
||
import argparse
|
||
import csv
|
||
import json
|
||
import sys
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import numpy as np
|
||
import torch
|
||
|
||
from .data import RobustStats, apply_robust_stats, load_aligned
|
||
from .mofe import EXPERT_NAMES
|
||
from .mofe import MixtureOfFusionExperts
|
||
from .train_mofe import (
|
||
MOFE7_MLP,
|
||
SEEDS,
|
||
_conditions,
|
||
_metric_dict,
|
||
_predict,
|
||
_sha256,
|
||
_write_csv,
|
||
)
|
||
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
REFERENCE_DIR = ROOT / "outputs" / "mofe_7experts"
|
||
DEFAULT_DIAGNOSTIC_OUTPUT = ROOT / "outputs" / "followups" / "D0_task_preference"
|
||
MODEL_CONFIG: dict[str, Any] = {
|
||
"router": "mlp",
|
||
"expert_names": EXPERT_NAMES,
|
||
"availability_mode": "hard",
|
||
}
|
||
|
||
|
||
def _load_model(seed: int, dims: tuple[int, int, int], device: torch.device) -> MixtureOfFusionExperts:
|
||
checkpoint = REFERENCE_DIR / "models" / MOFE7_MLP / f"seed_{seed}" / "model_best.pt"
|
||
saved = torch.load(checkpoint, map_location=device, weights_only=False)
|
||
if saved.get("config") != MODEL_CONFIG or tuple(saved.get("dims", ())) != dims:
|
||
raise ValueError(f"checkpoint does not match the selected single-router model: {checkpoint}")
|
||
if int(saved.get("seed", -1)) != seed:
|
||
raise ValueError(f"checkpoint seed mismatch: expected {seed}, found {saved.get('seed')}")
|
||
model = MixtureOfFusionExperts(dims=dims, **MODEL_CONFIG).to(device)
|
||
model.load_state_dict(saved["state_dict"])
|
||
return model.eval()
|
||
|
||
|
||
def _rank_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
groups: dict[tuple[int, str], list[dict[str, Any]]] = {}
|
||
for row in rows:
|
||
if row["expert"] == "learned_router":
|
||
continue
|
||
groups.setdefault((int(row["seed"]), str(row["condition"])), []).append(row)
|
||
|
||
result: list[dict[str, Any]] = []
|
||
for (seed, condition), values in sorted(groups.items()):
|
||
by_name = {str(row["expert"]): row for row in values}
|
||
ordered = [by_name[name] for name in EXPERT_NAMES]
|
||
f1 = [float(row["macro_f1"]) for row in ordered]
|
||
mae = [float(row["mae"]) for row in ordered]
|
||
pearson = [float(row["pearson"]) for row in ordered]
|
||
result.append({
|
||
"seed": seed,
|
||
"condition": condition,
|
||
"spearman_macro_f1_vs_mae": _spearman(f1, mae),
|
||
"spearman_macro_f1_vs_pearson": _spearman(f1, pearson),
|
||
"best_macro_f1_expert": EXPERT_NAMES[int(np.argmax(f1))],
|
||
"best_mae_expert": EXPERT_NAMES[int(np.argmin(mae))],
|
||
"best_pearson_expert": EXPERT_NAMES[int(np.argmax(pearson))],
|
||
"macro_f1_order_best_to_worst": ">".join(EXPERT_NAMES[i] for i in np.argsort(-np.asarray(f1), kind="stable")),
|
||
"mae_order_best_to_worst": ">".join(EXPERT_NAMES[i] for i in np.argsort(np.asarray(mae), kind="stable")),
|
||
"pearson_order_best_to_worst": ">".join(EXPERT_NAMES[i] for i in np.argsort(-np.asarray(pearson), kind="stable")),
|
||
})
|
||
return result
|
||
|
||
|
||
def _spearman(left: list[float], right: list[float]) -> float:
|
||
def average_ranks(values: list[float]) -> np.ndarray:
|
||
array = np.asarray(values, dtype=np.float64)
|
||
order = np.argsort(array, kind="stable")
|
||
ranks = np.empty(len(array), dtype=np.float64)
|
||
start = 0
|
||
while start < len(array):
|
||
end = start + 1
|
||
while end < len(array) and array[order[end]] == array[order[start]]:
|
||
end += 1
|
||
ranks[order[start:end]] = (start + 1 + end) / 2
|
||
start = end
|
||
return ranks
|
||
|
||
left_ranks = average_ranks(left)
|
||
right_ranks = average_ranks(right)
|
||
if np.std(left_ranks) == 0 or np.std(right_ranks) == 0:
|
||
return 0.0
|
||
return float(np.corrcoef(left_ranks, right_ranks)[0, 1])
|
||
|
||
|
||
def _summary_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
result = []
|
||
for expert in (*EXPERT_NAMES, "learned_router"):
|
||
matching = [row for row in rows if row["expert"] == expert and row["condition"] != "clean"]
|
||
per_seed: dict[int, list[dict[str, Any]]] = {}
|
||
for row in matching:
|
||
per_seed.setdefault(int(row["seed"]), []).append(row)
|
||
seed_means = []
|
||
for seed, seed_rows in sorted(per_seed.items()):
|
||
seed_means.append({
|
||
metric: float(np.mean([float(row[metric]) for row in seed_rows]))
|
||
for metric in ("macro_f1", "mae", "pearson", "available_position_fraction")
|
||
})
|
||
if not seed_means:
|
||
continue
|
||
out: dict[str, Any] = {"expert": expert, "n_seeds": len(seed_means), "conditions_averaged": len(matching) // len(seed_means)}
|
||
for metric in ("macro_f1", "mae", "pearson", "available_position_fraction"):
|
||
values = [item[metric] for item in seed_means]
|
||
out[f"corrupt_{metric}_mean"] = float(np.mean(values))
|
||
out[f"corrupt_{metric}_seed_sd"] = float(np.std(values, ddof=1)) if len(values) > 1 else 0.0
|
||
result.append(out)
|
||
return result
|
||
|
||
|
||
def _write_readout(output: Path, summary: list[dict[str, Any]], ranks: list[dict[str, Any]]) -> None:
|
||
rho_f1_mae = np.asarray([float(row["spearman_macro_f1_vs_mae"]) for row in ranks])
|
||
rho_f1_pearson = np.asarray([float(row["spearman_macro_f1_vs_pearson"]) for row in ranks])
|
||
best_f1 = {name: sum(row["best_macro_f1_expert"] == name for row in ranks) for name in EXPERT_NAMES}
|
||
best_mae = {name: sum(row["best_mae_expert"] == name for row in ranks) for name in EXPERT_NAMES}
|
||
best_pearson = {name: sum(row["best_pearson_expert"] == name for row in ranks) for name in EXPERT_NAMES}
|
||
rank_by_condition: dict[str, list[dict[str, Any]]] = {}
|
||
for row in ranks:
|
||
rank_by_condition.setdefault(str(row["condition"]), []).append(row)
|
||
|
||
def winners(condition: str, column: str) -> str:
|
||
matching = rank_by_condition[condition]
|
||
counts = {name: sum(row[column] == name for row in matching) for name in EXPERT_NAMES}
|
||
max_count = max(counts.values())
|
||
names = [name for name, count in counts.items() if count == max_count]
|
||
return ", ".join(f"{name} ({max_count}/{len(matching)})" for name in names)
|
||
|
||
by_name = {str(row["expert"]): row for row in summary}
|
||
lines = [
|
||
"# Single-router MoFE 任务偏好诊断",
|
||
"",
|
||
"本诊断使用保留的 single-router 检查点和验证集，用于判断是否值得增加第二个 router；它不是测试集估计。",
|
||
"",
|
||
"## Forced-expert 规则",
|
||
"",
|
||
"所选模态子集可用的位置强制使用对应 expert；该子集不可用时，由已训练 router 在其他可用 expert 中选择；全模态缺失时沿用 learned missing token。可用率表示所选 expert 能被强制使用的位置比例。",
|
||
"",
|
||
"## 缺失条件平均指标",
|
||
"",
|
||
"下表先在每个 seed 内对 15 种连续块缺失条件求平均，再汇总三个 seed；seed 标准差见 CSV。",
|
||
"",
|
||
"| Expert | Macro-F1 ↑ | MAE ↓ | Pearson ↑ | 可强制使用比例 |",
|
||
"| --- | ---: | ---: | ---: | ---: |",
|
||
]
|
||
for name in (*EXPERT_NAMES, "learned_router"):
|
||
row = by_name[name]
|
||
lines.append(
|
||
f"| {name} | {float(row['corrupt_macro_f1_mean']):.3f} | {float(row['corrupt_mae_mean']):.3f} | "
|
||
f"{float(row['corrupt_pearson_mean']):.3f} | {float(row['corrupt_available_position_fraction_mean']):.3f} |"
|
||
)
|
||
lines.extend([
|
||
"",
|
||
"## 两个任务的 expert 偏好",
|
||
"",
|
||
f"在 {len(ranks)} 个 seed—条件组合中，分类 Macro-F1 与回归 MAE 的平均 Spearman ρ 为 **{rho_f1_mae.mean():.3f}**。MAE 越低越好，因此负相关表示两个指标倾向于选中相似的 expert。Macro-F1 与 Pearson 的平均 ρ 为 **{rho_f1_pearson.mean():.3f}**。",
|
||
"",
|
||
"六个重点条件下的相关性先按三个 seed 求平均；最优 expert 一栏显示三个 seed 中的多数结果：",
|
||
"",
|
||
"| 条件 | ρ(Macro-F1, MAE) | ρ(Macro-F1, Pearson) | Macro-F1 最优 | MAE 最优 | Pearson 最优 |",
|
||
"| --- | ---: | ---: | --- | --- | --- |",
|
||
])
|
||
key_conditions = (
|
||
("clean", "Clean"),
|
||
("text_30", "Text 30%"),
|
||
("audio_30", "Audio 30%"),
|
||
("vision_30", "Vision 30%"),
|
||
("audio_vision_30", "Audio+Vision 30%"),
|
||
("all_modalities_30", "All-modal 30%"),
|
||
)
|
||
for condition, label in key_conditions:
|
||
condition_rows = rank_by_condition[condition]
|
||
rho_mae = float(np.mean([float(row["spearman_macro_f1_vs_mae"]) for row in condition_rows]))
|
||
rho_pearson = float(np.mean([float(row["spearman_macro_f1_vs_pearson"]) for row in condition_rows]))
|
||
lines.append(
|
||
f"| {label} | {rho_mae:.3f} | {rho_pearson:.3f} | "
|
||
f"{winners(condition, 'best_macro_f1_expert')} | {winners(condition, 'best_mae_expert')} | "
|
||
f"{winners(condition, 'best_pearson_expert')} |"
|
||
)
|
||
lines.extend([
|
||
"",
|
||
f"各指标的最优 expert 次数：Macro-F1（{_format_counts(best_f1)}）；MAE（{_format_counts(best_mae)}）；Pearson（{_format_counts(best_pearson)}）。",
|
||
"",
|
||
"当前排名没有显示稳定的分类—回归 expert 分工：Macro-F1 较高通常同时对应较低 MAE 和较高 Pearson；文本 expert 在分类与回归指标上都是最常见的赢家。因此，这项诊断**没有提供增加第二个 router 所需的任务特异模态偏好证据**。目前保留 single-router 作为活动参照；这不代表两个任务在任何数据或设置下都不可能需要不同路由。",
|
||
"",
|
||
"## 结论范围",
|
||
"",
|
||
"输入是官方提供的 50 个有序 wordpiece 位置。结果只反映这些位置及本次缺失掩码下的任务与 expert 关系，不表示物理时间可靠性。",
|
||
"",
|
||
"逐条件结果见 `forced_expert_metrics.csv` 和 `rank_concordance.csv`；跨 seed 汇总见 `expert_task_preference_summary.csv`。",
|
||
])
|
||
(output / "task_preference_diagnostic.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||
|
||
|
||
def _format_counts(counts: dict[str, int]) -> str:
|
||
return ", ".join(f"{name}: {count}" for name, count in counts.items())
|
||
|
||
|
||
def run(args: argparse.Namespace) -> None:
|
||
output = args.output_dir.resolve()
|
||
output.mkdir(parents=True, exist_ok=True)
|
||
device = torch.device("cuda" if args.device == "auto" and torch.cuda.is_available() else "cpu") if args.device == "auto" else torch.device(args.device)
|
||
torch.set_num_threads(args.threads)
|
||
|
||
raw = load_aligned()
|
||
scaler_path = REFERENCE_DIR / "aligned_robust_stats.npz"
|
||
stats = RobustStats.load(scaler_path)
|
||
valid = apply_robust_stats(raw["valid"], stats)
|
||
dims = tuple(int(x.shape[-1]) for x in valid.x)
|
||
|
||
metric_rows: list[dict[str, Any]] = []
|
||
for seed in args.seeds:
|
||
model = _load_model(seed, dims, device)
|
||
conditions = _conditions(valid, seed)
|
||
for condition, rate, masks in conditions:
|
||
predictions = {
|
||
"learned_router": _predict(model, valid, masks, device, args.batch_size),
|
||
**{
|
||
expert: _predict(model, valid, masks, device, args.batch_size, force_expert=expert)
|
||
for expert in EXPERT_NAMES
|
||
},
|
||
}
|
||
for expert, prediction in predictions.items():
|
||
coverage = 1.0
|
||
if expert != "learned_router":
|
||
expert_index = EXPERT_NAMES.index(expert)
|
||
coverage = float(model._availability(torch.as_tensor(masks, dtype=torch.bool, device=device), EXPERT_NAMES)[..., expert_index].float().mean().item())
|
||
else:
|
||
coverage = float(prediction["availability"].astype(bool).any(axis=-1).mean())
|
||
metric_rows.append({
|
||
"method": MOFE7_MLP,
|
||
"seed": seed,
|
||
"condition": condition,
|
||
"missing_rate": rate,
|
||
"expert": expert,
|
||
"n_valid": valid.n,
|
||
"available_position_fraction": coverage,
|
||
**_metric_dict(valid.y_cls, valid.y_reg, prediction["logits"], prediction["intensity"]),
|
||
})
|
||
print(f"forced-expert diagnostic complete for seed={seed} on {device}", flush=True)
|
||
del model
|
||
if torch.cuda.is_available():
|
||
torch.cuda.empty_cache()
|
||
|
||
rank_rows = _rank_rows(metric_rows)
|
||
summary = _summary_rows(metric_rows)
|
||
_write_csv(output / "forced_expert_metrics.csv", metric_rows)
|
||
_write_csv(output / "expert_task_preference_summary.csv", summary)
|
||
_write_csv(output / "rank_concordance.csv", rank_rows)
|
||
_write_readout(output, summary, rank_rows)
|
||
feature_path = ROOT.parents[1] / "E题数据" / "附件2-数据集特征文件" / "aligned_50.pkl"
|
||
metadata = {
|
||
"diagnostic": "forced-expert task preference for the retained single-router MoFE-7",
|
||
"checkpoint_dir": str(REFERENCE_DIR / "models" / MOFE7_MLP),
|
||
"checkpoint_seeds": list(args.seeds),
|
||
"feature_file": str(feature_path),
|
||
"feature_sha256": _sha256(feature_path),
|
||
"scaler_file": str(scaler_path),
|
||
"scaler_sha256": _sha256(scaler_path),
|
||
"device": str(device),
|
||
"cuda_device": torch.cuda.get_device_name(0) if device.type == "cuda" else None,
|
||
"python_version": sys.version,
|
||
"torch_version": torch.__version__,
|
||
"numpy_version": np.__version__,
|
||
"valid_examples": valid.n,
|
||
"conditions": [condition for condition, _, _ in _conditions(valid, args.seeds[0])],
|
||
"corruption_seed_protocol": "seed + 13 + pattern_index*101 + int(rate*1000)",
|
||
"forced_expert_policy": "use the requested expert where its modality subset is available; fall back to the trained single router at positions where it is unavailable; all-missing positions use the learned missing token",
|
||
"scope_note": "50 official ordered wordpiece positions; no claim about physical-time reliability",
|
||
"interpretation_note": "Ranking agreement is descriptive on the supplied validation split; it is a motivation diagnostic, not an unbiased test-set estimate.",
|
||
}
|
||
(output / "run_manifest.json").write_text(json.dumps(metadata, indent=2, ensure_ascii=False), encoding="utf-8")
|
||
print(f"saved forced-expert diagnostic to {output}", flush=True)
|
||
|
||
|
||
def main() -> None:
|
||
parser = argparse.ArgumentParser(description="Measure classification/regression preferences across existing MoFE experts.")
|
||
parser.add_argument("--seeds", type=int, nargs="+", default=list(SEEDS))
|
||
parser.add_argument("--batch-size", type=int, default=128)
|
||
parser.add_argument("--threads", type=int, default=4)
|
||
parser.add_argument("--device", default="auto")
|
||
parser.add_argument("--output-dir", type=Path, default=DEFAULT_DIAGNOSTIC_OUTPUT)
|
||
run(parser.parse_args())
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|