1011 lines
54 KiB
Python
1011 lines
54 KiB
Python
"""Shared-time/private-content experiment for Q1.
|
||
|
||
The frozen M4_sourceTime checkpoints provide a common temporal coordinate.
|
||
Text, audio, and vision feature vectors remain modality-specific. Optional
|
||
same-slot products/differences are task input features, not a shared semantic
|
||
space. All learned PCA transforms and emotion probes are fitted inside each
|
||
training fold.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import csv
|
||
import json
|
||
import platform
|
||
import random
|
||
import time
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
from typing import Any, Mapping, Sequence
|
||
|
||
import matplotlib
|
||
|
||
matplotlib.use("Agg")
|
||
import matplotlib.pyplot as plt
|
||
import numpy as np
|
||
import sklearn
|
||
import torch
|
||
from sklearn.decomposition import PCA
|
||
from sklearn.linear_model import LogisticRegression, Ridge
|
||
from sklearn.metrics import accuracy_score, f1_score
|
||
from sklearn.preprocessing import StandardScaler
|
||
|
||
from .correspondence_eval import _write_csv
|
||
from .experiment_data import (
|
||
FeatureSample,
|
||
fit_feature_stats,
|
||
load_feature_samples,
|
||
)
|
||
from .m4_shared_latent_eval import _normalize_attention, _pair_metrics
|
||
from .tsfa_emotion_probe import CLASS_NAMES, _class_from_sentiment, _pearson
|
||
from .tsfa_experiment import GRID_SIZE, _collect_fold_features
|
||
from .tsfa_shared_private import _pool_five, _pool_original_source
|
||
from .types import MODALITIES
|
||
|
||
|
||
MODS = tuple(MODALITIES)
|
||
PAIRINGS = (("text", "audio"), ("text", "vision"), ("audio", "vision"))
|
||
OUTPUT_NAME = "shared_time_private_content"
|
||
SLOT_PCA_SIZE = 256
|
||
MODAL_PCA_DIMS = {"text": 64, "audio": 24, "vision": 64}
|
||
INTERACTION_DIM = 24
|
||
CLASS_SIGNS = np.asarray([-1.0, 0.0, 1.0], dtype=np.float64)
|
||
|
||
|
||
def _fit_pca(train_values: Sequence[np.ndarray], input_dim: int, seed: int) -> PCA:
|
||
stacked = np.concatenate([np.asarray(value, dtype=np.float32).reshape(-1, input_dim) for value in train_values])
|
||
n_components = min(SLOT_PCA_SIZE, input_dim, len(stacked) - 1)
|
||
if n_components < 1:
|
||
raise ValueError("not enough training slots to fit a PCA projection")
|
||
return PCA(n_components=n_components, svd_solver="auto", random_state=seed).fit(stacked)
|
||
|
||
|
||
def _transform_padded(values: np.ndarray, pca: PCA, size: int = SLOT_PCA_SIZE) -> np.ndarray:
|
||
transformed = pca.transform(np.asarray(values, dtype=np.float32).reshape(-1, values.shape[-1]))
|
||
out = np.zeros((len(transformed), size), dtype=np.float32)
|
||
out[:, : transformed.shape[1]] = transformed.astype(np.float32, copy=False)
|
||
return out.reshape(*values.shape[:-1], size)
|
||
|
||
|
||
def _fit_transform_map(
|
||
train_ids: Sequence[str],
|
||
values_by_id: Mapping[str, np.ndarray],
|
||
*,
|
||
seed: int,
|
||
requested_dim: int = SLOT_PCA_SIZE,
|
||
) -> tuple[PCA, dict[str, np.ndarray]]:
|
||
input_dim = int(next(iter(values_by_id.values())).shape[-1])
|
||
pca = _fit_pca([values_by_id[sid] for sid in train_ids], input_dim, seed)
|
||
transformed = {sid: _transform_padded(value, pca, requested_dim) for sid, value in values_by_id.items()}
|
||
return pca, transformed
|
||
|
||
|
||
def _flatten_five(values: np.ndarray) -> np.ndarray:
|
||
return _pool_five(values)
|
||
|
||
|
||
def _pearson_safe(actual: np.ndarray, predicted: np.ndarray) -> float:
|
||
value = _pearson(np.asarray(actual, dtype=np.float64), np.asarray(predicted, dtype=np.float64))
|
||
return float(value) if np.isfinite(value) else 0.0
|
||
|
||
|
||
def _metric_values(rows: Sequence[Mapping[str, Any]], class_key: str, value_key: str) -> dict[str, float]:
|
||
true_class = np.asarray([int(row["true_class_id"]) for row in rows], dtype=np.int64)
|
||
predicted_class = np.asarray([int(row[class_key]) for row in rows], dtype=np.int64)
|
||
actual = np.asarray([float(row["true_label"]) for row in rows], dtype=np.float64)
|
||
predicted = np.asarray([float(row[value_key]) for row in rows], dtype=np.float64)
|
||
return {
|
||
"accuracy": float(accuracy_score(true_class, predicted_class)),
|
||
"macro_f1": float(f1_score(true_class, predicted_class, labels=[0, 1, 2], average="macro", zero_division=0)),
|
||
"mae": float(np.mean(np.abs(actual - predicted))),
|
||
"pearson": _pearson_safe(actual, predicted),
|
||
}
|
||
|
||
|
||
def _training_probe(train_x: np.ndarray, train_class: np.ndarray, train_y: np.ndarray, seed: int):
|
||
scaler_cls = StandardScaler()
|
||
train_class_x = scaler_cls.fit_transform(train_x)
|
||
classifier = LogisticRegression(C=0.05, max_iter=5000, solver="lbfgs", random_state=seed)
|
||
classifier.fit(train_class_x, train_class)
|
||
|
||
scaler_direct = StandardScaler()
|
||
train_direct_x = scaler_direct.fit_transform(train_x)
|
||
direct = Ridge(alpha=25.0).fit(train_direct_x, train_y)
|
||
|
||
scaler_intensity = StandardScaler()
|
||
train_intensity_x = scaler_intensity.fit_transform(train_x)
|
||
intensity = Ridge(alpha=25.0).fit(train_intensity_x, np.abs(train_y))
|
||
return (scaler_cls, classifier), (scaler_direct, direct), (scaler_intensity, intensity)
|
||
|
||
|
||
def _predict_probe(probe, values: np.ndarray) -> np.ndarray:
|
||
scaler, model = probe
|
||
return model.predict(scaler.transform(values))
|
||
|
||
|
||
def _build_interaction_slots(
|
||
by_id: Mapping[str, Mapping[str, np.ndarray]],
|
||
included_pairs: Sequence[tuple[str, str]],
|
||
) -> dict[str, np.ndarray]:
|
||
result = {}
|
||
for sid, modalities in by_id.items():
|
||
base = np.concatenate([modalities[name] for name in MODS], axis=-1)
|
||
blocks = [base]
|
||
for left, right in included_pairs:
|
||
left_value = modalities[left][:, :INTERACTION_DIM]
|
||
right_value = modalities[right][:, :INTERACTION_DIM]
|
||
blocks.append(np.concatenate([left_value * right_value, np.abs(left_value - right_value)], axis=-1))
|
||
result[sid] = np.concatenate(blocks, axis=-1).astype(np.float32, copy=False)
|
||
return result
|
||
|
||
|
||
def _interaction_sets() -> dict[str, tuple[tuple[str, str], ...]]:
|
||
pairs = {"TA": PAIRINGS[0], "TV": PAIRINGS[1], "AV": PAIRINGS[2]}
|
||
names = ("TA", "TV", "AV")
|
||
return {f"I{index}": tuple(pairs[name] for bit, name in enumerate(names) if index & (1 << bit)) for index in range(8)}
|
||
|
||
|
||
def _make_fold_representations(
|
||
train_ids: Sequence[str],
|
||
pooled: Mapping[str, Mapping[str, np.ndarray]],
|
||
seed: int,
|
||
) -> tuple[dict[str, dict[str, np.ndarray]], list[dict[str, Any]], dict[str, Any]]:
|
||
ids = list(pooled)
|
||
raw_by_id = {
|
||
sid: np.concatenate([pooled[sid][name] for name in MODS], axis=-1).astype(np.float32, copy=False)
|
||
for sid in ids
|
||
}
|
||
pca_log: list[dict[str, Any]] = []
|
||
modality_pcas: dict[str, PCA] = {}
|
||
modality_by_id: dict[str, dict[str, np.ndarray]] = {sid: {} for sid in ids}
|
||
for name in MODS:
|
||
input_dim = int(pooled[ids[0]][name].shape[-1])
|
||
requested = min(MODAL_PCA_DIMS[name], input_dim)
|
||
train_matrix = np.concatenate([pooled[sid][name] for sid in train_ids], axis=0)
|
||
pca = PCA(n_components=requested, svd_solver="auto", random_state=seed).fit(train_matrix)
|
||
modality_pcas[name] = pca
|
||
for sid in ids:
|
||
modality_by_id[sid][name] = pca.transform(pooled[sid][name]).astype(np.float32, copy=False)
|
||
pca_log.append({
|
||
"projection": f"modality_{name}",
|
||
"input_dim": input_dim,
|
||
"components": int(pca.n_components_),
|
||
"explained_variance_ratio_sum": float(pca.explained_variance_ratio_.sum()),
|
||
"train_slot_count": int(len(train_matrix)),
|
||
})
|
||
|
||
full_pca, full_projected = _fit_transform_map(train_ids, raw_by_id, seed=seed)
|
||
pca_log.append({
|
||
"projection": "all_modalities_joint_256",
|
||
"input_dim": int(raw_by_id[ids[0]].shape[-1]),
|
||
"components": int(full_pca.n_components_),
|
||
"explained_variance_ratio_sum": float(full_pca.explained_variance_ratio_.sum()),
|
||
"train_slot_count": int(sum(len(raw_by_id[sid]) for sid in train_ids)),
|
||
})
|
||
|
||
representations: dict[str, dict[str, np.ndarray]] = {}
|
||
raw_vector = {sid: _flatten_five(value) for sid, value in raw_by_id.items()}
|
||
representations["M4-TimeOnly-full-raw"] = raw_vector
|
||
representations["Private-Identity"] = raw_vector
|
||
pca_private = {
|
||
sid: np.concatenate([modality_by_id[sid][name] for name in MODS], axis=-1)
|
||
for sid in ids
|
||
}
|
||
representations["Private-PCA-T64-A24-V64"] = {sid: _flatten_five(value) for sid, value in pca_private.items()}
|
||
representations["Private-DimMatched-PCA256"] = {sid: _flatten_five(value) for sid, value in full_projected.items()}
|
||
|
||
subset_map = {
|
||
"T": ("text",),
|
||
"A": ("audio",),
|
||
"V": ("vision",),
|
||
"TA": ("text", "audio"),
|
||
"TV": ("text", "vision"),
|
||
"AV": ("audio", "vision"),
|
||
"TAV": MODS,
|
||
}
|
||
subset_metadata: dict[str, dict[str, Any]] = {}
|
||
for tag, names in subset_map.items():
|
||
subset_raw = {sid: np.concatenate([pooled[sid][name] for name in names], axis=-1) for sid in ids}
|
||
subset_pca, subset_projected = _fit_transform_map(train_ids, subset_raw, seed=seed + len(tag))
|
||
label = f"Subset-{tag}-PCA256"
|
||
representations[label] = {sid: _flatten_five(value) for sid, value in subset_projected.items()}
|
||
subset_metadata[tag] = {"modalities": names, "components": int(subset_pca.n_components_), "input_dim": int(subset_raw[ids[0]].shape[-1])}
|
||
pca_log.append({
|
||
"projection": label,
|
||
"input_dim": subset_metadata[tag]["input_dim"],
|
||
"components": int(subset_pca.n_components_),
|
||
"explained_variance_ratio_sum": float(subset_pca.explained_variance_ratio_.sum()),
|
||
"train_slot_count": int(sum(len(subset_raw[sid]) for sid in train_ids)),
|
||
})
|
||
|
||
interaction_details: dict[str, dict[str, Any]] = {}
|
||
for index, (key, pairs) in enumerate(_interaction_sets().items()):
|
||
interaction_raw = _build_interaction_slots(modality_by_id, pairs)
|
||
if pairs:
|
||
interaction_pca, interaction_projected = _fit_transform_map(
|
||
train_ids, interaction_raw, seed=seed + 100 + index
|
||
)
|
||
components = int(interaction_pca.n_components_)
|
||
ev = float(interaction_pca.explained_variance_ratio_.sum())
|
||
else:
|
||
# I0 is the train-fold modality PCA representation with zero padding
|
||
# to the same 256-slot output width as the interaction candidates.
|
||
interaction_projected = {}
|
||
for sid, value in interaction_raw.items():
|
||
out = np.zeros((GRID_SIZE, SLOT_PCA_SIZE), dtype=np.float32)
|
||
out[:, : min(value.shape[-1], SLOT_PCA_SIZE)] = value[:, :SLOT_PCA_SIZE]
|
||
interaction_projected[sid] = out
|
||
components = int(interaction_raw[ids[0]].shape[-1])
|
||
ev = float("nan")
|
||
interaction_pca = None
|
||
label = f"TAV-{key}"
|
||
representations[label] = {sid: _flatten_five(value) for sid, value in interaction_projected.items()}
|
||
interaction_details[key] = {"pairs": [f"{left}-{right}" for left, right in pairs], "components": components}
|
||
pca_log.append({
|
||
"projection": label,
|
||
"input_dim": int(interaction_raw[ids[0]].shape[-1]),
|
||
"components": components,
|
||
"explained_variance_ratio_sum": ev,
|
||
"train_slot_count": int(sum(len(interaction_raw[sid]) for sid in train_ids)),
|
||
})
|
||
|
||
metadata = {
|
||
"subset_views": subset_map,
|
||
"subset_metadata": subset_metadata,
|
||
"interaction_details": interaction_details,
|
||
"raw_slot_dim": int(raw_by_id[ids[0]].shape[-1]),
|
||
"private_pca_slot_dim": int(pca_private[ids[0]].shape[-1]),
|
||
"dimension_matched_slot_dim": SLOT_PCA_SIZE,
|
||
"time_only_and_private_identity_are_identical": True,
|
||
}
|
||
return representations, pca_log, metadata
|
||
|
||
|
||
def _temporal_rows(
|
||
fold: int,
|
||
samples: Sequence[FeatureSample],
|
||
temporal_by_id: Mapping[str, Mapping[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
rows: list[dict[str, Any]] = []
|
||
grid = (np.arange(GRID_SIZE, dtype=np.float64) + 0.5) / GRID_SIZE
|
||
for sample in samples:
|
||
record = temporal_by_id[sample.sample_id]
|
||
expected: dict[str, np.ndarray] = {}
|
||
for modality in MODS:
|
||
valid = np.asarray(record["valid"][modality], dtype=bool)
|
||
times = np.asarray(record["times"][modality], dtype=np.float64)
|
||
weights = np.asarray(record["weights"][modality], dtype=np.float64)
|
||
mu = weights @ times
|
||
expected[modality] = mu
|
||
violations = int(np.sum(np.diff(mu) < -0.02))
|
||
corr = _pearson_safe(grid, mu)
|
||
rows.append({
|
||
"fold": fold,
|
||
"sample_id": sample.sample_id,
|
||
"video_id": sample.group_id,
|
||
"kind": "modality",
|
||
"modality": modality,
|
||
"mvr_eps_0_02": violations / max(len(mu) - 1, 1),
|
||
"span_signed": float(mu[-1] - mu[0]),
|
||
"span_absolute": float(abs(mu[-1] - mu[0])),
|
||
"expected_time_correlation": corr,
|
||
"valid_source_positions": int(valid.sum()),
|
||
"mean_attention_entropy": float(np.mean(-np.sum(weights * np.log(np.maximum(weights, 1e-12)), axis=1))),
|
||
})
|
||
for left, right in (*PAIRINGS, *((b, a) for a, b in PAIRINGS)):
|
||
src_to_slot, src_idx, _ = _normalize_attention(record["weights"][left], record["valid"][left])
|
||
dst_idx = np.flatnonzero(record["valid"][right])
|
||
mapping = src_to_slot.T @ np.asarray(record["weights"][right], dtype=np.float64)[:, dst_idx]
|
||
mapping /= np.maximum(mapping.sum(axis=1, keepdims=True), 1e-12)
|
||
metric = _pair_metrics(mapping, record["times"][left][src_idx], record["times"][right][dst_idx])
|
||
rows.append({
|
||
"fold": fold,
|
||
"sample_id": sample.sample_id,
|
||
"video_id": sample.group_id,
|
||
"kind": "pairwise",
|
||
"direction": f"{left}-to-{right}",
|
||
"pairwise_time_mae": metric["pairwise_time_mae"],
|
||
"pairwise_signed_lag": metric["pairwise_signed_lag"],
|
||
"pairwise_time_corr": metric["pairwise_time_corr"],
|
||
})
|
||
return rows
|
||
|
||
|
||
def _prediction_metrics(representation: str, rows: Sequence[Mapping[str, Any]], dimension: int) -> dict[str, Any]:
|
||
class_values = _metric_values(rows, "predicted_class_id", "predicted_dual_label")
|
||
direct_actual = np.asarray([float(row["true_label"]) for row in rows], dtype=np.float64)
|
||
direct_pred = np.asarray([float(row["predicted_direct_label"]) for row in rows], dtype=np.float64)
|
||
dual_pred = np.asarray([float(row["predicted_dual_label"]) for row in rows], dtype=np.float64)
|
||
return {
|
||
"method_family": "SharedTime-PrivateContent",
|
||
"representation": representation,
|
||
"feature_dimension": dimension,
|
||
"accuracy": class_values["accuracy"],
|
||
"macro_f1": class_values["macro_f1"],
|
||
"direct_ridge_mae": float(np.mean(np.abs(direct_actual - direct_pred))),
|
||
"direct_ridge_pearson": _pearson_safe(direct_actual, direct_pred),
|
||
"dual_head_mae": float(np.mean(np.abs(direct_actual - dual_pred))),
|
||
"dual_head_pearson": _pearson_safe(direct_actual, dual_pred),
|
||
}
|
||
|
||
|
||
def _external_rows(path: Path, selectors: Sequence[tuple[str, str, str]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||
"""Normalize prior fixed-OOF predictions to the current result schema."""
|
||
rows: list[dict[str, Any]] = []
|
||
refs: list[dict[str, Any]] = []
|
||
if not path.is_file():
|
||
return rows, refs
|
||
with path.open("r", encoding="utf-8-sig", newline="") as handle:
|
||
source = list(csv.DictReader(handle))
|
||
for method, view, display in selectors:
|
||
selected = [row for row in source if row.get("method") == method and row.get("view") == view]
|
||
if not selected:
|
||
continue
|
||
for row in selected:
|
||
rows.append({
|
||
"method_family": "Historical-Control",
|
||
"representation": display,
|
||
"probe": "historical",
|
||
"fold": int(row.get("fold", 0) or 0),
|
||
"sample_id": row["sample_id"],
|
||
"video_id": row.get("video_id", row["sample_id"].split("/")[0]),
|
||
"true_class_id": int(row["true_class_id"]),
|
||
"true_label": float(row["true_label"]),
|
||
"predicted_class_id": int(row["predicted_class_id"]),
|
||
"predicted_direct_label": float(row["predicted_label"]),
|
||
"predicted_dual_label": float(row["predicted_label"]),
|
||
})
|
||
refs.append({"representation": display, "rows": selected, "feature_dimension": int(selected[0].get("feature_dimension", 0) or 0)})
|
||
return rows, refs
|
||
|
||
|
||
def _summarize_historical(refs: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]:
|
||
summary = []
|
||
for ref in refs:
|
||
rows = ref["rows"]
|
||
true_class = np.asarray([int(row["true_class_id"]) for row in rows], dtype=np.int64)
|
||
pred_class = np.asarray([int(row["predicted_class_id"]) for row in rows], dtype=np.int64)
|
||
actual = np.asarray([float(row["true_label"]) for row in rows], dtype=np.float64)
|
||
pred = np.asarray([float(row["predicted_label"]) for row in rows], dtype=np.float64)
|
||
summary.append({
|
||
"method_family": "Historical-Control",
|
||
"representation": ref["representation"],
|
||
"feature_dimension": ref["feature_dimension"],
|
||
"accuracy": float(accuracy_score(true_class, pred_class)),
|
||
"macro_f1": float(f1_score(true_class, pred_class, labels=[0, 1, 2], average="macro", zero_division=0)),
|
||
"direct_ridge_mae": float(np.mean(np.abs(actual - pred))),
|
||
"direct_ridge_pearson": _pearson_safe(actual, pred),
|
||
"dual_head_mae": float("nan"),
|
||
"dual_head_pearson": float("nan"),
|
||
})
|
||
return summary
|
||
|
||
|
||
def _group_bootstrap_pair(
|
||
model_a: str,
|
||
model_b: str,
|
||
prediction_maps: Mapping[str, Mapping[str, Mapping[str, Any]]],
|
||
groups_by_id: Mapping[str, str],
|
||
*,
|
||
seed: int,
|
||
draws: int = 2000,
|
||
) -> list[dict[str, Any]]:
|
||
ids = sorted(set(prediction_maps[model_a]) & set(prediction_maps[model_b]))
|
||
groups = sorted({groups_by_id[sid] for sid in ids})
|
||
if len(ids) != 100 or not groups:
|
||
return []
|
||
metric_specs = (
|
||
("accuracy", "predicted_class_id", "class"),
|
||
("macro_f1", "predicted_class_id", "class"),
|
||
("direct_ridge_mae", "predicted_direct_label", "direct"),
|
||
("direct_ridge_pearson", "predicted_direct_label", "direct"),
|
||
("dual_head_mae", "predicted_dual_label", "dual"),
|
||
("dual_head_pearson", "predicted_dual_label", "dual"),
|
||
)
|
||
def calc(model: str, samples: Sequence[str], key: str, kind: str) -> float:
|
||
rows = [prediction_maps[model][sid] for sid in samples]
|
||
if kind == "class":
|
||
actual = [int(row["true_class_id"]) for row in rows]
|
||
pred = [int(row[key]) for row in rows]
|
||
if key == "predicted_class_id":
|
||
if len(actual) == 0:
|
||
return float("nan")
|
||
if metric == "accuracy":
|
||
return float(accuracy_score(actual, pred))
|
||
return float(f1_score(actual, pred, labels=[0, 1, 2], average="macro", zero_division=0))
|
||
else:
|
||
actual = np.asarray([float(row["true_label"]) for row in rows])
|
||
pred = np.asarray([float(row[key]) for row in rows])
|
||
if metric.endswith("mae"):
|
||
return float(np.mean(np.abs(actual - pred)))
|
||
return _pearson_safe(actual, pred)
|
||
return float("nan")
|
||
|
||
result = []
|
||
rng = np.random.default_rng(seed)
|
||
selected_ids = {group: [sid for sid in ids if groups_by_id[sid] == group] for group in groups}
|
||
for metric, key, kind in metric_specs:
|
||
if kind == "dual" and any(
|
||
"predicted_intensity_abs" not in prediction_maps[name][ids[0]]
|
||
for name in (model_a, model_b)
|
||
):
|
||
continue
|
||
observed_a = calc(model_a, ids, key, kind)
|
||
observed_b = calc(model_b, ids, key, kind)
|
||
point = observed_a - observed_b
|
||
differences = []
|
||
for _ in range(draws):
|
||
sampled_groups = rng.choice(groups, size=len(groups), replace=True)
|
||
boot_ids = [sid for group in sampled_groups for sid in selected_ids[str(group)]]
|
||
if not boot_ids:
|
||
continue
|
||
a = calc(model_a, boot_ids, key, kind)
|
||
b = calc(model_b, boot_ids, key, kind)
|
||
if np.isfinite(a) and np.isfinite(b):
|
||
differences.append(a - b)
|
||
low, high = np.quantile(differences, [0.025, 0.975]) if differences else (float("nan"), float("nan"))
|
||
result.append({
|
||
"model_a": model_a,
|
||
"model_b": model_b,
|
||
"metric": metric,
|
||
"difference_a_minus_b": point,
|
||
"bootstrap_95_low": float(low),
|
||
"bootstrap_95_high": float(high),
|
||
"video_group_count": len(groups),
|
||
"bootstrap_draws": draws,
|
||
"note": "Paired resampling of original video_id groups; exploratory intervals, no multiple-comparison correction.",
|
||
})
|
||
return result
|
||
|
||
|
||
def _plot_architecture(path: Path) -> None:
|
||
fig, ax = plt.subplots(figsize=(12, 4.8))
|
||
ax.axis("off")
|
||
boxes = [
|
||
(0.02, "BERT text\n768-D"), (0.02, "eGeMAPS audio\n25-D"), (0.02, "DeiT vision\n192-D"),
|
||
]
|
||
for index, (y, label) in enumerate(boxes):
|
||
ax.text(0.08, 0.77 - index * 0.27, label, ha="center", va="center", fontsize=11,
|
||
bbox={"boxstyle": "round,pad=0.6", "fc": "#e8eff8", "ec": "#3c5a78"})
|
||
ax.annotate("", xy=(0.30, 0.77 - index * 0.27), xytext=(0.19, 0.77 - index * 0.27),
|
||
arrowprops={"arrowstyle": "->", "lw": 1.6})
|
||
ax.text(0.42, 0.50, "Frozen M4 source-time weights\n(shared temporal coordinate, 50 slots)", ha="center", va="center", fontsize=11,
|
||
bbox={"boxstyle": "round,pad=0.8", "fc": "#fff0ce", "ec": "#ad7b19"})
|
||
ax.annotate("", xy=(0.61, 0.50), xytext=(0.53, 0.50), arrowprops={"arrowstyle": "->", "lw": 1.6})
|
||
ax.text(0.72, 0.68, "T private content", ha="center", va="center", fontsize=10,
|
||
bbox={"boxstyle": "round,pad=0.5", "fc": "#edf7ed", "ec": "#4e7d51"})
|
||
ax.text(0.72, 0.50, "A private content", ha="center", va="center", fontsize=10,
|
||
bbox={"boxstyle": "round,pad=0.5", "fc": "#edf7ed", "ec": "#4e7d51"})
|
||
ax.text(0.72, 0.32, "V private content", ha="center", va="center", fontsize=10,
|
||
bbox={"boxstyle": "round,pad=0.5", "fc": "#edf7ed", "ec": "#4e7d51"})
|
||
ax.annotate("", xy=(0.86, 0.50), xytext=(0.80, 0.50), arrowprops={"arrowstyle": "->", "lw": 1.6})
|
||
ax.text(0.93, 0.50, "Task probe\n+ optional same-slot\ninteractions", ha="center", va="center", fontsize=10,
|
||
bbox={"boxstyle": "round,pad=0.7", "fc": "#f4ebf7", "ec": "#7a4c80"})
|
||
ax.set_title("Shared-time, modality-private-content representation", fontsize=14, pad=12)
|
||
fig.tight_layout()
|
||
fig.savefig(path, dpi=180, bbox_inches="tight")
|
||
plt.close(fig)
|
||
|
||
|
||
def _plot_metrics(output: Path, metrics: Sequence[Mapping[str, Any]], predictions: Sequence[Mapping[str, Any]], temporal: Sequence[Mapping[str, Any]], sample_payload: Mapping[str, Any]) -> list[str]:
|
||
names = [row["representation"] for row in metrics]
|
||
new = [row for row in metrics if row["method_family"] == "SharedTime-PrivateContent"]
|
||
by_name = {row["representation"]: row for row in new}
|
||
generated = []
|
||
|
||
_plot_architecture(output / "01_shared_time_private_content_structure.png")
|
||
generated.append("01_shared_time_private_content_structure.png")
|
||
|
||
subset_order = ["Subset-T-PCA256", "Subset-A-PCA256", "Subset-V-PCA256", "Subset-TA-PCA256", "Subset-TV-PCA256", "Subset-AV-PCA256", "Subset-TAV-PCA256"]
|
||
fig, axes = plt.subplots(1, 2, figsize=(14, 5))
|
||
x = np.arange(len(subset_order))
|
||
axes[0].bar(x, [by_name[n]["macro_f1"] for n in subset_order], color="#4678a8")
|
||
axes[0].set_ylabel("Macro-F1 (higher is better)")
|
||
axes[1].bar(x, [by_name[n]["direct_ridge_mae"] for n in subset_order], color="#d18a44")
|
||
axes[1].set_ylabel("Direct Ridge MAE (lower is better)")
|
||
for ax in axes:
|
||
ax.set_xticks(x, [n.replace("Subset-", "").replace("-PCA256", "") for n in subset_order], rotation=25)
|
||
ax.grid(axis="y", alpha=0.25)
|
||
fig.suptitle("Modality subset contribution (equal 256-slot output width)")
|
||
fig.tight_layout()
|
||
fig.savefig(output / "02_modality_subset_contribution.png", dpi=180)
|
||
plt.close(fig)
|
||
generated.append("02_modality_subset_contribution.png")
|
||
|
||
interactions = [f"TAV-I{i}" for i in range(8)]
|
||
fig, axes = plt.subplots(1, 2, figsize=(13, 4.8))
|
||
x = np.arange(8)
|
||
axes[0].bar(x, [by_name[n]["macro_f1"] for n in interactions], color="#487da7")
|
||
axes[0].set_ylabel("Macro-F1")
|
||
axes[1].bar(x, [by_name[n]["dual_head_mae"] for n in interactions], color="#ba7049")
|
||
axes[1].set_ylabel("Dual-head MAE")
|
||
for ax in axes:
|
||
ax.set_xticks(x, [f"I{i}" for i in range(8)])
|
||
ax.grid(axis="y", alpha=0.25)
|
||
fig.suptitle("Same-slot task-specific interaction ablation")
|
||
fig.tight_layout()
|
||
fig.savefig(output / "03_interaction_ablation.png", dpi=180)
|
||
plt.close(fig)
|
||
generated.append("03_interaction_ablation.png")
|
||
|
||
unitary = ["Subset-T-PCA256", "Subset-A-PCA256", "Subset-V-PCA256"]
|
||
fig, ax1 = plt.subplots(figsize=(9, 5))
|
||
x = np.arange(3)
|
||
f1 = [by_name[name]["macro_f1"] for name in unitary]
|
||
mae = [by_name[name]["direct_ridge_mae"] for name in unitary]
|
||
ax1.bar(x - 0.18, f1, width=0.36, label="Polarity Macro-F1", color="#497ca8")
|
||
ax2 = ax1.twinx()
|
||
ax2.bar(x + 0.18, mae, width=0.36, label="Intensity MAE", color="#d58c48")
|
||
ax1.set_xticks(x, ["Text", "Audio", "Vision"])
|
||
ax1.set_ylabel("Macro-F1")
|
||
ax2.set_ylabel("Direct Ridge MAE")
|
||
ax1.grid(axis="y", alpha=0.25)
|
||
ax1.legend(loc="upper left")
|
||
ax2.legend(loc="upper right")
|
||
ax1.set_title("Which modality helps polarity versus intensity?")
|
||
fig.tight_layout()
|
||
fig.savefig(output / "04_polarity_vs_intensity_modality.png", dpi=180)
|
||
plt.close(fig)
|
||
generated.append("04_polarity_vs_intensity_modality.png")
|
||
|
||
focus = ["Private-DimMatched-PCA256", "TAV-I0", "TAV-I7"]
|
||
fig, ax = plt.subplots(figsize=(9, 5))
|
||
x = np.arange(len(focus))
|
||
direct = [by_name[name]["direct_ridge_mae"] for name in focus]
|
||
dual = [by_name[name]["dual_head_mae"] for name in focus]
|
||
ax.bar(x - 0.18, direct, width=0.36, label="Direct signed Ridge", color="#9b7653")
|
||
ax.bar(x + 0.18, dual, width=0.36, label="Polarity × |intensity|", color="#468b75")
|
||
ax.set_xticks(x, ["Private PCA", "I0", "I7"])
|
||
ax.set_ylabel("MAE (lower is better)")
|
||
ax.set_title("Dual-head composition versus direct signed regression")
|
||
ax.legend()
|
||
ax.grid(axis="y", alpha=0.25)
|
||
fig.tight_layout()
|
||
fig.savefig(output / "05_dual_head_vs_direct_ridge.png", dpi=180)
|
||
plt.close(fig)
|
||
generated.append("05_dual_head_vs_direct_ridge.png")
|
||
|
||
dim_names = ["M4-TimeOnly-full-raw", "Private-PCA-T64-A24-V64", "Private-DimMatched-PCA256"]
|
||
fig, axes = plt.subplots(1, 2, figsize=(11, 4.7))
|
||
x = np.arange(len(dim_names))
|
||
axes[0].bar(x, [by_name[n]["macro_f1"] for n in dim_names], color="#547eaa")
|
||
axes[0].set_ylabel("Macro-F1")
|
||
axes[1].bar(x, [by_name[n]["direct_ridge_mae"] for n in dim_names], color="#c78448")
|
||
axes[1].set_ylabel("MAE")
|
||
for ax in axes:
|
||
ax.set_xticks(x, ["Raw 985/slot", "Per-modality PCA", "Joint PCA 256/slot"], rotation=15)
|
||
ax.grid(axis="y", alpha=0.25)
|
||
fig.suptitle("Dimension-control check")
|
||
fig.tight_layout()
|
||
fig.savefig(output / "06_dimension_match_comparison.png", dpi=180)
|
||
plt.close(fig)
|
||
generated.append("06_dimension_match_comparison.png")
|
||
|
||
target = "Private-DimMatched-PCA256"
|
||
pred_rows = [row for row in predictions if row["method_family"] == "SharedTime-PrivateContent" and row["representation"] == target]
|
||
temporal_by_id: dict[str, list[float]] = {}
|
||
for row in temporal:
|
||
if row.get("kind") == "pairwise":
|
||
temporal_by_id.setdefault(row["sample_id"], []).append(float(row["pairwise_time_mae"]))
|
||
xs, ys, colors = [], [], []
|
||
for row in pred_rows:
|
||
if row["sample_id"] in temporal_by_id:
|
||
xs.append(float(np.mean(temporal_by_id[row["sample_id"]])))
|
||
ys.append(abs(float(row["true_label"]) - float(row["predicted_dual_label"])))
|
||
colors.append(int(row["true_class_id"]))
|
||
fig, ax = plt.subplots(figsize=(8, 5.5))
|
||
scatter = ax.scatter(xs, ys, c=colors, cmap="viridis", s=42, alpha=0.8, edgecolors="white", linewidths=0.3)
|
||
ax.set_xlabel("Frozen M4 pairwise time MAE (normalized video time)")
|
||
ax.set_ylabel("Dual-head absolute emotion error")
|
||
ax.set_title("Temporal quality and task error are related but distinct")
|
||
fig.colorbar(scatter, ax=ax, ticks=[0, 1, 2], label="True polarity class")
|
||
ax.grid(alpha=0.2)
|
||
fig.tight_layout()
|
||
fig.savefig(output / "07_temporal_quality_vs_task.png", dpi=180)
|
||
plt.close(fig)
|
||
generated.append("07_temporal_quality_vs_task.png")
|
||
|
||
fig, axes = plt.subplots(3, 1, figsize=(12, 8.5), sharex=True)
|
||
centers = sample_payload["grid"]
|
||
for ax, name in zip(axes, MODS):
|
||
values = sample_payload["modal_pca"][name]
|
||
image = ax.imshow(values.T, aspect="auto", interpolation="nearest", extent=[0, 1, values.shape[1], 0], cmap="coolwarm")
|
||
ax.set_ylabel(f"{name.title()}\nPCA component")
|
||
fig.colorbar(image, ax=ax, fraction=0.015, pad=0.01)
|
||
axes[-1].set_xlabel("Shared temporal coordinate (normalized video time)")
|
||
fig.suptitle(f"Private modality content by shared time slot\n{sample_payload['sample_id']} | {sample_payload['text'][:140]}")
|
||
fig.tight_layout()
|
||
fig.savefig(output / "08_representative_private_content.png", dpi=180)
|
||
plt.close(fig)
|
||
generated.append("08_representative_private_content.png")
|
||
|
||
fig, axes = plt.subplots(3, 1, figsize=(12, 8.5), sharex=True)
|
||
for ax, name in zip(axes, MODS):
|
||
weights = sample_payload["weights"][name]
|
||
times = sample_payload["times"][name]
|
||
image = ax.imshow(weights, aspect="auto", interpolation="nearest", extent=[float(times[0]), float(times[-1]), GRID_SIZE, 0], cmap="magma")
|
||
ax.set_ylabel(f"{name.title()} source index")
|
||
fig.colorbar(image, ax=ax, fraction=0.015, pad=0.01)
|
||
axes[-1].set_xlabel("Normalized source time")
|
||
fig.suptitle(f"Frozen M4 time weights (example {sample_payload['sample_id']})")
|
||
fig.tight_layout()
|
||
fig.savefig(output / "09_representative_m4_weights.png", dpi=180)
|
||
plt.close(fig)
|
||
generated.append("09_representative_m4_weights.png")
|
||
return generated
|
||
|
||
|
||
def _run(args: argparse.Namespace) -> None:
|
||
started = time.time()
|
||
random.seed(args.seed)
|
||
np.random.seed(args.seed)
|
||
torch.manual_seed(args.seed)
|
||
if torch.cuda.is_available():
|
||
torch.cuda.manual_seed_all(args.seed)
|
||
device = torch.device("cuda" if args.device == "auto" and torch.cuda.is_available() else ("cpu" if args.device == "auto" else args.device))
|
||
if device.type == "cuda" and not torch.cuda.is_available():
|
||
raise RuntimeError("CUDA was requested, but this WSL PyTorch environment has no visible GPU")
|
||
|
||
q1_root = Path(__file__).resolve().parents[1]
|
||
output = args.output_dir or q1_root / "outputs" / OUTPUT_NAME
|
||
output.mkdir(parents=True, exist_ok=True)
|
||
samples = load_feature_samples(args.feature_dir, args.manifest)
|
||
samples_by_id = {sample.sample_id: sample for sample in samples}
|
||
splits = json.loads(args.splits.read_text(encoding="utf-8"))
|
||
if len(splits) != 5:
|
||
raise ValueError(f"expected exactly five fixed folds, found {len(splits)}")
|
||
valid_ids = [sid for split in splits for sid in split["validation_sample_ids"]]
|
||
if len(samples) != 100 or len(valid_ids) != 100 or set(valid_ids) != set(samples_by_id) or len(set(valid_ids)) != 100:
|
||
raise ValueError("the five fixed validation folds must cover all 100 extracted samples exactly once")
|
||
for sample in samples:
|
||
if _class_from_sentiment(sample.sentiment) != sample.polarity:
|
||
raise ValueError(f"strict-sign class and annotation disagree for {sample.sample_id}")
|
||
|
||
prediction_rows: list[dict[str, Any]] = []
|
||
temporal_rows: list[dict[str, Any]] = []
|
||
fold_rows: list[dict[str, Any]] = []
|
||
pca_rows: list[dict[str, Any]] = []
|
||
representations_by_name: dict[str, dict[str, np.ndarray]] = {}
|
||
dimensions_by_name: dict[str, int] = {}
|
||
example_payload: dict[str, Any] | None = None
|
||
|
||
for split in splits:
|
||
fold = int(split["fold"])
|
||
train_ids = list(split["train_sample_ids"])
|
||
validation_ids = list(split["validation_sample_ids"])
|
||
train_samples = [samples_by_id[sid] for sid in train_ids]
|
||
validation_samples = [samples_by_id[sid] for sid in validation_ids]
|
||
train_groups = {sample.group_id for sample in train_samples}
|
||
heldout_groups = {sample.group_id for sample in validation_samples}
|
||
if train_groups & heldout_groups:
|
||
raise ValueError(f"video group leakage in fold {fold}: {sorted(train_groups & heldout_groups)}")
|
||
if set(train_ids) & set(validation_ids) or set(train_ids) | set(validation_ids) != set(samples_by_id):
|
||
raise ValueError(f"sample split mismatch in fold {fold}")
|
||
stats = fit_feature_stats(train_samples)
|
||
_, _, temporal_by_id = _collect_fold_features(
|
||
fold=fold,
|
||
train_samples=train_samples,
|
||
validation_samples=validation_samples,
|
||
feature_stats=stats,
|
||
checkpoint_root=args.checkpoint_root,
|
||
device=device,
|
||
batch_size=args.batch_size,
|
||
)
|
||
all_samples = [*train_samples, *validation_samples]
|
||
pooled = _pool_original_source(all_samples, stats, temporal_by_id)
|
||
representations, fold_pca_rows, rep_meta = _make_fold_representations(train_ids, pooled, args.seed + fold)
|
||
pca_rows.extend({"fold": fold, **row} for row in fold_pca_rows)
|
||
for name, values in representations.items():
|
||
dimension = int(next(iter(values.values())).shape[-1])
|
||
if name in dimensions_by_name and dimensions_by_name[name] != dimension:
|
||
raise ValueError(f"representation width changed across folds for {name}")
|
||
dimensions_by_name[name] = dimension
|
||
representations_by_name.setdefault(name, {}).update(values)
|
||
|
||
train_class = np.asarray([_class_from_sentiment(sample.sentiment) for sample in train_samples], dtype=np.int64)
|
||
train_y = np.asarray([sample.sentiment for sample in train_samples], dtype=np.float64)
|
||
x_train_by_name = {name: np.stack([values[sid] for sid in train_ids]) for name, values in representations.items()}
|
||
x_val_by_name = {name: np.stack([values[sid] for sid in validation_ids]) for name, values in representations.items()}
|
||
for name in representations:
|
||
classifier, direct, intensity = _training_probe(x_train_by_name[name], train_class, train_y, args.seed)
|
||
predicted_class = classifier[1].predict(classifier[0].transform(x_val_by_name[name]))
|
||
direct_prediction = np.clip(_predict_probe(direct, x_val_by_name[name]), -3.0, 3.0)
|
||
magnitude = np.clip(_predict_probe(intensity, x_val_by_name[name]), 0.0, 3.0)
|
||
dual_prediction = CLASS_SIGNS[predicted_class] * magnitude
|
||
for index, sample in enumerate(validation_samples):
|
||
prediction_rows.append({
|
||
"method_family": "SharedTime-PrivateContent",
|
||
"representation": name,
|
||
"fold": fold,
|
||
"sample_id": sample.sample_id,
|
||
"video_id": sample.group_id,
|
||
"true_class_id": int(_class_from_sentiment(sample.sentiment)),
|
||
"true_class": CLASS_NAMES[_class_from_sentiment(sample.sentiment)],
|
||
"predicted_class_id": int(predicted_class[index]),
|
||
"predicted_class": CLASS_NAMES[int(predicted_class[index])],
|
||
"true_label": float(sample.sentiment),
|
||
"predicted_direct_label": float(direct_prediction[index]),
|
||
"predicted_intensity_abs": float(magnitude[index]),
|
||
"predicted_dual_label": float(dual_prediction[index]),
|
||
"feature_dimension": dimensions_by_name[name],
|
||
})
|
||
temporal_rows.extend(_temporal_rows(fold, validation_samples, temporal_by_id))
|
||
fold_rows.append({
|
||
"fold": fold,
|
||
"train_count": len(train_samples),
|
||
"validation_count": len(validation_samples),
|
||
"train_video_count": len(train_groups),
|
||
"validation_video_count": len(heldout_groups),
|
||
"video_group_overlap": 0,
|
||
"feature_standardization_fit": "training fold only",
|
||
"PCA_fit": "training fold 50-slot observations only",
|
||
})
|
||
if example_payload is None:
|
||
sample = validation_samples[0]
|
||
pca_values = {name: np.stack([representations[f"Private-PCA-T64-A24-V64"][sid] for sid in [sample.sample_id]])}
|
||
# retain each modality's fold-local projected sequence for the sample figure
|
||
# (recompute from the same already-fitted projections used above).
|
||
modal_data = {}
|
||
for name in MODS:
|
||
dim = min(MODAL_PCA_DIMS[name], pooled[sample.sample_id][name].shape[-1])
|
||
train_matrix = np.concatenate([pooled[sid][name] for sid in train_ids], axis=0)
|
||
pca = PCA(n_components=dim, svd_solver="auto", random_state=args.seed + fold).fit(train_matrix)
|
||
modal_data[name] = pca.transform(pooled[sample.sample_id][name]).astype(np.float32)
|
||
manifest = {row["video_id"] + "/" + row["clip_id"]: row for row in csv.DictReader(args.manifest.open("r", encoding="utf-8-sig"))}
|
||
example_payload = {
|
||
"sample_id": sample.sample_id,
|
||
"text": manifest.get(sample.sample_id, {}).get("text", ""),
|
||
"grid": (np.arange(GRID_SIZE) + 0.5) / GRID_SIZE,
|
||
"modal_pca": modal_data,
|
||
"weights": temporal_by_id[sample.sample_id]["weights"],
|
||
"times": temporal_by_id[sample.sample_id]["times"],
|
||
}
|
||
print(f"[shared-time/private-content fold {fold}/5] train={len(train_samples)} heldout={len(validation_samples)} representations={len(representations)} device={device}", flush=True)
|
||
del pooled, temporal_by_id, representations
|
||
if device.type == "cuda":
|
||
torch.cuda.empty_cache()
|
||
|
||
expected_count = 100 * len(dimensions_by_name)
|
||
if len(prediction_rows) != expected_count:
|
||
raise ValueError(f"OOF coverage mismatch: expected {expected_count} prediction rows, got {len(prediction_rows)}")
|
||
|
||
metric_rows = []
|
||
for name in dimensions_by_name:
|
||
rows = [row for row in prediction_rows if row["representation"] == name]
|
||
metric = _prediction_metrics(name, rows, dimensions_by_name[name])
|
||
fold_f1 = []
|
||
fold_accuracy = []
|
||
for fold in range(1, 6):
|
||
fold_subset = [row for row in rows if int(row["fold"]) == fold]
|
||
labels = [int(row["true_class_id"]) for row in fold_subset]
|
||
pred = [int(row["predicted_class_id"]) for row in fold_subset]
|
||
fold_f1.append(float(f1_score(labels, pred, labels=[0, 1, 2], average="macro", zero_division=0)))
|
||
fold_accuracy.append(float(accuracy_score(labels, pred)))
|
||
metric.update({
|
||
"accuracy_fold_mean": float(np.mean(fold_accuracy)),
|
||
"accuracy_fold_sd": float(np.std(fold_accuracy, ddof=1)),
|
||
"macro_f1_fold_mean": float(np.mean(fold_f1)),
|
||
"macro_f1_fold_sd": float(np.std(fold_f1, ddof=1)),
|
||
})
|
||
metric_rows.append(metric)
|
||
|
||
prior_root = args.prior_output_dir
|
||
old_path = prior_root / "tsfa_shared_private" / "emotion_probe_predictions.csv"
|
||
shared_path = prior_root / "shared_definition_comparison" / "emotion_probe_predictions.csv"
|
||
historical_rows: list[dict[str, Any]] = []
|
||
historical_refs: list[dict[str, Any]] = []
|
||
selected_old = [
|
||
("TSFA-old", "all_modalities", "TSFA-old"),
|
||
("TSFA+RawPrivate", "all_modalities", "TSFA+RawPrivate"),
|
||
("RawPrivate-PCA", "RawPrivate-PCA", "RawPrivate-PCA-old"),
|
||
("SPR", "private_all", "SPR-private-all"),
|
||
("SPR-dim-matched", "SPR-dim-matched", "SPR-dim-matched"),
|
||
]
|
||
rows, refs = _external_rows(old_path, selected_old)
|
||
historical_rows.extend(rows)
|
||
historical_refs.extend(refs)
|
||
selected_shared = [
|
||
("GCCA", "main_1280d", "GCCA-old"),
|
||
("Predictive-Same", "main_1280d", "Predictive-Same-old"),
|
||
("Predictive-Local1", "main_1280d", "Predictive-Local1-old"),
|
||
("Similarity-SPR", "main_1280d", "Similarity-SPR-old"),
|
||
("Math-B0", "math_all_modalities", "Math-B0"),
|
||
("Math-B4", "math_all_modalities", "Math-B4"),
|
||
]
|
||
rows, refs = _external_rows(shared_path, selected_shared)
|
||
historical_rows.extend(rows)
|
||
historical_refs.extend(refs)
|
||
historical_predictions = {row["representation"]: [] for row in historical_rows}
|
||
for row in historical_rows:
|
||
historical_predictions[row["representation"]].append(row)
|
||
for name, rows in historical_predictions.items():
|
||
if len(rows) == 100:
|
||
prediction_rows.extend(rows)
|
||
metric_rows.extend(_summarize_historical(refs=historical_refs))
|
||
|
||
prediction_by_model: dict[str, dict[str, Mapping[str, Any]]] = {}
|
||
for row in prediction_rows:
|
||
prediction_by_model.setdefault(row["representation"], {})[row["sample_id"]] = row
|
||
group_by_id = {sample.sample_id: sample.group_id for sample in samples}
|
||
bootstrap_pairs = [
|
||
("TAV-I7", "TAV-I0"),
|
||
("Subset-A-PCA256", "Math-B4"),
|
||
("Subset-TV-PCA256", "Math-B0"),
|
||
("M4-TimeOnly-full-raw", "TSFA+RawPrivate"),
|
||
("Private-DimMatched-PCA256", "TSFA+RawPrivate"),
|
||
("Private-DimMatched-PCA256", "RawPrivate-PCA-old"),
|
||
("Private-DimMatched-PCA256", "Math-B0"),
|
||
("Private-DimMatched-PCA256", "Math-B4"),
|
||
]
|
||
bootstrap_rows = []
|
||
for index, (model_a, model_b) in enumerate(bootstrap_pairs):
|
||
if model_a in prediction_by_model and model_b in prediction_by_model:
|
||
bootstrap_rows.extend(_group_bootstrap_pair(
|
||
model_a, model_b, prediction_by_model, group_by_id,
|
||
seed=args.seed + 500 + index, draws=args.bootstrap_draws,
|
||
))
|
||
|
||
temporal_modality = [row for row in temporal_rows if row.get("kind") == "modality"]
|
||
temporal_pair = [row for row in temporal_rows if row.get("kind") == "pairwise"]
|
||
temporal_summary = {
|
||
"mvr_eps_0_02_mean": float(np.mean([row["mvr_eps_0_02"] for row in temporal_modality])),
|
||
"span_abs_mean": float(np.mean([row["span_absolute"] for row in temporal_modality])),
|
||
"expected_time_correlation_mean": float(np.mean([row["expected_time_correlation"] for row in temporal_modality])),
|
||
"pairwise_time_mae_mean": float(np.mean([row["pairwise_time_mae"] for row in temporal_pair])),
|
||
"pairwise_time_correlation_mean": float(np.mean([row["pairwise_time_corr"] for row in temporal_pair])),
|
||
"sample_count": len({row["sample_id"] for row in temporal_modality}),
|
||
}
|
||
plot_files = _plot_metrics(output, metric_rows, prediction_rows, temporal_rows, example_payload or {})
|
||
|
||
_write_csv(output / "predictions_oof.csv", prediction_rows)
|
||
_write_csv(output / "metrics_oof.csv", metric_rows)
|
||
_write_csv(output / "temporal_monitoring.csv", temporal_rows)
|
||
_write_csv(output / "fold_manifest.csv", fold_rows)
|
||
_write_csv(output / "pca_fold_diagnostics.csv", pca_rows)
|
||
_write_csv(output / "group_bootstrap_contrasts.csv", bootstrap_rows)
|
||
config = {
|
||
"experiment": OUTPUT_NAME,
|
||
"generated_utc": datetime.now(timezone.utc).isoformat(),
|
||
"sample_count": len(samples),
|
||
"video_group_count": len({sample.group_id for sample in samples}),
|
||
"class_counts": {name: sum(_class_from_sentiment(s.sentiment) == idx for s in samples) for idx, name in enumerate(CLASS_NAMES)},
|
||
"folds": 5,
|
||
"fold_source": str(args.splits),
|
||
"checkpoint_root": str(args.checkpoint_root),
|
||
"device": str(device),
|
||
"gpu_name": torch.cuda.get_device_name(0) if device.type == "cuda" else None,
|
||
"seed": args.seed,
|
||
"bootstrap_draws": args.bootstrap_draws,
|
||
"features": "frozen BERT / eGeMAPS / DeiT from Q1 feature extraction; fold-standardized before M4 weighting",
|
||
"frozen_time_model": "M4_sourceTime checkpoint trained for each fixed grouped fold",
|
||
"public_coordinate": "50 normalized video-time slots; shared coordinate only",
|
||
"private_content": "modality-specific pooled BERT/eGeMAPS/DeiT vectors; no cross-modal semantic equality or shared contrastive objective",
|
||
"pca_protocol": "PCA fitted on the 50 pooled slots of training clips only; at most 256 components per slot; zero-pad to a common width when input rank is lower; downstream five-bin vector is 1280D for PCA256 views",
|
||
"private_modality_pca_dimensions": MODAL_PCA_DIMS,
|
||
"interaction_definition": "per same slot [p_m * p_n ; abs(p_m - p_n)] with d=24 (limited by audio feature width), only supplied as task-specific features; bit mask: I1=TA, I2=TV, I3=TA+TV, I4=AV, I5=TA+AV, I6=TV+AV, I7=TA+TV+AV; no shared semantic encoder",
|
||
"dual_head": "LogisticRegression(C=0.05) predicts strict sign class; Ridge(alpha=25) predicts abs(label), clipped to [0,3]; signed output=class sign times magnitude; neutral gives exactly zero",
|
||
"direct_regression": "Ridge(alpha=25) predicts signed label directly and is clipped to [-3,3]",
|
||
"feature_and_probe_scalers": "fit on each training fold only",
|
||
"labels_used_in_alignment_or_pca": False,
|
||
"time_only_equals_private_identity": True,
|
||
"temporal_summary": temporal_summary,
|
||
"plots": plot_files,
|
||
"runtime_seconds": round(time.time() - started, 2),
|
||
"versions": {"python": platform.python_version(), "torch": torch.__version__, "sklearn": sklearn.__version__, "numpy": np.__version__},
|
||
"notes": [
|
||
"M4 weights provide temporal coordinate estimates; this experiment treats them as a frozen temporal basis and does not claim semantic equivalence across modalities.",
|
||
"TimeOnly-full-raw and Private-Identity have identical numerical features by definition; duplicate rows make the conceptual alias explicit.",
|
||
"Interaction dimension d=24 is capped by the 25D eGeMAPS source; the same 24D PCA coordinates are used in TA, TV, and AV interactions.",
|
||
"Bootstrap intervals resample the 37 source video groups and do not apply multiple-comparison correction.",
|
||
"Historical methods are read from their previous fixed-OOF prediction files; math files are read-only.",
|
||
],
|
||
}
|
||
(output / "config.json").write_text(json.dumps(config, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
_write_report(output, metric_rows, temporal_summary, bootstrap_rows, plot_files, config)
|
||
print(f"[complete] output={output} samples={len(samples)} groups={config['video_group_count']} models={len(dimensions_by_name)} device={device}", flush=True)
|
||
|
||
|
||
def _fmt(value: Any) -> str:
|
||
try:
|
||
value = float(value)
|
||
except (TypeError, ValueError):
|
||
return "—"
|
||
return f"{value:.3f}" if np.isfinite(value) else "—"
|
||
|
||
|
||
def _write_report(output: Path, metrics: Sequence[Mapping[str, Any]], temporal: Mapping[str, Any], bootstrap: Sequence[Mapping[str, Any]], plots: Sequence[str], config: Mapping[str, Any]) -> None:
|
||
by_name = {row["representation"]: row for row in metrics}
|
||
experiment_rows = [row for row in metrics if row["method_family"] == "SharedTime-PrivateContent"]
|
||
best_f1 = max(experiment_rows, key=lambda row: float(row["macro_f1"]))
|
||
best_mae = min(experiment_rows, key=lambda row: float(row["direct_ridge_mae"]))
|
||
best_dual = min(experiment_rows, key=lambda row: float(row["dual_head_mae"]))
|
||
chosen = ["Private-DimMatched-PCA256", "Subset-T-PCA256", "Subset-A-PCA256", "Subset-V-PCA256", "Subset-TAV-PCA256", "TAV-I0", "TAV-I1", "TAV-I2", "TAV-I3", "TAV-I4", "TAV-I5", "TAV-I6", "TAV-I7"]
|
||
lines = [
|
||
"# Shared Time, Private Content: Q1 experiment",
|
||
"",
|
||
"## Core definition",
|
||
"",
|
||
"The shared object is the **temporal coordinate**: all modalities are indexed by 50 normalized video-time slots. The content at a slot remains private to its modality (BERT text, eGeMAPS audio, or DeiT vision). Same-slot products and absolute differences are optional inputs to an emotion task probe; they are not treated as evidence that modalities share semantics.",
|
||
"",
|
||
"M4_sourceTime weights are frozen per grouped fold. They provide a common temporal basis, after which the original standardized modality features are pooled. All scalers and PCA bases are fit on training folds only. Emotion labels are used only by the downstream LogisticRegression/Ridge probes.",
|
||
"",
|
||
"## Protocol",
|
||
"",
|
||
f"- Samples/groups: {config['sample_count']} clips from {config['video_group_count']} source videos; fixed five-fold GroupKFold predictions cover each clip once.",
|
||
f"- Runtime device: `{config['device']}` ({config.get('gpu_name') or 'CPU'}).",
|
||
"- Feature dimensions: BERT 768, eGeMAPS 25, DeiT 192. Each fold standardizes from training clips, then pools with its frozen M4_sourceTime checkpoint.",
|
||
"- Private PCA: Text 64, Audio 24, Vision 64 components. The joint dimension-matched comparison uses up to 256 slot components, then pools five consecutive slot bins to 1280 clip features.",
|
||
"- Emotion probes: strict sign classes {-1,0,+1}; LogisticRegression C=0.05; direct Ridge alpha=25; dual-head intensity Ridge alpha=25 on |label|, clipped to [0,3], then multiplied by predicted polarity. Direct Ridge output is clipped to [-3,3].",
|
||
"- Cross-modal interactions use `[p_m ⊙ p_n ; |p_m-p_n|]` on the same slot, with d=24. This is a task-specific interaction. No shared encoder, contrastive loss, orthogonality loss, or semantic reconstruction objective is used.",
|
||
"- Confidence intervals use 2,000 paired bootstrap draws over source-video groups. No multiple-comparison correction is applied; treat small metric differences as exploratory.",
|
||
"",
|
||
"## Main results",
|
||
"",
|
||
"| Representation | Dim | Accuracy | Macro-F1 | Direct MAE | Direct Pearson | Dual-head MAE | Dual Pearson |",
|
||
"|---|---:|---:|---:|---:|---:|---:|---:|",
|
||
]
|
||
for name in chosen:
|
||
row = by_name.get(name)
|
||
if row:
|
||
lines.append(f"| {name} | {row['feature_dimension']} | {_fmt(row['accuracy'])} | {_fmt(row['macro_f1'])} | {_fmt(row['direct_ridge_mae'])} | {_fmt(row['direct_ridge_pearson'])} | {_fmt(row['dual_head_mae'])} | {_fmt(row['dual_head_pearson'])} |")
|
||
lines.extend([
|
||
"",
|
||
f"Highest exploratory OOF Macro-F1: **{best_f1['representation']}** ({_fmt(best_f1['macro_f1'])}).",
|
||
f"Lowest direct Ridge OOF MAE: **{best_mae['representation']}** ({_fmt(best_mae['direct_ridge_mae'])}).",
|
||
f"Lowest dual-head OOF MAE: **{best_dual['representation']}** ({_fmt(best_dual['dual_head_mae'])}).",
|
||
"",
|
||
"The strict sign class distribution is 18 Negative, 25 Neutral, and 57 Positive; an always-Positive classifier has 0.57 accuracy. Macro-F1 reports the three fixed classes with zero F1 for a class that is never predicted.",
|
||
"",
|
||
"## Temporal monitoring (frozen M4; not an emotion score)",
|
||
"",
|
||
f"- Mean MVR at ε=0.02: {_fmt(temporal['mvr_eps_0_02_mean'])}.",
|
||
f"- Mean absolute start-to-end span: {_fmt(temporal['span_abs_mean'])} of normalized clip time.",
|
||
f"- Mean correlation of expected source time with the 50-slot coordinate: {_fmt(temporal['expected_time_correlation_mean'])}.",
|
||
f"- Mean pairwise source-time MAE: {_fmt(temporal['pairwise_time_mae_mean'])}; mean pairwise time correlation: {_fmt(temporal['pairwise_time_correlation_mean'])}.",
|
||
"These values describe the frozen coordinate map. They do not establish that the three feature streams encode the same meaning.",
|
||
"",
|
||
"## Files",
|
||
"",
|
||
"- `predictions_oof.csv`: one row per clip and new representation, with polarity, direct-regression, and dual-head predictions.",
|
||
"- `metrics_oof.csv`: OOF metrics for new representations plus fixed historical controls.",
|
||
"- `group_bootstrap_contrasts.csv`: paired group-bootstrap differences, where matching prior OOF rows exist.",
|
||
"- `temporal_monitoring.csv`: clip-level MVR, span, expected-time correlation, and pairwise temporal error.",
|
||
"- `pca_fold_diagnostics.csv`: component counts and explained variance by fold.",
|
||
"- `fold_manifest.csv`: split and group-leakage audit.",
|
||
"",
|
||
"## Figures",
|
||
"",
|
||
])
|
||
for name in plots:
|
||
lines.append(f"- `{name}`")
|
||
lines.extend([
|
||
"",
|
||
"## Interpretation limits",
|
||
"",
|
||
"This is a 100-clip exploratory evaluation from 37 source videos. The OOF probes reuse an existing fixed split, and the 50 slots within a clip are dependent observations for PCA. Slot-level PCA is fitted only to training clips, but confidence intervals resample videos rather than slots. We do not use the emotion labels to train the temporal model or PCA. Historical comparisons retain their original feature and probe definitions; they are controls, not perfectly matched re-runs.",
|
||
"",
|
||
"M4-TimeOnly-full-raw and Private-Identity have identical numerical content because Private-Identity means independently retained modality feature streams after the same frozen temporal pooling. Their duplicate rows are an explicit conceptual alias, not an independent ablation.",
|
||
])
|
||
(output / "RESULTS.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||
|
||
|
||
def main() -> None:
|
||
q1_root = Path(__file__).resolve().parents[1]
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument("--device", default="auto", choices=("auto", "cpu", "cuda"))
|
||
parser.add_argument("--seed", type=int, default=42)
|
||
parser.add_argument("--batch-size", type=int, default=8)
|
||
parser.add_argument("--bootstrap-draws", type=int, default=2000)
|
||
parser.add_argument("--feature-dir", type=Path, default=q1_root / "outputs" / "q1_features" / "features")
|
||
parser.add_argument("--manifest", type=Path, default=q1_root / "outputs" / "audit" / "manifest.csv")
|
||
parser.add_argument("--splits", type=Path, default=q1_root / "outputs" / "method_comparison" / "splits.json")
|
||
parser.add_argument("--checkpoint-root", type=Path, default=q1_root / "outputs" / "alignment_debug" / "heldout")
|
||
parser.add_argument("--prior-output-dir", type=Path, default=q1_root / "outputs")
|
||
parser.add_argument("--output-dir", type=Path, default=None)
|
||
_run(parser.parse_args())
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|