152 lines
7.6 KiB
Python
152 lines
7.6 KiB
Python
"""Full Q1 physical and official unaligned split audit; no model training."""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
ROOT = HERE.parent
|
|
|
|
from ..adapter import Q1AlignmentAdapter
|
|
from ..q2.math.data import restricted_load
|
|
|
|
MODALITIES = ("text", "audio", "vision")
|
|
RESULTS = ROOT / "output" / "q1" / "unified_adapter"
|
|
from ..data_paths import ATTACHMENT2
|
|
|
|
OFFICIAL = ATTACHMENT2 / "unaligned_50.pkl"
|
|
|
|
|
|
def _distribution(values: list[float]) -> dict[str, float | int]:
|
|
arr = np.asarray(values, np.float64)
|
|
return {"count": int(len(arr)), "min": float(arr.min()) if len(arr) else 0.0,
|
|
"p25": float(np.percentile(arr, 25)) if len(arr) else 0.0,
|
|
"median": float(np.median(arr)) if len(arr) else 0.0,
|
|
"p75": float(np.percentile(arr, 75)) if len(arr) else 0.0,
|
|
"max": float(arr.max()) if len(arr) else 0.0,
|
|
"mean": float(arr.mean()) if len(arr) else 0.0}
|
|
|
|
|
|
def _audit_split(name: str, split: dict, baseline: dict) -> dict:
|
|
adapter = Q1AlignmentAdapter()
|
|
n = len(split["id"])
|
|
hashes = {m: hashlib.sha256() for m in MODALITIES}
|
|
mask_hash = hashlib.sha256()
|
|
lengths = {m: [] for m in MODALITIES}
|
|
observed_bins = {m: [] for m in MODALITIES}
|
|
all_missing_bins = Counter()
|
|
coverages = {m: [] for m in MODALITIES}
|
|
relations = {m: Counter() for m in MODALITIES}
|
|
all_missing = Counter()
|
|
conflicts = Counter()
|
|
text_padding_nonzero = 0
|
|
max_provenance_error = 0.0
|
|
max_nonfinite = 0
|
|
modes = Counter()
|
|
shapes = {m: Counter() for m in MODALITIES}
|
|
for i in range(n):
|
|
sample = adapter.from_unaligned_record(split, i)
|
|
modes[sample.metadata["coordinate_mode"]] += 1
|
|
stacked_mask = np.stack([sample.observed[m] for m in MODALITIES], axis=-1)
|
|
mask_hash.update(stacked_mask.tobytes(order="C"))
|
|
attention = np.asarray(split["text_bert"][i, 1], bool)
|
|
text_padding_nonzero += int(np.count_nonzero(np.any(split["text"][i] != 0, axis=1) & ~attention))
|
|
for m in MODALITIES:
|
|
x = sample.features[m]
|
|
p = sample.provenance[m]
|
|
hashes[m].update(x.tobytes(order="C"))
|
|
shapes[m][str(list(x.shape))] += 1
|
|
lengths[m].append(p.source_span)
|
|
observed_bins[m].append(int(sample.observed[m].sum()))
|
|
all_missing_bins[m] += int((~sample.observed[m]).sum())
|
|
coverages[m].extend(sample.coverage[m].tolist())
|
|
relations[m]["L<K" if p.source_span < 50 else "L=K" if p.source_span == 50 else "L>K"] += 1
|
|
all_missing[m] += int(not sample.observed[m].any())
|
|
conflicts[m] += int(p.length_conflict)
|
|
if m == "vision":
|
|
all_missing["vision_tail_ambiguous"] += int(p.tail_ambiguous)
|
|
row_sums = np.asarray(p.source_weights.sum(axis=1)).reshape(-1)
|
|
if sample.observed[m].any():
|
|
max_provenance_error = max(max_provenance_error,
|
|
float(np.max(np.abs(row_sums[sample.observed[m]] - 1.0))))
|
|
max_provenance_error = max(max_provenance_error,
|
|
float(np.max(np.abs(row_sums[~sample.observed[m]]))) if (~sample.observed[m]).any() else 0.0)
|
|
max_nonfinite += int(np.count_nonzero(~np.isfinite(x)))
|
|
actual = {m: h.hexdigest() for m, h in hashes.items()}
|
|
expected = baseline[name]["sha256"]
|
|
return {"samples": n, "coordinate_modes": dict(modes), "feature_shapes": {m: dict(v) for m, v in shapes.items()},
|
|
"source_length": {m: _distribution(v) for m, v in lengths.items()},
|
|
"source_length_vs_K": {m: dict(v) for m, v in relations.items()},
|
|
"observed_target_bins_per_sample": {m: _distribution(v) for m, v in observed_bins.items()},
|
|
"coverage_per_target_bin": {m: _distribution(v) for m, v in coverages.items()},
|
|
"text_nonzero_rows_outside_attention": text_padding_nonzero,
|
|
"length_conflict_samples": dict(conflicts),
|
|
"tail_ambiguous_samples": all_missing["vision_tail_ambiguous"],
|
|
"all_missing_target_bins": dict(all_missing_bins),
|
|
"all_missing_modality_samples": {m: all_missing[m] for m in MODALITIES},
|
|
"nonfinite_output_values": max_nonfinite,
|
|
"max_provenance_row_sum_error": max_provenance_error,
|
|
"sha256": actual, "legacy_sha256": expected,
|
|
"exact_feature_equivalence": {m: actual[m] == expected[m] for m in MODALITIES},
|
|
"mask_sha256": mask_hash.hexdigest(),
|
|
"exact_mask_equivalence": mask_hash.hexdigest() == baseline[name]["mask_sha256"]}
|
|
|
|
|
|
def _audit_physical() -> dict:
|
|
manifest = HERE / "features_v2" / "manifest_q1.jsonl"
|
|
ids = [json.loads(line)["sample_id"] for line in manifest.read_text(encoding="utf-8").splitlines()]
|
|
adapter = Q1AlignmentAdapter()
|
|
modes = Counter()
|
|
nonfinite = 0
|
|
provenance_error = 0.0
|
|
shapes = {m: Counter() for m in MODALITIES}
|
|
duration = []
|
|
for sample_id in ids:
|
|
sample = adapter.from_q1_sample(sample_id)
|
|
modes[sample.metadata["coordinate_mode"]] += 1
|
|
duration.append(sample.metadata["duration_s"])
|
|
for m in MODALITIES:
|
|
x = sample.features[m]
|
|
shapes[m][str(list(x.shape))] += 1
|
|
nonfinite += int(np.count_nonzero(~np.isfinite(x)))
|
|
sums = np.asarray(sample.provenance[m].source_weights.sum(axis=1)).ravel()
|
|
observed = sample.observed[m]
|
|
if observed.any():
|
|
provenance_error = max(provenance_error, float(np.max(np.abs(sums[observed] - 1))))
|
|
return {"samples": len(ids), "coordinate_modes": dict(modes), "feature_shapes": {m: dict(v) for m, v in shapes.items()},
|
|
"duration_s": _distribution(duration), "nonfinite_output_values": nonfinite,
|
|
"max_provenance_row_sum_error": provenance_error,
|
|
"stored_dense_0_1_s_views_untouched": True}
|
|
|
|
|
|
def main() -> None:
|
|
RESULTS.mkdir(parents=True, exist_ok=True)
|
|
baseline = json.loads((RESULTS / "legacy_relative_baseline.json").read_text(encoding="utf-8"))
|
|
obj = restricted_load(OFFICIAL)
|
|
report = {"adapter": "q1-unified-1", "official_input": str(OFFICIAL.relative_to(ROOT)),
|
|
"relative": {name: _audit_split(name, obj[name], baseline) for name in ("train", "valid", "test")}}
|
|
del obj
|
|
report["physical"] = _audit_physical()
|
|
(RESULTS / "full_audit.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
equivalence = {"comparison": "SHA-256 of full float32 feature arrays and boolean masks against frozen old adapter output",
|
|
"splits": {name: {"samples": v["samples"], "exact_feature_equivalence": v["exact_feature_equivalence"],
|
|
"exact_mask_equivalence": v["exact_mask_equivalence"],
|
|
"old_feature_sha256": v["legacy_sha256"], "new_feature_sha256": v["sha256"],
|
|
"old_mask_sha256": baseline[name]["mask_sha256"],
|
|
"new_mask_sha256": v["mask_sha256"]}
|
|
for name, v in report["relative"].items()}}
|
|
(RESULTS / "equivalence_report.json").write_text(json.dumps(equivalence, indent=2), encoding="utf-8")
|
|
print(json.dumps({"relative": {name: {"samples": v["samples"],
|
|
"exact_feature_equivalence": v["exact_feature_equivalence"],
|
|
"exact_mask_equivalence": v["exact_mask_equivalence"]} for name, v in report["relative"].items()},
|
|
"physical_samples": report["physical"]["samples"]}, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|