260 lines
9.9 KiB
JSON
260 lines
9.9 KiB
JSON
{
|
|
"created_utc": "2026-09-23T14:26:26.904353+00:00",
|
|
"experiment": "TSFA: Temporal-Semantic Factorized Alignment",
|
|
"sample_count": 100,
|
|
"fold_count": 5,
|
|
"heldout_partition_count": 100,
|
|
"unique_video_id_count": 37,
|
|
"folds": [
|
|
{
|
|
"fold": 1,
|
|
"train_count": 80,
|
|
"heldout_count": 20,
|
|
"train_video_id_count": 31,
|
|
"heldout_video_id_count": 6,
|
|
"video_id_overlap": [],
|
|
"validation_sample_ids": [
|
|
"-3g5yACwYnA/13",
|
|
"-3g5yACwYnA/3",
|
|
"-3g5yACwYnA/2",
|
|
"-3g5yACwYnA/9",
|
|
"-egA8-b7-3M/26",
|
|
"-egA8-b7-3M/17",
|
|
"-egA8-b7-3M/18",
|
|
"-egA8-b7-3M/16",
|
|
"-egA8-b7-3M/13",
|
|
"-egA8-b7-3M/1",
|
|
"-egA8-b7-3M/6",
|
|
"-egA8-b7-3M/9",
|
|
"-egA8-b7-3M/20",
|
|
"-I_e4mIh0yE/1",
|
|
"-I_e4mIh0yE/3",
|
|
"-qDkUB0GgYY/6",
|
|
"-9y-fZ3swSY/0",
|
|
"-9y-fZ3swSY/4",
|
|
"-9y-fZ3swSY/8",
|
|
"-UUCSKoHeMA/0"
|
|
]
|
|
},
|
|
{
|
|
"fold": 2,
|
|
"train_count": 80,
|
|
"heldout_count": 20,
|
|
"train_video_id_count": 30,
|
|
"heldout_video_id_count": 7,
|
|
"video_id_overlap": [],
|
|
"validation_sample_ids": [
|
|
"-THoVjtIkeU/12",
|
|
"-THoVjtIkeU/2",
|
|
"-THoVjtIkeU/6",
|
|
"-UuX1xuaiiE/1",
|
|
"-UuX1xuaiiE/0",
|
|
"-UuX1xuaiiE/3",
|
|
"-UuX1xuaiiE/6",
|
|
"-lzEya4AM_4/5",
|
|
"-lzEya4AM_4/6",
|
|
"-mqbVkbCndg/0",
|
|
"-wny0OAz3g8/1",
|
|
"-wny0OAz3g8/0",
|
|
"-wny0OAz3g8/3",
|
|
"-wny0OAz3g8/2",
|
|
"-wny0OAz3g8/5",
|
|
"-wny0OAz3g8/7",
|
|
"-wny0OAz3g8/9",
|
|
"-571d8cVauQ/0",
|
|
"-571d8cVauQ/5",
|
|
"-HeZS2-Prhc/2"
|
|
]
|
|
},
|
|
{
|
|
"fold": 3,
|
|
"train_count": 80,
|
|
"heldout_count": 20,
|
|
"train_video_id_count": 29,
|
|
"heldout_video_id_count": 8,
|
|
"video_id_overlap": [],
|
|
"validation_sample_ids": [
|
|
"-HwX2H8Z4hY/2",
|
|
"-HwX2H8Z4hY/5",
|
|
"-HwX2H8Z4hY/6",
|
|
"-HwX2H8Z4hY/9",
|
|
"-aqamKhZ1Ec/0",
|
|
"-vxjVxOeScU/4",
|
|
"-wMB_hJL-3o/7",
|
|
"-hnBHBN8p5A/7",
|
|
"-hnBHBN8p5A/6",
|
|
"-AUZQgSxyPQ/2",
|
|
"-RfYyzHpjk4/11",
|
|
"-RfYyzHpjk4/8",
|
|
"-RfYyzHpjk4/2",
|
|
"-s9qJ7ATP7w/1",
|
|
"-s9qJ7ATP7w/0",
|
|
"-s9qJ7ATP7w/5",
|
|
"-s9qJ7ATP7w/4",
|
|
"-s9qJ7ATP7w/7",
|
|
"-s9qJ7ATP7w/6",
|
|
"-s9qJ7ATP7w/8"
|
|
]
|
|
},
|
|
{
|
|
"fold": 4,
|
|
"train_count": 80,
|
|
"heldout_count": 20,
|
|
"train_video_id_count": 29,
|
|
"heldout_video_id_count": 8,
|
|
"video_id_overlap": [],
|
|
"validation_sample_ids": [
|
|
"-aNfi7CP8vM/7",
|
|
"-iRBcNs9oI8/3",
|
|
"-iRBcNs9oI8/7",
|
|
"-iRBcNs9oI8/6",
|
|
"-iRBcNs9oI8/9",
|
|
"-iRBcNs9oI8/8",
|
|
"-t217m2on-s/2",
|
|
"-t217m2on-s/7",
|
|
"-tPCytz4rww/11",
|
|
"-tPCytz4rww/10",
|
|
"-tPCytz4rww/12",
|
|
"-tPCytz4rww/16",
|
|
"-tPCytz4rww/18",
|
|
"-UacrmKiTn4/10",
|
|
"-UacrmKiTn4/4",
|
|
"-uywlfIYOS8/4",
|
|
"-6rXp3zJ3kc/8",
|
|
"-ri04Z7vwnc/0",
|
|
"-ri04Z7vwnc/2",
|
|
"-ri04Z7vwnc/5"
|
|
]
|
|
},
|
|
{
|
|
"fold": 5,
|
|
"train_count": 80,
|
|
"heldout_count": 20,
|
|
"train_video_id_count": 29,
|
|
"heldout_video_id_count": 8,
|
|
"video_id_overlap": [],
|
|
"validation_sample_ids": [
|
|
"-3nNcZdcdvU/5",
|
|
"-NFrJFQijFE/1",
|
|
"-NFrJFQijFE/2",
|
|
"-a55Q6RWvTA/3",
|
|
"-dxfTGcXJoc/1",
|
|
"-dxfTGcXJoc/0",
|
|
"-dxfTGcXJoc/2",
|
|
"-dxfTGcXJoc/6",
|
|
"-mJ2ud6oKI8/1",
|
|
"-mJ2ud6oKI8/2",
|
|
"-mJ2ud6oKI8/6",
|
|
"-mJ2ud6oKI8/9",
|
|
"-mJ2ud6oKI8/8",
|
|
"-tANM6ETl_M/3",
|
|
"-MeTTeMJBNc/0",
|
|
"-MeTTeMJBNc/13",
|
|
"-MeTTeMJBNc/7",
|
|
"-yRb-Jum7EQ/1",
|
|
"-yRb-Jum7EQ/5",
|
|
"-yRb-Jum7EQ/6"
|
|
]
|
|
}
|
|
],
|
|
"alignment_models_retrained": false,
|
|
"temporal_branch": "frozen M4_sourceTime D5 checkpoint per grouped fold",
|
|
"semantic_branch_trained": true,
|
|
"emotion_labels_used_for_alignment": false,
|
|
"feature_extractors_changed": false,
|
|
"feature_encoder_note": "Existing BERT text and DeiT image features are reused unchanged.",
|
|
"example_sample_id": "-3g5yACwYnA/13",
|
|
"parameters": {
|
|
"seed": 42,
|
|
"grid_size": 50,
|
|
"hidden_size": 128,
|
|
"heads_in_frozen_alignment": 4,
|
|
"delta_normalized_time": 0.1,
|
|
"hard_negative_offsets_slots": [
|
|
-5,
|
|
-3,
|
|
-2,
|
|
2,
|
|
3,
|
|
5
|
|
],
|
|
"semantic_epochs": 40,
|
|
"semantic_learning_rate": 0.001,
|
|
"local_contrastive_temperature": 0.1,
|
|
"content_probe_epochs": 40,
|
|
"content_probe_temperature": 0.1,
|
|
"reconstruction_probe_epochs": 40,
|
|
"shuffle_repeats": 20,
|
|
"random_window_repeats": 20,
|
|
"window_ablation_status": "delta sweep and 80-percent attention-mass window deferred until the seven-method MVP is reviewed"
|
|
},
|
|
"methods": [
|
|
"M3_noSourceTime",
|
|
"M3_sourceTime",
|
|
"M4_sourceTime",
|
|
"TSFA-main",
|
|
"TSFA-multiply",
|
|
"TSFA-random",
|
|
"TSFA-global"
|
|
],
|
|
"protocol": {
|
|
"semantic_query_key_inputs": "Frozen M4 attention-pooled text value output is the semantic query input; M4 pre-attention projected Audio/Vision content is the semantic key/value input. No explicit source-time code, absolute PE, slot ID, or latent positional vector is passed to semantic Q/K. Temporal selection can still encode time indirectly.",
|
|
"baseline_content_values": "M4 and M3 Audio/Vision use each checkpoint's actual MHA attention output. M3 Text uses attention-pooled pre-attention text features to remove its source-time query residual.",
|
|
"temporal_role": "Frozen M4 attention supplies candidate masks; TSFA-multiply additionally multiplies semantic probabilities by M4 temporal attention",
|
|
"local_contrastive": "same-slot positives with within-clip slot-offset negatives at +/-2, +/-3, +/-5; symmetric Text-Audio and Text-Vision loss",
|
|
"random_candidate": "same delta-width windows with independently sampled centers at inference; 20 held-out random draws, one train draw for the train-only probe",
|
|
"global_candidate": "all valid source positions available to semantic attention",
|
|
"content_probe": "same train-only symmetric within-clip InfoNCE probe applied to each method; held-out negatives are positions more than two slots away",
|
|
"content_shuffle": "independently permute each modality's 50 projected content rows within each held-out clip; M4 and TSFA-main",
|
|
"reconstruction": "fold-train decoder predicts one aligned content representation from the other two; shift one partner by signed offsets +/-1,2,5,10 on identical target support",
|
|
"confidence_intervals": "95-percent video_id-cluster bootstrap; 2,000 repetitions for CSV summaries"
|
|
},
|
|
"input_paths": {
|
|
"feature_dir": "/home/gloamxun/modeling_zhaocui/deep_learning/Q1/outputs/q1_features/features",
|
|
"feature_manifest": "/home/gloamxun/modeling_zhaocui/deep_learning/Q1/outputs/audit/manifest.csv",
|
|
"grouped_splits": "/home/gloamxun/modeling_zhaocui/deep_learning/Q1/outputs/method_comparison/splits.json",
|
|
"checkpoint_root": "/home/gloamxun/modeling_zhaocui/deep_learning/Q1/outputs/alignment_debug/heldout",
|
|
"m3_m4_checkpoint_variants": [
|
|
"M3_noSourceTime",
|
|
"M3_sourceTime",
|
|
"M4_sourceTime"
|
|
]
|
|
},
|
|
"device": "cuda",
|
|
"gpu_name": "NVIDIA GeForce RTX 5070 Ti",
|
|
"python": "3.14.7",
|
|
"torch": "2.14.0+cu130",
|
|
"elapsed_seconds": 129.8167963027954,
|
|
"interpretation_limits": [
|
|
"Same-slot labels are a temporal training convention, not independent semantic ground truth.",
|
|
"Frozen M4 temporal attention was trained with a timestamp-derived Gaussian prior; time may enter content indirectly through candidate selection.",
|
|
"AUC, retrieval, and shifted similarity measure within-clip content matchability, not human event alignment.",
|
|
"Reconstruction predicts M4/TSFA pooled content values, not raw waveform or image pixels.",
|
|
"Five folds use one alignment-model seed; bootstrap intervals quantify video-group sampling uncertainty, not seed uncertainty.",
|
|
"Human event IoU/center error remains pending event annotation."
|
|
],
|
|
"input_sha256": {
|
|
"feature_manifest": "bc07d5fb1ff104144cf845abf856c4e04da3caadda0c05d4b798a64a581bb35a",
|
|
"grouped_splits": "ccdcab416488fd3d94b762696f70411ebcd02346417cb7f1a5be70e7a1a6cf4d",
|
|
"uv_lock": "28ae0c736eef130a380022da82c79c473f467e2cb8dad4c199a85fd0b1faadf1",
|
|
"features_combined": "ed99618a0f4ac943fd21c214a331aeff11ca6aa7419b48622ab25f8f8d8ae985",
|
|
"feature_file_count": 100,
|
|
"frozen_alignment_checkpoints": {
|
|
"fold_01/M3_noSourceTime": "e54088fbc4432f41e9a09b167cd90a752c6bf678461e4d4ca553e1a7eb23fadc",
|
|
"fold_01/M3_sourceTime": "377772c546ec1cb79359b4792de76f67df9700a1ab48f0dfc66451640733ef67",
|
|
"fold_01/M4_sourceTime": "ad2e5fb0db3a8891a831ad3d2c85dde59f0563bc41dc05fff10655589d37d1fc",
|
|
"fold_02/M3_noSourceTime": "2bd24ca12a8c6eb4335b54db5de7b0f9963752279f0bf4daa9c7641b251e00ae",
|
|
"fold_02/M3_sourceTime": "64eca35e24962055c8e2d1520f62af291f395f8243438bc1609925d62794c673",
|
|
"fold_02/M4_sourceTime": "f77c6a46ec6424b7a52907bca81a0842ec0066ba9dec38b8e10bfbbf455a1a62",
|
|
"fold_03/M3_noSourceTime": "76dff4f2643a4fd46dc6d7a5d7d6a74ce8e119da53fed81d98b4c22d90af0052",
|
|
"fold_03/M3_sourceTime": "f52e47bdbe8a3b3b360ad053a202570170f45cdfc6d07066f6e8558d31182125",
|
|
"fold_03/M4_sourceTime": "de6fcfbaa57eab745b6ccfd7a19860589fccaa30b497813a3afdecb15956ac6c",
|
|
"fold_04/M3_noSourceTime": "735354a326ab23a832370449862493bcab2452551858a6825c099c27ff982b58",
|
|
"fold_04/M3_sourceTime": "304c7287f4d777b27a5354e4316802cdc6de75dd1dd20a419314255b67618a4e",
|
|
"fold_04/M4_sourceTime": "d0e50814377b5b98fbf70c3ca48431756b22391823629dab74c3192b8bfe1a3d",
|
|
"fold_05/M3_noSourceTime": "0f8d57bae2af04e0a7432a42a68b02b09c60bac4da68d4b7595905a31e1c8fe2",
|
|
"fold_05/M3_sourceTime": "47bccf271f6046e695fd7ad7e16d9a018d944df5773b5a8e15af9be80bd9af41",
|
|
"fold_05/M4_sourceTime": "787e28df330b9891f8ff24c6522d808bd0a1bac269f29c6f11dff55b8ba95d4e"
|
|
}
|
|
}
|
|
} |