{ "created_utc": "2026-09-23T14:26:26.904353+00:00", "experiment": "TSFA: Temporal-Semantic Factorized Alignment", "sample_count": 100, "fold_count": 5, "heldout_partition_count": 100, "unique_video_id_count": 37, "folds": [ { "fold": 1, "train_count": 80, "heldout_count": 20, "train_video_id_count": 31, "heldout_video_id_count": 6, "video_id_overlap": [], "validation_sample_ids": [ "-3g5yACwYnA/13", "-3g5yACwYnA/3", "-3g5yACwYnA/2", "-3g5yACwYnA/9", "-egA8-b7-3M/26", "-egA8-b7-3M/17", "-egA8-b7-3M/18", "-egA8-b7-3M/16", "-egA8-b7-3M/13", "-egA8-b7-3M/1", "-egA8-b7-3M/6", "-egA8-b7-3M/9", "-egA8-b7-3M/20", "-I_e4mIh0yE/1", "-I_e4mIh0yE/3", "-qDkUB0GgYY/6", "-9y-fZ3swSY/0", "-9y-fZ3swSY/4", "-9y-fZ3swSY/8", "-UUCSKoHeMA/0" ] }, { "fold": 2, "train_count": 80, "heldout_count": 20, "train_video_id_count": 30, "heldout_video_id_count": 7, "video_id_overlap": [], "validation_sample_ids": [ "-THoVjtIkeU/12", "-THoVjtIkeU/2", "-THoVjtIkeU/6", "-UuX1xuaiiE/1", "-UuX1xuaiiE/0", "-UuX1xuaiiE/3", "-UuX1xuaiiE/6", "-lzEya4AM_4/5", "-lzEya4AM_4/6", "-mqbVkbCndg/0", "-wny0OAz3g8/1", "-wny0OAz3g8/0", "-wny0OAz3g8/3", "-wny0OAz3g8/2", "-wny0OAz3g8/5", "-wny0OAz3g8/7", "-wny0OAz3g8/9", "-571d8cVauQ/0", "-571d8cVauQ/5", "-HeZS2-Prhc/2" ] }, { "fold": 3, "train_count": 80, "heldout_count": 20, "train_video_id_count": 29, "heldout_video_id_count": 8, "video_id_overlap": [], "validation_sample_ids": [ "-HwX2H8Z4hY/2", "-HwX2H8Z4hY/5", "-HwX2H8Z4hY/6", "-HwX2H8Z4hY/9", "-aqamKhZ1Ec/0", "-vxjVxOeScU/4", "-wMB_hJL-3o/7", "-hnBHBN8p5A/7", "-hnBHBN8p5A/6", "-AUZQgSxyPQ/2", "-RfYyzHpjk4/11", "-RfYyzHpjk4/8", "-RfYyzHpjk4/2", "-s9qJ7ATP7w/1", "-s9qJ7ATP7w/0", "-s9qJ7ATP7w/5", "-s9qJ7ATP7w/4", "-s9qJ7ATP7w/7", "-s9qJ7ATP7w/6", "-s9qJ7ATP7w/8" ] }, { "fold": 4, "train_count": 80, "heldout_count": 20, "train_video_id_count": 29, "heldout_video_id_count": 8, "video_id_overlap": [], "validation_sample_ids": [ "-aNfi7CP8vM/7", "-iRBcNs9oI8/3", "-iRBcNs9oI8/7", "-iRBcNs9oI8/6", "-iRBcNs9oI8/9", "-iRBcNs9oI8/8", "-t217m2on-s/2", "-t217m2on-s/7", "-tPCytz4rww/11", "-tPCytz4rww/10", "-tPCytz4rww/12", "-tPCytz4rww/16", "-tPCytz4rww/18", "-UacrmKiTn4/10", "-UacrmKiTn4/4", "-uywlfIYOS8/4", "-6rXp3zJ3kc/8", "-ri04Z7vwnc/0", "-ri04Z7vwnc/2", "-ri04Z7vwnc/5" ] }, { "fold": 5, "train_count": 80, "heldout_count": 20, "train_video_id_count": 29, "heldout_video_id_count": 8, "video_id_overlap": [], "validation_sample_ids": [ "-3nNcZdcdvU/5", "-NFrJFQijFE/1", "-NFrJFQijFE/2", "-a55Q6RWvTA/3", "-dxfTGcXJoc/1", "-dxfTGcXJoc/0", "-dxfTGcXJoc/2", "-dxfTGcXJoc/6", "-mJ2ud6oKI8/1", "-mJ2ud6oKI8/2", "-mJ2ud6oKI8/6", "-mJ2ud6oKI8/9", "-mJ2ud6oKI8/8", "-tANM6ETl_M/3", "-MeTTeMJBNc/0", "-MeTTeMJBNc/13", "-MeTTeMJBNc/7", "-yRb-Jum7EQ/1", "-yRb-Jum7EQ/5", "-yRb-Jum7EQ/6" ] } ], "alignment_models_retrained": false, "temporal_branch": "frozen M4_sourceTime D5 checkpoint per grouped fold", "semantic_branch_trained": true, "emotion_labels_used_for_alignment": false, "feature_extractors_changed": false, "feature_encoder_note": "Existing BERT text and DeiT image features are reused unchanged.", "example_sample_id": "-3g5yACwYnA/13", "parameters": { "seed": 42, "grid_size": 50, "hidden_size": 128, "heads_in_frozen_alignment": 4, "delta_normalized_time": 0.1, "hard_negative_offsets_slots": [ -5, -3, -2, 2, 3, 5 ], "semantic_epochs": 40, "semantic_learning_rate": 0.001, "local_contrastive_temperature": 0.1, "content_probe_epochs": 40, "content_probe_temperature": 0.1, "reconstruction_probe_epochs": 40, "shuffle_repeats": 20, "random_window_repeats": 20, "window_ablation_status": "delta sweep and 80-percent attention-mass window deferred until the seven-method MVP is reviewed" }, "methods": [ "M3_noSourceTime", "M3_sourceTime", "M4_sourceTime", "TSFA-main", "TSFA-multiply", "TSFA-random", "TSFA-global" ], "protocol": { "semantic_query_key_inputs": "Frozen M4 attention-pooled text value output is the semantic query input; M4 pre-attention projected Audio/Vision content is the semantic key/value input. No explicit source-time code, absolute PE, slot ID, or latent positional vector is passed to semantic Q/K. Temporal selection can still encode time indirectly.", "baseline_content_values": "M4 and M3 Audio/Vision use each checkpoint's actual MHA attention output. M3 Text uses attention-pooled pre-attention text features to remove its source-time query residual.", "temporal_role": "Frozen M4 attention supplies candidate masks; TSFA-multiply additionally multiplies semantic probabilities by M4 temporal attention", "local_contrastive": "same-slot positives with within-clip slot-offset negatives at +/-2, +/-3, +/-5; symmetric Text-Audio and Text-Vision loss", "random_candidate": "same delta-width windows with independently sampled centers at inference; 20 held-out random draws, one train draw for the train-only probe", "global_candidate": "all valid source positions available to semantic attention", "content_probe": "same train-only symmetric within-clip InfoNCE probe applied to each method; held-out negatives are positions more than two slots away", "content_shuffle": "independently permute each modality's 50 projected content rows within each held-out clip; M4 and TSFA-main", "reconstruction": "fold-train decoder predicts one aligned content representation from the other two; shift one partner by signed offsets +/-1,2,5,10 on identical target support", "confidence_intervals": "95-percent video_id-cluster bootstrap; 2,000 repetitions for CSV summaries" }, "input_paths": { "feature_dir": "/home/gloamxun/modeling_zhaocui/deep_learning/Q1/outputs/q1_features/features", "feature_manifest": "/home/gloamxun/modeling_zhaocui/deep_learning/Q1/outputs/audit/manifest.csv", "grouped_splits": "/home/gloamxun/modeling_zhaocui/deep_learning/Q1/outputs/method_comparison/splits.json", "checkpoint_root": "/home/gloamxun/modeling_zhaocui/deep_learning/Q1/outputs/alignment_debug/heldout", "m3_m4_checkpoint_variants": [ "M3_noSourceTime", "M3_sourceTime", "M4_sourceTime" ] }, "device": "cuda", "gpu_name": "NVIDIA GeForce RTX 5070 Ti", "python": "3.14.7", "torch": "2.14.0+cu130", "elapsed_seconds": 129.8167963027954, "interpretation_limits": [ "Same-slot labels are a temporal training convention, not independent semantic ground truth.", "Frozen M4 temporal attention was trained with a timestamp-derived Gaussian prior; time may enter content indirectly through candidate selection.", "AUC, retrieval, and shifted similarity measure within-clip content matchability, not human event alignment.", "Reconstruction predicts M4/TSFA pooled content values, not raw waveform or image pixels.", "Five folds use one alignment-model seed; bootstrap intervals quantify video-group sampling uncertainty, not seed uncertainty.", "Human event IoU/center error remains pending event annotation." ], "input_sha256": { "feature_manifest": "bc07d5fb1ff104144cf845abf856c4e04da3caadda0c05d4b798a64a581bb35a", "grouped_splits": "ccdcab416488fd3d94b762696f70411ebcd02346417cb7f1a5be70e7a1a6cf4d", "uv_lock": "28ae0c736eef130a380022da82c79c473f467e2cb8dad4c199a85fd0b1faadf1", "features_combined": "ed99618a0f4ac943fd21c214a331aeff11ca6aa7419b48622ab25f8f8d8ae985", "feature_file_count": 100, "frozen_alignment_checkpoints": { "fold_01/M3_noSourceTime": "e54088fbc4432f41e9a09b167cd90a752c6bf678461e4d4ca553e1a7eb23fadc", "fold_01/M3_sourceTime": "377772c546ec1cb79359b4792de76f67df9700a1ab48f0dfc66451640733ef67", "fold_01/M4_sourceTime": "ad2e5fb0db3a8891a831ad3d2c85dde59f0563bc41dc05fff10655589d37d1fc", "fold_02/M3_noSourceTime": "2bd24ca12a8c6eb4335b54db5de7b0f9963752279f0bf4daa9c7641b251e00ae", "fold_02/M3_sourceTime": "64eca35e24962055c8e2d1520f62af291f395f8243438bc1609925d62794c673", "fold_02/M4_sourceTime": "f77c6a46ec6424b7a52907bca81a0842ec0066ba9dec38b8e10bfbbf455a1a62", "fold_03/M3_noSourceTime": "76dff4f2643a4fd46dc6d7a5d7d6a74ce8e119da53fed81d98b4c22d90af0052", "fold_03/M3_sourceTime": "f52e47bdbe8a3b3b360ad053a202570170f45cdfc6d07066f6e8558d31182125", "fold_03/M4_sourceTime": "de6fcfbaa57eab745b6ccfd7a19860589fccaa30b497813a3afdecb15956ac6c", "fold_04/M3_noSourceTime": "735354a326ab23a832370449862493bcab2452551858a6825c099c27ff982b58", "fold_04/M3_sourceTime": "304c7287f4d777b27a5354e4316802cdc6de75dd1dd20a419314255b67618a4e", "fold_04/M4_sourceTime": "d0e50814377b5b98fbf70c3ca48431756b22391823629dab74c3192b8bfe1a3d", "fold_05/M3_noSourceTime": "0f8d57bae2af04e0a7432a42a68b02b09c60bac4da68d4b7595905a31e1c8fe2", "fold_05/M3_sourceTime": "47bccf271f6046e695fd7ad7e16d9a018d944df5773b5a8e15af9be80bd9af41", "fold_05/M4_sourceTime": "787e28df330b9891f8ff24c6522d808bd0a1bac269f29c6f11dff55b8ba95d4e" } } }