{ "alignment": [ { "method": "M1", "modality": "text", "n": 100, "mvr_mean": 0.0, "mvr_std": 0.0, "normalized_entropy_mean": 0.0008249471336603165, "normalized_entropy_std": 0.005233484169432736, "width80_source_positions_mean": 1.004799998998642, "width80_source_positions_std": 0.0312849358458409, "expected_time_start_s_mean": 0.373650000790672, "expected_time_start_s_std": 0.28760723340035366, "expected_time_end_s_mean": 7.3229000100512325, "expected_time_end_s_std": 4.567867965796829, "trajectory_span_fraction_mean": 0.8612861529795919, "trajectory_span_fraction_std": 0.07875035859337948 }, { "method": "M1", "modality": "audio", "n": 100, "mvr_mean": 0.0, "mvr_std": 0.0, "normalized_entropy_mean": 0.35824004098773005, "normalized_entropy_std": 0.05432258900057586, "width80_source_positions_mean": 12.056200053691864, "width80_source_positions_std": 7.307551875172266, "expected_time_start_s_mean": 0.3177500062027157, "expected_time_start_s_std": 0.2794568898376313, "expected_time_end_s_mean": 7.441361117547106, "expected_time_end_s_std": 4.533387053269832, "trajectory_span_fraction_mean": 0.8937758523714728, "trajectory_span_fraction_std": 0.06338502509117576 }, { "method": "M1", "modality": "vision", "n": 100, "mvr_mean": 0.0, "mvr_std": 0.0, "normalized_entropy_mean": 0.02091189981205389, "normalized_entropy_std": 0.03423249778642653, "width80_source_positions_mean": 1.144999988079071, "width80_source_positions_std": 0.27983220085257804, "expected_time_start_s_mean": 0.30618889875375566, "expected_time_start_s_std": 0.2956530091396046, "expected_time_end_s_mean": 7.43663269007148, "expected_time_end_s_std": 4.535796288658362, "trajectory_span_fraction_mean": 0.8956127409730107, "trajectory_span_fraction_std": 0.0656268624353691 }, { "method": "M2", "modality": "text", "n": 100, "mvr_mean": 0.0, "mvr_std": 0.0, "normalized_entropy_mean": 0.003971234038472176, "normalized_entropy_std": 0.010095580232451877, "width80_source_positions_mean": 1.0215999960899353, "width80_source_positions_std": 0.060615033764357146, "expected_time_start_s_mean": 0.3753062509072731, "expected_time_start_s_std": 0.2858492171110105, "expected_time_end_s_mean": 7.3229000100512325, "expected_time_end_s_std": 4.567867965796829, "trajectory_span_fraction_mean": 0.8611038638930768, "trajectory_span_fraction_std": 0.07866203063075565 }, { "method": "M2", "modality": "audio", "n": 100, "mvr_mean": 0.0, "mvr_std": 0.0, "normalized_entropy_mean": 0.39199826538562776, "normalized_entropy_std": 0.04856653297051989, "width80_source_positions_mean": 12.777399990558624, "width80_source_positions_std": 7.437441491467199, "expected_time_start_s_mean": 0.08135000219190694, "expected_time_start_s_std": 0.046356409936463026, "expected_time_end_s_mean": 7.716148159798975, "expected_time_end_s_std": 4.594836781525238, "trajectory_span_fraction_mean": 0.968455815287307, "trajectory_span_fraction_std": 0.009448427161791137 }, { "method": "M2", "modality": "vision", "n": 100, "mvr_mean": 0.0, "mvr_std": 0.0, "normalized_entropy_mean": 0.015714890849776567, "normalized_entropy_std": 0.04120196183036711, "width80_source_positions_mean": 1.108399999141693, "width80_source_positions_std": 0.3072087474461631, "expected_time_start_s_mean": 0.024000000567226686, "expected_time_start_s_std": 0.0514830577428781, "expected_time_end_s_mean": 7.7077327980137875, "expected_time_end_s_std": 4.617512750249008, "trajectory_span_fraction_mean": 0.9744537002407014, "trajectory_span_fraction_std": 0.016307502074103464 }, { "method": "M3", "modality": "text", "n": 300, "mvr_mean": 0.0, "mvr_std": 0.0, "normalized_entropy_mean": 0.0004981429129838944, "normalized_entropy_std": 0.004964740878146852, "width80_source_positions_mean": 1.0029999995231629, "width80_source_positions_std": 0.02989949245257891, "expected_time_start_s_mean": 0.37365000079067195, "expected_time_start_s_std": 0.28664372237282515, "expected_time_end_s_mean": 7.320650003693404, "expected_time_end_s_std": 4.545633324884766, "trajectory_span_fraction_mean": 0.8611844960658346, "trajectory_span_fraction_std": 0.07838045740132776 }, { "method": "M3", "modality": "audio", "n": 300, "mvr_mean": 0.01857142824679613, "mvr_std": 0.028241165293696132, "normalized_entropy_mean": 0.992265993754069, "normalized_entropy_std": 0.00445452263162519, "width80_source_positions_mean": 607.8159977213542, "width80_source_positions_std": 365.5015286317014, "expected_time_start_s_mean": 3.883805515007902, "expected_time_start_s_std": 2.328319755482711, "expected_time_end_s_mean": 3.8684255829138965, "expected_time_end_s_std": 2.2993181536785716, "trajectory_span_fraction_mean": -0.001562372346719104, "trajectory_span_fraction_std": 0.016018112979249148 }, { "method": "M3", "modality": "vision", "n": 300, "mvr_mean": 0.06843537279715141, "mvr_std": 0.06722724739800441, "normalized_entropy_mean": 0.9827406370639801, "normalized_entropy_std": 0.021948411615917433, "width80_source_positions_mean": 31.175333293279014, "width80_source_positions_std": 18.019629268682312, "expected_time_start_s_mean": 3.8615792202155004, "expected_time_start_s_std": 2.266734235588232, "expected_time_end_s_mean": 3.834497800808173, "expected_time_end_s_std": 2.259828297195092, "trajectory_span_fraction_mean": -0.004611149926980335, "trajectory_span_fraction_std": 0.0433981312652511 }, { "method": "M4", "modality": "text", "n": 300, "mvr_mean": 0.0, "mvr_std": 0.0, "normalized_entropy_mean": 0.9999777108430863, "normalized_entropy_std": 4.937972466967483e-06, "width80_source_positions_mean": 15.891133365631104, "width80_source_positions_std": 8.431963588888667, "expected_time_start_s_mean": 3.83314954836363, "expected_time_start_s_std": 2.345445678904365, "expected_time_end_s_mean": 3.8348252384682473, "expected_time_end_s_std": 2.3463801144674252, "trajectory_span_fraction_mean": 0.00020046850045522113, "trajectory_span_fraction_std": 0.0014442569299196843 }, { "method": "M4", "modality": "audio", "n": 300, "mvr_mean": 0.0, "mvr_std": 0.0, "normalized_entropy_mean": 0.9999900078773498, "normalized_entropy_std": 3.4153207662384735e-06, "width80_source_positions_mean": 618.1359326680501, "width80_source_positions_std": 369.4452964156476, "expected_time_start_s_mean": 3.867822667326802, "expected_time_start_s_std": 2.309874647070878, "expected_time_end_s_mean": 3.868319556290074, "expected_time_end_s_std": 2.3104860442974764, "trajectory_span_fraction_mean": 4.607309897740703e-05, "trajectory_span_fraction_std": 0.0006428369960135046 }, { "method": "M4", "modality": "vision", "n": 300, "mvr_mean": 0.0, "mvr_std": 0.0, "normalized_entropy_mean": 0.9999960112571716, "normalized_entropy_std": 4.416330070963206e-06, "width80_source_positions_mean": 32.02326666514079, "width80_source_positions_std": 18.480166711480305, "expected_time_start_s_mean": 3.8529649544991638, "expected_time_start_s_std": 2.3080516925693866, "expected_time_end_s_mean": 3.8542076376005214, "expected_time_end_s_std": 2.3106234287813843, "trajectory_span_fraction_mean": 0.00010548800230026034, "trajectory_span_fraction_std": 0.001267114074095688 } ], "retrieval": [ { "method": "M1", "direction": "text_to_audio", "n": 5, "r_at_1_mean": 0.0012000000569969416, "r_at_1_std": 0.001643167750561775, "r_at_5_mean": 0.007000000309199094, "r_at_5_std": 0.0015811389419980182, "mrr_mean": 0.008951337542384863, "mrr_std": 0.0015105744211016399 }, { "method": "M1", "direction": "audio_to_text", "n": 5, "r_at_1_mean": 0.0028000001097097994, "r_at_1_std": 0.0014832398031884779, "r_at_5_mean": 0.006000000238418579, "r_at_5_std": 0.0021213204717563106, "mrr_mean": 0.009594007954001426, "mrr_std": 0.0008820339883026645 }, { "method": "M1", "direction": "text_to_vision", "n": 5, "r_at_1_mean": 0.0020000000949949026, "r_at_1_std": 0.0020000001240987344, "r_at_5_mean": 0.007000000402331352, "r_at_5_std": 0.0031622779576235397, "mrr_mean": 0.009382870234549045, "mrr_std": 0.0038973032823256706 }, { "method": "M1", "direction": "vision_to_text", "n": 5, "r_at_1_mean": 0.005000000214204192, "r_at_1_std": 0.0038729835451934104, "r_at_5_mean": 0.007000000402331352, "r_at_5_std": 0.004301162959612636, "mrr_mean": 0.010874488297849894, "mrr_std": 0.0033759486781861315 }, { "method": "M1", "direction": "audio_to_vision", "n": 5, "r_at_1_mean": 0.0010000000474974513, "r_at_1_std": 0.0010000000474974513, "r_at_5_mean": 0.008600000198930502, "r_at_5_std": 0.004037325898109636, "mrr_mean": 0.010010761767625808, "mrr_std": 0.0028143308073864106 }, { "method": "M1", "direction": "vision_to_audio", "n": 5, "r_at_1_mean": 0.002000000048428774, "r_at_1_std": 0.0009999999892897908, "r_at_5_mean": 0.008000000286847353, "r_at_5_std": 0.0036742348045336777, "mrr_mean": 0.009903422184288501, "mrr_std": 0.0023425375089251867 }, { "method": "M2", "direction": "text_to_audio", "n": 5, "r_at_1_mean": 0.0014000000432133675, "r_at_1_std": 0.0011401754384133685, "r_at_5_mean": 0.009200000390410423, "r_at_5_std": 0.0022803508359855447, "mrr_mean": 0.010239988192915916, "mrr_std": 0.0015598764796833854 }, { "method": "M2", "direction": "audio_to_text", "n": 5, "r_at_1_mean": 0.004000000143423677, "r_at_1_std": 0.0015811388683705022, "r_at_5_mean": 0.009200000483542681, "r_at_5_std": 0.0030331502218969453, "mrr_mean": 0.011443204618990422, "mrr_std": 0.0018186032524210385 }, { "method": "M2", "direction": "text_to_vision", "n": 5, "r_at_1_mean": 0.003600000194273889, "r_at_1_std": 0.003209361472308511, "r_at_5_mean": 0.008800000487826764, "r_at_5_std": 0.004764451893141854, "mrr_mean": 0.011315117496997118, "mrr_std": 0.004998444639556679 }, { "method": "M2", "direction": "vision_to_text", "n": 5, "r_at_1_mean": 0.00680000027641654, "r_at_1_std": 0.001923538612521058, "r_at_5_mean": 0.009200000576674939, "r_at_5_std": 0.0013038406947566386, "mrr_mean": 0.013902883604168892, "mrr_std": 0.0007593274807152007 }, { "method": "M2", "direction": "audio_to_vision", "n": 5, "r_at_1_mean": 0.0014000000664964318, "r_at_1_std": 0.0005477225835205916, "r_at_5_mean": 0.006800000183284282, "r_at_5_std": 0.003346640098121672, "mrr_mean": 0.009431831166148185, "mrr_std": 0.0017045066129491178 }, { "method": "M2", "direction": "vision_to_audio", "n": 5, "r_at_1_mean": 0.002000000118277967, "r_at_1_std": 0.0018708288289165209, "r_at_5_mean": 0.00640000025741756, "r_at_5_std": 0.0032093615938253205, "mrr_mean": 0.00967191495001316, "mrr_std": 0.002293345042131113 }, { "method": "M3", "direction": "text_to_audio", "n": 15, "r_at_1_mean": 0.01966666733690848, "r_at_1_std": 0.009707631019479971, "r_at_5_mean": 0.043000001646578315, "r_at_5_std": 0.020511321663901527, "mrr_mean": 0.03803744154671828, "mrr_std": 0.014634778360297347 }, { "method": "M3", "direction": "audio_to_text", "n": 15, "r_at_1_mean": 0.009066667081788183, "r_at_1_std": 0.009353889531597692, "r_at_5_mean": 0.020933334335374334, "r_at_5_std": 0.013084706401165278, "mrr_mean": 0.023117945653696854, "mrr_std": 0.011741696228606028 }, { "method": "M3", "direction": "text_to_vision", "n": 15, "r_at_1_mean": 0.0062666668944681685, "r_at_1_std": 0.00413118239876218, "r_at_5_mean": 0.013800000709791978, "r_at_5_std": 0.007466114240091005, "mrr_mean": 0.01513038482517004, "mrr_std": 0.005506734651604392 }, { "method": "M3", "direction": "vision_to_text", "n": 15, "r_at_1_mean": 0.00506666685299327, "r_at_1_std": 0.003634490987677242, "r_at_5_mean": 0.011266667147477467, "r_at_5_std": 0.005573748401698712, "mrr_mean": 0.01275980711604158, "mrr_std": 0.003981040361844423 }, { "method": "M3", "direction": "audio_to_vision", "n": 15, "r_at_1_mean": 0.006400000179807345, "r_at_1_std": 0.0033974781354081164, "r_at_5_mean": 0.0142666673908631, "r_at_5_std": 0.007215723836767552, "mrr_mean": 0.016017937225600083, "mrr_std": 0.005224730877211012 }, { "method": "M3", "direction": "vision_to_audio", "n": 15, "r_at_1_mean": 0.00426666682275633, "r_at_1_std": 0.0024630605290560273, "r_at_5_mean": 0.00913333367401113, "r_at_5_std": 0.005111145810859167, "mrr_mean": 0.012493464909493924, "mrr_std": 0.0045251231363594895 }, { "method": "M4", "direction": "text_to_audio", "n": 15, "r_at_1_mean": 0.007066666963510215, "r_at_1_std": 0.01601546933258453, "r_at_5_mean": 0.01226666709408164, "r_at_5_std": 0.019414526078011158, "mrr_mean": 0.014511969638988376, "mrr_std": 0.018944261404425476 }, { "method": "M4", "direction": "audio_to_text", "n": 15, "r_at_1_mean": 0.00046666668883214396, "r_at_1_std": 0.0005163978040219006, "r_at_5_mean": 0.002266666820893685, "r_at_5_std": 0.00252039318928005, "mrr_mean": 0.005223872559145093, "mrr_std": 0.0023624842688679767 }, { "method": "M4", "direction": "text_to_vision", "n": 15, "r_at_1_mean": 0.0016000000604738792, "r_at_1_std": 0.0011832159931418852, "r_at_5_mean": 0.00760000046963493, "r_at_5_std": 0.005852960507593743, "mrr_mean": 0.0100666221541663, "mrr_std": 0.005020802122090484 }, { "method": "M4", "direction": "vision_to_text", "n": 15, "r_at_1_mean": 0.0014666667130465308, "r_at_1_std": 0.0010600988416483645, "r_at_5_mean": 0.007333333759258191, "r_at_5_std": 0.005300494367735815, "mrr_mean": 0.009886658145114779, "mrr_std": 0.004472077112598579 }, { "method": "M4", "direction": "audio_to_vision", "n": 15, "r_at_1_mean": 0.002133333380334079, "r_at_1_std": 0.0009154754048125681, "r_at_5_mean": 0.010533333849161863, "r_at_5_std": 0.004405624197684311, "mrr_mean": 0.012994256491462389, "mrr_std": 0.00403471312250859 }, { "method": "M4", "direction": "vision_to_audio", "n": 15, "r_at_1_mean": 0.00206666673378398, "r_at_1_std": 0.0007037315398609183, "r_at_5_mean": 0.010266667200873296, "r_at_5_std": 0.003712270612174926, "mrr_mean": 0.012846407542626063, "mrr_std": 0.003972973272038369 } ], "reconstruction": [ { "method": "M1", "target_modality": "text", "n": 5, "mae_standardized_mean": 0.8308010935783386, "mae_standardized_std": 0.028634584948895196, "smooth_l1_standardized_mean": 0.4546235024929047, "smooth_l1_standardized_std": 0.025151457325080215 }, { "method": "M1", "target_modality": "audio", "n": 5, "mae_standardized_mean": 0.6908806562423706, "mae_standardized_std": 0.06720606353302543, "smooth_l1_standardized_mean": 0.34603241086006165, "smooth_l1_standardized_std": 0.06541831705153776 }, { "method": "M1", "target_modality": "vision", "n": 5, "mae_standardized_mean": 0.9209612965583801, "mae_standardized_std": 0.04009937912435952, "smooth_l1_standardized_mean": 0.5347834944725036, "smooth_l1_standardized_std": 0.036821434614539225 }, { "method": "M2", "target_modality": "text", "n": 5, "mae_standardized_mean": 0.8190697312355042, "mae_standardized_std": 0.03563334190353864, "smooth_l1_standardized_mean": 0.44534851908683776, "smooth_l1_standardized_std": 0.030773561031959877 }, { "method": "M2", "target_modality": "audio", "n": 5, "mae_standardized_mean": 0.6621044635772705, "mae_standardized_std": 0.05689547728439605, "smooth_l1_standardized_mean": 0.32293344140052793, "smooth_l1_standardized_std": 0.05414120404134669 }, { "method": "M2", "target_modality": "vision", "n": 5, "mae_standardized_mean": 0.9195392370223999, "mae_standardized_std": 0.03744298345037278, "smooth_l1_standardized_mean": 0.5340398073196411, "smooth_l1_standardized_std": 0.0346960350528028 }, { "method": "M3", "target_modality": "text", "n": 15, "mae_standardized_mean": 0.8311106324195862, "mae_standardized_std": 0.025205971351551344, "smooth_l1_standardized_mean": 0.45519693891207375, "smooth_l1_standardized_std": 0.021717713583393992 }, { "method": "M3", "target_modality": "audio", "n": 15, "mae_standardized_mean": 0.38449583053588865, "mae_standardized_std": 0.050754623523549675, "smooth_l1_standardized_mean": 0.14107545365889868, "smooth_l1_standardized_std": 0.05381054674326515 }, { "method": "M3", "target_modality": "vision", "n": 15, "mae_standardized_mean": 0.868217929204305, "mae_standardized_std": 0.03802189028391118, "smooth_l1_standardized_mean": 0.48890710473060606, "smooth_l1_standardized_std": 0.03450529817617854 }, { "method": "M4", "target_modality": "text", "n": 15, "mae_standardized_mean": 0.4924671709537506, "mae_standardized_std": 0.011169848357006705, "smooth_l1_standardized_mean": 0.18583014905452727, "smooth_l1_standardized_std": 0.008262501869688806 }, { "method": "M4", "target_modality": "audio", "n": 15, "mae_standardized_mean": 0.37839911778767904, "mae_standardized_std": 0.05503994105535867, "smooth_l1_standardized_mean": 0.13528098662694296, "smooth_l1_standardized_std": 0.05838370221384358 }, { "method": "M4", "target_modality": "vision", "n": 15, "mae_standardized_mean": 0.8630666335423788, "mae_standardized_std": 0.03867536107911695, "smooth_l1_standardized_mean": 0.48466227054595945, "smooth_l1_standardized_std": 0.03512809212998346 } ], "emotion": [ { "method": "M1", "n": 5, "accuracy_mean": 0.5900000000000001, "accuracy_std": 0.1673320053068151, "macro_f1_mean": 0.43700352733686065, "macro_f1_std": 0.22785227549938045, "mae_mean": 0.5521364179253578, "mae_std": 0.12144893660249381, "pearson_mean": 0.5305163430956757, "pearson_std": 0.09576372130151391 }, { "method": "M2", "n": 5, "accuracy_mean": 0.55, "accuracy_std": 0.1767766952966369, "macro_f1_mean": 0.3434301254301254, "macro_f1_std": 0.14084701638388486, "mae_mean": 0.5691992919147014, "mae_std": 0.11997495602498047, "pearson_mean": 0.5082439027512137, "pearson_std": 0.09789171743073 }, { "method": "M3", "n": 15, "accuracy_mean": 0.5766666666666667, "accuracy_std": 0.18503538915054937, "macro_f1_mean": 0.43789820467598245, "macro_f1_std": 0.21990438278507196, "mae_mean": 0.5534388141334057, "mae_std": 0.11230839313313754, "pearson_mean": 0.5457241196963684, "pearson_std": 0.09265676731571848 }, { "method": "M4", "n": 15, "accuracy_mean": 0.56, "accuracy_std": 0.15142183273415838, "macro_f1_mean": 0.41446068467573843, "macro_f1_std": 0.13578816810911878, "mae_mean": 0.6231901267170906, "mae_std": 0.13326862372482587, "pearson_mean": 0.5067829543507287, "pearson_std": 0.1021700904550588 } ] }