{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:WZ34WRPFD24Y2BKDMFRBPVSYWP","short_pith_number":"pith:WZ34WRPF","schema_version":"1.0","canonical_sha256":"b677cb45e51eb98d0543616217d658b3dd0c9b77e6a47833ac9b256373655b97","source":{"kind":"arxiv","id":"2412.16855","version":2},"attestation_state":"computed","paper":{"title":"GME: Improving Universal Multimodal Retrieval by Multimodal LLMs","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"Training an MLLM on synthetically balanced fused text-image data produces a single dense retriever that leads on universal multimodal search tasks.","cross_cats":["cs.IR"],"primary_cat":"cs.CL","authors_text":"Dingkun Long, Meishan Zhang, Mingxin Li, Min Zhang, Pengjun Xie, Wenjie Li, Wen Xie, Xin Zhang, Yanzhao Zhang, Ziqi Dai","submitted_at":"2024-12-22T04:40:24Z","abstract_excerpt":"Universal Multimodal Retrieval (UMR) aims to enable search across various modalities using a unified model, where queries and candidates can consist of pure text, images, or a combination of both. Previous work has attempted to adopt multimodal large language models (MLLMs) to realize UMR using only text data. However, our preliminary experiments demonstrate that more diverse multimodal training data can further unlock the potential of MLLMs. Despite its effectiveness, the existing multimodal training data is highly imbalanced in terms of modality, which motivates us to develop a training data"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":true,"formal_links_present":true},"canonical_record":{"source":{"id":"2412.16855","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-12-22T04:40:24Z","cross_cats_sorted":["cs.IR"],"title_canon_sha256":"ea6af16b54e7eb7912e00c33bd2c62ddb3a35dd38ab597b5206a3f4fbb5d0b62","abstract_canon_sha256":"748847fa281d19d5e0a56770ec05a9bba758fa4e7ad976dc2000ca0e11f1851a"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:38:53.248036Z","signature_b64":"2NlEVssAlVJgCmLNge9ovOAAbZn+SW9l80xSurnVwoWqDjWzJ6dgCDxxOXmhNx6uB8zui0fG+42y2cp3LPT7Bg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b677cb45e51eb98d0543616217d658b3dd0c9b77e6a47833ac9b256373655b97","last_reissued_at":"2026-05-17T23:38:53.247396Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:38:53.247396Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"GME: Improving Universal Multimodal Retrieval by Multimodal LLMs","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"Training an MLLM on synthetically balanced fused text-image data produces a single dense retriever that leads on universal multimodal search tasks.","cross_cats":["cs.IR"],"primary_cat":"cs.CL","authors_text":"Dingkun Long, Meishan Zhang, Mingxin Li, Min Zhang, Pengjun Xie, Wenjie Li, Wen Xie, Xin Zhang, Yanzhao Zhang, Ziqi Dai","submitted_at":"2024-12-22T04:40:24Z","abstract_excerpt":"Universal Multimodal Retrieval (UMR) aims to enable search across various modalities using a unified model, where queries and candidates can consist of pure text, images, or a combination of both. Previous work has attempted to adopt multimodal large language models (MLLMs) to realize UMR using only text data. However, our preliminary experiments demonstrate that more diverse multimodal training data can further unlock the potential of MLLMs. Despite its effectiveness, the existing multimodal training data is highly imbalanced in terms of modality, which motivates us to develop a training data"},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"Experimental results show that our method achieves state-of-the-art performance among existing UMR methods.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"That the synthetic fused-modal training dataset is of high quality and sufficiently diverse to unlock the full potential of MLLMs for universal multimodal retrieval without introducing biases or artifacts.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"GME achieves state-of-the-art results in universal multimodal retrieval by training on a balanced synthetic multimodal dataset.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"Training an MLLM on synthetically balanced fused text-image data produces a single dense retriever that leads on universal multimodal search tasks.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"7391d1dcc42778fcf2b5a7552aaa16498aa1df4333a5ae5f014102366d3a3f30"},"source":{"id":"2412.16855","kind":"arxiv","version":2},"verdict":{"id":"70f17f27-5ab2-405f-861c-74df3ec6f8c6","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-15T06:31:17.683583Z","strongest_claim":"Experimental results show that our method achieves state-of-the-art performance among existing UMR methods.","one_line_summary":"GME achieves state-of-the-art results in universal multimodal retrieval by training on a balanced synthetic multimodal dataset.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"That the synthetic fused-modal training dataset is of high quality and sufficiently diverse to unlock the full potential of MLLMs for universal multimodal retrieval without introducing biases or artifacts.","pith_extraction_headline":"Training an MLLM on synthetically balanced fused text-image data produces a single dense retriever that leads on universal multimodal search tasks."},"references":{"count":86,"sample":[{"doi":"","year":2020,"title":"Overview of touch ´e 2020: Argument retrieval - extended abstract","work_id":"6f0e6b93-d715-48fb-a68b-eea2dd6f658f","ref_index":1,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2016,"title":"A full-text learning to rank dataset for medical information retrieval","work_id":"1ffb6db3-12fa-4a54-a937-d43eb8fe19ae","ref_index":2,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2020,"title":"Tom B. Brown, Benjamin Mann, Nick Ryder, Melanie Sub- biah, Jared Kaplan, Prafulla Dhariwal, Arvind Neelakan- tan, Pranav Shyam, Girish Sastry, Amanda Askell, Sand- hini Agarwal, Ariel Herbert-V oss, ","work_id":"625c70f2-e4ac-4dfb-89e1-1b062618e14d","ref_index":3,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2022,"title":"Webqa: Multihop and multimodal QA","work_id":"62462bb5-05db-482f-844b-c2ab8a75925d","ref_index":4,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2016,"title":"Training Deep Nets with Sublinear Memory Cost","work_id":"f2c5c287-a500-40e4-a136-e7e3172db1d7","ref_index":5,"cited_arxiv_id":"1604.06174","is_internal_anchor":true}],"resolved_work":86,"snapshot_sha256":"545c259cae8b81c6a2a04e5d7160b1de7ea401847645d347a3a47e11a27c1f5f","internal_anchors":9},"formal_canon":{"evidence_count":2,"snapshot_sha256":"e91c7ab870fe8bc20fedae355cc788a12da726078e63b71a75da8ece9325dc3e"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2412.16855","created_at":"2026-05-17T23:38:53.247497+00:00"},{"alias_kind":"arxiv_version","alias_value":"2412.16855v2","created_at":"2026-05-17T23:38:53.247497+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.16855","created_at":"2026-05-17T23:38:53.247497+00:00"},{"alias_kind":"pith_short_12","alias_value":"WZ34WRPFD24Y","created_at":"2026-05-18T12:33:37.589309+00:00"},{"alias_kind":"pith_short_16","alias_value":"WZ34WRPFD24Y2BKD","created_at":"2026-05-18T12:33:37.589309+00:00"},{"alias_kind":"pith_short_8","alias_value":"WZ34WRPF","created_at":"2026-05-18T12:33:37.589309+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":33,"internal_anchor_count":33,"sample":[{"citing_arxiv_id":"2605.23310","citing_title":"From Head to Tail: Asymmetric Knowledge Transfer in Long-tail Recommendation with Generative Semantic IDs","ref_index":24,"is_internal_anchor":true},{"citing_arxiv_id":"2602.10445","citing_title":"End-to-End Semantic ID Generation for Generative Advertisement Recommendation","ref_index":34,"is_internal_anchor":true},{"citing_arxiv_id":"2605.21832","citing_title":"FLUID: From Ephemeral IDs to Multimodal Semantic Codes for Industrial-Scale Livestreaming Recommendation","ref_index":48,"is_internal_anchor":true},{"citing_arxiv_id":"2605.05249","citing_title":"TriAlignGR: Triangular Multitask Alignment with Multimodal Deep Interest Mining for Generative Recommendation","ref_index":3,"is_internal_anchor":true},{"citing_arxiv_id":"2605.14787","citing_title":"Do Composed Image Retrieval Benchmarks Require Multimodal Composition?","ref_index":8,"is_internal_anchor":true},{"citing_arxiv_id":"2605.16638","citing_title":"TTE-Flash: Accelerating Reasoning-based Multimodal Representations via Think-Then-Embed Tokens","ref_index":10,"is_internal_anchor":true},{"citing_arxiv_id":"2605.18434","citing_title":"TIGER-FG: Text-Guided Implicit Fine-Grained Grounding for E-commerce Retrieval","ref_index":16,"is_internal_anchor":true},{"citing_arxiv_id":"2605.14311","citing_title":"Beyond Binary: Reframing GUI Critique as Continuous Semantic Alignment","ref_index":129,"is_internal_anchor":true},{"citing_arxiv_id":"2509.00798","citing_title":"Progressive Multimodal Search and Reasoning for Knowledge-Intensive Visual Question Answering","ref_index":56,"is_internal_anchor":true},{"citing_arxiv_id":"2509.24621","citing_title":"FreeRet: MLLMs as Training-Free Retrievers","ref_index":28,"is_internal_anchor":true},{"citing_arxiv_id":"2512.13511","citing_title":"Adapting MLLMs for Nuanced Video Retrieval","ref_index":87,"is_internal_anchor":true},{"citing_arxiv_id":"2601.21262","citing_title":"CausalEmbed: Auto-Regressive Multi-Vector Generation in Latent Space for Visual Document Embedding","ref_index":35,"is_internal_anchor":true},{"citing_arxiv_id":"2605.14311","citing_title":"Beyond Binary: Reframing GUI Critique as Continuous Semantic Alignment","ref_index":129,"is_internal_anchor":true},{"citing_arxiv_id":"2605.14581","citing_title":"A Picture is Worth a Thousand Words? An Empirical Study of Aggregation Strategies for Visual Financial Document Retrieval","ref_index":14,"is_internal_anchor":true},{"citing_arxiv_id":"2604.02073","citing_title":"PLUME: Latent Reasoning Based Universal Multimodal Embedding","ref_index":54,"is_internal_anchor":true},{"citing_arxiv_id":"2605.08384","citing_title":"jina-embeddings-v5-omni: Geometry-preserving Embeddings via Locked Aligned Towers","ref_index":47,"is_internal_anchor":true},{"citing_arxiv_id":"2604.26565","citing_title":"DenseStep2M: A Scalable, Training-Free Pipeline for Dense Instructional Video Annotation","ref_index":87,"is_internal_anchor":true},{"citing_arxiv_id":"2605.08384","citing_title":"jina-embeddings-v5-omni: Geometry-preserving Embeddings via Locked Aligned Towers","ref_index":47,"is_internal_anchor":true},{"citing_arxiv_id":"2604.13710","citing_title":"SLQ: Bridging Modalities via Shared Latent Queries for Retrieval with Frozen MLLMs","ref_index":45,"is_internal_anchor":true},{"citing_arxiv_id":"2604.25273","citing_title":"Combating Visual Neglect and Semantic Drift in Large Multimodal Models for Enhanced Cross-Modal Retrieval","ref_index":37,"is_internal_anchor":true},{"citing_arxiv_id":"2604.23321","citing_title":"MMEB-V3: Measuring the Performance Gaps of Omni-Modality Embedding Models","ref_index":39,"is_internal_anchor":true},{"citing_arxiv_id":"2604.22280","citing_title":"Beyond Chain-of-Thought: Rewrite as a Universal Interface for Generative Multimodal Embeddings","ref_index":53,"is_internal_anchor":true},{"citing_arxiv_id":"2604.21326","citing_title":"MiMIC: Mitigating Visual Modality Collapse in Universal Multimodal Retrieval While Avoiding Semantic Misalignment","ref_index":109,"is_internal_anchor":true},{"citing_arxiv_id":"2604.12148","citing_title":"ViLL-E: Video LLM Embeddings for Retrieval","ref_index":65,"is_internal_anchor":true},{"citing_arxiv_id":"2604.11095","citing_title":"Bottleneck Tokens for Unified Multimodal Retrieval","ref_index":27,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":2,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/WZ34WRPFD24Y2BKDMFRBPVSYWP","json":"https://pith.science/pith/WZ34WRPFD24Y2BKDMFRBPVSYWP.json","graph_json":"https://pith.science/api/pith-number/WZ34WRPFD24Y2BKDMFRBPVSYWP/graph.json","events_json":"https://pith.science/api/pith-number/WZ34WRPFD24Y2BKDMFRBPVSYWP/events.json","paper":"https://pith.science/paper/WZ34WRPF"},"agent_actions":{"view_html":"https://pith.science/pith/WZ34WRPFD24Y2BKDMFRBPVSYWP","download_json":"https://pith.science/pith/WZ34WRPFD24Y2BKDMFRBPVSYWP.json","view_paper":"https://pith.science/paper/WZ34WRPF","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2412.16855&json=true","fetch_graph":"https://pith.science/api/pith-number/WZ34WRPFD24Y2BKDMFRBPVSYWP/graph.json","fetch_events":"https://pith.science/api/pith-number/WZ34WRPFD24Y2BKDMFRBPVSYWP/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/WZ34WRPFD24Y2BKDMFRBPVSYWP/action/timestamp_anchor","attest_storage":"https://pith.science/pith/WZ34WRPFD24Y2BKDMFRBPVSYWP/action/storage_attestation","attest_author":"https://pith.science/pith/WZ34WRPFD24Y2BKDMFRBPVSYWP/action/author_attestation","sign_citation":"https://pith.science/pith/WZ34WRPFD24Y2BKDMFRBPVSYWP/action/citation_signature","submit_replication":"https://pith.science/pith/WZ34WRPFD24Y2BKDMFRBPVSYWP/action/replication_record"}},"created_at":"2026-05-17T23:38:53.247497+00:00","updated_at":"2026-05-17T23:38:53.247497+00:00"}