{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:S4UB4TFUYLYC5PYCXMTVDQ2FE5","short_pith_number":"pith:S4UB4TFU","schema_version":"1.0","canonical_sha256":"97281e4cb4c2f02ebf02bb2751c345277b16573f08deba03a7a3e3d904c85b6d","source":{"kind":"arxiv","id":"2504.15521","version":1},"attestation_state":"computed","paper":{"title":"The Bitter Lesson Learned from 2,000+ Multilingual Benchmarks","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Chenyang Lyu, Huifeng Yin, Kaifu Zhang, Longyue Wang, Minghao Wu, Sinuo Liu, Weihua Luo, Weixuan Wang, Xintong Wang, Yu Zhao","submitted_at":"2025-04-22T01:47:37Z","abstract_excerpt":"As large language models (LLMs) continue to advance in linguistic capabilities, robust multilingual evaluation has become essential for promoting equitable technological progress. This position paper examines over 2,000 multilingual (non-English) benchmarks from 148 countries, published between 2021 and 2024, to evaluate past, present, and future practices in multilingual benchmarking. Our findings reveal that, despite significant investments amounting to tens of millions of dollars, English remains significantly overrepresented in these benchmarks. Additionally, most benchmarks rely on origin"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2504.15521","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-04-22T01:47:37Z","cross_cats_sorted":[],"title_canon_sha256":"2ed0358a589295804aa699d9cadedaeb020261cfeb14a54c39c1c964c15c66a6","abstract_canon_sha256":"f6506cca950706619fb5c5459a6171ee92d36aec7710e10116a435d0de31929e"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:52:25.938835Z","signature_b64":"Ryteu5LkdF6ZodVRg9Dcx7NDlhYM9ixNe3e2v95I4Cf4KrCFqMmfeZWtM/VhEqeI98oA8P5QY0+m8TFAp/UIAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"97281e4cb4c2f02ebf02bb2751c345277b16573f08deba03a7a3e3d904c85b6d","last_reissued_at":"2026-07-05T10:52:25.938339Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:52:25.938339Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"The Bitter Lesson Learned from 2,000+ Multilingual Benchmarks","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Chenyang Lyu, Huifeng Yin, Kaifu Zhang, Longyue Wang, Minghao Wu, Sinuo Liu, Weihua Luo, Weixuan Wang, Xintong Wang, Yu Zhao","submitted_at":"2025-04-22T01:47:37Z","abstract_excerpt":"As large language models (LLMs) continue to advance in linguistic capabilities, robust multilingual evaluation has become essential for promoting equitable technological progress. This position paper examines over 2,000 multilingual (non-English) benchmarks from 148 countries, published between 2021 and 2024, to evaluate past, present, and future practices in multilingual benchmarking. Our findings reveal that, despite significant investments amounting to tens of millions of dollars, English remains significantly overrepresented in these benchmarks. Additionally, most benchmarks rely on origin"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2504.15521","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2504.15521/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2504.15521","created_at":"2026-07-05T10:52:25.938398+00:00"},{"alias_kind":"arxiv_version","alias_value":"2504.15521v1","created_at":"2026-07-05T10:52:25.938398+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.15521","created_at":"2026-07-05T10:52:25.938398+00:00"},{"alias_kind":"pith_short_12","alias_value":"S4UB4TFUYLYC","created_at":"2026-07-05T10:52:25.938398+00:00"},{"alias_kind":"pith_short_16","alias_value":"S4UB4TFUYLYC5PYC","created_at":"2026-07-05T10:52:25.938398+00:00"},{"alias_kind":"pith_short_8","alias_value":"S4UB4TFU","created_at":"2026-07-05T10:52:25.938398+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":7,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.30788","citing_title":"XLGoBench: Detecting cross-lingual skill gaps with algorithmic tasks","ref_index":6,"is_internal_anchor":false},{"citing_arxiv_id":"2601.10791","citing_title":"OmniMol: Transferring Particle Physics Knowledge to Molecular Dynamics with Point-Edge Transformers","ref_index":44,"is_internal_anchor":false},{"citing_arxiv_id":"2605.14040","citing_title":"Physics-R1: An Audited Olympiad Corpus and Recipe for Visual Physics Reasoning","ref_index":49,"is_internal_anchor":false},{"citing_arxiv_id":"2605.11206","citing_title":"Instructions Shape Production of Language, not Processing","ref_index":262,"is_internal_anchor":false},{"citing_arxiv_id":"2605.11206","citing_title":"Instructions Shape Production of Language, not Processing","ref_index":262,"is_internal_anchor":false},{"citing_arxiv_id":"2604.19262","citing_title":"CulturALL: Benchmarking Multilingual and Multicultural Competence of LLMs on Grounded Tasks","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2605.07093","citing_title":"The Translation Tax Is Not a Scalar: A Counterfactual Audit of English-Source Cue Inheritance in Chinese Multilingual Benchmarks","ref_index":10,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/S4UB4TFUYLYC5PYCXMTVDQ2FE5","json":"https://pith.science/pith/S4UB4TFUYLYC5PYCXMTVDQ2FE5.json","graph_json":"https://pith.science/api/pith-number/S4UB4TFUYLYC5PYCXMTVDQ2FE5/graph.json","events_json":"https://pith.science/api/pith-number/S4UB4TFUYLYC5PYCXMTVDQ2FE5/events.json","paper":"https://pith.science/paper/S4UB4TFU"},"agent_actions":{"view_html":"https://pith.science/pith/S4UB4TFUYLYC5PYCXMTVDQ2FE5","download_json":"https://pith.science/pith/S4UB4TFUYLYC5PYCXMTVDQ2FE5.json","view_paper":"https://pith.science/paper/S4UB4TFU","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2504.15521&json=true","fetch_graph":"https://pith.science/api/pith-number/S4UB4TFUYLYC5PYCXMTVDQ2FE5/graph.json","fetch_events":"https://pith.science/api/pith-number/S4UB4TFUYLYC5PYCXMTVDQ2FE5/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/S4UB4TFUYLYC5PYCXMTVDQ2FE5/action/timestamp_anchor","attest_storage":"https://pith.science/pith/S4UB4TFUYLYC5PYCXMTVDQ2FE5/action/storage_attestation","attest_author":"https://pith.science/pith/S4UB4TFUYLYC5PYCXMTVDQ2FE5/action/author_attestation","sign_citation":"https://pith.science/pith/S4UB4TFUYLYC5PYCXMTVDQ2FE5/action/citation_signature","submit_replication":"https://pith.science/pith/S4UB4TFUYLYC5PYCXMTVDQ2FE5/action/replication_record"}},"created_at":"2026-07-05T10:52:25.938398+00:00","updated_at":"2026-07-05T10:52:25.938398+00:00"}