{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:5GDS2KFJCMPQTHTH77CECN74GL","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"1696edc59f38e6f209087be5c2d11c03690759b7653b8d48a1db9bb01ab8a9ca","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-04T13:27:09Z","title_canon_sha256":"a80f5478de9a966b86b324ddc0c24d9f9daa956b38a8afc0e961139a7837788e"},"schema_version":"1.0","source":{"id":"2412.03304","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.03304","created_at":"2026-07-05T10:16:39Z"},{"alias_kind":"arxiv_version","alias_value":"2412.03304v2","created_at":"2026-07-05T10:16:39Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.03304","created_at":"2026-07-05T10:16:39Z"},{"alias_kind":"pith_short_12","alias_value":"5GDS2KFJCMPQ","created_at":"2026-07-05T10:16:39Z"},{"alias_kind":"pith_short_16","alias_value":"5GDS2KFJCMPQTHTH","created_at":"2026-07-05T10:16:39Z"},{"alias_kind":"pith_short_8","alias_value":"5GDS2KFJ","created_at":"2026-07-05T10:16:39Z"}],"graph_snapshots":[{"event_id":"sha256:2b3e3a0f471ffc38eacf8a4a89fb3a9a06c2629a992312ee07b2714e4b9e1434","target":"graph","created_at":"2026-07-05T10:16:39Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2412.03304/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Cultural biases in multilingual datasets pose significant challenges for their effectiveness as global benchmarks. These biases stem not only from differences in language but also from the cultural knowledge required to interpret questions, reducing the practical utility of translated datasets like MMLU. Furthermore, translation often introduces artefacts that can distort the meaning or clarity of questions in the target language. A common practice in multilingual evaluation is to rely on machine-translated evaluation sets, but simply translating a dataset is insufficient to address these chal","authors_text":"Alice Oh, Andre F. T. Martins, Angelika Romanou, Antoine Bosselut, Beyza Ermis, Cl\\'ementine Fourrier, Daniel Vila-Suero, Daphne Ippolito, David I. Adelani, Enzo Ferrante, Jian Gang Ngui, Kelly Marchisio, Leshem Choshen, Madeline Smith, Marzieh Fadaee, Peerat Limkonchotiwat, Raymond Ng, Sara Hooker, Sebastian Ruder, Shayne Longpre, Shivalika Singh, Wei Qi Leong, Wei-Yin Ko, Yosephine Susanto","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-04T13:27:09Z","title":"Global MMLU: Understanding and Addressing Cultural and Linguistic Biases in Multilingual Evaluation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.03304","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1a5bdbeae7b54c52b7a478fa58fe0775f57bde2955a7578f6ae7e72acb47c8a3","target":"record","created_at":"2026-07-05T10:16:39Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"1696edc59f38e6f209087be5c2d11c03690759b7653b8d48a1db9bb01ab8a9ca","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-04T13:27:09Z","title_canon_sha256":"a80f5478de9a966b86b324ddc0c24d9f9daa956b38a8afc0e961139a7837788e"},"schema_version":"1.0","source":{"id":"2412.03304","kind":"arxiv","version":2}},"canonical_sha256":"e9872d28a9131f099e67ffc44137fc32edf9f7000095cc072f159f51f8e01372","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"e9872d28a9131f099e67ffc44137fc32edf9f7000095cc072f159f51f8e01372","first_computed_at":"2026-07-05T10:16:39.958543Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:16:39.958543Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"81daTC9xT20oK3DY2Kc2PZzzHiIIpXynMo8YJcqc3b4K6nmOcPaNlf3E9T/YfeqtQ6rJOiBGVEZN2IYj4gQ/Ag==","signature_status":"signed_v1","signed_at":"2026-07-05T10:16:39.959051Z","signed_message":"canonical_sha256_bytes"},"source_id":"2412.03304","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1a5bdbeae7b54c52b7a478fa58fe0775f57bde2955a7578f6ae7e72acb47c8a3","sha256:2b3e3a0f471ffc38eacf8a4a89fb3a9a06c2629a992312ee07b2714e4b9e1434"],"state_sha256":"349757bbad215b60569f1d6ede16162cea05df94d6b1941a250b6ebde1b9a49e"}