{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:DR3QNHMJLWWGL5WKPJP2FCHQGI","short_pith_number":"pith:DR3QNHMJ","schema_version":"1.0","canonical_sha256":"1c77069d895dac65f6ca7a5fa288f03210e436a90b3b113e3bf188460c81db13","source":{"kind":"arxiv","id":"2406.20052","version":3},"attestation_state":"computed","paper":{"title":"Understanding and Mitigating Language Confusion in LLMs","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Alexandre B\\'erard, Kelly Marchisio, Sebastian Ruder, Th\\'eo Dehaze, Wei-Yin Ko","submitted_at":"2024-06-28T17:03:51Z","abstract_excerpt":"We investigate a surprising limitation of LLMs: their inability to consistently generate text in a user's desired language. We create the Language Confusion Benchmark (LCB) to evaluate such failures, covering 15 typologically diverse languages with existing and newly-created English and multilingual prompts. We evaluate a range of LLMs on monolingual and cross-lingual generation reflecting practical use cases, finding that Llama Instruct and Mistral models exhibit high degrees of language confusion and even the strongest models fail to consistently respond in the correct language. We observe t"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2406.20052","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-28T17:03:51Z","cross_cats_sorted":[],"title_canon_sha256":"62bc991d9352f9c5ba226f75663340bb0ab422a913e3fee339d651b4a076270b","abstract_canon_sha256":"e0e63cb4035e71bbaa5dc171699e15e8d9863bc836601ef8a645a02313471c43"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:44:14.989725Z","signature_b64":"s1sYGSm2tCd6ikNse5hC+XjxffCuEPyzR5QR8M3BsAI0KeqaVvo3uawbogRFjnEtU79I+043nMi5eOhXOsifCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1c77069d895dac65f6ca7a5fa288f03210e436a90b3b113e3bf188460c81db13","last_reissued_at":"2026-07-05T10:44:14.989243Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:44:14.989243Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Understanding and Mitigating Language Confusion in LLMs","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Alexandre B\\'erard, Kelly Marchisio, Sebastian Ruder, Th\\'eo Dehaze, Wei-Yin Ko","submitted_at":"2024-06-28T17:03:51Z","abstract_excerpt":"We investigate a surprising limitation of LLMs: their inability to consistently generate text in a user's desired language. We create the Language Confusion Benchmark (LCB) to evaluate such failures, covering 15 typologically diverse languages with existing and newly-created English and multilingual prompts. We evaluate a range of LLMs on monolingual and cross-lingual generation reflecting practical use cases, finding that Llama Instruct and Mistral models exhibit high degrees of language confusion and even the strongest models fail to consistently respond in the correct language. We observe t"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.20052","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.20052/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2406.20052","created_at":"2026-07-05T10:44:14.989299+00:00"},{"alias_kind":"arxiv_version","alias_value":"2406.20052v3","created_at":"2026-07-05T10:44:14.989299+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.20052","created_at":"2026-07-05T10:44:14.989299+00:00"},{"alias_kind":"pith_short_12","alias_value":"DR3QNHMJLWWG","created_at":"2026-07-05T10:44:14.989299+00:00"},{"alias_kind":"pith_short_16","alias_value":"DR3QNHMJLWWGL5WK","created_at":"2026-07-05T10:44:14.989299+00:00"},{"alias_kind":"pith_short_8","alias_value":"DR3QNHMJ","created_at":"2026-07-05T10:44:14.989299+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2604.16235","citing_title":"Optimizing Korean-Centric LLMs via Token Pruning","ref_index":8,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/DR3QNHMJLWWGL5WKPJP2FCHQGI","json":"https://pith.science/pith/DR3QNHMJLWWGL5WKPJP2FCHQGI.json","graph_json":"https://pith.science/api/pith-number/DR3QNHMJLWWGL5WKPJP2FCHQGI/graph.json","events_json":"https://pith.science/api/pith-number/DR3QNHMJLWWGL5WKPJP2FCHQGI/events.json","paper":"https://pith.science/paper/DR3QNHMJ"},"agent_actions":{"view_html":"https://pith.science/pith/DR3QNHMJLWWGL5WKPJP2FCHQGI","download_json":"https://pith.science/pith/DR3QNHMJLWWGL5WKPJP2FCHQGI.json","view_paper":"https://pith.science/paper/DR3QNHMJ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2406.20052&json=true","fetch_graph":"https://pith.science/api/pith-number/DR3QNHMJLWWGL5WKPJP2FCHQGI/graph.json","fetch_events":"https://pith.science/api/pith-number/DR3QNHMJLWWGL5WKPJP2FCHQGI/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/DR3QNHMJLWWGL5WKPJP2FCHQGI/action/timestamp_anchor","attest_storage":"https://pith.science/pith/DR3QNHMJLWWGL5WKPJP2FCHQGI/action/storage_attestation","attest_author":"https://pith.science/pith/DR3QNHMJLWWGL5WKPJP2FCHQGI/action/author_attestation","sign_citation":"https://pith.science/pith/DR3QNHMJLWWGL5WKPJP2FCHQGI/action/citation_signature","submit_replication":"https://pith.science/pith/DR3QNHMJLWWGL5WKPJP2FCHQGI/action/replication_record"}},"created_at":"2026-07-05T10:44:14.989299+00:00","updated_at":"2026-07-05T10:44:14.989299+00:00"}