{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:7VALWHU23N7TFV7BUYBKCVGBDM","short_pith_number":"pith:7VALWHU2","schema_version":"1.0","canonical_sha256":"fd40bb1e9adb7f32d7e1a602a154c11b028bb850cc48a11b517f1fd50a79534e","source":{"kind":"arxiv","id":"2411.05504","version":1},"attestation_state":"computed","paper":{"title":"LBPE: Long-token-first Tokenization to Improve Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Guiguang Ding, Haoran Lian, Hui Chen, Jianwei Niu, Peng Liu, Shasha Mo, Yizhe Xiong, Zijia Lin","submitted_at":"2024-11-08T12:03:36Z","abstract_excerpt":"The prevalent use of Byte Pair Encoding (BPE) in Large Language Models (LLMs) facilitates robust handling of subword units and avoids issues of out-of-vocabulary words. Despite its success, a critical challenge persists: long tokens, rich in semantic information, have fewer occurrences in tokenized datasets compared to short tokens, which can result in imbalanced learning issue across different tokens. To address that, we propose LBPE, which prioritizes long tokens during the encoding process. LBPE generates tokens according to their reverse ranks of token length rather than their ranks in the"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2411.05504","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-11-08T12:03:36Z","cross_cats_sorted":[],"title_canon_sha256":"4175f451c064b1c90637e2283528a46188ec3b22c17676f9f268b0fe8afe6715","abstract_canon_sha256":"b4966b1fbe4dcf8ca919030ba4b48894183a5da0e70687d0b323ed2db4d8983f"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:33:02.221597Z","signature_b64":"o/p7rrfGP8Pkl/TKfrDmMOVuixApST9ylutRE3UbzaIgQbn6RD7dtybvG4OIittcy8GJ/BU43SbEUHPx4+OwCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"fd40bb1e9adb7f32d7e1a602a154c11b028bb850cc48a11b517f1fd50a79534e","last_reissued_at":"2026-07-05T09:33:02.221033Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:33:02.221033Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"LBPE: Long-token-first Tokenization to Improve Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Guiguang Ding, Haoran Lian, Hui Chen, Jianwei Niu, Peng Liu, Shasha Mo, Yizhe Xiong, Zijia Lin","submitted_at":"2024-11-08T12:03:36Z","abstract_excerpt":"The prevalent use of Byte Pair Encoding (BPE) in Large Language Models (LLMs) facilitates robust handling of subword units and avoids issues of out-of-vocabulary words. Despite its success, a critical challenge persists: long tokens, rich in semantic information, have fewer occurrences in tokenized datasets compared to short tokens, which can result in imbalanced learning issue across different tokens. To address that, we propose LBPE, which prioritizes long tokens during the encoding process. LBPE generates tokens according to their reverse ranks of token length rather than their ranks in the"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.05504","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2411.05504/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2411.05504","created_at":"2026-07-05T09:33:02.221100+00:00"},{"alias_kind":"arxiv_version","alias_value":"2411.05504v1","created_at":"2026-07-05T09:33:02.221100+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.05504","created_at":"2026-07-05T09:33:02.221100+00:00"},{"alias_kind":"pith_short_12","alias_value":"7VALWHU23N7T","created_at":"2026-07-05T09:33:02.221100+00:00"},{"alias_kind":"pith_short_16","alias_value":"7VALWHU23N7TFV7B","created_at":"2026-07-05T09:33:02.221100+00:00"},{"alias_kind":"pith_short_8","alias_value":"7VALWHU2","created_at":"2026-07-05T09:33:02.221100+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2412.07171","citing_title":"Breaking the Stage Barrier: A Novel Single-Stage Approach to Long Context Extension for Large Language Models","ref_index":36,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/7VALWHU23N7TFV7BUYBKCVGBDM","json":"https://pith.science/pith/7VALWHU23N7TFV7BUYBKCVGBDM.json","graph_json":"https://pith.science/api/pith-number/7VALWHU23N7TFV7BUYBKCVGBDM/graph.json","events_json":"https://pith.science/api/pith-number/7VALWHU23N7TFV7BUYBKCVGBDM/events.json","paper":"https://pith.science/paper/7VALWHU2"},"agent_actions":{"view_html":"https://pith.science/pith/7VALWHU23N7TFV7BUYBKCVGBDM","download_json":"https://pith.science/pith/7VALWHU23N7TFV7BUYBKCVGBDM.json","view_paper":"https://pith.science/paper/7VALWHU2","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2411.05504&json=true","fetch_graph":"https://pith.science/api/pith-number/7VALWHU23N7TFV7BUYBKCVGBDM/graph.json","fetch_events":"https://pith.science/api/pith-number/7VALWHU23N7TFV7BUYBKCVGBDM/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/7VALWHU23N7TFV7BUYBKCVGBDM/action/timestamp_anchor","attest_storage":"https://pith.science/pith/7VALWHU23N7TFV7BUYBKCVGBDM/action/storage_attestation","attest_author":"https://pith.science/pith/7VALWHU23N7TFV7BUYBKCVGBDM/action/author_attestation","sign_citation":"https://pith.science/pith/7VALWHU23N7TFV7BUYBKCVGBDM/action/citation_signature","submit_replication":"https://pith.science/pith/7VALWHU23N7TFV7BUYBKCVGBDM/action/replication_record"}},"created_at":"2026-07-05T09:33:02.221100+00:00","updated_at":"2026-07-05T09:33:02.221100+00:00"}