{"as_of":"2026-08-18T23:04:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:d9785682ede3fac7d076005e725457a3e24db2873458f9bc86a41d2c95f2da4d","coverage":[{"denominator":36,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":36,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-11T19:25:22.152544Z","state":"measured"},{"denominator":38,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":38,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-18T06:34:40.430872+00:00","state":"measured"},{"denominator":2,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":2,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-16T10:55:55.460756Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-07-04T04:39:34.656602Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.06926","snapshot_observed_at":"2026-08-16T10:55:55.460756Z","title":"When every token counts: Optimal segmentation for low-resource language models,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.16977","last_updated":"2025-04-23T17:28:38Z","snapshot_observed_at":"2026-08-18T21:46:11.534440Z","submitted_at":"2025-04-23T17:28:38Z","title":"Tokenization Matters: Improving Zero-Shot NER for Indic Languages","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-16T10:55:55.460756Z"},"links":{"cited_paper":"/paper/2412.06926","citing_paper":"/paper/2504.16977"},"observation_digest":"sha256:c8c9ec4a8da7485d3b8d7cd915d74ed59d19ed2a61cf38be7fef5284c11f798f","observation_id":"472eac41-6e95-4b58-9343-b1c85a10bd84","resolution":{"observed_at":"2026-08-16T10:55:55.460756Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"cited_work":{"arxiv_id":"2412.06926","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2412.06926","snapshot_observed_at":"2026-07-04T04:39:34.656602Z","title":null,"venue":null,"work_id":"2f32031e-b01e-407f-802c-1faabd0dbe82","year":2024},"citing_paper":{"arxiv_id":"2606.20993","last_updated":"2026-06-18T23:50:54Z","snapshot_observed_at":"2026-08-05T09:19:18.829577Z","submitted_at":"2026-06-18T23:50:54Z","title":"Phonemes to the Rescue: Multilingual Tokenization Based on International Phonetic Alphabet","version":1},"reference_index":54,"source":"arxiv_source","source_observed_at":"2026-06-26T16:53:25.556222Z"},"links":{"cited_paper":"/paper/2412.06926","citing_paper":"/paper/2606.20993"},"observation_digest":"sha256:73852dba1e24ca1e222fa22f0f54272fca2d3ce3e48ef651961c755c89092f5d","observation_id":"7adebcde-5119-423a-a6da-c2bca8560b34","resolution":{"observed_at":"2026-07-04T04:39:34.657942Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2412.06926/citation-record","integrity":"/paper/2412.06926/integrity","json":"/paper/2412.06926/citation-record.json","paper":"/paper/2412.06926"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2305.13707","last_updated":"2023-05-23T05:46:45Z","snapshot_observed_at":"2026-08-17T20:00:45.281755Z","submitted_at":"2023-05-23T05:46:45Z","title":"Do All Languages Cost the Same? Tokenization in the Era of Commercial Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.13707","snapshot_observed_at":"2026-08-11T19:25:22.056655Z","title":"Mortensen, Noah A","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.056655Z"},"links":{"cited_paper":"/paper/2305.13707","citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:5bfef6e20998965b41200be4b25c7af1683192264024f2dab93d6ad9b8665654","observation_id":"51720b38-9869-4158-ae8f-f9bb8beb6575","resolution":{"observed_at":"2026-08-11T19:25:22.056655Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2005.14165","last_updated":"2020-07-22T19:47:17Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2020-05-28T17:29:03Z","title":"Language Models are Few-Shot Learners","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2005.14165","snapshot_observed_at":"2026-08-11T19:25:22.060360Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.060360Z"},"links":{"cited_paper":"/paper/2005.14165","citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:1b00fd0d489b3185a62f62c001ba2d459ff66598ccd33d03aeb132034a1aebb9","observation_id":"17cf01e3-9573-4395-a4c5-70f20dfccd92","resolution":{"observed_at":"2026-08-11T19:25:22.060360Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.478023Z","title":"Clark, Eunsol Choi, Michael Collins, Dan Garrette, Tom Kwiatkowski, Vitaly Nikolaev, and Jennimaria Palomaki","venue":null,"work_id":"f6880764-e709-412c-aa60-bac76e506f0b","year":2020},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.063543Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:095e0145207592c631af50d933f846620890c8a3aebe8836524f77cd76d8216c","observation_id":"6043def0-88ca-44b4-bcd1-e126b0660993","resolution":{"observed_at":"2026-08-11T19:25:22.480560Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.066887Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.066887Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:971d685a59fb01dbb566a375b335d28b744cf6a322835c55b90f8e5797ec2209","observation_id":"8545b9be-fb32-46ea-8015-d5721aaa00b7","resolution":{"observed_at":"2026-08-11T19:25:22.066887Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.01035","last_updated":"2024-02-07T10:51:11Z","snapshot_observed_at":"2026-08-18T02:33:35.025376Z","submitted_at":"2024-02-01T21:49:34Z","title":"Getting the most out of your tokenizer for pre-training and domain adaptation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.01035","snapshot_observed_at":"2026-08-11T19:25:22.070092Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.070092Z"},"links":{"cited_paper":"/paper/2402.01035","citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:068780306e8592851a9d447ab19a15c5ff953d6406c5bbe76e37cf9d947c60e6","observation_id":"14a6a898-b4ef-4761-b192-cd8d6c3ac452","resolution":{"observed_at":"2026-08-11T19:25:22.070092Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1810.04805","last_updated":"2019-05-24T20:37:26Z","snapshot_observed_at":"2026-08-14T18:16:28.847993Z","submitted_at":"2018-10-11T00:50:01Z","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1810.04805","snapshot_observed_at":"2026-08-11T19:25:22.073299Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.073299Z"},"links":{"cited_paper":"/paper/1810.04805","citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:1e96442d7921760e9f0cda16649264cc9933ee0b298ffbdd945f99b97efb68e7","observation_id":"3e7ce535-6458-4f8c-8b6a-af2d7a9614df","resolution":{"observed_at":"2026-08-11T19:25:22.073299Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.470615Z","title":null,"venue":null,"work_id":"eb4f40f2-8bf2-4f18-8f0a-1c32e34d3c8d","year":2024},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.076728Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:ec37f109091a600afe58f8fbb0c09e90f1009de79b7587e7b9045cfe7e7c66c0","observation_id":"887604cf-d2a5-43f5-a6ff-bdd32e6b1c6d","resolution":{"observed_at":"2026-08-11T19:25:22.472945Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.079698Z","title":null,"venue":null,"work_id":null,"year":1973},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.079698Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:aa3ef95cbb9e989a8c1f25c93b939709ffa77911fdbfbc3b487e7e2454f3f25d","observation_id":"d9a96d13-12fd-4b5b-a2d4-0017a7478305","resolution":{"observed_at":"2026-08-11T19:25:22.079698Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.082230Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.082230Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:1e685a4ce049b610667bc6452199a9cca75b884c8b7f2f8a240cff02ae30f7df","observation_id":"c68e6069-a91a-4f68-bdc4-909124d1e8eb","resolution":{"observed_at":"2026-08-11T19:25:22.082230Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.084667Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.084667Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:fb9fa1939fb8a07b4d8dcaa2968f8ef4ff83fdac3b522e89871f7ea83a3f7323","observation_id":"da4b618c-e179-4e97-9fc2-39c8889b919b","resolution":{"observed_at":"2026-08-11T19:25:22.084667Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.21783","last_updated":"2024-11-23T23:27:33Z","snapshot_observed_at":"2026-08-13T17:20:44.002518Z","submitted_at":"2024-07-31T17:54:27Z","title":"The Llama 3 Herd of Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.21783","snapshot_observed_at":"2026-08-11T19:25:22.087056Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.087056Z"},"links":{"cited_paper":"/paper/2407.21783","citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:2323c7d5824b107cf909016f182e8c022cb62d41e8e9378616e2fd5f7ebfc9dd","observation_id":"d7e50150-09ab-48db-a0b1-8b271f46c416","resolution":{"observed_at":"2026-08-11T19:25:22.087056Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1804.10959","last_updated":"2018-04-29T16:13:44Z","snapshot_observed_at":"2026-08-17T19:21:06.288840Z","submitted_at":"2018-04-29T16:13:44Z","title":"Subword Regularization: Improving Neural Network Translation Models with Multiple Subword Candidates","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1804.10959","snapshot_observed_at":"2026-08-11T19:25:22.090169Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.090169Z"},"links":{"cited_paper":"/paper/1804.10959","citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:99e13b1d5d00f872216369c5ceaee4e5903a35576aa568b33b520dc4a09a888e","observation_id":"795d1b2d-ea62-4a05-b114-c34f2442cbbc","resolution":{"observed_at":"2026-08-11T19:25:22.090169Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.462313Z","title":null,"venue":null,"work_id":"f90fcc8e-e867-4ea7-852b-c440e6bf1076","year":2018},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.092780Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:91d84959a41ccec1ff923dedc5c35a20ce01fc89d2a9e4e5597c43717a0e5d23","observation_id":"18ee15c1-4447-45a4-9d49-d709d12cb944","resolution":{"observed_at":"2026-08-11T19:25:22.465279Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2006.07264","last_updated":"2020-06-12T15:21:57Z","snapshot_observed_at":"2026-08-13T05:37:12.453613Z","submitted_at":"2020-06-12T15:21:57Z","title":"Low-resource Languages: A Review of Past Work and Future Challenges","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2006.07264","snapshot_observed_at":"2026-08-11T19:25:22.095097Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.095097Z"},"links":{"cited_paper":"/paper/2006.07264","citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:00ca51a6ced0842abfaa05585435f3e21c8f2bcf643847ba3911f16012a6d416","observation_id":"ed137759-f8af-484b-84ca-ed6ab4f02134","resolution":{"observed_at":"2026-08-11T19:25:22.095097Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.452834Z","title":"Marcus, Mary Ann Marcinkiewicz, and Beatrice Santorini","venue":null,"work_id":"4346c59f-ce5f-40bf-9e61-e4e121108577","year":1993},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.097769Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:da0875c2300bb8858f383a4609282d18070061b9044431efc1989306df043de3","observation_id":"28e134b2-db04-4550-8691-f0a7fa367d64","resolution":{"observed_at":"2026-08-11T19:25:22.456538Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2023.findings-acl.230","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.187864Z","title":null,"venue":null,"work_id":"41944aa7-e63e-4a46-bc81-767cfcfcc4e4","year":2023},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.100121Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:0eb084af6f6c6c52def21caa21a9f6e13349a924e7ef5efe984b725148366e35","observation_id":"f7158363-8079-4502-b296-89e553b3ccb9","resolution":{"observed_at":"2026-08-11T19:25:22.191554Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1604.01696","last_updated":"2016-04-06T17:15:10Z","snapshot_observed_at":"2026-08-18T08:04:34.502335Z","submitted_at":"2016-04-06T17:15:10Z","title":"A Corpus and Evaluation Framework for Deeper Understanding of Commonsense Stories","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1604.01696","snapshot_observed_at":"2026-08-11T19:25:22.102716Z","title":null,"venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.102716Z"},"links":{"cited_paper":"/paper/1604.01696","citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:12f9faace212ee279f9fb482b4717a30b26eaed86bffa89b310cb9fbd32e95a0","observation_id":"b8c417db-9ab5-422c-8fb2-e32e59599c64","resolution":{"observed_at":"2026-08-11T19:25:22.102716Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.444520Z","title":null,"venue":null,"work_id":"5615c964-f9cc-4ec2-92c8-32bcff1f7202","year":2023},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.105358Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:38c82951bc9d92922a13a68f9306c0e9d793d9bbcff2ca840b2014cfa79bf878","observation_id":"c37c32ba-e990-455c-960b-bb511e0f5a95","resolution":{"observed_at":"2026-08-11T19:25:22.447015Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-17T09:58:46.058102Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-08-11T19:25:22.107648Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.107648Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:95f28acca23b4219f78535b41c645bc7d7a997e20e7fc020511af07a5b06ea05","observation_id":"fa61c109-b017-491e-af8c-6168aea6e493","resolution":{"observed_at":"2026-08-11T19:25:22.107648Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1606.06031","last_updated":"2016-06-20T09:37:17Z","snapshot_observed_at":"2026-08-17T07:59:38.221655Z","submitted_at":"2016-06-20T09:37:17Z","title":"The LAMBADA dataset: Word prediction requiring a broad discourse context","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1606.06031","snapshot_observed_at":"2026-08-11T19:25:22.110282Z","title":null,"venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.110282Z"},"links":{"cited_paper":"/paper/1606.06031","citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:1d44c98b1d8a03b83a9e5d3b1d3aadaf8816617e09f535f603fd0840f1c718c0","observation_id":"f25817aa-4ee6-4b60-b2b6-49c50c437dc3","resolution":{"observed_at":"2026-08-11T19:25:22.110282Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.15425","last_updated":"2023-10-20T10:09:55Z","snapshot_observed_at":"2026-08-16T15:32:19.964286Z","submitted_at":"2023-05-17T14:17:57Z","title":"Language Model Tokenizers Introduce Unfairness Between Languages","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.15425","snapshot_observed_at":"2026-08-11T19:25:22.112975Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.112975Z"},"links":{"cited_paper":"/paper/2305.15425","citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:959c33439539c47949823ebb727e5870e94684822d19fd5c78c9544544bc096e","observation_id":"8f3d0465-aa73-4646-aace-d3067162310b","resolution":{"observed_at":"2026-08-11T19:25:22.112975Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.436748Z","title":null,"venue":null,"work_id":"1b559c45-9460-40f3-8f29-126809ad3b13","year":2019},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.115428Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:f1054a2273e5e1f3d4ddd4fd2f9977ec4bf413620a9c70cf80930206974be712","observation_id":"679ce153-a688-4453-b7c4-f7c1063e7ae3","resolution":{"observed_at":"2026-08-11T19:25:22.439237Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.117661Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.117661Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:22b13be333b84581dfcb5f4a7d7d4cf912b5f67d62813ab5d4bffb9ceef8c4da","observation_id":"c878b7df-5fb4-4505-b0fb-5617c76bb5f9","resolution":{"observed_at":"2026-08-11T19:25:22.117661Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.120049Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.120049Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:ff1630059f1ca60a6d8eac35cd0008ee3b54319e4ac4773c66d10699330e9f77","observation_id":"33fdf193-adb9-4028-b21b-50f1956cecad","resolution":{"observed_at":"2026-08-11T19:25:22.120049Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.122282Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.122282Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:13b36e58b9ab1b80978b4d7ecb06dbf442f3f356794c3e91f9bc32e1a4644634","observation_id":"0c3fc772-0f05-4fce-86cc-faade0aab335","resolution":{"observed_at":"2026-08-11T19:25:22.122282Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.428911Z","title":null,"venue":null,"work_id":"2ea9e114-4c6a-40fe-85bf-86906a1206a1","year":2018},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.125122Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:045d5a4ef8ffb02913a0b39856c14f0f31ed23fbd6e31fa2e60bed6b199e1bbd","observation_id":"afd4083e-b090-46fe-8f22-d2ab3ccb1d34","resolution":{"observed_at":"2026-08-11T19:25:22.431371Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.18376","last_updated":"2024-10-07T13:17:03Z","snapshot_observed_at":"2026-08-17T18:25:11.014021Z","submitted_at":"2024-02-28T14:52:15Z","title":"Tokenization Is More Than Compression","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.18376","snapshot_observed_at":"2026-08-11T19:25:22.128101Z","title":"Schmidt, Varshini Reddy, Haoran Zhang, Alec Alameddine, Omri Uzan, Yuval Pinter, and Chris Tanner","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.128101Z"},"links":{"cited_paper":"/paper/2402.18376","citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:1cd893d473450d3cd1828f990e2755bc19facf4857541e906594a94f5ca82b6a","observation_id":"032d1d6d-e205-49f8-a007-6722a45a082b","resolution":{"observed_at":"2026-08-11T19:25:22.128101Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.420978Z","title":null,"venue":null,"work_id":"f63a4eb2-94ac-46db-a9bb-85e22cdbd9b1","year":2012},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.131350Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:3c65bc00241edeb7115bfbe8c1bcf432a4601dbba4522d9feccfe4580465615b","observation_id":"a58f0b5d-74f9-4135-9e9a-3fc4309e54bf","resolution":{"observed_at":"2026-08-11T19:25:22.423545Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.413105Z","title":null,"venue":null,"work_id":"6ba6dab1-d640-42f1-85e0-8bd87aaa492b","year":2018},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.134149Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:5b7240cc127c6b8aa9db1da65bb71dc8cc35dc546c3018bf890496fe9633d7d2","observation_id":"d8506af5-1699-4158-907c-e3d8d2866223","resolution":{"observed_at":"2026-08-11T19:25:22.415568Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.01289","last_updated":"2024-05-31T08:07:45Z","snapshot_observed_at":"2026-08-16T14:13:27.288883Z","submitted_at":"2024-03-02T19:01:40Z","title":"Greed is All You Need: An Evaluation of Tokenizer Inference Methods","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.01289","snapshot_observed_at":"2026-08-11T19:25:22.136962Z","title":"Schmidt, Chris Tanner, and Yuval Pinter","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":30,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.136962Z"},"links":{"cited_paper":"/paper/2403.01289","citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:ef6d86eaeefabbbb8bb91fd7206a9ec73fb8cbd2cda520c802c9443f7443d016","observation_id":"1ad9fc31-0882-4ba0-8f4c-311a49ae3cc3","resolution":{"observed_at":"2026-08-11T19:25:22.136962Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2409.11501","last_updated":"2024-09-17T19:05:37Z","snapshot_observed_at":"2026-08-16T13:17:42.755759Z","submitted_at":"2024-09-17T19:05:37Z","title":"Egalitarian Language Representation in Language Models: It All Begins with Tokenizers","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.11501","snapshot_observed_at":"2026-08-11T19:25:22.140105Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.140105Z"},"links":{"cited_paper":"/paper/2409.11501","citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:cca3e59987fa02362de1ed82969d456e7669f206b0a76be2019536e26f2bc867","observation_id":"82ac2e93-56b0-485e-a80c-a8017d75d31e","resolution":{"observed_at":"2026-08-11T19:25:22.140105Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.142947Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.142947Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:dba6ccfa2b1d872e0e61943b14ff93caeded30ec72c201d2af2fc2bd0a03a4bd","observation_id":"ccb193a0-747a-4e40-8aed-aef7dcf40493","resolution":{"observed_at":"2026-08-11T19:25:22.142947Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.145180Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.145180Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:8593e044eda2b021704ed1af0f7884f2f64038e3b8a99d528fdfd086fe4cc1b2","observation_id":"ff1acca1-06b5-43db-a08b-42d9dc54bc1e","resolution":{"observed_at":"2026-08-11T19:25:22.145180Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.395056Z","title":null,"venue":null,"work_id":"5acad900-64fa-418f-b0af-432243803a55","year":2022},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.147457Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:19a81971e47bb04204415fa5346d6baffb097da99bc5f433d88ecf8cb3258f36","observation_id":"e1ce7be9-2059-42e0-a329-203afb4b2c6d","resolution":{"observed_at":"2026-08-11T19:25:22.397573Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-18T06:34:40.430872+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.149689Z","title":"online\" 'onlinestring :=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.149689Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:85e5ce0e3de72935715e3d30c7d725e0e7466feef3a920faf15df35fdb799421","observation_id":"ea23ced9-9282-4f0e-8776-7c6ebd66f0ca","resolution":{"observed_at":"2026-08-11T19:25:22.149689Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T19:25:22.152544Z","title":"write newline","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models","version":5},"reference_index":36,"source":"arxiv_source","source_observed_at":"2026-08-11T19:25:22.152544Z"},"links":{"citing_paper":"/paper/2412.06926"},"observation_digest":"sha256:d84523c53b1432023961591bfd11d91140f8a1975cecc91d3c4d3f35a6e3fa12","observation_id":"df4a8c44-be81-498c-a11f-39a5ff5ef582","resolution":{"observed_at":"2026-08-11T19:25:22.152544Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2412.06926","last_updated":"2025-05-01T18:15:36Z","latest_version":5,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-17T23:56:37.032564Z","submitted_at":"2024-12-09T19:11:54Z","title":"When Every Token Counts: Optimal Segmentation for Low-Resource Language Models"},"reference_resolution":{"displayed":36,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":33,"verified_exact":1,"verified_fuzzy":2},"total_outbound_references":36},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-18T06:34:40.430872+00:00","source":"crossref"},{"observed_at":"2026-08-18T06:34:34.496301+00:00","source":"retraction_watch"}],"thesis":"As of 18 August 2026, this Paper Citation Record lists 36 of 36 outbound references and 2 inbound Pith citation observations for arXiv:2412.06926."}