{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:LIBB27GCH7LFPTTSZAO2L5OBUV","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"dd70a8869eafdffec0ff60fcb66a72f7260efa0a8ce9a7af51423505d5d44e01","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-06-14T16:05:58Z","title_canon_sha256":"ea31aed7e63cb55470912fa252b0f9c45f431236641677666be8a8ca255a74aa"},"schema_version":"1.0","source":{"id":"2106.07540","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2106.07540","created_at":"2026-07-05T03:18:28Z"},{"alias_kind":"arxiv_version","alias_value":"2106.07540v2","created_at":"2026-07-05T03:18:28Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2106.07540","created_at":"2026-07-05T03:18:28Z"},{"alias_kind":"pith_short_12","alias_value":"LIBB27GCH7LF","created_at":"2026-07-05T03:18:28Z"},{"alias_kind":"pith_short_16","alias_value":"LIBB27GCH7LFPTTS","created_at":"2026-07-05T03:18:28Z"},{"alias_kind":"pith_short_8","alias_value":"LIBB27GC","created_at":"2026-07-05T03:18:28Z"}],"graph_snapshots":[{"event_id":"sha256:3583e6bfdce0cfef300067d1dc14ce18edacd8e0f988af9f0feb262c614e7de8","target":"graph","created_at":"2026-07-05T03:18:28Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2106.07540/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The first step in any NLP pipeline is to split the text into individual tokens. The most obvious and straightforward approach is to use words as tokens. However, given a large text corpus, representing all the words is not efficient in terms of vocabulary size. In the literature, many tokenization algorithms have emerged to tackle this problem by creating subwords which in turn limits the vocabulary size in a given text corpus. Most tokenization techniques are language-agnostic i.e they don't incorporate the linguistic features of a given language. Not to mention the difficulty of evaluating s","authors_text":"Irfan Ahmad, Maged S. Al-shaibani, Mustafa Ghaleb, Zaid Alyafeai","cross_cats":["cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-06-14T16:05:58Z","title":"Evaluating Various Tokenizers for Arabic Text Classification"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2106.07540","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b7872cbb672c2f9eb333ada61bc7533eb93542d6def98e8e1f7360670f9375e0","target":"record","created_at":"2026-07-05T03:18:28Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"dd70a8869eafdffec0ff60fcb66a72f7260efa0a8ce9a7af51423505d5d44e01","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-06-14T16:05:58Z","title_canon_sha256":"ea31aed7e63cb55470912fa252b0f9c45f431236641677666be8a8ca255a74aa"},"schema_version":"1.0","source":{"id":"2106.07540","kind":"arxiv","version":2}},"canonical_sha256":"5a021d7cc23fd657ce72c81da5f5c1a54b538fcf169448034380ad848f3d608a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"5a021d7cc23fd657ce72c81da5f5c1a54b538fcf169448034380ad848f3d608a","first_computed_at":"2026-07-05T03:18:28.171997Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T03:18:28.171997Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"ZZXBgb+tI7sEc5W+f6Sshp/vxFv8gby9dPRUeZXHufgoii2C5wMNQD21i8l4mXM3HdM0uKgL96lQKN24QJ3OBg==","signature_status":"signed_v1","signed_at":"2026-07-05T03:18:28.172581Z","signed_message":"canonical_sha256_bytes"},"source_id":"2106.07540","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b7872cbb672c2f9eb333ada61bc7533eb93542d6def98e8e1f7360670f9375e0","sha256:3583e6bfdce0cfef300067d1dc14ce18edacd8e0f988af9f0feb262c614e7de8"],"state_sha256":"f4faeb6b1ddca9b933f1a16cdb2adaa52cee996250b223cb26b7be7bcaa104b8"}