{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:CLM3OUGSKCD5LSDDI7GKTT4DWT","short_pith_number":"pith:CLM3OUGS","schema_version":"1.0","canonical_sha256":"12d9b750d25087d5c86347cca9cf83b4efcd8867442ae33366f5fa19027f006f","source":{"kind":"arxiv","id":"2409.18164","version":2},"attestation_state":"computed","paper":{"title":"Data-Prep-Kit: getting your data ready for LLM application development","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL","cs.LG"],"primary_cat":"cs.AI","authors_text":"Abdulhamid Adebayo, Alexy Roytman, Boris Lublinsky, Constantin Adam, Daiki Tsuzuku, David Wood, Dhiraj Joshi, Hajar Emami-Gohari, Hima Patel, Michele Dolfi, Mohammad Nassar, Nirmit Desai, Paramesvaran Selvam, Petros Zerfos, Revital Eres, Saptha Surendran, Shahrokh Daijavad, Shivdeep Singh, Sungeun An, Takuya Goto, Xuan-Hong Dang, Yan Koyfman, Yousaf Shah, Yuan Chi Chang","submitted_at":"2024-09-26T17:30:28Z","abstract_excerpt":"Data preparation is the first and a very important step towards any Large Language Model (LLM) development. This paper introduces an easy-to-use, extensible, and scale-flexible open-source data preparation toolkit called Data Prep Kit (DPK). DPK is architected and designed to enable users to scale their data preparation to their needs. With DPK they can prepare data on a local machine or effortlessly scale to run on a cluster with thousands of CPU Cores. DPK comes with a highly scalable, yet extensible set of modules that transform natural language and code data. If the user needs additional t"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2409.18164","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-09-26T17:30:28Z","cross_cats_sorted":["cs.CL","cs.LG"],"title_canon_sha256":"4c2ee2d98060064d1dbbcfd122562922619bb17ebd6dda2430bbfa9e19475ded","abstract_canon_sha256":"bef5644940b85537c41cbb7e42ead354eccbc6f25f9f3ec07f0793eba6fc75d5"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:34:38.338904Z","signature_b64":"Lh47HgWUEcmVE5Zrc42zVd6T4C+WBD1wa7g04Z3FT/MqMb7N+HiF5IDNp8HcrTKm7gYiUsK7kzoJMrH6DkQkCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"12d9b750d25087d5c86347cca9cf83b4efcd8867442ae33366f5fa19027f006f","last_reissued_at":"2026-07-05T09:34:38.338488Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:34:38.338488Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Data-Prep-Kit: getting your data ready for LLM application development","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL","cs.LG"],"primary_cat":"cs.AI","authors_text":"Abdulhamid Adebayo, Alexy Roytman, Boris Lublinsky, Constantin Adam, Daiki Tsuzuku, David Wood, Dhiraj Joshi, Hajar Emami-Gohari, Hima Patel, Michele Dolfi, Mohammad Nassar, Nirmit Desai, Paramesvaran Selvam, Petros Zerfos, Revital Eres, Saptha Surendran, Shahrokh Daijavad, Shivdeep Singh, Sungeun An, Takuya Goto, Xuan-Hong Dang, Yan Koyfman, Yousaf Shah, Yuan Chi Chang","submitted_at":"2024-09-26T17:30:28Z","abstract_excerpt":"Data preparation is the first and a very important step towards any Large Language Model (LLM) development. This paper introduces an easy-to-use, extensible, and scale-flexible open-source data preparation toolkit called Data Prep Kit (DPK). DPK is architected and designed to enable users to scale their data preparation to their needs. With DPK they can prepare data on a local machine or effortlessly scale to run on a cluster with thousands of CPU Cores. DPK comes with a highly scalable, yet extensible set of modules that transform natural language and code data. If the user needs additional t"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.18164","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2409.18164/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2409.18164","created_at":"2026-07-05T09:34:38.338546+00:00"},{"alias_kind":"arxiv_version","alias_value":"2409.18164v2","created_at":"2026-07-05T09:34:38.338546+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.18164","created_at":"2026-07-05T09:34:38.338546+00:00"},{"alias_kind":"pith_short_12","alias_value":"CLM3OUGSKCD5","created_at":"2026-07-05T09:34:38.338546+00:00"},{"alias_kind":"pith_short_16","alias_value":"CLM3OUGSKCD5LSDD","created_at":"2026-07-05T09:34:38.338546+00:00"},{"alias_kind":"pith_short_8","alias_value":"CLM3OUGS","created_at":"2026-07-05T09:34:38.338546+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/CLM3OUGSKCD5LSDDI7GKTT4DWT","json":"https://pith.science/pith/CLM3OUGSKCD5LSDDI7GKTT4DWT.json","graph_json":"https://pith.science/api/pith-number/CLM3OUGSKCD5LSDDI7GKTT4DWT/graph.json","events_json":"https://pith.science/api/pith-number/CLM3OUGSKCD5LSDDI7GKTT4DWT/events.json","paper":"https://pith.science/paper/CLM3OUGS"},"agent_actions":{"view_html":"https://pith.science/pith/CLM3OUGSKCD5LSDDI7GKTT4DWT","download_json":"https://pith.science/pith/CLM3OUGSKCD5LSDDI7GKTT4DWT.json","view_paper":"https://pith.science/paper/CLM3OUGS","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2409.18164&json=true","fetch_graph":"https://pith.science/api/pith-number/CLM3OUGSKCD5LSDDI7GKTT4DWT/graph.json","fetch_events":"https://pith.science/api/pith-number/CLM3OUGSKCD5LSDDI7GKTT4DWT/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/CLM3OUGSKCD5LSDDI7GKTT4DWT/action/timestamp_anchor","attest_storage":"https://pith.science/pith/CLM3OUGSKCD5LSDDI7GKTT4DWT/action/storage_attestation","attest_author":"https://pith.science/pith/CLM3OUGSKCD5LSDDI7GKTT4DWT/action/author_attestation","sign_citation":"https://pith.science/pith/CLM3OUGSKCD5LSDDI7GKTT4DWT/action/citation_signature","submit_replication":"https://pith.science/pith/CLM3OUGSKCD5LSDDI7GKTT4DWT/action/replication_record"}},"created_at":"2026-07-05T09:34:38.338546+00:00","updated_at":"2026-07-05T09:34:38.338546+00:00"}