{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:CB5Y2YTMFBWE7RQDZZNWJG2I5B","short_pith_number":"pith:CB5Y2YTM","canonical_record":{"source":{"id":"2407.06380","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-08T20:47:58Z","cross_cats_sorted":[],"title_canon_sha256":"e6b5dd6c48b2b5bbd865a22b0b71ad7b5e8b4ecd396b29d216691b1f30137351","abstract_canon_sha256":"6b0b94a709e3ef45a10fe06bdfef80fb67d463d093f57769eb743857b8615179"},"schema_version":"1.0"},"canonical_sha256":"107b8d626c286c4fc603ce5b649b48e8702c3bf28a08fe69c45cebb4f8df9d6c","source":{"kind":"arxiv","id":"2407.06380","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.06380","created_at":"2026-07-05T09:22:41Z"},{"alias_kind":"arxiv_version","alias_value":"2407.06380v2","created_at":"2026-07-05T09:22:41Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.06380","created_at":"2026-07-05T09:22:41Z"},{"alias_kind":"pith_short_12","alias_value":"CB5Y2YTMFBWE","created_at":"2026-07-05T09:22:41Z"},{"alias_kind":"pith_short_16","alias_value":"CB5Y2YTMFBWE7RQD","created_at":"2026-07-05T09:22:41Z"},{"alias_kind":"pith_short_8","alias_value":"CB5Y2YTM","created_at":"2026-07-05T09:22:41Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:CB5Y2YTMFBWE7RQDZZNWJG2I5B","target":"record","payload":{"canonical_record":{"source":{"id":"2407.06380","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-08T20:47:58Z","cross_cats_sorted":[],"title_canon_sha256":"e6b5dd6c48b2b5bbd865a22b0b71ad7b5e8b4ecd396b29d216691b1f30137351","abstract_canon_sha256":"6b0b94a709e3ef45a10fe06bdfef80fb67d463d093f57769eb743857b8615179"},"schema_version":"1.0"},"canonical_sha256":"107b8d626c286c4fc603ce5b649b48e8702c3bf28a08fe69c45cebb4f8df9d6c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:22:41.167884Z","signature_b64":"iOjjN85XEpf2SICaddbOLTTv+pjWcWw0IMWFUGHhp1wCcazhitO4jL7ZwMnGjz3SVgSvokVLhf0SN5lzOYYUBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"107b8d626c286c4fc603ce5b649b48e8702c3bf28a08fe69c45cebb4f8df9d6c","last_reissued_at":"2026-07-05T09:22:41.167298Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:22:41.167298Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2407.06380","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:22:41Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"gu7jQM2mNYaLop2j83SdYcUD47bgCxOEjate/8iABkRdzhq9ORJ66kNjgOa2Z+/EIObgstSSaAdyivhuXQqRDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T06:28:48.906745Z"},"content_sha256":"c23ba02571be92010e95622c2e5dd968e6690c874cbe599d05a7db6ead0ef138","schema_version":"1.0","event_id":"sha256:c23ba02571be92010e95622c2e5dd968e6690c874cbe599d05a7db6ead0ef138"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:CB5Y2YTMFBWE7RQDZZNWJG2I5B","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Data, Data Everywhere: A Guide for Pretraining Dataset Construction","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Aastha Jhunjhunwala, Bo Liu, Bryan Catanzaro, Joseph Jennings, Jupinder Parmar, Mohammad Shoeybi, Mostofa Patwary, Shrimai Prabhumoye, Zhilin Wang","submitted_at":"2024-07-08T20:47:58Z","abstract_excerpt":"The impressive capabilities of recent language models can be largely attributed to the multi-trillion token pretraining datasets that they are trained on. However, model developers fail to disclose their construction methodology which has lead to a lack of open information on how to develop effective pretraining sets. To address this issue, we perform the first systematic study across the entire pipeline of pretraining set construction. First, we run ablations on existing techniques for pretraining set development to identify which methods translate to the largest gains in model accuracy on do"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.06380","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2407.06380/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:22:41Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"f04CmePi2KoevumzGpscrL3Uowf3QtXkg7b4roqr4K4OktADcnnDSrKY/K7rIbwMOGI3Q0LatAamN3vUvZAIAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T06:28:48.907327Z"},"content_sha256":"ee3f16fd6f64880cfe747ac81baca42a71a09d149267e65c1608cccb3696a840","schema_version":"1.0","event_id":"sha256:ee3f16fd6f64880cfe747ac81baca42a71a09d149267e65c1608cccb3696a840"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/CB5Y2YTMFBWE7RQDZZNWJG2I5B/bundle.json","state_url":"https://pith.science/pith/CB5Y2YTMFBWE7RQDZZNWJG2I5B/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/CB5Y2YTMFBWE7RQDZZNWJG2I5B/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-21T06:28:48Z","links":{"resolver":"https://pith.science/pith/CB5Y2YTMFBWE7RQDZZNWJG2I5B","bundle":"https://pith.science/pith/CB5Y2YTMFBWE7RQDZZNWJG2I5B/bundle.json","state":"https://pith.science/pith/CB5Y2YTMFBWE7RQDZZNWJG2I5B/state.json","well_known_bundle":"https://pith.science/.well-known/pith/CB5Y2YTMFBWE7RQDZZNWJG2I5B/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:CB5Y2YTMFBWE7RQDZZNWJG2I5B","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"6b0b94a709e3ef45a10fe06bdfef80fb67d463d093f57769eb743857b8615179","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-08T20:47:58Z","title_canon_sha256":"e6b5dd6c48b2b5bbd865a22b0b71ad7b5e8b4ecd396b29d216691b1f30137351"},"schema_version":"1.0","source":{"id":"2407.06380","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.06380","created_at":"2026-07-05T09:22:41Z"},{"alias_kind":"arxiv_version","alias_value":"2407.06380v2","created_at":"2026-07-05T09:22:41Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.06380","created_at":"2026-07-05T09:22:41Z"},{"alias_kind":"pith_short_12","alias_value":"CB5Y2YTMFBWE","created_at":"2026-07-05T09:22:41Z"},{"alias_kind":"pith_short_16","alias_value":"CB5Y2YTMFBWE7RQD","created_at":"2026-07-05T09:22:41Z"},{"alias_kind":"pith_short_8","alias_value":"CB5Y2YTM","created_at":"2026-07-05T09:22:41Z"}],"graph_snapshots":[{"event_id":"sha256:ee3f16fd6f64880cfe747ac81baca42a71a09d149267e65c1608cccb3696a840","target":"graph","created_at":"2026-07-05T09:22:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2407.06380/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The impressive capabilities of recent language models can be largely attributed to the multi-trillion token pretraining datasets that they are trained on. However, model developers fail to disclose their construction methodology which has lead to a lack of open information on how to develop effective pretraining sets. To address this issue, we perform the first systematic study across the entire pipeline of pretraining set construction. First, we run ablations on existing techniques for pretraining set development to identify which methods translate to the largest gains in model accuracy on do","authors_text":"Aastha Jhunjhunwala, Bo Liu, Bryan Catanzaro, Joseph Jennings, Jupinder Parmar, Mohammad Shoeybi, Mostofa Patwary, Shrimai Prabhumoye, Zhilin Wang","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-08T20:47:58Z","title":"Data, Data Everywhere: A Guide for Pretraining Dataset Construction"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.06380","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:c23ba02571be92010e95622c2e5dd968e6690c874cbe599d05a7db6ead0ef138","target":"record","created_at":"2026-07-05T09:22:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"6b0b94a709e3ef45a10fe06bdfef80fb67d463d093f57769eb743857b8615179","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-08T20:47:58Z","title_canon_sha256":"e6b5dd6c48b2b5bbd865a22b0b71ad7b5e8b4ecd396b29d216691b1f30137351"},"schema_version":"1.0","source":{"id":"2407.06380","kind":"arxiv","version":2}},"canonical_sha256":"107b8d626c286c4fc603ce5b649b48e8702c3bf28a08fe69c45cebb4f8df9d6c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"107b8d626c286c4fc603ce5b649b48e8702c3bf28a08fe69c45cebb4f8df9d6c","first_computed_at":"2026-07-05T09:22:41.167298Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:22:41.167298Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"iOjjN85XEpf2SICaddbOLTTv+pjWcWw0IMWFUGHhp1wCcazhitO4jL7ZwMnGjz3SVgSvokVLhf0SN5lzOYYUBw==","signature_status":"signed_v1","signed_at":"2026-07-05T09:22:41.167884Z","signed_message":"canonical_sha256_bytes"},"source_id":"2407.06380","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:c23ba02571be92010e95622c2e5dd968e6690c874cbe599d05a7db6ead0ef138","sha256:ee3f16fd6f64880cfe747ac81baca42a71a09d149267e65c1608cccb3696a840"],"state_sha256":"8e1b94837a4aebf320286f5fcd301eeb626ab3dff880eb461f81b9a048361366"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jv9Ko0dFSOHOAbe9F/2nYMyAJHbqsW2iHMImOXWrscwx31bek6RA+37JVV/illoh+8UBbFUCjC8UXCEZ/TtaDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-21T06:28:48.912496Z","bundle_sha256":"4e352202323602c089c1d42dac82a6af54ca399c1a12abc9d20e1712c109e9c6"}}