{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2026:XAW366NP42VQE364EK27H34OZX","short_pith_number":"pith:XAW366NP","schema_version":"1.0","canonical_sha256":"b82dbf79afe6ab026fdc22b5f3ef8ecdccd31b111204a086288adf69ad803152","source":{"kind":"arxiv","id":"2601.21725","version":2},"attestation_state":"computed","paper":{"title":"Procedural Pretraining: Warming Up Language Models with Abstract Data","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Anton van den Hengel, Damien Teney, Hemanth Saratchandran, Liangze Jiang, Zachary Shinnick","submitted_at":"2026-01-29T13:48:43Z","abstract_excerpt":"Pretraining language models directly on web-scale corpora is the de facto paradigm. We study an alternative where the model is initially exposed to abstract structured data to ease the subsequent acquisition of rich semantic knowledge, much like humans learning simple logic and mathematics before higher reasoning. We focus on procedural data, generated by formal languages and other simple algorithms, as such abstract data. We first diagnose the algorithmic skills that different forms of procedural data can improve, often significantly. For example, the accuracy of context recall (Needle-in-a-h"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2601.21725","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2026-01-29T13:48:43Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"2220e88d330656d813945ac6c6ba6f2d3e3c3b04eed256c69c0aa39653b1f41d","abstract_canon_sha256":"674e7c60ca153f66799cf4b4e228fa4ead44fc3727b8d1a5d32e5417616fcb1e"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-29T01:04:36.680197Z","signature_b64":"TNz9IKNmU7RniCtWWd2HU6Mc4oiNeTKstc1xvD/Z33GJth8DICrSjn/kWE90cl4ihZHjGxN3z2rRo23cIeO5CQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b82dbf79afe6ab026fdc22b5f3ef8ecdccd31b111204a086288adf69ad803152","last_reissued_at":"2026-05-29T01:04:36.679609Z","signature_status":"signed_v1","first_computed_at":"2026-05-29T01:04:36.679609Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Procedural Pretraining: Warming Up Language Models with Abstract Data","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Anton van den Hengel, Damien Teney, Hemanth Saratchandran, Liangze Jiang, Zachary Shinnick","submitted_at":"2026-01-29T13:48:43Z","abstract_excerpt":"Pretraining language models directly on web-scale corpora is the de facto paradigm. We study an alternative where the model is initially exposed to abstract structured data to ease the subsequent acquisition of rich semantic knowledge, much like humans learning simple logic and mathematics before higher reasoning. We focus on procedural data, generated by formal languages and other simple algorithms, as such abstract data. We first diagnose the algorithmic skills that different forms of procedural data can improve, often significantly. For example, the accuracy of context recall (Needle-in-a-h"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2601.21725","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2601.21725/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2601.21725","created_at":"2026-05-29T01:04:36.679686+00:00"},{"alias_kind":"arxiv_version","alias_value":"2601.21725v2","created_at":"2026-05-29T01:04:36.679686+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2601.21725","created_at":"2026-05-29T01:04:36.679686+00:00"},{"alias_kind":"pith_short_12","alias_value":"XAW366NP42VQ","created_at":"2026-05-29T01:04:36.679686+00:00"},{"alias_kind":"pith_short_16","alias_value":"XAW366NP42VQE364","created_at":"2026-05-29T01:04:36.679686+00:00"},{"alias_kind":"pith_short_8","alias_value":"XAW366NP","created_at":"2026-05-29T01:04:36.679686+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2604.15503","citing_title":"Brain Score Tracks Shared Properties of Languages: Evidence from Many Natural Languages and Structured Sequences","ref_index":12,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/XAW366NP42VQE364EK27H34OZX","json":"https://pith.science/pith/XAW366NP42VQE364EK27H34OZX.json","graph_json":"https://pith.science/api/pith-number/XAW366NP42VQE364EK27H34OZX/graph.json","events_json":"https://pith.science/api/pith-number/XAW366NP42VQE364EK27H34OZX/events.json","paper":"https://pith.science/paper/XAW366NP"},"agent_actions":{"view_html":"https://pith.science/pith/XAW366NP42VQE364EK27H34OZX","download_json":"https://pith.science/pith/XAW366NP42VQE364EK27H34OZX.json","view_paper":"https://pith.science/paper/XAW366NP","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2601.21725&json=true","fetch_graph":"https://pith.science/api/pith-number/XAW366NP42VQE364EK27H34OZX/graph.json","fetch_events":"https://pith.science/api/pith-number/XAW366NP42VQE364EK27H34OZX/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/XAW366NP42VQE364EK27H34OZX/action/timestamp_anchor","attest_storage":"https://pith.science/pith/XAW366NP42VQE364EK27H34OZX/action/storage_attestation","attest_author":"https://pith.science/pith/XAW366NP42VQE364EK27H34OZX/action/author_attestation","sign_citation":"https://pith.science/pith/XAW366NP42VQE364EK27H34OZX/action/citation_signature","submit_replication":"https://pith.science/pith/XAW366NP42VQE364EK27H34OZX/action/replication_record"}},"created_at":"2026-05-29T01:04:36.679686+00:00","updated_at":"2026-05-29T01:04:36.679686+00:00"}