{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:647CWRTCVSWMAGFIJFPGPMMYWA","short_pith_number":"pith:647CWRTC","schema_version":"1.0","canonical_sha256":"f73e2b4662acacc018a8495e67b198b0288e669ba39fee2cae4b847469edc3de","source":{"kind":"arxiv","id":"2309.14322","version":2},"attestation_state":"computed","paper":{"title":"Small-scale proxies for large-scale Transformer training instabilities","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Abhishek Kumar, Alex Alemi, Ben Adlam, Izzeddin Gur, Jaehoon Lee, Jascha Sohl-Dickstein, Jeffrey Pennington, John D. Co-Reyes, Justin Gilmer, Katie Everett, Kelvin Xu, Lechao Xiao, Mitchell Wortsman, Peter J. Liu, Roman Novak, Simon Kornblith","submitted_at":"2023-09-25T17:48:51Z","abstract_excerpt":"Teams that have trained large Transformer-based models have reported training instabilities at large scale that did not appear when training with the same hyperparameters at smaller scales. Although the causes of such instabilities are of scientific interest, the amount of resources required to reproduce them has made investigation difficult. In this work, we seek ways to reproduce and study training stability and instability at smaller scales. First, we focus on two sources of training instability described in previous work: the growth of logits in attention layers (Dehghani et al., 2023) and"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2309.14322","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-09-25T17:48:51Z","cross_cats_sorted":[],"title_canon_sha256":"692522a05eafd3228d7544e2a06deaf5da8ba56e40d3abf861602d862b391de5","abstract_canon_sha256":"e3a198a9dc3da292a34f42811b74e4af7ee2aa1c10bbe952465d9c4039e7d793"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:01:25.755306Z","signature_b64":"fJMzcPk7n86j3NFHrdaBFHVlvqUJOneYRjXscmbJxFDdjQZgmIV93JSYThErXSdszRZHQk9e51xipvRf2w3FDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f73e2b4662acacc018a8495e67b198b0288e669ba39fee2cae4b847469edc3de","last_reissued_at":"2026-07-05T07:01:25.754842Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:01:25.754842Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Small-scale proxies for large-scale Transformer training instabilities","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Abhishek Kumar, Alex Alemi, Ben Adlam, Izzeddin Gur, Jaehoon Lee, Jascha Sohl-Dickstein, Jeffrey Pennington, John D. Co-Reyes, Justin Gilmer, Katie Everett, Kelvin Xu, Lechao Xiao, Mitchell Wortsman, Peter J. Liu, Roman Novak, Simon Kornblith","submitted_at":"2023-09-25T17:48:51Z","abstract_excerpt":"Teams that have trained large Transformer-based models have reported training instabilities at large scale that did not appear when training with the same hyperparameters at smaller scales. Although the causes of such instabilities are of scientific interest, the amount of resources required to reproduce them has made investigation difficult. In this work, we seek ways to reproduce and study training stability and instability at smaller scales. First, we focus on two sources of training instability described in previous work: the growth of logits in attention layers (Dehghani et al., 2023) and"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.14322","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2309.14322/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2309.14322","created_at":"2026-07-05T07:01:25.754896+00:00"},{"alias_kind":"arxiv_version","alias_value":"2309.14322v2","created_at":"2026-07-05T07:01:25.754896+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.14322","created_at":"2026-07-05T07:01:25.754896+00:00"},{"alias_kind":"pith_short_12","alias_value":"647CWRTCVSWM","created_at":"2026-07-05T07:01:25.754896+00:00"},{"alias_kind":"pith_short_16","alias_value":"647CWRTCVSWMAGFI","created_at":"2026-07-05T07:01:25.754896+00:00"},{"alias_kind":"pith_short_8","alias_value":"647CWRTC","created_at":"2026-07-05T07:01:25.754896+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":15,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.25971","citing_title":"Improving Neural Network Training by Decoupling the Magnitude and Direction of Weight Vectors","ref_index":159,"is_internal_anchor":false},{"citing_arxiv_id":"2503.19786","citing_title":"Gemma 3 Technical Report","ref_index":45,"is_internal_anchor":false},{"citing_arxiv_id":"2605.22820","citing_title":"Integrable Elasticity via Neural Demand Potentials","ref_index":19,"is_internal_anchor":false},{"citing_arxiv_id":"2605.20314","citing_title":"Less Data, Faster Training: repeating smaller datasets speeds up learning via sampling biases","ref_index":30,"is_internal_anchor":false},{"citing_arxiv_id":"2507.06261","citing_title":"Gemini 2.5: Pushing the Frontier with Advanced Reasoning, Multimodality, Long Context, and Next Generation Agentic Capabilities","ref_index":90,"is_internal_anchor":false},{"citing_arxiv_id":"2406.11794","citing_title":"DataComp-LM: In search of the next generation of training sets for language models","ref_index":200,"is_internal_anchor":false},{"citing_arxiv_id":"2410.10781","citing_title":"When Attention Sink Emerges in Language Models: An Empirical View","ref_index":52,"is_internal_anchor":false},{"citing_arxiv_id":"2402.17762","citing_title":"Massive Activations in Large Language Models","ref_index":154,"is_internal_anchor":false},{"citing_arxiv_id":"2404.06395","citing_title":"MiniCPM: Unveiling the Potential of Small Language Models with Scalable Training Strategies","ref_index":42,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12492","citing_title":"Pion: A Spectrum-Preserving Optimizer via Orthogonal Equivalence Transformation","ref_index":80,"is_internal_anchor":false},{"citing_arxiv_id":"2408.12528","citing_title":"Show-o: One Single Transformer to Unify Multimodal Understanding and Generation","ref_index":22,"is_internal_anchor":false},{"citing_arxiv_id":"2405.09818","citing_title":"Chameleon: Mixed-Modal Early-Fusion Foundation Models","ref_index":33,"is_internal_anchor":false},{"citing_arxiv_id":"2604.11707","citing_title":"Representations Before Pixels: Semantics-Guided Hierarchical Video Prediction","ref_index":79,"is_internal_anchor":false},{"citing_arxiv_id":"2501.03575","citing_title":"Cosmos World Foundation Model Platform for Physical AI","ref_index":219,"is_internal_anchor":false},{"citing_arxiv_id":"2507.20534","citing_title":"Kimi K2: Open Agentic Intelligence","ref_index":86,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/647CWRTCVSWMAGFIJFPGPMMYWA","json":"https://pith.science/pith/647CWRTCVSWMAGFIJFPGPMMYWA.json","graph_json":"https://pith.science/api/pith-number/647CWRTCVSWMAGFIJFPGPMMYWA/graph.json","events_json":"https://pith.science/api/pith-number/647CWRTCVSWMAGFIJFPGPMMYWA/events.json","paper":"https://pith.science/paper/647CWRTC"},"agent_actions":{"view_html":"https://pith.science/pith/647CWRTCVSWMAGFIJFPGPMMYWA","download_json":"https://pith.science/pith/647CWRTCVSWMAGFIJFPGPMMYWA.json","view_paper":"https://pith.science/paper/647CWRTC","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2309.14322&json=true","fetch_graph":"https://pith.science/api/pith-number/647CWRTCVSWMAGFIJFPGPMMYWA/graph.json","fetch_events":"https://pith.science/api/pith-number/647CWRTCVSWMAGFIJFPGPMMYWA/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/647CWRTCVSWMAGFIJFPGPMMYWA/action/timestamp_anchor","attest_storage":"https://pith.science/pith/647CWRTCVSWMAGFIJFPGPMMYWA/action/storage_attestation","attest_author":"https://pith.science/pith/647CWRTCVSWMAGFIJFPGPMMYWA/action/author_attestation","sign_citation":"https://pith.science/pith/647CWRTCVSWMAGFIJFPGPMMYWA/action/citation_signature","submit_replication":"https://pith.science/pith/647CWRTCVSWMAGFIJFPGPMMYWA/action/replication_record"}},"created_at":"2026-07-05T07:01:25.754896+00:00","updated_at":"2026-07-05T07:01:25.754896+00:00"}