{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:LINTUUTQIOZXILZRO7TIZLBY5X","short_pith_number":"pith:LINTUUTQ","schema_version":"1.0","canonical_sha256":"5a1b3a527043b3742f3177e68cac38edf7a0daed1c0638a022e99de4fc059431","source":{"kind":"arxiv","id":"2309.08586","version":2},"attestation_state":"computed","paper":{"title":"Replacing softmax with ReLU in Vision Transformers","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CV","authors_text":"Jaehoon Lee, Justin Gilmer, Mitchell Wortsman, Simon Kornblith","submitted_at":"2023-09-15T17:43:40Z","abstract_excerpt":"Previous research observed accuracy degradation when replacing the attention softmax with a point-wise activation such as ReLU. In the context of vision transformers, we find that this degradation is mitigated when dividing by sequence length. Our experiments training small to large vision transformers on ImageNet-21k indicate that ReLU-attention can approach or match the performance of softmax-attention in terms of scaling behavior as a function of compute."},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2309.08586","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2023-09-15T17:43:40Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"494c189225aeb24c14450140ac40e51270005ed6a04fafc76170fde049807a85","abstract_canon_sha256":"7a5a4618e56161ae1e907003c05b4405afa8320187e5fe6775ee98f8b3e64911"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:01:25.684475Z","signature_b64":"To8Pbc5AX+WmQUqwH5WX6OAj2On+nQimC+0Hy6KZQxUntp6Fa5FSwwa9yyq9wrDH202jI30myO2z9Neujz9HBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"5a1b3a527043b3742f3177e68cac38edf7a0daed1c0638a022e99de4fc059431","last_reissued_at":"2026-07-05T07:01:25.684007Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:01:25.684007Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Replacing softmax with ReLU in Vision Transformers","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CV","authors_text":"Jaehoon Lee, Justin Gilmer, Mitchell Wortsman, Simon Kornblith","submitted_at":"2023-09-15T17:43:40Z","abstract_excerpt":"Previous research observed accuracy degradation when replacing the attention softmax with a point-wise activation such as ReLU. In the context of vision transformers, we find that this degradation is mitigated when dividing by sequence length. Our experiments training small to large vision transformers on ImageNet-21k indicate that ReLU-attention can approach or match the performance of softmax-attention in terms of scaling behavior as a function of compute."},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.08586","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2309.08586/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2309.08586","created_at":"2026-07-05T07:01:25.684062+00:00"},{"alias_kind":"arxiv_version","alias_value":"2309.08586v2","created_at":"2026-07-05T07:01:25.684062+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.08586","created_at":"2026-07-05T07:01:25.684062+00:00"},{"alias_kind":"pith_short_12","alias_value":"LINTUUTQIOZX","created_at":"2026-07-05T07:01:25.684062+00:00"},{"alias_kind":"pith_short_16","alias_value":"LINTUUTQIOZXILZR","created_at":"2026-07-05T07:01:25.684062+00:00"},{"alias_kind":"pith_short_8","alias_value":"LINTUUTQ","created_at":"2026-07-05T07:01:25.684062+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":7,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.08105","citing_title":"A Unifying View of Attention Sinks: Two Algorithms, Two Solutions","ref_index":43,"is_internal_anchor":false},{"citing_arxiv_id":"2605.06501","citing_title":"Cubit: Token Mixer with Kernel Ridge Regression","ref_index":88,"is_internal_anchor":false},{"citing_arxiv_id":"2605.13507","citing_title":"Efficient Implementation of an Adaptive Transformer Accelerator for Massive MIMO Outdoor Localization","ref_index":25,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10775","citing_title":"On the global convergence of gradient descent for wide shallow models with bounded nonlinearities","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2605.06501","citing_title":"Cubit: Token Mixer with Kernel Ridge Regression","ref_index":88,"is_internal_anchor":false},{"citing_arxiv_id":"2605.04279","citing_title":"Gradient Flow Structure and Quantitative Dynamics of Multi-Head Self-Attention","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2605.04279","citing_title":"Gradient Flow Structure and Quantitative Dynamics of Multi-Head Self-Attention","ref_index":20,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/LINTUUTQIOZXILZRO7TIZLBY5X","json":"https://pith.science/pith/LINTUUTQIOZXILZRO7TIZLBY5X.json","graph_json":"https://pith.science/api/pith-number/LINTUUTQIOZXILZRO7TIZLBY5X/graph.json","events_json":"https://pith.science/api/pith-number/LINTUUTQIOZXILZRO7TIZLBY5X/events.json","paper":"https://pith.science/paper/LINTUUTQ"},"agent_actions":{"view_html":"https://pith.science/pith/LINTUUTQIOZXILZRO7TIZLBY5X","download_json":"https://pith.science/pith/LINTUUTQIOZXILZRO7TIZLBY5X.json","view_paper":"https://pith.science/paper/LINTUUTQ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2309.08586&json=true","fetch_graph":"https://pith.science/api/pith-number/LINTUUTQIOZXILZRO7TIZLBY5X/graph.json","fetch_events":"https://pith.science/api/pith-number/LINTUUTQIOZXILZRO7TIZLBY5X/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/LINTUUTQIOZXILZRO7TIZLBY5X/action/timestamp_anchor","attest_storage":"https://pith.science/pith/LINTUUTQIOZXILZRO7TIZLBY5X/action/storage_attestation","attest_author":"https://pith.science/pith/LINTUUTQIOZXILZRO7TIZLBY5X/action/author_attestation","sign_citation":"https://pith.science/pith/LINTUUTQIOZXILZRO7TIZLBY5X/action/citation_signature","submit_replication":"https://pith.science/pith/LINTUUTQIOZXILZRO7TIZLBY5X/action/replication_record"}},"created_at":"2026-07-05T07:01:25.684062+00:00","updated_at":"2026-07-05T07:01:25.684062+00:00"}