{"work":{"id":"eb3dbae4-931f-40ab-a37b-507a35f42712","openalex_id":"https://openalex.org/W2994673210","doi":"10.48550/arxiv.2001.04451","arxiv_id":"2001.04451","raw_key":null,"title":"Reformer: The Efficient Transformer","authors":null,"authors_text":"Nikita Kitaev, {\\L}ukasz Kaiser, Anselm Levskaya","year":2020,"venue":"cs.LG","abstract":"Large Transformer models routinely achieve state-of-the-art results on a number of tasks but training these models can be prohibitively costly, especially on long sequences. We introduce two techniques to improve the efficiency of Transformers. For one, we replace dot-product attention by one that uses locality-sensitive hashing, changing its complexity from O($L^2$) to O($L\\log L$), where $L$ is the length of the sequence. Furthermore, we use reversible residual layers instead of the standard residuals, which allows storing activations only once in the training process instead of $N$ times, where $N$ is the number of layers. The resulting model, the Reformer, performs on par with Transformer models while being much more memory-efficient and much faster on long sequences.","external_url":"https://arxiv.org/abs/2001.04451","cited_by_count":323,"metadata_source":"pith","metadata_fetched_at":"2026-08-05T02:28:24.338817+00:00","pith_arxiv_id":"2001.04451","created_at":"2026-05-09T06:15:37.712922+00:00","updated_at":"2026-08-05T02:28:24.338817+00:00","title_quality_ok":false,"display_title":"Reformer: The Efficient Transformer","render_title":"Reformer: The Efficient Transformer"},"hub":{"state":{"work_id":"eb3dbae4-931f-40ab-a37b-507a35f42712","tier":"hub","tier_reason":"10+ Pith inbound or 1,000+ external citations","pith_inbound_count":64,"external_cited_by_count":323,"distinct_field_count":14,"first_pith_cited_at":"2019-10-09T03:23:22+00:00","last_pith_cited_at":"2026-07-08T03:35:59+00:00","author_build_status":"not_needed","summary_status":"needed","contexts_status":"needed","graph_status":"needed","ask_index_status":"not_needed","reader_status":"not_needed","recognition_status":"not_needed","updated_at":"2026-08-23T10:59:29.419378+00:00","tier_text":"hub"},"tier":"hub","role_counts":[{"context_role":"background","n":9},{"context_role":"baseline","n":1},{"context_role":"dataset","n":1},{"context_role":"method","n":1}],"polarity_counts":[{"context_polarity":"background","n":9},{"context_polarity":"baseline","n":1},{"context_polarity":"unclear","n":1},{"context_polarity":"use_dataset","n":1}],"runs":{},"summary":{},"graph":{},"authors":[]}}