{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2022:3B54A3DOLNHAT5WITHY3PCMF65","short_pith_number":"pith:3B54A3DO","schema_version":"1.0","canonical_sha256":"d87bc06c6e5b4e09f6c899f1b78985f76550e84e2b3824dde2b1f349120321d0","source":{"kind":"arxiv","id":"2211.17135","version":1},"attestation_state":"computed","paper":{"title":"BudgetLongformer: Can we Cheaply Pretrain a SotA Legal Language Model From Scratch?","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Daniele Giofr\\'e, Joel Niklaus","submitted_at":"2022-11-30T16:09:20Z","abstract_excerpt":"Pretrained transformer models have achieved state-of-the-art results in many tasks and benchmarks recently. Many state-of-the-art Language Models (LMs), however, do not scale well above the threshold of 512 input tokens. In specialized domains though (such as legal, scientific or biomedical), models often need to process very long text (sometimes well above 10000 tokens). Even though many efficient transformers have been proposed (such as Longformer, BigBird or FNet), so far, only very few such efficient models are available for specialized domains. Additionally, since the pretraining process "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2211.17135","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2022-11-30T16:09:20Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"389ee33f52ba5a47a1edd33ef7e50e84921403392ab001bcdea8bbb6bb5f245f","abstract_canon_sha256":"71f30fc70129691f3ac080b03af0e92bf3dd597b4711822bc228fd043c94b434"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:21:15.516521Z","signature_b64":"v5iuCXDc4Cblh08Kc1Uf/mV6fT5Pyct7vD5xveQV/jtjYzMkqhDhUgSGYegX9N3f/+dUspO+lAWL+kPf8Oa+CQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d87bc06c6e5b4e09f6c899f1b78985f76550e84e2b3824dde2b1f349120321d0","last_reissued_at":"2026-07-05T05:21:15.516125Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:21:15.516125Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"BudgetLongformer: Can we Cheaply Pretrain a SotA Legal Language Model From Scratch?","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Daniele Giofr\\'e, Joel Niklaus","submitted_at":"2022-11-30T16:09:20Z","abstract_excerpt":"Pretrained transformer models have achieved state-of-the-art results in many tasks and benchmarks recently. Many state-of-the-art Language Models (LMs), however, do not scale well above the threshold of 512 input tokens. In specialized domains though (such as legal, scientific or biomedical), models often need to process very long text (sometimes well above 10000 tokens). Even though many efficient transformers have been proposed (such as Longformer, BigBird or FNet), so far, only very few such efficient models are available for specialized domains. Additionally, since the pretraining process "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2211.17135","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2211.17135/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2211.17135","created_at":"2026-07-05T05:21:15.516182+00:00"},{"alias_kind":"arxiv_version","alias_value":"2211.17135v1","created_at":"2026-07-05T05:21:15.516182+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2211.17135","created_at":"2026-07-05T05:21:15.516182+00:00"},{"alias_kind":"pith_short_12","alias_value":"3B54A3DOLNHA","created_at":"2026-07-05T05:21:15.516182+00:00"},{"alias_kind":"pith_short_16","alias_value":"3B54A3DOLNHAT5WI","created_at":"2026-07-05T05:21:15.516182+00:00"},{"alias_kind":"pith_short_8","alias_value":"3B54A3DO","created_at":"2026-07-05T05:21:15.516182+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/3B54A3DOLNHAT5WITHY3PCMF65","json":"https://pith.science/pith/3B54A3DOLNHAT5WITHY3PCMF65.json","graph_json":"https://pith.science/api/pith-number/3B54A3DOLNHAT5WITHY3PCMF65/graph.json","events_json":"https://pith.science/api/pith-number/3B54A3DOLNHAT5WITHY3PCMF65/events.json","paper":"https://pith.science/paper/3B54A3DO"},"agent_actions":{"view_html":"https://pith.science/pith/3B54A3DOLNHAT5WITHY3PCMF65","download_json":"https://pith.science/pith/3B54A3DOLNHAT5WITHY3PCMF65.json","view_paper":"https://pith.science/paper/3B54A3DO","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2211.17135&json=true","fetch_graph":"https://pith.science/api/pith-number/3B54A3DOLNHAT5WITHY3PCMF65/graph.json","fetch_events":"https://pith.science/api/pith-number/3B54A3DOLNHAT5WITHY3PCMF65/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/3B54A3DOLNHAT5WITHY3PCMF65/action/timestamp_anchor","attest_storage":"https://pith.science/pith/3B54A3DOLNHAT5WITHY3PCMF65/action/storage_attestation","attest_author":"https://pith.science/pith/3B54A3DOLNHAT5WITHY3PCMF65/action/author_attestation","sign_citation":"https://pith.science/pith/3B54A3DOLNHAT5WITHY3PCMF65/action/citation_signature","submit_replication":"https://pith.science/pith/3B54A3DOLNHAT5WITHY3PCMF65/action/replication_record"}},"created_at":"2026-07-05T05:21:15.516182+00:00","updated_at":"2026-07-05T05:21:15.516182+00:00"}