{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:7ZKTROEYPUT3OHQRT22M4P2O2E","short_pith_number":"pith:7ZKTROEY","schema_version":"1.0","canonical_sha256":"fe5538b8987d27b71e119eb4ce3f4ed1270d9ee4b9302811473a31b6c09ebabf","source":{"kind":"arxiv","id":"2402.08268","version":4},"attestation_state":"computed","paper":{"title":"World Model on Million-Length Video And Language With Blockwise RingAttention","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"7B parameter models process video and language sequences exceeding 1 million tokens.","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Hao Liu, Matei Zaharia, Pieter Abbeel, Wilson Yan","submitted_at":"2024-02-13T07:47:36Z","abstract_excerpt":"Enabling long-context understanding remains a key challenge in scaling existing sequence models -- a crucial component in developing generally intelligent models that can process and operate over long temporal horizons that potentially consist of millions of tokens. In this paper, we aim to address these challenges by providing a comprehensive exploration of the full development process for producing 1M context language models and video-language models, setting new benchmarks in language retrieval and new capabilities in long video understanding. We detail our long context data curation proces"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":true,"formal_links_present":true},"canonical_record":{"source":{"id":"2402.08268","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-02-13T07:47:36Z","cross_cats_sorted":[],"title_canon_sha256":"a17f2d3c39ca5c2a6aec7c5170eff5fd2cf874427dfa86450ed3bb835872d23b","abstract_canon_sha256":"dca7c76a40ebd7467d053db24b9fac5090178337ce8421c47f2ee989a5b189d7"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:38:48.813790Z","signature_b64":"h0voKelRup+ILIcNqMLfKpMkIX32ONQbujQOrzAdrbfoDEe+CBlulh+35hdQJnLFqNeZzUFRrxaiqdlGfKhICA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"fe5538b8987d27b71e119eb4ce3f4ed1270d9ee4b9302811473a31b6c09ebabf","last_reissued_at":"2026-05-17T23:38:48.813238Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:38:48.813238Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"World Model on Million-Length Video And Language With Blockwise RingAttention","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"7B parameter models process video and language sequences exceeding 1 million tokens.","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Hao Liu, Matei Zaharia, Pieter Abbeel, Wilson Yan","submitted_at":"2024-02-13T07:47:36Z","abstract_excerpt":"Enabling long-context understanding remains a key challenge in scaling existing sequence models -- a crucial component in developing generally intelligent models that can process and operate over long temporal horizons that potentially consist of millions of tokens. In this paper, we aim to address these challenges by providing a comprehensive exploration of the full development process for producing 1M context language models and video-language models, setting new benchmarks in language retrieval and new capabilities in long video understanding. We detail our long context data curation proces"},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"We open-source a family of 7B parameter models capable of processing long text documents and videos exceeding 1M tokens, setting new benchmarks in language retrieval and new capabilities in long video understanding.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"That the Blockwise RingAttention mechanism combined with progressive context extension from 4K to 1M tokens enables effective utilization of the full context length without prohibitive computational costs or performance loss.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"Presents open-source 7B models for million-token video and language understanding via Blockwise RingAttention, setting new benchmarks in retrieval and long video tasks.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"7B parameter models process video and language sequences exceeding 1 million tokens.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"aeecb379bdff3ba1a5bb08a27b1c3b63f69bae0ac603dd068a30ecf1c010ac14"},"source":{"id":"2402.08268","kind":"arxiv","version":4},"verdict":{"id":"6374161b-ebcd-483d-98e6-c6c767d8879d","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-16T06:32:36.859746Z","strongest_claim":"We open-source a family of 7B parameter models capable of processing long text documents and videos exceeding 1M tokens, setting new benchmarks in language retrieval and new capabilities in long video understanding.","one_line_summary":"Presents open-source 7B models for million-token video and language understanding via Blockwise RingAttention, setting new benchmarks in retrieval and long video tasks.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"That the Blockwise RingAttention mechanism combined with progressive context extension from 4K to 1M tokens enables effective utilization of the full context length without prohibitive computational costs or performance loss.","pith_extraction_headline":"7B parameter models process video and language sequences exceeding 1 million tokens."},"references":{"count":36,"sample":[{"doi":"","year":null,"title":"Jointly training large autoregressive multimodal models","work_id":"d6fca016-5cb9-4bbe-8e83-c1333222097e","ref_index":1,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":null,"title":"OpenFlamingo: An Open-Source Framework for Training Large Autoregressive Vision-Language Models","work_id":"87bfa84a-e663-4165-806f-93ef439d88d0","ref_index":2,"cited_arxiv_id":"2308.01390","is_internal_anchor":true},{"doi":"","year":2004,"title":"Longformer: The Long-Document Transformer","work_id":"abea7a44-6668-4de7-aab6-f53a6e5aa088","ref_index":3,"cited_arxiv_id":"2004.05150","is_internal_anchor":true},{"doi":"","year":null,"title":"Striped attention: Faster ring attention for causal transformers","work_id":"7cc4e141-1418-4a62-bc6a-4e3270436eec","ref_index":4,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":1901,"title":"Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared D Kaplan, Prafulla Dhariwal, Arvind Neelakantan, Pranav Shyam, Girish Sastry, Amanda Askell, et al","work_id":"0cfb0982-a2c1-413b-87eb-f784fefa6518","ref_index":5,"cited_arxiv_id":"","is_internal_anchor":false}],"resolved_work":36,"snapshot_sha256":"6704dbb0e5dc8e38c64853e13334e7ffa68523f39882096af3ae1bb67c779095","internal_anchors":23},"formal_canon":{"evidence_count":3,"snapshot_sha256":"73287facd96eb55d6898c0a4d98d798e125534b6d1fff6125b0d2c85f006dedf"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2402.08268","created_at":"2026-05-17T23:38:48.813332+00:00"},{"alias_kind":"arxiv_version","alias_value":"2402.08268v4","created_at":"2026-05-17T23:38:48.813332+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.08268","created_at":"2026-05-17T23:38:48.813332+00:00"},{"alias_kind":"pith_short_12","alias_value":"7ZKTROEYPUT3","created_at":"2026-05-18T12:33:37.589309+00:00"},{"alias_kind":"pith_short_16","alias_value":"7ZKTROEYPUT3OHQR","created_at":"2026-05-18T12:33:37.589309+00:00"},{"alias_kind":"pith_short_8","alias_value":"7ZKTROEY","created_at":"2026-05-18T12:33:37.589309+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":29,"internal_anchor_count":29,"sample":[{"citing_arxiv_id":"2501.05067","citing_title":"LLaVA-Octopus: Unlocking Instruction-Driven Adaptive Projector Fusion for Video Understanding","ref_index":42,"is_internal_anchor":true},{"citing_arxiv_id":"2412.14171","citing_title":"Thinking in Space: How Multimodal Large Language Models See, Remember, and Recall Spaces","ref_index":50,"is_internal_anchor":true},{"citing_arxiv_id":"2605.18678","citing_title":"Lance: Unified Multimodal Modeling by Multi-Task Synergy","ref_index":70,"is_internal_anchor":true},{"citing_arxiv_id":"2407.08608","citing_title":"FlashAttention-3: Fast and Accurate Attention with Asynchrony and Low-precision","ref_index":32,"is_internal_anchor":true},{"citing_arxiv_id":"2605.18115","citing_title":"WinTok: A Win-Win Hybrid Tokenizer via Decomposing Visual Understanding and Generation with Transferable Tokens","ref_index":57,"is_internal_anchor":true},{"citing_arxiv_id":"2605.18678","citing_title":"Lance: Unified Multimodal Modeling by Multi-Task Synergy","ref_index":69,"is_internal_anchor":true},{"citing_arxiv_id":"2406.08035","citing_title":"LVBench: An Extreme Long Video Understanding Benchmark","ref_index":20,"is_internal_anchor":true},{"citing_arxiv_id":"2506.15155","citing_title":"eLLM: Elastic Memory Management Framework for Efficient LLM Serving","ref_index":20,"is_internal_anchor":true},{"citing_arxiv_id":"2412.14164","citing_title":"MetaMorph: Multimodal Understanding and Generation via Instruction Tuning","ref_index":39,"is_internal_anchor":true},{"citing_arxiv_id":"2410.17434","citing_title":"LongVU: Spatiotemporal Adaptive Compression for Long Video-Language Understanding","ref_index":17,"is_internal_anchor":true},{"citing_arxiv_id":"2601.15507","citing_title":"A Unified and Controllable Framework for Layered Image Generation with Visual Effects","ref_index":28,"is_internal_anchor":true},{"citing_arxiv_id":"2409.04429","citing_title":"VILA-U: a Unified Foundation Model Integrating Visual Understanding and Generation","ref_index":13,"is_internal_anchor":true},{"citing_arxiv_id":"2404.14396","citing_title":"SEED-X: Multimodal Models with Unified Multi-granularity Comprehension and Generation","ref_index":42,"is_internal_anchor":true},{"citing_arxiv_id":"2410.13848","citing_title":"Janus: Decoupling Visual Encoding for Unified Multimodal Understanding and Generation","ref_index":52,"is_internal_anchor":true},{"citing_arxiv_id":"2504.01805","citing_title":"SpaceR: Reinforcing MLLMs in Video Spatial Reasoning","ref_index":21,"is_internal_anchor":true},{"citing_arxiv_id":"2410.17247","citing_title":"PyramidDrop: Accelerating Your Large Vision-Language Models via Pyramid Visual Redundancy Reduction","ref_index":32,"is_internal_anchor":true},{"citing_arxiv_id":"2407.04620","citing_title":"Learning to (Learn at Test Time): RNNs with Expressive Hidden States","ref_index":52,"is_internal_anchor":true},{"citing_arxiv_id":"2603.27259","citing_title":"Seeing the Scene Matters: Revealing Forgetting in Video Understanding Models with a Scene-Aware Long-Video Benchmark","ref_index":29,"is_internal_anchor":true},{"citing_arxiv_id":"2406.04264","citing_title":"MLVU: Benchmarking Multi-task Long Video Understanding","ref_index":30,"is_internal_anchor":true},{"citing_arxiv_id":"2404.14469","citing_title":"SnapKV: LLM Knows What You are Looking for Before Generation","ref_index":16,"is_internal_anchor":true},{"citing_arxiv_id":"2604.26934","citing_title":"World2VLM: Distilling World Model Imagination into VLMs for Dynamic Spatial Reasoning","ref_index":33,"is_internal_anchor":true},{"citing_arxiv_id":"2604.25636","citing_title":"Refinement via Regeneration: Enlarging Modification Space Boosts Image Refinement in Unified Multimodal Models","ref_index":29,"is_internal_anchor":true},{"citing_arxiv_id":"2605.05646","citing_title":"MUSE: Resolving Manifold Misalignment in Visual Tokenization via Topological Orthogonality","ref_index":124,"is_internal_anchor":true},{"citing_arxiv_id":"2604.21926","citing_title":"Seeing Without Eyes: 4D Human-Scene Understanding from Wearable IMUs","ref_index":55,"is_internal_anchor":true},{"citing_arxiv_id":"2405.21060","citing_title":"Transformers are SSMs: Generalized Models and Efficient Algorithms Through Structured State Space Duality","ref_index":61,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":3,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/7ZKTROEYPUT3OHQRT22M4P2O2E","json":"https://pith.science/pith/7ZKTROEYPUT3OHQRT22M4P2O2E.json","graph_json":"https://pith.science/api/pith-number/7ZKTROEYPUT3OHQRT22M4P2O2E/graph.json","events_json":"https://pith.science/api/pith-number/7ZKTROEYPUT3OHQRT22M4P2O2E/events.json","paper":"https://pith.science/paper/7ZKTROEY"},"agent_actions":{"view_html":"https://pith.science/pith/7ZKTROEYPUT3OHQRT22M4P2O2E","download_json":"https://pith.science/pith/7ZKTROEYPUT3OHQRT22M4P2O2E.json","view_paper":"https://pith.science/paper/7ZKTROEY","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2402.08268&json=true","fetch_graph":"https://pith.science/api/pith-number/7ZKTROEYPUT3OHQRT22M4P2O2E/graph.json","fetch_events":"https://pith.science/api/pith-number/7ZKTROEYPUT3OHQRT22M4P2O2E/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/7ZKTROEYPUT3OHQRT22M4P2O2E/action/timestamp_anchor","attest_storage":"https://pith.science/pith/7ZKTROEYPUT3OHQRT22M4P2O2E/action/storage_attestation","attest_author":"https://pith.science/pith/7ZKTROEYPUT3OHQRT22M4P2O2E/action/author_attestation","sign_citation":"https://pith.science/pith/7ZKTROEYPUT3OHQRT22M4P2O2E/action/citation_signature","submit_replication":"https://pith.science/pith/7ZKTROEYPUT3OHQRT22M4P2O2E/action/replication_record"}},"created_at":"2026-05-17T23:38:48.813332+00:00","updated_at":"2026-05-17T23:38:48.813332+00:00"}