{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:ZHH5XUCE4TARUOMMQP65NIXVLE","short_pith_number":"pith:ZHH5XUCE","canonical_record":{"source":{"id":"2406.15567","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-06-21T18:05:35Z","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"title_canon_sha256":"d0812956ec3f51becf3298521d116f0fd53d61ed7c552cb9eb331aad0843a0d5","abstract_canon_sha256":"232ce36b592d72dc3bc999bc971367b55465879dcda78f42c03bdbc29bead5f7"},"schema_version":"1.0"},"canonical_sha256":"c9cfdbd044e4c11a398c83fdd6a2f55931a86bdbdb37e694b49585f57c7371e7","source":{"kind":"arxiv","id":"2406.15567","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.15567","created_at":"2026-07-05T08:35:24Z"},{"alias_kind":"arxiv_version","alias_value":"2406.15567v1","created_at":"2026-07-05T08:35:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.15567","created_at":"2026-07-05T08:35:24Z"},{"alias_kind":"pith_short_12","alias_value":"ZHH5XUCE4TAR","created_at":"2026-07-05T08:35:24Z"},{"alias_kind":"pith_short_16","alias_value":"ZHH5XUCE4TARUOMM","created_at":"2026-07-05T08:35:24Z"},{"alias_kind":"pith_short_8","alias_value":"ZHH5XUCE","created_at":"2026-07-05T08:35:24Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:ZHH5XUCE4TARUOMMQP65NIXVLE","target":"record","payload":{"canonical_record":{"source":{"id":"2406.15567","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-06-21T18:05:35Z","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"title_canon_sha256":"d0812956ec3f51becf3298521d116f0fd53d61ed7c552cb9eb331aad0843a0d5","abstract_canon_sha256":"232ce36b592d72dc3bc999bc971367b55465879dcda78f42c03bdbc29bead5f7"},"schema_version":"1.0"},"canonical_sha256":"c9cfdbd044e4c11a398c83fdd6a2f55931a86bdbdb37e694b49585f57c7371e7","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:35:24.558610Z","signature_b64":"b3xNfxYDmE/PAwA9KOHyOufdPjQNszy0CGXlZqSjhnrJimiBo9JoQqrZntSwtXQfYscRUwCyFLD2xLhQbjGbDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c9cfdbd044e4c11a398c83fdd6a2f55931a86bdbdb37e694b49585f57c7371e7","last_reissued_at":"2026-07-05T08:35:24.558164Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:35:24.558164Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.15567","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:35:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"c+j+zbLLsld3lRygeQTv+QOusbVQkUC2040gz+UZ9+VaWC4KatDILejmnfkv4opUQEizAtQfmnnViWhV9KgnCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-16T05:03:51.260544Z"},"content_sha256":"15a3cfb220f38d9e3f12b06eec68e573d98fc3565099e0a19118974609bd996c","schema_version":"1.0","event_id":"sha256:15a3cfb220f38d9e3f12b06eec68e573d98fc3565099e0a19118974609bd996c"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:ZHH5XUCE4TARUOMMQP65NIXVLE","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"SAIL: Self-Improving Efficient Online Alignment of Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL","stat.ML"],"primary_cat":"cs.LG","authors_text":"Alec Koppel, Amrit Bedi, Furong Huang, Mengdi Wang, Mucong Ding, Souradip Chakraborty, Vibhu Agrawal, Zora Che","submitted_at":"2024-06-21T18:05:35Z","abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) is a key method for aligning large language models (LLMs) with human preferences. However, current offline alignment approaches like DPO, IPO, and SLiC rely heavily on fixed preference datasets, which can lead to sub-optimal performance. On the other hand, recent literature has focused on designing online RLHF methods but still lacks a unified conceptual formulation and suffers from distribution shift issues. To address this, we establish that online LLM alignment is underpinned by bilevel optimization. By reducing this formulation to an effici"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.15567","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.15567/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:35:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jPkEkOvc84XSPh66t2ETTkmuM8NubCBI9iZYHtRV0Ej88JnkXQNehXHSfRfZCva8nGaBEcJVcNAPAjDPe4T7Ag==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-16T05:03:51.261046Z"},"content_sha256":"f3825771e188b08c8d9263cd44a4bd766aa77ed373436130ccff0bd91a63446a","schema_version":"1.0","event_id":"sha256:f3825771e188b08c8d9263cd44a4bd766aa77ed373436130ccff0bd91a63446a"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ZHH5XUCE4TARUOMMQP65NIXVLE/bundle.json","state_url":"https://pith.science/pith/ZHH5XUCE4TARUOMMQP65NIXVLE/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ZHH5XUCE4TARUOMMQP65NIXVLE/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-16T05:03:51Z","links":{"resolver":"https://pith.science/pith/ZHH5XUCE4TARUOMMQP65NIXVLE","bundle":"https://pith.science/pith/ZHH5XUCE4TARUOMMQP65NIXVLE/bundle.json","state":"https://pith.science/pith/ZHH5XUCE4TARUOMMQP65NIXVLE/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ZHH5XUCE4TARUOMMQP65NIXVLE/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:ZHH5XUCE4TARUOMMQP65NIXVLE","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"232ce36b592d72dc3bc999bc971367b55465879dcda78f42c03bdbc29bead5f7","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-06-21T18:05:35Z","title_canon_sha256":"d0812956ec3f51becf3298521d116f0fd53d61ed7c552cb9eb331aad0843a0d5"},"schema_version":"1.0","source":{"id":"2406.15567","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.15567","created_at":"2026-07-05T08:35:24Z"},{"alias_kind":"arxiv_version","alias_value":"2406.15567v1","created_at":"2026-07-05T08:35:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.15567","created_at":"2026-07-05T08:35:24Z"},{"alias_kind":"pith_short_12","alias_value":"ZHH5XUCE4TAR","created_at":"2026-07-05T08:35:24Z"},{"alias_kind":"pith_short_16","alias_value":"ZHH5XUCE4TARUOMM","created_at":"2026-07-05T08:35:24Z"},{"alias_kind":"pith_short_8","alias_value":"ZHH5XUCE","created_at":"2026-07-05T08:35:24Z"}],"graph_snapshots":[{"event_id":"sha256:f3825771e188b08c8d9263cd44a4bd766aa77ed373436130ccff0bd91a63446a","target":"graph","created_at":"2026-07-05T08:35:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.15567/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement Learning from Human Feedback (RLHF) is a key method for aligning large language models (LLMs) with human preferences. However, current offline alignment approaches like DPO, IPO, and SLiC rely heavily on fixed preference datasets, which can lead to sub-optimal performance. On the other hand, recent literature has focused on designing online RLHF methods but still lacks a unified conceptual formulation and suffers from distribution shift issues. To address this, we establish that online LLM alignment is underpinned by bilevel optimization. By reducing this formulation to an effici","authors_text":"Alec Koppel, Amrit Bedi, Furong Huang, Mengdi Wang, Mucong Ding, Souradip Chakraborty, Vibhu Agrawal, Zora Che","cross_cats":["cs.AI","cs.CL","stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-06-21T18:05:35Z","title":"SAIL: Self-Improving Efficient Online Alignment of Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.15567","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:15a3cfb220f38d9e3f12b06eec68e573d98fc3565099e0a19118974609bd996c","target":"record","created_at":"2026-07-05T08:35:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"232ce36b592d72dc3bc999bc971367b55465879dcda78f42c03bdbc29bead5f7","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-06-21T18:05:35Z","title_canon_sha256":"d0812956ec3f51becf3298521d116f0fd53d61ed7c552cb9eb331aad0843a0d5"},"schema_version":"1.0","source":{"id":"2406.15567","kind":"arxiv","version":1}},"canonical_sha256":"c9cfdbd044e4c11a398c83fdd6a2f55931a86bdbdb37e694b49585f57c7371e7","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c9cfdbd044e4c11a398c83fdd6a2f55931a86bdbdb37e694b49585f57c7371e7","first_computed_at":"2026-07-05T08:35:24.558164Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:35:24.558164Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"b3xNfxYDmE/PAwA9KOHyOufdPjQNszy0CGXlZqSjhnrJimiBo9JoQqrZntSwtXQfYscRUwCyFLD2xLhQbjGbDw==","signature_status":"signed_v1","signed_at":"2026-07-05T08:35:24.558610Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.15567","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:15a3cfb220f38d9e3f12b06eec68e573d98fc3565099e0a19118974609bd996c","sha256:f3825771e188b08c8d9263cd44a4bd766aa77ed373436130ccff0bd91a63446a"],"state_sha256":"49d9a647af374a82983d4545e4e0c5b71bb5a4ef402c652d721fda8ffa3da052"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"B+VjokWIKf+mP13sNFRy1HPDCqCa6s++6d97evOUPtm/9XaVNu6K5THvA/LPhGer3Y6doGvC1G/TIJlQaFAdAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-16T05:03:51.265841Z","bundle_sha256":"f65e2b9dcd6aa3384b3da890aef3e7da9208f782cd0699956969baef9f925ff0"}}