{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:E4QUA3B2JHRHNVLHR6YPIGJYOI","short_pith_number":"pith:E4QUA3B2","schema_version":"1.0","canonical_sha256":"2721406c3a49e276d5678fb0f41938723f16b48d5b068d06b8bae8c1e663c950","source":{"kind":"arxiv","id":"2405.21046","version":1},"attestation_state":"computed","paper":{"title":"Exploratory Preference Optimization: Harnessing Implicit Q*-Approximation for Sample-Efficient RLHF","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL","stat.ML"],"primary_cat":"cs.LG","authors_text":"Ahmed Awadallah, Akshay Krishnamurthy, Alexander Rakhlin, Corby Rosset, Dylan J. Foster, Tengyang Xie","submitted_at":"2024-05-31T17:39:06Z","abstract_excerpt":"Reinforcement learning from human feedback (RLHF) has emerged as a central tool for language model alignment. We consider online exploration in RLHF, which exploits interactive access to human or AI feedback by deliberately encouraging the model to produce diverse, maximally informative responses. By allowing RLHF to confidently stray from the pre-trained model, online exploration offers the possibility of novel, potentially super-human capabilities, but its full potential as a paradigm for language model training has yet to be realized, owing to computational and statistical bottlenecks in di"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2405.21046","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-05-31T17:39:06Z","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"title_canon_sha256":"8f35f97726880ae0163d48be1bfce4184027da95ec710de30c496afc899402ce","abstract_canon_sha256":"3679310d79fd4241200f20e01ea48846c85d96730aa814732d327ceadc794532"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:25:45.298486Z","signature_b64":"InrYMR8EkA2bXKb4rgXbIOMIjVBKRqKANh4TewHrQwRjTv56RLLP974FIPqx4fur5V+AFKqV/1GktP5MKn//CA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2721406c3a49e276d5678fb0f41938723f16b48d5b068d06b8bae8c1e663c950","last_reissued_at":"2026-07-05T08:25:45.297973Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:25:45.297973Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Exploratory Preference Optimization: Harnessing Implicit Q*-Approximation for Sample-Efficient RLHF","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL","stat.ML"],"primary_cat":"cs.LG","authors_text":"Ahmed Awadallah, Akshay Krishnamurthy, Alexander Rakhlin, Corby Rosset, Dylan J. Foster, Tengyang Xie","submitted_at":"2024-05-31T17:39:06Z","abstract_excerpt":"Reinforcement learning from human feedback (RLHF) has emerged as a central tool for language model alignment. We consider online exploration in RLHF, which exploits interactive access to human or AI feedback by deliberately encouraging the model to produce diverse, maximally informative responses. By allowing RLHF to confidently stray from the pre-trained model, online exploration offers the possibility of novel, potentially super-human capabilities, but its full potential as a paradigm for language model training has yet to be realized, owing to computational and statistical bottlenecks in di"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.21046","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.21046/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2405.21046","created_at":"2026-07-05T08:25:45.298029+00:00"},{"alias_kind":"arxiv_version","alias_value":"2405.21046v1","created_at":"2026-07-05T08:25:45.298029+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.21046","created_at":"2026-07-05T08:25:45.298029+00:00"},{"alias_kind":"pith_short_12","alias_value":"E4QUA3B2JHRH","created_at":"2026-07-05T08:25:45.298029+00:00"},{"alias_kind":"pith_short_16","alias_value":"E4QUA3B2JHRHNVLH","created_at":"2026-07-05T08:25:45.298029+00:00"},{"alias_kind":"pith_short_8","alias_value":"E4QUA3B2","created_at":"2026-07-05T08:25:45.298029+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":12,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.22600","citing_title":"On the Position Bias of On-Policy Distillation","ref_index":40,"is_internal_anchor":false},{"citing_arxiv_id":"2606.06053","citing_title":"Online KL-Regularized Reinforcement Learning with Function Approximation under Misspecification","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2606.22600","citing_title":"On the Position Bias of On-Policy Distillation","ref_index":40,"is_internal_anchor":false},{"citing_arxiv_id":"2602.06239","citing_title":"Provably avoiding over-optimization in Direct Preference Optimization without knowing the data distribution","ref_index":33,"is_internal_anchor":false},{"citing_arxiv_id":"2509.20265","citing_title":"Failure Modes of Maximum Entropy RLHF","ref_index":54,"is_internal_anchor":false},{"citing_arxiv_id":"2509.23102","citing_title":"Multiplayer Nash Preference Optimization","ref_index":32,"is_internal_anchor":false},{"citing_arxiv_id":"2602.06239","citing_title":"Provably avoiding over-optimization in Direct Preference Optimization without knowing the data distribution","ref_index":33,"is_internal_anchor":false},{"citing_arxiv_id":"2605.09214","citing_title":"Fast Rates for Offline Contextual Bandits with Forward-KL Regularization under Single-Policy Concentrability","ref_index":42,"is_internal_anchor":false},{"citing_arxiv_id":"2605.09867","citing_title":"Continuous Latent Contexts Enable Efficient Online Learning in Transformers","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2605.01123","citing_title":"PERSA: Reinforcement Learning for Professor-Style Personalized Feedback with LLMs","ref_index":64,"is_internal_anchor":false},{"citing_arxiv_id":"2605.07331","citing_title":"Rethinking Importance Sampling in LLM Policy Optimization: A Cumulative Token Perspective","ref_index":21,"is_internal_anchor":false},{"citing_arxiv_id":"2604.04410","citing_title":"Relative Density Ratio Optimization for Stable and Statistically Consistent Model Alignment","ref_index":18,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/E4QUA3B2JHRHNVLHR6YPIGJYOI","json":"https://pith.science/pith/E4QUA3B2JHRHNVLHR6YPIGJYOI.json","graph_json":"https://pith.science/api/pith-number/E4QUA3B2JHRHNVLHR6YPIGJYOI/graph.json","events_json":"https://pith.science/api/pith-number/E4QUA3B2JHRHNVLHR6YPIGJYOI/events.json","paper":"https://pith.science/paper/E4QUA3B2"},"agent_actions":{"view_html":"https://pith.science/pith/E4QUA3B2JHRHNVLHR6YPIGJYOI","download_json":"https://pith.science/pith/E4QUA3B2JHRHNVLHR6YPIGJYOI.json","view_paper":"https://pith.science/paper/E4QUA3B2","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2405.21046&json=true","fetch_graph":"https://pith.science/api/pith-number/E4QUA3B2JHRHNVLHR6YPIGJYOI/graph.json","fetch_events":"https://pith.science/api/pith-number/E4QUA3B2JHRHNVLHR6YPIGJYOI/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/E4QUA3B2JHRHNVLHR6YPIGJYOI/action/timestamp_anchor","attest_storage":"https://pith.science/pith/E4QUA3B2JHRHNVLHR6YPIGJYOI/action/storage_attestation","attest_author":"https://pith.science/pith/E4QUA3B2JHRHNVLHR6YPIGJYOI/action/author_attestation","sign_citation":"https://pith.science/pith/E4QUA3B2JHRHNVLHR6YPIGJYOI/action/citation_signature","submit_replication":"https://pith.science/pith/E4QUA3B2JHRHNVLHR6YPIGJYOI/action/replication_record"}},"created_at":"2026-07-05T08:25:45.298029+00:00","updated_at":"2026-07-05T08:25:45.298029+00:00"}