{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:E4QUA3B2JHRHNVLHR6YPIGJYOI","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"3679310d79fd4241200f20e01ea48846c85d96730aa814732d327ceadc794532","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-05-31T17:39:06Z","title_canon_sha256":"8f35f97726880ae0163d48be1bfce4184027da95ec710de30c496afc899402ce"},"schema_version":"1.0","source":{"id":"2405.21046","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.21046","created_at":"2026-07-05T08:25:45Z"},{"alias_kind":"arxiv_version","alias_value":"2405.21046v1","created_at":"2026-07-05T08:25:45Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.21046","created_at":"2026-07-05T08:25:45Z"},{"alias_kind":"pith_short_12","alias_value":"E4QUA3B2JHRH","created_at":"2026-07-05T08:25:45Z"},{"alias_kind":"pith_short_16","alias_value":"E4QUA3B2JHRHNVLH","created_at":"2026-07-05T08:25:45Z"},{"alias_kind":"pith_short_8","alias_value":"E4QUA3B2","created_at":"2026-07-05T08:25:45Z"}],"graph_snapshots":[{"event_id":"sha256:163225f802f1990e299cec6e9bd80f20a0d47cf259bee2dba8a00e7126b9dc94","target":"graph","created_at":"2026-07-05T08:25:45Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.21046/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement learning from human feedback (RLHF) has emerged as a central tool for language model alignment. We consider online exploration in RLHF, which exploits interactive access to human or AI feedback by deliberately encouraging the model to produce diverse, maximally informative responses. By allowing RLHF to confidently stray from the pre-trained model, online exploration offers the possibility of novel, potentially super-human capabilities, but its full potential as a paradigm for language model training has yet to be realized, owing to computational and statistical bottlenecks in di","authors_text":"Ahmed Awadallah, Akshay Krishnamurthy, Alexander Rakhlin, Corby Rosset, Dylan J. Foster, Tengyang Xie","cross_cats":["cs.AI","cs.CL","stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-05-31T17:39:06Z","title":"Exploratory Preference Optimization: Harnessing Implicit Q*-Approximation for Sample-Efficient RLHF"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.21046","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:d57ad669af2341690074f8f77e9bde1e11cfedfab8a9452697b97a85bec2e3b7","target":"record","created_at":"2026-07-05T08:25:45Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"3679310d79fd4241200f20e01ea48846c85d96730aa814732d327ceadc794532","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-05-31T17:39:06Z","title_canon_sha256":"8f35f97726880ae0163d48be1bfce4184027da95ec710de30c496afc899402ce"},"schema_version":"1.0","source":{"id":"2405.21046","kind":"arxiv","version":1}},"canonical_sha256":"2721406c3a49e276d5678fb0f41938723f16b48d5b068d06b8bae8c1e663c950","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"2721406c3a49e276d5678fb0f41938723f16b48d5b068d06b8bae8c1e663c950","first_computed_at":"2026-07-05T08:25:45.297973Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:25:45.297973Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"InrYMR8EkA2bXKb4rgXbIOMIjVBKRqKANh4TewHrQwRjTv56RLLP974FIPqx4fur5V+AFKqV/1GktP5MKn//CA==","signature_status":"signed_v1","signed_at":"2026-07-05T08:25:45.298486Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.21046","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:d57ad669af2341690074f8f77e9bde1e11cfedfab8a9452697b97a85bec2e3b7","sha256:163225f802f1990e299cec6e9bd80f20a0d47cf259bee2dba8a00e7126b9dc94"],"state_sha256":"f417eb041d82d4dc43a753f176e58ac97f502588aab0c7cb4c2b95c680b7f821"}