{"as_of":"2026-08-16T01:40:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:368ca1ebd54eb41a784d7e2c79954f775bb97e5f48e182b0eee8a428b57ad827","coverage":[{"denominator":31,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":31,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-11T15:55:20.449861Z","state":"measured"},{"denominator":32,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":32,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-15T06:32:42.880941+00:00","state":"measured"},{"denominator":1,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":1,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-16T01:00:59.114689Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"pith","source_observed_at":"2026-08-16T01:00:59.301176Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"cited_work":{"arxiv_id":"2412.10616","doi":null,"metadata_source":"pith","pith_arxiv_id":"2412.10616","snapshot_observed_at":"2026-08-16T01:00:59.301176Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","venue":"cs.LG","work_id":"8b4c9074-db54-46af-b39c-6d3e5b0d6741","year":2024},"citing_paper":{"arxiv_id":"2505.02363","last_updated":"2025-05-05T04:54:44Z","snapshot_observed_at":"2026-08-16T00:51:08.375551Z","submitted_at":"2025-05-05T04:54:44Z","title":"SIMPLEMIX: Frustratingly Simple Mixing of Off- and On-policy Data in Language Model Preference Learning","version":1},"reference_index":2020,"source":"pdf_text","source_observed_at":"2026-08-16T01:00:59.114689Z"},"links":{"cited_paper":"/paper/2412.10616","citing_paper":"/paper/2505.02363"},"observation_digest":"sha256:8ee4674de5d6415880204c196a9007690d8b7f11ba9de871f71819afb352efef","observation_id":"99eb5fc0-a8a6-4d90-b54b-f04130792ae2","resolution":{"observed_at":"2026-08-16T01:00:59.305430Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2412.10616/citation-record","integrity":"/paper/2412.10616/integrity","json":"/paper/2412.10616/citation-record.json","paper":"/paper/2412.10616"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T15:55:20.778356Z","title":"Apply Azuma Hoeffding with its sample average overt + γ terms","venue":null,"work_id":"edc2e9bf-d7fa-4a7c-9a71-58695b440cde","year":2024},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.438177Z"},"links":{"citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:5e0d6f117be4952ce640fec3ba400f0a03dc3f6036033280db3b2425dbc47fd6","observation_id":"9be54856-dcf2-4041-92b5-06150ca4a498","resolution":{"observed_at":"2026-08-11T15:55:20.781548Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T15:55:20.767707Z","title":"For completeness, we first describe the preference model below: BTL-based Pairwise Preference (Dueling) Model:Consider a decision spaceD","venue":null,"work_id":"26f326b2-2be1-4725-943a-dfdc6f54b06f","year":2022},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.442345Z"},"links":{"citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:0e43b4987d5e68c956bae8e564e4b43827d8e4a9a682c29375071844f6093b6d","observation_id":"45a7588e-81aa-4a12-8963-8613679a9eff","resolution":{"observed_at":"2026-08-11T15:55:20.771727Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2404.08495","last_updated":"2024-04-16T17:36:39Z","snapshot_observed_at":"2026-08-13T15:18:03.060033Z","submitted_at":"2024-04-12T14:25:49Z","title":"Dataset Reset Policy Optimization for RLHF","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.08495","snapshot_observed_at":"2026-08-11T15:55:20.344726Z","title":"Dataset reset policy optimization for rlhf.arXiv preprint arXiv:2404.08495,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.344726Z"},"links":{"cited_paper":"/paper/2404.08495","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:c95db6b632534941a13233bf0e48838f21fc6424bc711a54f9b28ccfef9101b7","observation_id":"997853da-6905-415f-8ce0-a13a3c224fb1","resolution":{"observed_at":"2026-08-11T15:55:20.344726Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.01306","last_updated":"2024-11-19T18:12:45Z","snapshot_observed_at":"2026-08-12T21:18:02.964833Z","submitted_at":"2024-02-02T10:53:36Z","title":"KTO: Model Alignment as Prospect Theoretic Optimization","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.01306","snapshot_observed_at":"2026-08-11T15:55:20.353607Z","title":"KTO: Model alignment as prospect theoretic optimization.arXiv preprint arXiv:2402.01306,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.353607Z"},"links":{"cited_paper":"/paper/2402.01306","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:5476d3baa10f89e7fcaa441bd95f291b10eded4bd3d869bd38af18a3ed177e64","observation_id":"4952c080-e234-4766-866f-e4d7774a6aab","resolution":{"observed_at":"2026-08-11T15:55:20.353607Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.16730","last_updated":"2023-12-27T21:58:45Z","snapshot_observed_at":"2026-08-15T06:12:05.935323Z","submitted_at":"2023-12-27T21:58:45Z","title":"Foundations of Reinforcement Learning and Interactive Decision Making","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.16730","snapshot_observed_at":"2026-08-11T15:55:20.358418Z","title":"Foundations of reinforcement learning and interactive decision making","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.358418Z"},"links":{"cited_paper":"/paper/2312.16730","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:956d8240ee3a1d6015840b12cc903c257457341acb0afead00e2f9ab2e11fa34","observation_id":"3d883342-ed52-4d67-8bd6-d059d691205a","resolution":{"observed_at":"2026-08-11T15:55:20.358418Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.04792","last_updated":"2024-02-29T20:59:17Z","snapshot_observed_at":"2026-08-15T00:47:58.611457Z","submitted_at":"2024-02-07T12:31:13Z","title":"Direct Language Model Alignment from Online AI Feedback","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.04792","snapshot_observed_at":"2026-08-11T15:55:20.366546Z","title":"Direct language model alignment from online AI feedback","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.366546Z"},"links":{"cited_paper":"/paper/2402.04792","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:9b2db095ab00f2744021961bf0ee6d566a034e5d6f972741a1c7c03fb16478bb","observation_id":"d906bcd4-5162-4f85-a9e9-caef7db45874","resolution":{"observed_at":"2026-08-11T15:55:20.366546Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.14734","last_updated":"2024-11-01T20:05:19Z","snapshot_observed_at":"2026-08-15T19:24:05.829199Z","submitted_at":"2024-05-23T16:01:46Z","title":"SimPO: Simple Preference Optimization with a Reference-Free Reward","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.14734","snapshot_observed_at":"2026-08-11T15:55:20.375425Z","title":"Simpo: Simple preference optimization with a reference-free reward","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.375425Z"},"links":{"cited_paper":"/paper/2405.14734","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:8832bf8e7edc6806543bfa6d191c05bef2e5f135da5003a8f875b61b2ccb58af","observation_id":"f16a6eec-9670-4ec8-95b9-68c29bf7a740","resolution":{"observed_at":"2026-08-11T15:55:20.375425Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.00886","last_updated":"2024-06-11T16:25:52Z","snapshot_observed_at":"2026-08-13T05:12:12.105399Z","submitted_at":"2023-12-01T19:26:23Z","title":"Nash Learning from Human Feedback","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.00886","snapshot_observed_at":"2026-08-11T15:55:20.379126Z","title":"Nash learning from human feedback.arXiv preprint arXiv:2312.00886,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.379126Z"},"links":{"cited_paper":"/paper/2312.00886","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:dec04be2b15618e4f242ec6766a4bc0dd15ca555af92b0e345bd1e0f383ae98c","observation_id":"76aac503-48f1-477c-b8d0-d529221e938e","resolution":{"observed_at":"2026-08-11T15:55:20.379126Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.12358","last_updated":"2024-08-12T21:13:35Z","snapshot_observed_at":"2026-08-14T17:04:58.098057Z","submitted_at":"2024-04-18T17:37:02Z","title":"From $r$ to $Q^*$: Your Language Model is Secretly a Q-Function","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.12358","snapshot_observed_at":"2026-08-11T15:55:20.386884Z","title":"Fromr to Q⋆: Your language model is secretly a Q-function","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.386884Z"},"links":{"cited_paper":"/paper/2404.12358","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:7533624776bd68f0025df129f2b89f90755ef6a8d26f2eb8730e33db475515b9","observation_id":"6587a5a8-a90c-488f-9a01-024d1a61ee8c","resolution":{"observed_at":"2026-08-11T15:55:20.386884Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.03715","last_updated":"2024-04-04T17:56:41Z","snapshot_observed_at":"2026-08-14T12:56:25.698035Z","submitted_at":"2024-04-04T17:56:41Z","title":"Direct Nash Optimization: Teaching Language Models to Self-Improve with General Preferences","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.03715","snapshot_observed_at":"2026-08-11T15:55:20.390981Z","title":"Direct Nash Optimization: Teaching language models to self-improve with general preferences.arXiv preprint arXiv:2404.03715,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.390981Z"},"links":{"cited_paper":"/paper/2404.03715","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:b3baa204f2690383b4cffdf4ab9d90027274a1b7e110944db2e574e06b7aaf72","observation_id":"063a77eb-5e87-4049-8d7b-eb8c0558ba4b","resolution":{"observed_at":"2026-08-11T15:55:20.390981Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.01462","last_updated":"2024-07-16T16:51:38Z","snapshot_observed_at":"2026-08-15T14:28:07.822547Z","submitted_at":"2024-06-03T15:51:04Z","title":"The Importance of Online Data: Understanding Preference Fine-tuning via Coverage","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.01462","snapshot_observed_at":"2026-08-11T15:55:20.398995Z","title":"Understanding preference fine-tuning through the lens of coverage.arXiv preprint arXiv:2406.01462,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.398995Z"},"links":{"cited_paper":"/paper/2406.01462","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:bfecb7c48c960c951efddf702fe45ba4d579f6f9cb037652f5675d7a368c14f9","observation_id":"0d8f4596-48c9-487e-98d3-a8d74250d44d","resolution":{"observed_at":"2026-08-11T15:55:20.398995Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.04526","last_updated":"2024-08-08T15:26:18Z","snapshot_observed_at":"2026-08-12T23:06:30.761577Z","submitted_at":"2024-08-08T15:26:18Z","title":"Hybrid Reinforcement Learning Breaks Sample Size Barriers in Linear MDPs","version":1},"cited_work":{"arxiv_id":"2408.04526","doi":null,"metadata_source":"pith","pith_arxiv_id":"2408.04526","snapshot_observed_at":"2026-08-11T15:55:20.561430Z","title":"Hybrid Reinforcement Learning Breaks Sample Size Barriers in Linear MDPs","venue":"stat.ML","work_id":"af6a0fdf-f63e-467f-96c1-6896d8c2eaa7","year":2024},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.402857Z"},"links":{"cited_paper":"/paper/2408.04526","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:f5f2092c4fbbbd537cedd0e4a9b9f87a8ee2724a4c666014712fb254d5bf7be9","observation_id":"8e75d090-37fc-4dae-b31b-168ccefad6f8","resolution":{"observed_at":"2026-08-11T15:55:20.567068Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2306.14111","last_updated":"2023-11-03T18:36:03Z","snapshot_observed_at":"2026-08-13T11:09:58.299528Z","submitted_at":"2023-06-25T03:18:15Z","title":"Is RLHF More Difficult than Standard RL?","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.14111","snapshot_observed_at":"2026-08-11T15:55:20.406641Z","title":"Is RLHF more difficult than standard RL?arXiv preprint arXiv:2306.14111,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.406641Z"},"links":{"cited_paper":"/paper/2306.14111","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:3d5ddf8fc6a7caf72125730243c41f34d07b250c134b396dafbb9224cb1ea908","observation_id":"f3d36ecf-ffc9-459b-9912-f7762ac9c58b","resolution":{"observed_at":"2026-08-11T15:55:20.406641Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.14554","last_updated":"2024-03-12T21:49:59Z","snapshot_observed_at":"2026-08-13T05:43:30.514751Z","submitted_at":"2023-10-23T04:19:35Z","title":"Making RL with Preference-based Feedback Efficient via Randomization","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.14554","snapshot_observed_at":"2026-08-11T15:55:20.409826Z","title":"Making RL with preference-based feedback efficient via randomization.arXiv preprint arXiv:2310.14554,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.409826Z"},"links":{"cited_paper":"/paper/2310.14554","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:7fa3b68ee2cf6d91a72e62f9e9bc6064866fe2276a9b776b4da87d051d9ff6ac","observation_id":"ad795cf1-619e-4876-ba73-0065c3d40990","resolution":{"observed_at":"2026-08-11T15:55:20.409826Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.21046","last_updated":"2024-05-31T17:39:06Z","snapshot_observed_at":"2026-08-12T23:52:56.673075Z","submitted_at":"2024-05-31T17:39:06Z","title":"Exploratory Preference Optimization: Harnessing Implicit Q*-Approximation for Sample-Efficient RLHF","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.21046","snapshot_observed_at":"2026-08-11T15:55:20.412841Z","title":"Exploratory preference optimization: Harnessing implicit q*-approximation for sample-efficient rlhf","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.412841Z"},"links":{"cited_paper":"/paper/2405.21046","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:6350f636b0f121e464dc4761036d25129abb05a64b3b9b7101af016d186e9c89","observation_id":"99ac4f1f-8235-43d8-9c22-2d2365db660b","resolution":{"observed_at":"2026-08-11T15:55:20.412841Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.16682","last_updated":"2024-04-22T22:51:32Z","snapshot_observed_at":"2026-08-14T14:26:41.039383Z","submitted_at":"2023-12-27T18:53:09Z","title":"Some things are more CRINGE than others: Iterative Preference Optimization with the Pairwise Cringe Loss","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.16682","snapshot_observed_at":"2026-08-11T15:55:20.415875Z","title":"Some things are more cringe than others: Preference optimization with the pairwise cringe loss.arXiv preprint arXiv:2312.16682,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.415875Z"},"links":{"cited_paper":"/paper/2312.16682","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:9fdebc335e7cae4041c0cadeea88d66615d01f0e49fe048d18272bc0c13f0c79","observation_id":"99af4eb9-24cf-440a-b6ac-f0f5bb14a70d","resolution":{"observed_at":"2026-08-11T15:55:20.415875Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.07314","last_updated":"2024-11-12T08:24:10Z","snapshot_observed_at":"2026-08-13T04:21:00.187589Z","submitted_at":"2024-02-11T21:44:21Z","title":"Online Iterative Reinforcement Learning from Human Feedback with General Preference Model","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.07314","snapshot_observed_at":"2026-08-11T15:55:20.419053Z","title":"A theoretical analysis of Nash learning from human feedback under general KL-regularized preference.arXiv preprint arXiv:2402.07314,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.419053Z"},"links":{"cited_paper":"/paper/2402.07314","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:5b8d9ff8e7c46519f37ab4187bbee0e63da1425fd0ada174de1a3558b06c8588","observation_id":"f56f94a3-d0d3-4a03-a3a8-816047500123","resolution":{"observed_at":"2026-08-11T15:55:20.419053Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.10425","last_updated":"2023-05-17T17:57:10Z","snapshot_observed_at":"2026-08-14T18:51:06.334767Z","submitted_at":"2023-05-17T17:57:10Z","title":"SLiC-HF: Sequence Likelihood Calibration with Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.10425","snapshot_observed_at":"2026-08-11T15:55:20.422799Z","title":"Slic-hf: Sequence likelihood calibration with human feedback.arXiv preprint arXiv:2305.10425,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.422799Z"},"links":{"cited_paper":"/paper/2305.10425","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:72c556a2615623b57b754392f3dc12809f32fa3d418f7dccd27389890466ce4d","observation_id":"162e90e5-149f-4d8d-aa28-12054b696c37","resolution":{"observed_at":"2026-08-11T15:55:20.422799Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.01962","last_updated":"2023-06-30T13:05:42Z","snapshot_observed_at":"2026-08-14T21:12:03.690652Z","submitted_at":"2022-11-03T16:42:40Z","title":"GEC: A Unified Framework for Interactive Decision Making in MDP, POMDP, and Beyond","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.01962","snapshot_observed_at":"2026-08-11T15:55:20.426332Z","title":"GEC: A unified framework for interactive decision making in MDP, POMDP, and beyond.arXiv preprint arXiv:2211.01962,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.426332Z"},"links":{"cited_paper":"/paper/2211.01962","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:200c725892312bc240d74be6ef2c9a5f39f9417b29b4a9da393516a059d03931","observation_id":"14a9f5f7-209c-4749-896e-6db03eacc56c","resolution":{"observed_at":"2026-08-11T15:55:20.426332Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.16335","last_updated":"2024-01-29T17:43:42Z","snapshot_observed_at":"2026-08-13T04:32:29.674931Z","submitted_at":"2024-01-29T17:43:42Z","title":"Iterative Data Smoothing: Mitigating Reward Overfitting and Overoptimization in RLHF","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.16335","snapshot_observed_at":"2026-08-11T15:55:20.430135Z","title":"Iterative data smoothing: Mitigating reward overfitting and overoptimization in rlhf.arXiv preprint arXiv:2401.16335,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.430135Z"},"links":{"cited_paper":"/paper/2401.16335","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:66122d960e97d92a3ec15dde93c035094c3d19769d2f53190109a56fe41154d5","observation_id":"88e81838-4d92-4525-9c47-3872bc995401","resolution":{"observed_at":"2026-08-11T15:55:20.430135Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T15:55:20.787903Z","title":"However due to the hybrid nature of our algorithm, we are able to derive concentration lemmas with a faster convergence rate","venue":null,"work_id":"e753212b-2678-4d6f-abd2-5877e3ecc320","year":2024},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.434238Z"},"links":{"citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:f7fe3c8c0fca4efc4ae78a1147f40bbb8d9479e7097578b5496fb2f59af21655","observation_id":"32e87d64-117b-4fec-88c9-6ca08592d1df","resolution":{"observed_at":"2026-08-11T15:55:20.790917Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T15:55:20.757032Z","title":"We will use the abbreviation linB for this feedback model","venue":null,"work_id":"b17457b9-fb3b-4136-985a-824fee00e031","year":2021},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.446202Z"},"links":{"citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:6498e01bef9f1428db4d76bcadabd01532c4b203385cca5246ad9a5c3c4653e4","observation_id":"5c1c55aa-9177-48aa-8dbd-32857511b00c","resolution":{"observed_at":"2026-08-11T15:55:20.760787Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T15:55:20.745836Z","title":"Interested readers are encouraged to go over the proof of Lemma 9 of Saha (2021) to see the proof of lemma 4 above","venue":null,"work_id":"c30cdbc0-34f5-44ee-8fa3-158439e4fada","year":2021},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.449861Z"},"links":{"citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:1568e319d9b444a0c575176ca83cb1f1fda2b4298176cb0185cfb89f05b6f3f9","observation_id":"845c7a81-4884-4e38-99ce-ab8bb0da3083","resolution":{"observed_at":"2026-08-11T15:55:20.749858Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2405.19320","last_updated":"2025-02-19T00:51:58Z","snapshot_observed_at":"2026-08-12T23:54:55.850326Z","submitted_at":"2024-05-29T17:51:42Z","title":"Value-Incentivized Preference Optimization: A Unified Approach to Online and Offline RLHF","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.19320","snapshot_observed_at":"2026-08-11T15:55:20.340571Z","title":"Value-incentivized preference optimization: A unified approach to online and offline rlhf","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":1952,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.340571Z"},"links":{"cited_paper":"/paper/2405.19320","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:08e9c6e1b6b0bfbb257b25da6184fa1bcac47199b2f7f9d759810e1db0cb2882","observation_id":"5dd43ec3-19c7-4bde-8508-5212ae2fb123","resolution":{"observed_at":"2026-08-11T15:55:20.340571Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.01857","last_updated":"2024-06-05T09:00:36Z","snapshot_observed_at":"2026-08-13T04:04:15.912440Z","submitted_at":"2024-03-04T09:13:14Z","title":"Reward Model Learning vs. Direct Policy Optimization: A Comparative Analysis of Learning from Human Preferences","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.01857","snapshot_observed_at":"2026-08-11T15:55:20.382821Z","title":"Reward model learning vs","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":2012,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.382821Z"},"links":{"cited_paper":"/paper/2403.01857","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:0e6c59c5662fb293cd6d95af819a72ab524a6a041baa9ad744e2cb804dbafaa1","observation_id":"b6545e01-b8f3-42fa-a2d0-b9a8683d8c68","resolution":{"observed_at":"2026-08-11T15:55:20.382821Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.10342","last_updated":"2024-07-15T04:19:50Z","snapshot_observed_at":"2026-08-13T21:34:19.949417Z","submitted_at":"2024-02-15T22:11:18Z","title":"Exploration-Driven Policy Optimization in RLHF: Theoretical Insights on Efficient Data Utilization","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.10342","snapshot_observed_at":"2026-08-11T15:55:20.349314Z","title":"Exploration-driven policy optimization in RLHF: Theoretical insights on efficient data utilization.arXiv preprint arXiv:2402.10342,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":2017,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.349314Z"},"links":{"cited_paper":"/paper/2402.10342","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:0b2dd2019856e9e971313f88c65f938a2f5edebe3d657d6dd8a957b3da4a6187","observation_id":"c6ffbf91-c158-4f97-8d6d-24765c3f4be9","resolution":{"observed_at":"2026-08-11T15:55:20.349314Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.09681","last_updated":"2024-06-04T21:19:10Z","snapshot_observed_at":"2026-08-13T04:40:13.072528Z","submitted_at":"2024-01-18T02:21:06Z","title":"Harnessing Density Ratios for Online Reinforcement Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.09681","snapshot_observed_at":"2026-08-11T15:55:20.333579Z","title":"Harnessing density ratios for online reinforcement learning.arXiv preprint arXiv:2401.09681,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":2019,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.333579Z"},"links":{"cited_paper":"/paper/2401.09681","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:127ae95e39dd3dbf4d36b45c11c5d378acc3a29ebcb5244bd02420eac08050df","observation_id":"5ef03d1e-af95-4195-b97e-9944ce9e8842","resolution":{"observed_at":"2026-08-11T15:55:20.333579Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2210.06718","last_updated":"2023-03-11T11:47:54Z","snapshot_observed_at":"2026-08-14T16:00:23.429878Z","submitted_at":"2022-10-13T04:19:05Z","title":"Hybrid RL: Using Both Offline and Online Data Can Make RL Efficient","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2210.06718","snapshot_observed_at":"2026-08-11T15:55:20.394661Z","title":"Hybrid rl: Using both offline and online data can make rl efficient.arXiv preprint arXiv:2210.06718,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":2021,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.394661Z"},"links":{"cited_paper":"/paper/2210.06718","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:a7bb720a0b468b65d2b03f6aed8c8e96e45f3f119856f479cf14fbe921bf57fc","observation_id":"c47d63c8-2b0c-490d-9c4d-c6be1bf33edb","resolution":{"observed_at":"2026-08-11T15:55:20.394661Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.06657","last_updated":"2024-01-23T23:16:11Z","snapshot_observed_at":"2026-08-14T22:06:49.895133Z","submitted_at":"2023-09-13T01:07:25Z","title":"Statistical Rejection Sampling Improves Preference Optimization","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.06657","snapshot_observed_at":"2026-08-11T15:55:20.371083Z","title":"Statistical rejection sampling improves preference optimization.arXiv preprint arXiv:2309.06657,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":2022,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.371083Z"},"links":{"cited_paper":"/paper/2309.06657","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:c04916053adad31a0c8bc801a3fd1eefb2cf02084abd1517d0ea11f12110474b","observation_id":"648433e8-5327-4257-bc40-6a092fb608d9","resolution":{"observed_at":"2026-08-11T15:55:20.371083Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.16767","last_updated":"2024-12-10T03:17:30Z","snapshot_observed_at":"2026-08-13T00:22:06.602100Z","submitted_at":"2024-04-25T17:20:45Z","title":"REBEL: Reinforcement Learning via Regressing Relative Rewards","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.16767","snapshot_observed_at":"2026-08-11T15:55:20.362579Z","title":"REBEL: Reinforcement learning via regressing relative rewards.arXiv preprint arXiv:2404.16767,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":2023,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.362579Z"},"links":{"cited_paper":"/paper/2404.16767","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:a09ba4da06ae1501f53b0bb9b3dc391a9c7935efde924f19e66ffc83ac2f1595","observation_id":"557f6430-e525-4608-ac5b-88bd1e4649fd","resolution":{"observed_at":"2026-08-11T15:55:20.362579Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2204.05862","last_updated":"2022-04-12T15:02:38Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-04-12T15:02:38Z","title":"Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2204.05862","snapshot_observed_at":"2026-08-11T15:55:20.337568Z","title":"Training a helpful and harmless assistant with reinforcement learning from human feedback.arXiv preprint arXiv:2204.05862,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration","version":1},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-11T15:55:20.337568Z"},"links":{"cited_paper":"/paper/2204.05862","citing_paper":"/paper/2412.10616"},"observation_digest":"sha256:c8cb8021ad4ba7ee4b38b0b309402d5126027ecbcc6d3084cbaa587bcbab49e9","observation_id":"b4eef0a0-1051-4805-ad27-5df50376f04b","resolution":{"observed_at":"2026-08-11T15:55:20.337568Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2412.10616","last_updated":"2024-12-13T23:42:24Z","latest_version":1,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-15T01:36:16.632123Z","submitted_at":"2024-12-13T23:42:24Z","title":"Hybrid Preference Optimization for Alignment: Provably Faster Convergence Rates by Combining Offline Preferences with Online Exploration"},"reference_resolution":{"displayed":31,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":25,"verified_exact":1,"verified_fuzzy":5},"total_outbound_references":31},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"thesis":"As of 16 August 2026, this Paper Citation Record lists 31 of 31 outbound references and 1 inbound Pith citation observation for arXiv:2412.10616."}