{"as_of":"2026-08-17T22:28:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:5829c560606d08a950b85abab2fb8c17dfa0e8116bd1d33b8892564444a13ded","coverage":[{"denominator":27,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":27,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-15T23:58:39.026249Z","state":"measured"},{"denominator":27,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":27,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-17T06:30:58.91139+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2505.06273/citation-record","integrity":"/paper/2505.06273/integrity","json":"/paper/2505.06273/citation-record.json","paper":"/paper/2505.06273"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-17T09:58:46.058102Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-08-15T23:58:38.547058Z","title":"L., Almeida, D., Altenschmidt, J., Altman, S., Anadkat, S., et al","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.547058Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:da8ad018f577cc11cd88f79c0a05a10ccf8e213a4a15b0b4f274ce0807b3da57","observation_id":"df2b8aca-e28b-480d-8132-4f4bb2f184ab","resolution":{"observed_at":"2026-08-15T23:58:38.547058Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2206.02231","last_updated":"2023-09-06T21:13:28Z","snapshot_observed_at":"2026-08-16T17:35:08.472912Z","submitted_at":"2022-06-05T17:58:02Z","title":"Models of human preference for learning reward functions","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2206.02231","snapshot_observed_at":"2026-08-15T23:58:38.566308Z","title":"B., Hatgis-Kessell, S., Booth, S., Niekum, S., Stone, P., and Allievi, A","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.566308Z"},"links":{"cited_paper":"/paper/2206.02231","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:f8a85d859e701a67c5825379dfa1ab95b890aeb317954bb3785529c105423a2e","observation_id":"b019230c-ff8d-4849-aafb-1951c269198e","resolution":{"observed_at":"2026-08-15T23:58:38.566308Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2110.06169","last_updated":"2021-10-12T17:05:05Z","snapshot_observed_at":"2026-08-13T07:57:56.087944Z","submitted_at":"2021-10-12T17:05:05Z","title":"Offline Reinforcement Learning with Implicit Q-Learning","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2110.06169","snapshot_observed_at":"2026-08-15T23:58:38.571583Z","title":"Offline reinforce- ment learning with implicit q-learning.arXiv preprint arXiv:2110.06169,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.571583Z"},"links":{"cited_paper":"/paper/2110.06169","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:de3e0462d07f102061eab633ce45c22b1ecd39185a2efa0928b43dffd096d0f4","observation_id":"7c08a05f-86c2-4223-a1cd-c779f152a9da","resolution":{"observed_at":"2026-08-15T23:58:38.571583Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.16436","last_updated":"2024-12-04T08:15:35Z","snapshot_observed_at":"2026-08-16T17:37:51.594191Z","submitted_at":"2024-05-26T05:38:50Z","title":"Provably Mitigating Overoptimization in RLHF: Your SFT Loss is Implicitly an Adversarial Regularizer","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.16436","snapshot_observed_at":"2026-08-15T23:58:38.580544Z","title":"Provably mitigating overopti- mization in rlhf: Your sft loss is implicitly an adversarial regularizer.arXiv preprint arXiv:2405.16436,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.580544Z"},"links":{"cited_paper":"/paper/2405.16436","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:5749a16055f362a0025e03aa02f1a110cc39da06b26ddfa182a26325605cb833","observation_id":"9e5458fd-c9ec-4ed2-9613-0ef9f29fe47b","resolution":{"observed_at":"2026-08-15T23:58:38.580544Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.14734","last_updated":"2024-11-01T20:05:19Z","snapshot_observed_at":"2026-08-16T13:50:07.192537Z","submitted_at":"2024-05-23T16:01:46Z","title":"SimPO: Simple Preference Optimization with a Reference-Free Reward","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.14734","snapshot_observed_at":"2026-08-15T23:58:38.584826Z","title":"Simpo: Simple preference optimization with a reference-free reward.arXiv preprint arXiv:2405.14734,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.584826Z"},"links":{"cited_paper":"/paper/2405.14734","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:21fe0279af234c161418e58b959224e8c88b9512afb08aff00861a7d7d2163ba","observation_id":"e416e5e9-a6b0-4958-b35c-d82664df16a0","resolution":{"observed_at":"2026-08-15T23:58:38.584826Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.18252","last_updated":"2025-04-26T08:33:32Z","snapshot_observed_at":"2026-08-16T14:31:19.921863Z","submitted_at":"2024-10-23T19:59:50Z","title":"Asynchronous RLHF: Faster and More Efficient Off-Policy RL for Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.18252","snapshot_observed_at":"2026-08-15T23:58:38.653997Z","title":"Asynchronous rlhf: Faster and more efficient off-policy rl for language models","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.653997Z"},"links":{"cited_paper":"/paper/2410.18252","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:da7650dfb0342a70c2be0a29fdcbdcc41f24f0b36f1fedc79e159e0f62ef6ca1","observation_id":"1f08dc77-e740-409b-9573-806f67b5233f","resolution":{"observed_at":"2026-08-15T23:58:38.653997Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2203.10050","last_updated":"2022-03-18T16:50:38Z","snapshot_observed_at":"2026-08-16T17:13:26.987016Z","submitted_at":"2022-03-18T16:50:38Z","title":"SURF: Semi-supervised Reward Learning with Data Augmentation for Feedback-efficient Preference-based Reinforcement Learning","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2203.10050","snapshot_observed_at":"2026-08-15T23:58:38.755725Z","title":"Surf: Semi-supervised reward learning with data augmen- tation for feedback-efficient preference-based reinforce- ment learning.arXiv preprint arXiv:2203.10050,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.755725Z"},"links":{"cited_paper":"/paper/2203.10050","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:2cfc27b9ecb0c9b77651b40aded56bb020dc0b0619b83786e971810892f3726d","observation_id":"e4bbaf28-01cd-47f8-ac80-a5bb00d5b50c","resolution":{"observed_at":"2026-08-15T23:58:38.755725Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1707.06347","last_updated":"2017-08-28T09:20:06Z","snapshot_observed_at":"2026-08-15T20:26:32.102285Z","submitted_at":"2017-07-20T02:32:33Z","title":"Proximal Policy Optimization Algorithms","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1707.06347","snapshot_observed_at":"2026-08-15T23:58:38.831743Z","title":"Proximal policy optimization algorithms","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.831743Z"},"links":{"cited_paper":"/paper/1707.06347","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:23b65f2d3fd4cfaba74a36be2d29c89b738c34c33f29d34583a279fefeb09913","observation_id":"1d151406-6d1b-4081-9e4b-4b1e04ecd226","resolution":{"observed_at":"2026-08-15T23:58:38.831743Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.00888","last_updated":"2025-04-18T19:45:34Z","snapshot_observed_at":"2026-08-16T17:37:54.172951Z","submitted_at":"2024-06-02T23:13:56Z","title":"Aligning Language Models with Demonstrated Feedback","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.00888","snapshot_observed_at":"2026-08-15T23:58:38.836609Z","title":"Show, don’t tell: Aligning language models with demonstrated feedback.arXiv preprint arXiv:2406.00888,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.836609Z"},"links":{"cited_paper":"/paper/2406.00888","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:b924a99fd6c4ff84cf6cb593ab7b446289c84be19a9cbc3ef7c80fb53a487f8c","observation_id":"fba3397f-af44-4305-abb2-056750b0d8d5","resolution":{"observed_at":"2026-08-15T23:58:38.836609Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2409.05622","last_updated":"2024-12-16T02:32:15Z","snapshot_observed_at":"2026-08-16T17:38:23.445134Z","submitted_at":"2024-09-09T13:56:03Z","title":"Forward KL Regularized Preference Optimization for Aligning Diffusion Policies","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.05622","snapshot_observed_at":"2026-08-15T23:58:38.843175Z","title":"Forward kl regularized preference optimization for aligning diffusion policies.arXiv preprint arXiv:2409.05622,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.843175Z"},"links":{"cited_paper":"/paper/2409.05622","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:fbd6ed718461a3f15ed99f07eb7404319cc8190108325455dc9410ac8f4b66b6","observation_id":"435de69b-165f-468f-839a-a7fb790939cf","resolution":{"observed_at":"2026-08-15T23:58:38.843175Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.16240","last_updated":"2023-09-28T08:29:44Z","snapshot_observed_at":"2026-08-16T17:38:24.418331Z","submitted_at":"2023-09-28T08:29:44Z","title":"Beyond Reverse KL: Generalizing Direct Preference Optimization with Diverse Divergence Constraints","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.16240","snapshot_observed_at":"2026-08-15T23:58:38.850516Z","title":"Be- yond reverse kl: Generalizing direct preference optimiza- tion with diverse divergence constraints.arXiv preprint arXiv:2309.16240,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.850516Z"},"links":{"cited_paper":"/paper/2309.16240","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:b7c4e89d815d3050ada36b2f682ed72df6c36c9b1b158b4604b1beacc0837913","observation_id":"ce3367d6-9549-4bf5-bd50-2407d0d19eb9","resolution":{"observed_at":"2026-08-15T23:58:38.850516Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.14516","last_updated":"2024-12-19T04:31:56Z","snapshot_observed_at":"2026-08-15T23:47:47.627897Z","submitted_at":"2024-12-19T04:31:56Z","title":"Cal-DPO: Calibrated Direct Preference Optimization for Language Model Alignment","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.14516","snapshot_observed_at":"2026-08-15T23:58:38.854695Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.854695Z"},"links":{"cited_paper":"/paper/2412.14516","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:b9a4efc494e4a2f31bf222c484f035634c248e34fda521105c593a0b57bb56aa","observation_id":"b2ecad65-726b-42cc-a8d2-a16489530897","resolution":{"observed_at":"2026-08-15T23:58:38.854695Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2210.13435","last_updated":"2022-10-24T17:49:56Z","snapshot_observed_at":"2026-08-16T17:39:00.376398Z","submitted_at":"2022-10-24T17:49:56Z","title":"Dichotomy of Control: Separating What You Can Control from What You Cannot","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2210.13435","snapshot_observed_at":"2026-08-15T23:58:38.858135Z","title":"Dichotomy of control: Separating what you can control from what you cannot.arXiv preprint arXiv:2210.13435,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.858135Z"},"links":{"cited_paper":"/paper/2210.13435","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:17d6d882952cd61cbcbd8db717f033225b8d6e70df792d19578f9bf32cd14ff6","observation_id":"d5af1267-c833-4a0b-b1f6-64ac3b030c33","resolution":{"observed_at":"2026-08-15T23:58:38.858135Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.11999","last_updated":"2024-08-30T03:39:57Z","snapshot_observed_at":"2026-08-16T17:34:11.389619Z","submitted_at":"2024-04-18T08:49:38Z","title":"Token-level Direct Preference Optimization","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.11999","snapshot_observed_at":"2026-08-15T23:58:38.862316Z","title":"Token-level direct preference optimization.arXiv preprint arXiv:2404.11999,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.862316Z"},"links":{"cited_paper":"/paper/2404.11999","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:9fa2985c5fbb97993abcaba20ab9cc4646676b3f1fc5195c31e1fdcb175ec580","observation_id":"c6c3cc89-936e-48cb-92b4-9bd3348890fb","resolution":{"observed_at":"2026-08-15T23:58:38.862316Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.18922","last_updated":"2025-05-21T15:34:02Z","snapshot_observed_at":"2026-08-16T17:38:22.018669Z","submitted_at":"2024-04-29T17:58:30Z","title":"DPO Meets PPO: Reinforced Token Optimization for RLHF","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.18922","snapshot_observed_at":"2026-08-15T23:58:38.865761Z","title":"Dpo meets ppo: Reinforced token optimization for rlhf.arXiv preprint arXiv:2404.18922,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.865761Z"},"links":{"cited_paper":"/paper/2404.18922","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:3ca641ee1adbf5442cd49baa3724ac5b6fa447f904dbf66ef3d5256675b5b7b1","observation_id":"c2b93ec8-e704-4b73-ba47-f0276b1a35dc","resolution":{"observed_at":"2026-08-15T23:58:38.865761Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T23:58:39.615029Z","title":null,"venue":null,"work_id":"012106d9-91e0-4197-ae8c-18ed9f4b1a44","year":1999},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.874698Z"},"links":{"citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:cbbbd2ca0951529448340127357f28c8b8de36e76e5fbdfbe88f8b1e8db544ec","observation_id":"900ad02b-2619-4c42-8ec3-433b09c10f34","resolution":{"observed_at":"2026-08-15T23:58:39.709877Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T23:58:39.534024Z","title":null,"venue":null,"work_id":"571f0ba9-58ed-4d4c-9985-61c021569e90","year":2023},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.922235Z"},"links":{"citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:9043992a2417aa67812d4377643d019288d42fc49f17db11243a6e3092c26e9b","observation_id":"75a12e4e-98c0-42b7-8eab-b6c62432f22b","resolution":{"observed_at":"2026-08-15T23:58:39.557001Z","resolver_source":"raw_fallback","status":"malformed_identifier"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.01857","last_updated":"2024-06-05T09:00:36Z","snapshot_observed_at":"2026-08-16T17:38:25.517399Z","submitted_at":"2024-03-04T09:13:14Z","title":"Reward Model Learning vs. Direct Policy Optimization: A Comparative Analysis of Learning from Human Preferences","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.01857","snapshot_observed_at":"2026-08-15T23:58:38.589423Z","title":"Reward model learning vs","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":1999,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.589423Z"},"links":{"cited_paper":"/paper/2403.01857","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:eb611a03297c96f201eb18e184f6c43f9411677b7cd833695346a1f495aeaae4","observation_id":"e42e0a09-fb6a-4394-a405-66e1f2dd15b0","resolution":{"observed_at":"2026-08-15T23:58:38.589423Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1909.08593","last_updated":"2020-01-08T23:02:36Z","snapshot_observed_at":"2026-08-16T00:15:57.597094Z","submitted_at":"2019-09-18T17:33:39Z","title":"Fine-Tuning Language Models from Human Preferences","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1909.08593","snapshot_observed_at":"2026-08-15T23:58:38.869659Z","title":"M., Stiennon, N., Wu, J., Brown, T","venue":null,"work_id":null,"year":1909},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":2010,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.869659Z"},"links":{"cited_paper":"/paper/1909.08593","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:93cdfd3c52e04b2b47c5d3c457583230926653d6a17a2a0d28902c51498de022","observation_id":"8f2a234c-0723-4e8a-b15d-4a65a4e5fafe","resolution":{"observed_at":"2026-08-15T23:58:38.869659Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1506.02438","last_updated":"2018-10-20T18:55:07Z","snapshot_observed_at":"2026-08-15T05:06:05.489107Z","submitted_at":"2015-06-08T11:12:48Z","title":"High-Dimensional Continuous Control Using Generalized Advantage Estimation","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1506.02438","snapshot_observed_at":"2026-08-15T23:58:38.827512Z","title":"High-dimensional continuous control using generalized advantage estimation.arXiv preprint arXiv:1506.02438,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":2015,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.827512Z"},"links":{"cited_paper":"/paper/1506.02438","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:d2f4e8d950efac3ac460f9d7e9f8026c19cf999c782c3809de5beaa138e6bf48","observation_id":"aa9d8647-3e96-4c3c-bb48-f39ce045d14c","resolution":{"observed_at":"2026-08-15T23:58:38.827512Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2006.13900","last_updated":"2021-03-17T21:54:55Z","snapshot_observed_at":"2026-08-16T13:18:39.507010Z","submitted_at":"2020-06-24T17:35:15Z","title":"Quantifying Differences in Reward Functions","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2006.13900","snapshot_observed_at":"2026-08-15T23:58:38.557414Z","title":"Quantifying differences in reward functions.arXiv preprint arXiv:2006.13900,","venue":null,"work_id":null,"year":2006},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":2017,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.557414Z"},"links":{"cited_paper":"/paper/2006.13900","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:882b830af0b8643c4fdc0f67be07a8953efe9f13d02906f44fc70eed48229b88","observation_id":"e6c00768-6b85-4f49-a442-c968e6367cfe","resolution":{"observed_at":"2026-08-15T23:58:38.557414Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T23:58:39.516903Z","title":"While following this data generation procedure, we found a step in the reference code where transitions following a success signal were explicitly truncated","venue":null,"work_id":"a77a8c08-1f18-43a9-9ec4-11d38da02323","year":2023},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":2018,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:39.026249Z"},"links":{"citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:c7a665b2cbb341ae6fb34e3ea9f9aee2cd91cad0af27c9b4be8b57a4868b7118","observation_id":"bca9145c-5e66-4938-878d-7c77704cfde4","resolution":{"observed_at":"2026-08-15T23:58:39.524396Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2307.09288","last_updated":"2023-07-19T17:08:59Z","snapshot_observed_at":"2026-08-07T12:56:43.323460Z","submitted_at":"2023-07-18T14:31:57Z","title":"Llama 2: Open Foundation and Fine-Tuned Chat Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.09288","snapshot_observed_at":"2026-08-15T23:58:38.846724Z","title":"Llama 2: Open foundation and fine- tuned chat models.arXiv preprint arXiv:2307.09288,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":2020,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.846724Z"},"links":{"cited_paper":"/paper/2307.09288","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:7f4c73f87d105fafe42b0261beed6342e927f3dba8f8076b868274e69ba005e2","observation_id":"dcf98345-91be-4a2b-9828-6b6570c368a3","resolution":{"observed_at":"2026-08-15T23:58:38.846724Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2106.05091","last_updated":"2021-06-09T14:10:50Z","snapshot_observed_at":"2026-08-16T18:18:23.717610Z","submitted_at":"2021-06-09T14:10:50Z","title":"PEBBLE: Feedback-Efficient Interactive Reinforcement Learning via Relabeling Experience and Unsupervised Pre-training","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2106.05091","snapshot_observed_at":"2026-08-15T23:58:38.576437Z","title":"Pebble: Feedback- efficient interactive reinforcement learning via relabeling experience and unsupervised pre-training.arXiv preprint arXiv:2106.05091,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":2021,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.576437Z"},"links":{"cited_paper":"/paper/2106.05091","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:ba909f654cc879b1f7116666762aeb90e459f938935d208a1e5bac1d715fd5b7","observation_id":"c92fb850-7e76-4000-90a7-4297541278da","resolution":{"observed_at":"2026-08-15T23:58:38.576437Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.12358","last_updated":"2024-08-12T21:13:35Z","snapshot_observed_at":"2026-08-16T13:59:41.097676Z","submitted_at":"2024-04-18T17:37:02Z","title":"From $r$ to $Q^*$: Your Language Model is Secretly a Q-Function","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.12358","snapshot_observed_at":"2026-08-15T23:58:38.822297Z","title":"From r to q∗: Your language model is secretly a q-function.arXiv preprint arXiv:2404.12358, 2024a","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":2022,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.822297Z"},"links":{"cited_paper":"/paper/2404.12358","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:0524f1f0e7e267c0671e13cdc190285736c25c3375ccdcc039c4996284203467","observation_id":"f97cbcf5-d39e-43e7-95e7-5ccb7806eade","resolution":{"observed_at":"2026-08-15T23:58:38.822297Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2204.05862","last_updated":"2022-04-12T15:02:38Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-04-12T15:02:38Z","title":"Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2204.05862","snapshot_observed_at":"2026-08-15T23:58:38.552673Z","title":"Training a helpful and harmless assistant with rein- forcement learning from human feedback.arXiv preprint arXiv:2204.05862,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":2023,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.552673Z"},"links":{"cited_paper":"/paper/2204.05862","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:40857906890156e11fca5cf8652d97ba2bc5590c3de231d62f94c0f278a56877","observation_id":"2e513878-51b6-4341-9784-b1f57a848530","resolution":{"observed_at":"2026-08-15T23:58:38.552673Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.13639","last_updated":"2024-04-30T14:36:26Z","snapshot_observed_at":"2026-08-16T17:38:24.988221Z","submitted_at":"2023-10-20T16:37:56Z","title":"Contrastive Preference Learning: Learning from Human Feedback without RL","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.13639","snapshot_observed_at":"2026-08-15T23:58:38.561818Z","title":"B., and Sadigh, D","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?","version":2},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-15T23:58:38.561818Z"},"links":{"cited_paper":"/paper/2310.13639","citing_paper":"/paper/2505.06273"},"observation_digest":"sha256:c6dfb295200075628db57b4838cedf05fe1e2e181c93890249400a9d0e82164f","observation_id":"b47366cf-38d5-4c56-8816-d23fdc18eec6","resolution":{"observed_at":"2026-08-15T23:58:38.561818Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2505.06273","last_updated":"2025-05-13T04:50:08Z","latest_version":2,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-16T17:34:35.199380Z","submitted_at":"2025-05-06T15:09:55Z","title":"Policy-labeled Preference Learning: Is Preference Enough for RLHF?"},"reference_resolution":{"displayed":27,"state_counts":{"malformed_identifier":1,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":25,"verified_exact":0,"verified_fuzzy":1},"total_outbound_references":27},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"thesis":"As of 17 August 2026, this Paper Citation Record lists 27 of 27 outbound references and 0 inbound Pith citation observations for arXiv:2505.06273."}