{"as_of":"2026-08-19T11:10:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:2ebc8293104e153d1329cb13ac9b28c388f42160a6df8f0b282dd3abd2814870","coverage":[{"denominator":58,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":58,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-15T19:08:14.574756Z","state":"measured"},{"denominator":58,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":58,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-19T06:32:44.657259+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2506.17828/citation-record","integrity":"/paper/2506.17828/integrity","json":"/paper/2506.17828/citation-record.json","paper":"/paper/2506.17828"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-17T09:58:46.058102Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-08-15T19:08:14.202527Z","title":"Gpt-4 technical report","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.202527Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:71d6f4d25c09e596238ba23e230f4e5a0be34d3a1b50adfc5501a326dce91ff0","observation_id":"ee611819-3f7e-465d-8532-0b70afc8d4e7","resolution":{"observed_at":"2026-08-15T19:08:14.202527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:15.788936Z","title":"Reinforcement learning: Theory and algorithms","venue":null,"work_id":"d2e6ef19-05b2-4647-ad9a-91d9897292eb","year":2019},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.211866Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:2c87eade756200ddf55211c70085554aa523d1310b73a6308645ef761f3aef19","observation_id":"50cbe863-13a2-45e1-a89f-22ae5b59ea57","resolution":{"observed_at":"2026-08-15T19:08:15.795787Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.14740","last_updated":"2024-02-26T18:26:25Z","snapshot_observed_at":"2026-08-09T14:30:33.899591Z","submitted_at":"2024-02-22T17:52:34Z","title":"Back to Basics: Revisiting REINFORCE Style Optimization for Learning from Human Feedback in LLMs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.14740","snapshot_observed_at":"2026-08-15T19:08:14.218608Z","title":"Back to basics: Revisiting reinforce style optimization for learning from human feedback in llms.arXiv preprint arXiv:2402.14740, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.218608Z"},"links":{"cited_paper":"/paper/2402.14740","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:c8845dd64ca5af922b0ed3f7f6bd79f4c9984956b12a1c07bb979ae7664efed5","observation_id":"744ad2b3-92b3-4073-a463-75b8c62d8368","resolution":{"observed_at":"2026-08-15T19:08:14.218608Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.12036","last_updated":"2023-11-22T00:02:49Z","snapshot_observed_at":"2026-08-18T18:51:43.695932Z","submitted_at":"2023-10-18T15:21:28Z","title":"A General Theoretical Paradigm to Understand Learning from Human Preferences","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.12036","snapshot_observed_at":"2026-08-15T19:08:14.225388Z","title":"A general theoretical paradigm to understand learning from human preferences","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.225388Z"},"links":{"cited_paper":"/paper/2310.12036","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:8dc88b0a58c190a71c4f78008fda6a4f24df9789fd735cc36bf6b79278d41573","observation_id":"d84473d0-5e33-4456-af2e-9c13dd60aa38","resolution":{"observed_at":"2026-08-15T19:08:14.225388Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.16609","last_updated":"2023-09-28T17:07:49Z","snapshot_observed_at":"2026-08-09T21:25:20.369782Z","submitted_at":"2023-09-28T17:07:49Z","title":"Qwen Technical Report","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.16609","snapshot_observed_at":"2026-08-15T19:08:14.232737Z","title":"Qwen technical report.arXiv preprint arXiv:2309.16609, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.232737Z"},"links":{"cited_paper":"/paper/2309.16609","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:8b3192f992293bf8bb3d324084391f7e9b60f22a51f6292aa3415ab1282980b9","observation_id":"a4f7cb5e-e5a5-433d-a631-1ae0fe61e43f","resolution":{"observed_at":"2026-08-15T19:08:14.232737Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2204.05862","last_updated":"2022-04-12T15:02:38Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-04-12T15:02:38Z","title":"Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2204.05862","snapshot_observed_at":"2026-08-15T19:08:14.239876Z","title":"Training a helpful and harmless assistant with reinforcement learning from human feedback.arXiv preprint arXiv:2204.05862, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.239876Z"},"links":{"cited_paper":"/paper/2204.05862","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:a6b676fa47eb1f633315a6b1d38f1f9cd6f79d6268fbb08195a222e58c3d25af","observation_id":"81b0bc1d-1216-4e58-8df1-c1690ef2a97d","resolution":{"observed_at":"2026-08-15T19:08:14.239876Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:14.245892Z","title":"A survey of monte carlo tree search methods.IEEE Transactions on Computational Intelligence and AI in games, 4(1):1–43, 2012","venue":null,"work_id":null,"year":2012},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.245892Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:5eec5766d290030484f11922a354fabe4b90e42b92de6ccbc147091e5dcfbc7d","observation_id":"f1a7550d-660c-4f82-afc0-44c3f8f2f27a","resolution":{"observed_at":"2026-08-15T19:08:14.245892Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.20495","last_updated":"2024-05-30T21:36:12Z","snapshot_observed_at":"2026-08-16T13:47:34.023079Z","submitted_at":"2024-05-30T21:36:12Z","title":"Transfer Q Star: Principled Decoding for LLM Alignment","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.20495","snapshot_observed_at":"2026-08-15T19:08:14.252424Z","title":"Transfer q star: Principled decoding for llm alignment","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.252424Z"},"links":{"cited_paper":"/paper/2405.20495","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:48b5e3cd0885862708943966235322e8705c7229ce5e491e021869545c9f1018","observation_id":"e2610822-92da-4561-a7b9-a48eb13a532f","resolution":{"observed_at":"2026-08-15T19:08:14.252424Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.08495","last_updated":"2024-04-16T17:36:39Z","snapshot_observed_at":"2026-08-18T11:13:22.645268Z","submitted_at":"2024-04-12T14:25:49Z","title":"Dataset Reset Policy Optimization for RLHF","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.08495","snapshot_observed_at":"2026-08-15T19:08:14.259906Z","title":"Dataset reset policy optimization for rlhf.arXiv preprint arXiv:2404.08495, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.259906Z"},"links":{"cited_paper":"/paper/2404.08495","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:53772fd52d9f8d7f8c5f0c275a2ceea1f8a06ce278447aa5a6b1d135f7985e91","observation_id":"a3997ffc-3c70-4c65-8f76-584313a894f9","resolution":{"observed_at":"2026-08-15T19:08:14.259906Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:14.267848Z","title":"Stop summation: Min-form credit assignment is all process reward model needs for reasoning.arXiv preprint arXiv:2504.15275, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.267848Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:87bac6a1bf33c2be327b6a144d6f062eb4e4fdea0f68121ae794632e705c9929","observation_id":"dd6790e4-3450-4c44-b3ef-67a9dd4cc1d6","resolution":{"observed_at":"2026-08-15T19:08:14.267848Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:14.274186Z","title":"Ultrafeedback: Boosting language models with high-quality feedback","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.274186Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:3f97983c9d0cd275b84e82ce9be292a180cbaa18c77b74f648528401d38ae342","observation_id":"4c16a787-d391-4da9-94da-d5f1daf4f6d3","resolution":{"observed_at":"2026-08-15T19:08:14.274186Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2304.06767","last_updated":"2023-12-01T14:28:06Z","snapshot_observed_at":"2026-08-07T04:02:54.504761Z","submitted_at":"2023-04-13T18:22:40Z","title":"RAFT: Reward rAnked FineTuning for Generative Foundation Model Alignment","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2304.06767","snapshot_observed_at":"2026-08-15T19:08:14.280121Z","title":"Raft: Reward ranked finetuning for generative foundation model alignment","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.280121Z"},"links":{"cited_paper":"/paper/2304.06767","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:ebd6e9d86793f42c417c0bab14a981c03ad17576b24cd66aff7f892930ad532f","observation_id":"79b5424d-9322-432d-889b-622567018dc3","resolution":{"observed_at":"2026-08-15T19:08:14.280121Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:14.286087Z","title":"Rlhf workflow: From reward modeling to online rlhf, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.286087Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:c3a617f343d11575beae688d6bb27f5490fd82911409a5317896e16bd4109437","observation_id":"c1fcae9a-7bbd-4ac7-99fc-69b3a4ae3da9","resolution":{"observed_at":"2026-08-15T19:08:14.286087Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.04475","last_updated":"2025-03-10T09:27:03Z","snapshot_observed_at":"2026-07-06T17:56:23.317089Z","submitted_at":"2024-04-06T02:29:02Z","title":"Length-Controlled AlpacaEval: A Simple Way to Debias Automatic Evaluators","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.04475","snapshot_observed_at":"2026-08-15T19:08:14.291859Z","title":"Length-controlled alpacaeval: A simple way to debias automatic evaluators.arXiv preprint arXiv:2404.04475, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.291859Z"},"links":{"cited_paper":"/paper/2404.04475","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:73c920c173341d3f718850fba6e65e32c8e2a81e5c4db46a1a0da05328a223d3","observation_id":"e719762f-c6b8-431c-99fb-b21406c357b9","resolution":{"observed_at":"2026-08-15T19:08:14.291859Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:15.736400Z","title":"Alpacafarm: A simulation framework for methods that learn from human feedback.Advances in Neural Information Processing Systems, 36:30039–30069, 2023","venue":null,"work_id":"f2975a05-a19e-4493-92db-b7a367274297","year":2023},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.297478Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:b59637bc8f4e85e9054daed5aabcef82ee312a78c847664e89af0b32a5f8d50a","observation_id":"2fc67dec-5e38-4bc8-9f2f-9125c1729522","resolution":{"observed_at":"2026-08-15T19:08:15.742734Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.03950","last_updated":"2024-03-06T18:55:47Z","snapshot_observed_at":"2026-08-18T11:13:29.535070Z","submitted_at":"2024-03-06T18:55:47Z","title":"Stop Regressing: Training Value Functions via Classification for Scalable Deep RL","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.03950","snapshot_observed_at":"2026-08-15T19:08:14.303868Z","title":"Stop regressing: Training value functions via classification for scalable deep rl.arXiv preprint arXiv:2403.03950, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.303868Z"},"links":{"cited_paper":"/paper/2403.03950","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:ca6864fa79ce214db7656a55c1deffd6118ae7cea4a411393aaca33afb064119","observation_id":"7284d86f-6f80-4f70-baec-d5b67ccac989","resolution":{"observed_at":"2026-08-15T19:08:14.303868Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:14.311009Z","title":"Scaling laws for reward model overoptimization","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.311009Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:861448980347977c8f296e4cae4f338f735650ed06a14316b76be0eec41e99a8","observation_id":"dedde5e3-c863-47fd-a951-c8cbe2efcce2","resolution":{"observed_at":"2026-08-15T19:08:14.311009Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.12948","last_updated":"2026-01-04T03:57:36Z","snapshot_observed_at":"2026-08-15T12:33:55.451951Z","submitted_at":"2025-01-22T15:19:35Z","title":"DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.12948","snapshot_observed_at":"2026-08-15T19:08:14.317597Z","title":"Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.317597Z"},"links":{"cited_paper":"/paper/2501.12948","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:091cd7749699d0150de582fda89775b4f5cd06b4e71cc27f4bd698b1ea7faaf9","observation_id":"89640b75-cedc-4d30-a588-1a24a1e32584","resolution":{"observed_at":"2026-08-15T19:08:14.317597Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.06639","last_updated":"2024-05-10T17:59:04Z","snapshot_observed_at":"2026-08-16T13:53:40.024766Z","submitted_at":"2024-05-10T17:59:04Z","title":"Value Augmented Sampling for Language Model Alignment and Personalization","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.06639","snapshot_observed_at":"2026-08-15T19:08:14.323757Z","title":"Value augmented sampling for language model alignment and personalization.arXiv preprint arXiv:2405.06639, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.323757Z"},"links":{"cited_paper":"/paper/2405.06639","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:a348fbf9898637f2711772ca8b11ae0751e40410532b801787d5c3ac22a6489d","observation_id":"36272bd8-8096-4d01-9205-4825fe02531c","resolution":{"observed_at":"2026-08-15T19:08:14.323757Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:15.706161Z","title":"Inverse preference learning: Preference-based rl without a reward function","venue":null,"work_id":"cd95bce9-9adb-496a-9fa9-bd00bcbb82af","year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.330108Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:3cf38ae459b5d81732eea36d72a56eb66ce8e6c80468b1cd72971ac69b982937","observation_id":"b1e45707-df66-4644-855f-3f21030122be","resolution":{"observed_at":"2026-08-15T19:08:15.712208Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:14.335859Z","title":"Deal: Decoding-time alignment for large language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.335859Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:c24f362580a9bbbaa0b6b47fd73016cbde7839e46d0cf9b03a8d9f00613c7813","observation_id":"b43fc796-254a-4e5c-ae5d-95b7debcbc94","resolution":{"observed_at":"2026-08-15T19:08:14.335859Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.17031","last_updated":"2024-03-24T02:59:27Z","snapshot_observed_at":"2026-08-18T11:13:28.954163Z","submitted_at":"2024-03-24T02:59:27Z","title":"The N+ Implementation Details of RLHF with PPO: A Case Study on TL;DR Summarization","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.17031","snapshot_observed_at":"2026-08-15T19:08:14.341846Z","title":"The n+ implementation details of rlhf with ppo: A case study on tl; dr summarization.arXiv preprint arXiv:2403.17031, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.341846Z"},"links":{"cited_paper":"/paper/2403.17031","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:6bcb10cf5a3e53dc70056232309fd22a493ae23451699fc929f20e325b79e246","observation_id":"bded4ea0-0e61-4bd1-9592-25d177f931d7","resolution":{"observed_at":"2026-08-15T19:08:14.341846Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.02561","last_updated":"2023-06-30T21:39:54Z","snapshot_observed_at":"2026-08-18T18:36:22.935192Z","submitted_at":"2023-06-05T03:32:26Z","title":"LLM-Blender: Ensembling Large Language Models with Pairwise Ranking and Generative Fusion","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.02561","snapshot_observed_at":"2026-08-15T19:08:14.347674Z","title":"Llm-blender: Ensembling large language models with pairwise ranking and generative fusion.arXiv preprint arXiv:2306.02561, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.347674Z"},"links":{"cited_paper":"/paper/2306.02561","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:d4a793a599675554aa383b73931f48493f73dd0fa283fb3129103a48bc279368","observation_id":"877e0763-f7ec-4693-8cf1-43712e2fe91b","resolution":{"observed_at":"2026-08-15T19:08:14.347674Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:14.355893Z","title":"A natural policy gradient.Advances in neural information processing systems, 14, 2001","venue":null,"work_id":null,"year":2001},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.355893Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:f8af2e23dac0b8f0afce65d15c3023e26e2e555e2291617d349f151cf76bb74a","observation_id":"2c522dfd-c841-4e6d-82de-c703bdd4596f","resolution":{"observed_at":"2026-08-15T19:08:14.355893Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.01694","last_updated":"2024-01-23T23:42:41Z","snapshot_observed_at":"2026-08-18T11:13:23.191794Z","submitted_at":"2024-01-23T23:42:41Z","title":"ARGS: Alignment as Reward-Guided Search","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.01694","snapshot_observed_at":"2026-08-15T19:08:14.361484Z","title":"Args: Alignment as reward-guided search.arXiv preprint arXiv:2402.01694, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.361484Z"},"links":{"cited_paper":"/paper/2402.01694","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:19ce4dbd472b04a3a80d3f533b6455ac044cb5e1121af71f53b1b7aac303b176","observation_id":"0db04d10-d604-4cc7-a8b4-31080f448bea","resolution":{"observed_at":"2026-08-15T19:08:14.361484Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.05954","last_updated":"2024-11-01T17:46:46Z","snapshot_observed_at":"2026-08-18T10:32:32.430827Z","submitted_at":"2024-06-10T01:21:31Z","title":"Aligning Large Language Models with Representation Editing: A Control Perspective","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.05954","snapshot_observed_at":"2026-08-15T19:08:14.368583Z","title":"Aligning large language models with representation editing: A control perspective","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.368583Z"},"links":{"cited_paper":"/paper/2406.05954","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:f54da9130d08a7e85110401cae8818648f0b6efdcc6d65543c67fd4bee06dd9e","observation_id":"c8aee78b-af36-4b8f-812f-3d206e6dae75","resolution":{"observed_at":"2026-08-15T19:08:14.368583Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2102.06234","last_updated":"2021-02-11T19:35:33Z","snapshot_observed_at":"2026-08-18T11:13:28.406377Z","submitted_at":"2021-02-11T19:35:33Z","title":"Optimization Issues in KL-Constrained Approximate Policy Iteration","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2102.06234","snapshot_observed_at":"2026-08-15T19:08:14.376134Z","title":"Optimization issues in kl-constrained approximate policy iteration.arXiv preprint arXiv:2102.06234, 2021","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.376134Z"},"links":{"cited_paper":"/paper/2102.06234","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:35f5ea29a9604e78e63ec7d291f76cfc458e8dc73eec4338a6ef290f51b7abe6","observation_id":"a8c9afe5-3b56-498f-9129-1c8a0280e79a","resolution":{"observed_at":"2026-08-15T19:08:14.376134Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.16306","last_updated":"2025-08-03T22:12:55Z","snapshot_observed_at":"2026-08-16T13:40:17.963736Z","submitted_at":"2024-06-24T04:08:35Z","title":"Cascade Reward Sampling for Efficient Decoding-Time Alignment","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.16306","snapshot_observed_at":"2026-08-15T19:08:14.383059Z","title":"Cascade reward sampling for efficient decoding-time alignment.arXiv preprint arXiv:2406.16306, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.383059Z"},"links":{"cited_paper":"/paper/2406.16306","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:1e7bd7e0d225b654317d523c99a8d60af2caf09b262f8a1c5f4c21e4279cc1ad","observation_id":"c418a9ce-776a-47e1-9aec-d7e8708e6590","resolution":{"observed_at":"2026-08-15T19:08:14.383059Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:14.389432Z","title":"Alpacaeval: An automatic evaluator of instruction-following models, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.389432Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:3357499516c26426409d09fdacf1c9f741349473f7fdf7ce2a9c0f57466346b6","observation_id":"14f6de3c-da9f-46e4-b8b5-55009934b19f","resolution":{"observed_at":"2026-08-15T19:08:14.389432Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:14.395816Z","title":"Let’s verify step by step","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.395816Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:4c37a137be813a5f9d20189b8318fb498fdc98169a968df4cfa5f02a6d877466","observation_id":"5f93dc1f-f411-4cc7-a57d-744f866108d0","resolution":{"observed_at":"2026-08-15T19:08:14.395816Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2409.17819","last_updated":"2024-09-26T13:15:18Z","snapshot_observed_at":"2026-08-16T13:15:08.877776Z","submitted_at":"2024-09-26T13:15:18Z","title":"Inference-Time Language Model Alignment via Integrated Value Guidance","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.17819","snapshot_observed_at":"2026-08-15T19:08:14.402192Z","title":"Inference-time language model alignment via integrated value guidance.arXiv preprint arXiv:2409.17819, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.402192Z"},"links":{"cited_paper":"/paper/2409.17819","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:a3b226397c21aac207960ec8e9abafea905d7ff44dad2fbbec05fa95b3a85f07","observation_id":"b7b2f6b1-c043-4007-97d4-cac7186fb370","resolution":{"observed_at":"2026-08-15T19:08:14.402192Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.17022","last_updated":"2024-06-03T20:50:26Z","snapshot_observed_at":"2026-08-16T14:48:48.486166Z","submitted_at":"2023-10-25T22:00:05Z","title":"Controlled Decoding from Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.17022","snapshot_observed_at":"2026-08-15T19:08:14.408376Z","title":"Controlled decoding from language models","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.408376Z"},"links":{"cited_paper":"/paper/2310.17022","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:1d5fac348e9ab82ed7d3dd0d1ec0c761121c66a31ab77c0e9647382c94461082","observation_id":"3ed8fc66-0ea9-4060-8d1c-948040777f5c","resolution":{"observed_at":"2026-08-15T19:08:14.408376Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2112.09332","last_updated":"2022-06-01T19:08:11Z","snapshot_observed_at":"2026-08-19T02:54:55.334346Z","submitted_at":"2021-12-17T05:43:43Z","title":"WebGPT: Browser-assisted question-answering with human feedback","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2112.09332","snapshot_observed_at":"2026-08-15T19:08:14.414389Z","title":"Webgpt: Browser-assisted question-answering with human feedback.arXiv preprint arXiv:2112.09332, 2021","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.414389Z"},"links":{"cited_paper":"/paper/2112.09332","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:e7d727489bc5b59903675468ac58d9c1d8782df2dd61a165f8d70f747a1bc5c8","observation_id":"87f5d692-b0d3-47fb-b503-187c38a5abe7","resolution":{"observed_at":"2026-08-15T19:08:14.414389Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:14.420029Z","title":"Training language models to follow instructions with human feedback.Advances in Neural Information Processing Systems, 35:27730–27744, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.420029Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:70d0ad7c3d612fec5e8d33dbc82ab4fed11421978df4f5ada099f942dbe408b7","observation_id":"f95e0cbb-e569-4a80-9640-34b721cd5cb6","resolution":{"observed_at":"2026-08-15T19:08:14.420029Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.16033","last_updated":"2025-09-03T03:41:08Z","snapshot_observed_at":"2026-08-16T13:08:10.343974Z","submitted_at":"2024-10-18T04:38:21Z","title":"TreeBoN: Enhancing Inference-Time Alignment with Speculative Tree-Search and Best-of-N Sampling","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.16033","snapshot_observed_at":"2026-08-15T19:08:14.426493Z","title":"Treebon: Enhancing inference-time alignment with speculative tree-search and best-of-n sampling","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.426493Z"},"links":{"cited_paper":"/paper/2410.16033","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:0fb5e36294755c5cec212e7fef0889524505c560951ce5a040648e041ef83aa8","observation_id":"ea6f7082-3051-4f71-9b59-378044cee4f9","resolution":{"observed_at":"2026-08-15T19:08:14.426493Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.12358","last_updated":"2024-08-12T21:13:35Z","snapshot_observed_at":"2026-08-16T13:59:41.097676Z","submitted_at":"2024-04-18T17:37:02Z","title":"From $r$ to $Q^*$: Your Language Model is Secretly a Q-Function","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.12358","snapshot_observed_at":"2026-08-15T19:08:14.432916Z","title":"Fromr to q∗: Your language model is secretly a q-function.arXiv preprint arXiv:2404.12358, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.432916Z"},"links":{"cited_paper":"/paper/2404.12358","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:c7a0feaf447111e42e4bcbf223e218b1ab29123a51ad76a65d9f9479907533f0","observation_id":"f814bcac-3f49-4600-8840-29fd9fe3bdbc","resolution":{"observed_at":"2026-08-15T19:08:14.432916Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:14.439663Z","title":"Direct preference optimization: Your language model is secretly a reward model.Advances in Neural Information Processing Systems, 36, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.439663Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:29d9e422eeec27b3a14974bbf68ddfbb52e3d774e139e30c610be0f4012883ab","observation_id":"ee815ad4-78de-4461-a074-aa116b52515c","resolution":{"observed_at":"2026-08-15T19:08:14.439663Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1502.05477","last_updated":"2017-04-20T18:04:12Z","snapshot_observed_at":"2026-08-15T12:05:11.299477Z","submitted_at":"2015-02-19T06:44:25Z","title":"Trust Region Policy Optimization","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1502.05477","snapshot_observed_at":"2026-08-15T19:08:14.445386Z","title":"Trust region policy optimization.arXiv preprint arXiv:1502.05477, 2015","venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.445386Z"},"links":{"cited_paper":"/paper/1502.05477","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:83b614e5a188eb81717747dbf74487d154768e4bd9c4bea3149c0c89a9bc3818","observation_id":"9b9875b7-c12e-4415-a1b4-fdc0f9ec9713","resolution":{"observed_at":"2026-08-15T19:08:14.445386Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1506.02438","last_updated":"2018-10-20T18:55:07Z","snapshot_observed_at":"2026-08-15T05:06:05.489107Z","submitted_at":"2015-06-08T11:12:48Z","title":"High-Dimensional Continuous Control Using Generalized Advantage Estimation","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1506.02438","snapshot_observed_at":"2026-08-15T19:08:14.450938Z","title":"High-dimensional continuous control using generalized advantage estimation.arXiv preprint arXiv:1506.02438, 2015","venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.450938Z"},"links":{"cited_paper":"/paper/1506.02438","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:79aeab6df5121b26f541b0a3629e12f148290f8de302f6fd9495fd80f0cab8ce","observation_id":"e7cf550e-8b3a-44d2-8867-db4b01af33c8","resolution":{"observed_at":"2026-08-15T19:08:14.450938Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1707.06347","last_updated":"2017-08-28T09:20:06Z","snapshot_observed_at":"2026-08-15T20:26:32.102285Z","submitted_at":"2017-07-20T02:32:33Z","title":"Proximal Policy Optimization Algorithms","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1707.06347","snapshot_observed_at":"2026-08-15T19:08:14.456949Z","title":"Proximal policy optimization algorithms","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.456949Z"},"links":{"cited_paper":"/paper/1707.06347","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:6f9d3036137120addc2d844c9cee33e740627b0e6f86431e74fb809329a6cef5","observation_id":"7637d207-7007-465c-97e8-b52741fe1e5f","resolution":{"observed_at":"2026-08-15T19:08:14.456949Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:15.630792Z","title":"Adaptive trust region policy optimization: Global convergence and faster rates for regularized mdps","venue":null,"work_id":"e5f9abea-801c-4c76-8663-0999ee887506","year":2020},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.462585Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:0d3a32b275869d8ace54f0bd313cc3e111896ef4c9d35bfcf841d1b545eb2037","observation_id":"f0d89a27-f388-4cf0-a80b-f992c655430f","resolution":{"observed_at":"2026-08-15T19:08:15.636908Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.03300","last_updated":"2024-04-27T15:25:53Z","snapshot_observed_at":"2026-08-06T14:58:42.911363Z","submitted_at":"2024-02-05T18:55:32Z","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.03300","snapshot_observed_at":"2026-08-15T19:08:14.468474Z","title":"Deepseekmath: Pushing the limits of mathematical reasoning in open language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.468474Z"},"links":{"cited_paper":"/paper/2402.03300","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:b822a498934ce222a3f051d5e2f83d21675bbff8783b384da75542be7b15a1fa","observation_id":"f9e936f6-3c4e-4d46-bac2-8476db04cf37","resolution":{"observed_at":"2026-08-15T19:08:14.468474Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.03314","last_updated":"2024-08-06T17:35:05Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-08-06T17:35:05Z","title":"Scaling LLM Test-Time Compute Optimally can be More Effective than Scaling Model Parameters","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.03314","snapshot_observed_at":"2026-08-15T19:08:14.475991Z","title":"Scaling llm test-time compute optimally can be more effective than scaling model parameters.arXiv preprint arXiv:2408.03314, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.475991Z"},"links":{"cited_paper":"/paper/2408.03314","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:8298e9c49bf9c137e36e4dbd2c5e837123589f799ee4c44a870c1a330e99ceff","observation_id":"e2b8e3f7-fe23-462d-825a-cd7375a0d7cf","resolution":{"observed_at":"2026-08-15T19:08:14.475991Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2210.06718","last_updated":"2023-03-11T11:47:54Z","snapshot_observed_at":"2026-08-16T16:24:37.265288Z","submitted_at":"2022-10-13T04:19:05Z","title":"Hybrid RL: Using Both Offline and Online Data Can Make RL Efficient","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2210.06718","snapshot_observed_at":"2026-08-15T19:08:14.482273Z","title":"Hybrid rl: Using both offline and online data can make rl efficient.arXiv preprint arXiv:2210.06718, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.482273Z"},"links":{"cited_paper":"/paper/2210.06718","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:bcbd173d945a2af4a3117bcc29382b68531f8359b81ab8be88edece27ad2cb8e","observation_id":"199d94c2-2516-4795-abfe-d4d2ee6e611e","resolution":{"observed_at":"2026-08-15T19:08:14.482273Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:14.488903Z","title":"Learning to summarize with human feedback.Advances in Neural Information Processing Systems, 33:3008–3021, 2020","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.488903Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:29c278610d39c423ecd23036dc07e32e0349ff3e90e8622c6fff5de5e967ddd2","observation_id":"3492c259-5465-469b-a9c4-d284785595eb","resolution":{"observed_at":"2026-08-15T19:08:14.488903Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:14.495629Z","title":"On the convergence rates of policy gradient methods.Journal of Machine Learning Research, 23(282):1–36, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.495629Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:576b5597e8f533c8291324b2444fd0db55bc0bd59d547a6b6b8c45c0236dcdee","observation_id":"bdc53125-8e08-49f7-80c8-b67f95e85451","resolution":{"observed_at":"2026-08-15T19:08:14.495629Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.00451","last_updated":"2024-06-17T22:11:49Z","snapshot_observed_at":"2026-08-18T09:48:11.842015Z","submitted_at":"2024-05-01T11:10:24Z","title":"Monte Carlo Tree Search Boosts Reasoning via Iterative Preference Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.00451","snapshot_observed_at":"2026-08-15T19:08:14.502275Z","title":"Lillicrap, Kenji Kawaguchi, and Michael Shieh","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.502275Z"},"links":{"cited_paper":"/paper/2405.00451","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:6ce8d1d37213c5eb2cac2ffce064fa1fb06fdf86fb28b3ba1a4e6df22828a0b2","observation_id":"5dfb5fc4-6623-405c-9e18-0c96c5b66eea","resolution":{"observed_at":"2026-08-15T19:08:14.502275Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:15.587787Z","title":"Iterative preference learning from human feedback: Bridging theory and practice for rlhf under kl- constraint","venue":null,"work_id":"5ef43632-42e5-4c0f-bb53-7bd5a0540761","year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.509171Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:146be1581c33c7cf2e274162763c5f17240f1a7464955afb3294f39f8f9cc70d","observation_id":"806b563c-2708-49fb-bfe1-a9a2e1ecfcbe","resolution":{"observed_at":"2026-08-15T19:08:15.594361Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2404.10719","last_updated":"2024-10-10T08:30:17Z","snapshot_observed_at":"2026-08-18T11:11:58.049967Z","submitted_at":"2024-04-16T16:51:53Z","title":"Is DPO Superior to PPO for LLM Alignment? A Comprehensive Study","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.10719","snapshot_observed_at":"2026-08-15T19:08:14.515292Z","title":"Is dpo superior to ppo for llm alignment? a comprehensive study.arXiv preprint arXiv:2404.10719, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.515292Z"},"links":{"cited_paper":"/paper/2404.10719","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:ff843ba406e5a39acdf5e3b89fa22fe300996db436dad5a817f3bdfee0a9d936","observation_id":"16303d39-c57b-4079-bc76-5640df34b622","resolution":{"observed_at":"2026-08-15T19:08:14.515292Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.08193","last_updated":"2025-07-15T00:32:25Z","snapshot_observed_at":"2026-08-18T11:13:29.901243Z","submitted_at":"2024-10-10T17:58:24Z","title":"GenARM: Reward Guided Generation with Autoregressive Reward Model for Test-time Alignment","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.08193","snapshot_observed_at":"2026-08-15T19:08:14.522546Z","title":"Genarm: Reward guided generation with autoregressive reward model for test-time alignment","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.522546Z"},"links":{"cited_paper":"/paper/2410.08193","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:8f5ef591c81b2cdf4688f65d35dd8453c06b88278a58d9ba1879be1ec78d23f9","observation_id":"c741543d-1997-4ca3-bf7b-67c408694c59","resolution":{"observed_at":"2026-08-15T19:08:14.522546Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2104.05218","last_updated":"2021-08-15T18:34:33Z","snapshot_observed_at":"2026-08-18T11:13:27.863404Z","submitted_at":"2021-04-12T05:59:53Z","title":"FUDGE: Controlled Text Generation With Future Discriminators","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2104.05218","snapshot_observed_at":"2026-08-15T19:08:14.529371Z","title":"Fudge: Controlled text generation with future discriminators.arXiv preprint arXiv:2104.05218, 2021","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.529371Z"},"links":{"cited_paper":"/paper/2104.05218","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:56ec52b911612bd543258b64faa5882e256a9f5b460695814780ed4c63467de6","observation_id":"d26b7b2a-f374-4450-9ec2-af2f58b57e35","resolution":{"observed_at":"2026-08-15T19:08:14.529371Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:15.568330Z","title":"Llm alignment through successive policy re-weighting (spr)","venue":null,"work_id":"1adede56-67c4-4756-99ae-c3ec2e002614","year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.535725Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:d29ca606dd1cad4e610039f7c43ef27ede305cb7b2004aba3aed87aec024d358","observation_id":"b5696eaf-536f-45ae-8674-e416f20989c5","resolution":{"observed_at":"2026-08-15T19:08:15.574252Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2405.19262","last_updated":"2024-11-19T13:27:30Z","snapshot_observed_at":"2026-08-16T13:48:05.926297Z","submitted_at":"2024-05-29T16:55:32Z","title":"Weak-to-Strong Search: Align Large Language Models via Searching over Small Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.19262","snapshot_observed_at":"2026-08-15T19:08:14.541348Z","title":"Weak-to-strong search: Align large language models via searching over small language models.arXiv preprint arXiv:2405.19262, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.541348Z"},"links":{"cited_paper":"/paper/2405.19262","citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:a5a9b3c2c81e62598eeb08b17fcf720277c66bf776e7ba9c478ebcf1c0b40592","observation_id":"ae25fa4c-a574-4b60-9738-0fd31626139b","resolution":{"observed_at":"2026-08-15T19:08:14.541348Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:15.546449Z","title":"Starling-7b: Improving llm helpfulness & harmlessness with rlaif.2023, 2023","venue":null,"work_id":"bcb203aa-f8da-45b7-95f3-a92bedabc32c","year":2023},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.546908Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:fd21cde4b37c2c32b00c05367e50ad745f124292f0d225ca1c0b2a91cf271389","observation_id":"70baa008-3fdb-4b7c-b6ef-c1acef246b2e","resolution":{"observed_at":"2026-08-15T19:08:15.553701Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:15.526533Z","title":"However, optimizing these algorithms for peak performance requires substantial effort and resources, which are often beyond the reach of the open-source community","venue":null,"work_id":"8e530e9e-67f7-4943-8f13-bc0ca51cb813","year":null},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.553567Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:5b6a2a242f165d0ffd7843dd53b5595ce9f53c42178c0167ca7d206d9dac7aeb","observation_id":"7ec3185e-4a7d-4051-88ce-a8b0f99235ff","resolution":{"observed_at":"2026-08-15T19:08:15.533244Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:15.508835Z","title":null,"venue":null,"work_id":"34e8ab2f-7375-48a1-a97a-8b4b18719e77","year":null},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.560736Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:acb01348ca333c7bf7db570e76f1ee5fd969842346b48f6a21977ffe5f04cda3","observation_id":"edf01e89-e595-4e80-acd8-c18e1853083e","resolution":{"observed_at":"2026-08-15T19:08:15.514386Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:15.490597Z","title":null,"venue":null,"work_id":"d2abe700-d687-43a4-8294-885d3e9d6a44","year":null},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.567968Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:d457690737c8563ecbd5a1323c02497f77771ebe42f4f54b5cfe31faaecebfe2","observation_id":"8b0a2116-d1f3-4836-ab67-a5de1c6dad5d","resolution":{"observed_at":"2026-08-15T19:08:15.496566Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:08:15.467994Z","title":"A\"or \"B\"to indicate your choice. Your response should use the format: Comparison: <one-sentence comparison and explanation> Preferred: <","venue":null,"work_id":"51cbaadd-f74f-4a2e-8b6e-d58a815ef258","year":null},"citing_paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach","version":2},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-15T19:08:14.574756Z"},"links":{"citing_paper":"/paper/2506.17828"},"observation_digest":"sha256:5ab7b894284621f019cdf60b7593ec3ca3ca2783aeb453a4921cce4df4683407","observation_id":"c279e0b0-8cb8-4173-8dee-b642b7fa5d73","resolution":{"observed_at":"2026-08-15T19:08:15.477168Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2506.17828","last_updated":"2025-07-03T05:12:51Z","latest_version":2,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-18T11:12:16.166349Z","submitted_at":"2025-06-21T21:49:02Z","title":"Aligning Frozen LLMs by Reinforcement Learning: An Iterative Reweight-then-Optimize Approach"},"reference_resolution":{"displayed":58,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":49,"verified_exact":0,"verified_fuzzy":9},"total_outbound_references":58},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"thesis":"As of 19 August 2026, this Paper Citation Record lists 58 of 58 outbound references and 0 inbound Pith citation observations for arXiv:2506.17828."}