{"as_of":"2026-08-09T07:55:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:16c355643165d31aef0252ad993d3b30046d537843a8d6b0d3528b47af493a10","coverage":[{"denominator":0,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":34,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":34,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-09T06:31:02.800959+00:00","state":"measured"},{"denominator":34,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":34,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-08T19:07:35.445137Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":1,"source":"arxiv_reference","source_observed_at":"2026-08-05T02:28:24.338817Z","state":"measured"}],"external_citation_measurements":[{"count":7,"observed_at":"2026-08-05T02:28:24.338817Z","source":"arxiv_reference"}],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2403.07691","last_updated":"2024-03-14T07:47:08Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-03-12T14:34:08Z","title":"ORPO: Monolithic Preference Optimization without Reference Model","version":2},"reference_index":53,"source":"arxiv_source","source_observed_at":"2026-05-16T09:34:04.394588Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2403.07691"},"observation_digest":"sha256:7ffbb7914d5334e7b922f6110d5c4c2fb3822fe00939897fa8f6c2696327e6ed","observation_id":"73fa1a52-2f38-4c64-a58c-de79b310f68f","resolution":{"observed_at":"2026-05-16T09:34:04.707958Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2411.10442","last_updated":"2025-04-07T09:09:39Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-11-15T18:59:27Z","title":"Enhancing the Reasoning Ability of Multimodal Large Language Models via Mixed Preference Optimization","version":2},"reference_index":97,"source":"pdf_text","source_observed_at":"2026-05-16T09:16:17.150383Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2411.10442"},"observation_digest":"sha256:b75d1fb4a884f7eb437d76b67a9fe8fa887e5aaeb64313c93b5215adc6367abc","observation_id":"0b09bcf4-e9eb-4ddc-840d-b79250750aaf","resolution":{"observed_at":"2026-05-16T09:16:17.483570Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2412.15115","last_updated":"2025-01-03T02:18:21Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-12-19T17:56:09Z","title":"Qwen2.5 Technical Report","version":2},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-05-23T06:25:00.376073Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2412.15115"},"observation_digest":"sha256:0ca358d7129b44f6c319b1d1131b7b18346bc8718684d870a859710e4a52daf6","observation_id":"6fc7a908-e2fc-48ce-bc07-3bf1539d8369","resolution":{"observed_at":"2026-05-23T06:25:27.899604Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2502.06387","last_updated":"2026-04-07T06:33:34Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2025-02-10T12:15:27Z","title":"How Humans Help LLMs: Assessing and Incentivizing Human Preference Annotators","version":2},"reference_index":88,"source":"arxiv_source","source_observed_at":"2026-05-23T03:56:18.703995Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2502.06387"},"observation_digest":"sha256:2a29755905a76ee7eb1e599232de50e34ae4d4bdc4ddff457591283d2355fee9","observation_id":"95f05551-4bd2-4d8b-8c57-24adc9473adc","resolution":{"observed_at":"2026-05-23T03:57:29.575352Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-08T19:07:35.445137Z","title":"Secrets of rlhf in large language models part ii: Reward modeling,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.08657","last_updated":"2025-02-08T09:54:47Z","snapshot_observed_at":"2026-08-08T19:01:10.611273Z","submitted_at":"2025-02-08T09:54:47Z","title":"Refining Positive and Toxic Samples for Dual Safety Self-Alignment of LLMs with Minimal Human Interventions","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-08T19:07:35.445137Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2502.08657"},"observation_digest":"sha256:59f425a119acfd2868e3093353c26703e4f2d4cd15044a4edb9b6c6686ff7316","observation_id":"67d604db-332f-4d2e-a985-bce45b67d65c","resolution":{"observed_at":"2026-08-08T19:07:35.445137Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-08T11:57:11.983807Z","title":"Secrets of rlhf in large language models part ii: Reward modeling","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2502.09650","last_updated":"2025-05-13T18:54:09Z","snapshot_observed_at":"2026-08-09T02:50:07.586622Z","submitted_at":"2025-02-11T17:01:11Z","title":"Principled Data Selection for Alignment: The Hidden Risks of Difficult Examples","version":2},"reference_index":71,"source":"arxiv_source","source_observed_at":"2026-08-08T11:57:11.983807Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2502.09650"},"observation_digest":"sha256:17feea099fe8ce472eb8acf32534785264a1a9688f0720f1b5e2f7a05c6297f5","observation_id":"b94a2143-d519-4a37-9fa7-fbb3ac0946b1","resolution":{"observed_at":"2026-08-08T11:57:11.983807Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2505.07062","last_updated":"2025-05-11T17:28:30Z","snapshot_observed_at":"2026-08-02T16:13:31.498470Z","submitted_at":"2025-05-11T17:28:30Z","title":"Seed1.5-VL Technical Report","version":1},"reference_index":138,"source":"pdf_text","source_observed_at":"2026-05-11T05:26:04.960844Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2505.07062"},"observation_digest":"sha256:3317d6b804f0e4e8e0f7ffc844a258d25dcfbc1624ad97259b162e4bb3887222","observation_id":"c93a8b98-a211-439d-86d1-bf05ae4a3959","resolution":{"observed_at":"2026-05-11T05:26:05.576663Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-07T15:34:52.624148Z","title":"Secrets of rlhf in large language models part ii: Reward modeling.arXiv preprint arXiv:2401.06080, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.14674","last_updated":"2025-05-20T17:58:03Z","snapshot_observed_at":"2026-08-07T23:14:31.982740Z","submitted_at":"2025-05-20T17:58:03Z","title":"Reward Reasoning Model","version":1},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-07T15:34:52.624148Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2505.14674"},"observation_digest":"sha256:827af363072c78d53069c89f3dd38ca0918f452f534da798b4e938d7ea7bb776","observation_id":"a812a115-e4e8-4789-8eca-a48adb5d8948","resolution":{"observed_at":"2026-08-07T15:34:52.624148Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-07T14:57:31.942943Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.16869","last_updated":"2025-05-22T16:24:51Z","snapshot_observed_at":"2026-08-09T02:24:14.170168Z","submitted_at":"2025-05-22T16:24:51Z","title":"MPO: Multilingual Safety Alignment via Reward Gap Optimization","version":1},"reference_index":68,"source":"arxiv_source","source_observed_at":"2026-08-07T14:57:31.942943Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2505.16869"},"observation_digest":"sha256:acbebb2fb018200b4b58d9d8a4b652470d850a8386c8a62a1b34f951cf3668bd","observation_id":"987cc69d-84de-4dab-9a7e-544ac5e73b2c","resolution":{"observed_at":"2026-08-07T14:57:31.942943Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2505.19134","last_updated":"2026-04-13T23:58:15Z","snapshot_observed_at":"2026-07-06T21:30:04.828669Z","submitted_at":"2025-05-25T13:11:55Z","title":"Incentivizing High-Quality Human Annotations with Golden Questions","version":2},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-05-19T13:41:26.730528Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2505.19134"},"observation_digest":"sha256:710f7950aef338a49a18919f627cb30f6a18c2ee136dd3d0622e9a4c5ace229d","observation_id":"5fea45f8-50fa-4f09-9407-96a73b02ce3a","resolution":{"observed_at":"2026-05-19T13:42:19.352622Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-07T13:15:55.913770Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.22298","last_updated":"2025-05-28T12:37:06Z","snapshot_observed_at":"2026-08-07T13:08:19.239441Z","submitted_at":"2025-05-28T12:37:06Z","title":"Adaptive Detoxification: Safeguarding General Capabilities of LLMs through Toxicity-Aware Knowledge Editing","version":1},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-07T13:15:55.913770Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2505.22298"},"observation_digest":"sha256:fd1d51de1ae41059e05ba2f9a0a5a533c4b6c9f2a8484766b17514e5d9919294","observation_id":"3952763d-ace7-4824-9572-3163cc78be12","resolution":{"observed_at":"2026-08-07T13:15:55.913770Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-07T00:33:56.965198Z","title":"Secrets of rlhf in large language models part ii: Reward modeling","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.13888","last_updated":"2025-06-16T18:10:51Z","snapshot_observed_at":"2026-08-07T00:24:17.297652Z","submitted_at":"2025-06-16T18:10:51Z","title":"VL-GenRM: Enhancing Vision-Language Verification via Vision Experts and Iterative Training","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-07T00:33:56.965198Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2506.13888"},"observation_digest":"sha256:1b2a3c5ad0699f22665ceadfb2c674393e572f0e3b61df47af44ada03c153bc0","observation_id":"712e93bf-a4be-4b77-8f3b-32299be42706","resolution":{"observed_at":"2026-08-07T00:33:56.965198Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-06T19:57:28.485425Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.04189","last_updated":"2025-07-13T22:06:13Z","snapshot_observed_at":"2026-08-07T23:44:39.022597Z","submitted_at":"2025-07-05T23:46:35Z","title":"SymbolicThought: Integrating Language Models and Symbolic Reasoning for Consistent and Interpretable Human Relationship Understanding","version":2},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-06T19:57:28.485425Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2507.04189"},"observation_digest":"sha256:2826e3755e5922ffe0d8b10c4b497d9f131fdb6034ff90cd47f201ed08374297","observation_id":"39dc39f2-8afa-497d-ba50-1883950b60d0","resolution":{"observed_at":"2026-08-06T19:57:28.485425Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-06T18:51:30.462615Z","title":"Secrets of rlhf in large language models part ii: Reward modeling","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.07375","last_updated":"2025-07-10T01:56:56Z","snapshot_observed_at":"2026-08-08T11:06:36.428118Z","submitted_at":"2025-07-10T01:56:56Z","title":"Bradley-Terry and Multi-Objective Reward Modeling Are Complementary","version":1},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-06T18:51:30.462615Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2507.07375"},"observation_digest":"sha256:c8773e3cc2367c55d2462f9057480ad2d63600fc66465f14e971a67dee0500f4","observation_id":"27fdfeb5-0952-486b-a2b7-02ddb351d597","resolution":{"observed_at":"2026-08-06T18:51:30.462615Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-04T14:52:22.806003Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.22851","last_updated":"2026-07-02T18:42:52Z","snapshot_observed_at":"2026-08-09T07:25:07.650510Z","submitted_at":"2025-09-26T19:03:24Z","title":"Adaptive Margin RLHF via Preference over Preferences","version":4},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-04T14:52:22.806003Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2509.22851"},"observation_digest":"sha256:efc57c2810f8b30566400d6cdbdab5d9d646563ca226ecc3c215f133212335bc","observation_id":"6e2155e5-e488-4033-a71f-79949deb1a4c","resolution":{"observed_at":"2026-08-04T14:52:22.806003Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2510.13830","last_updated":"2026-05-09T20:37:45Z","snapshot_observed_at":"2026-08-08T06:12:13.417479Z","submitted_at":"2025-10-10T08:57:34Z","title":"Users as Annotators: LLM Preference Learning from Comparison Mode","version":2},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-05-18T08:19:58.093621Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2510.13830"},"observation_digest":"sha256:e22517fe8564082371e10d3ae9e2e0d014d9df7fcb7d5ee3a81cdfc77e693fe6","observation_id":"b1810d4c-e22a-4a9e-b1f4-565668a357f8","resolution":{"observed_at":"2026-05-18T08:21:07.012995Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2604.11446","last_updated":"2026-04-13T13:28:12Z","snapshot_observed_at":"2026-07-06T22:59:49.614780Z","submitted_at":"2026-04-13T13:28:12Z","title":"Low-rank Optimization Trajectories Modeling for LLM RLVR Acceleration","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-05-10T16:07:20.180349Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2604.11446"},"observation_digest":"sha256:0ecdeb12e6bd9d744bf23ddef8da395715f97b1c27459e09cbc734f604bb6448","observation_id":"b3c752b3-7b51-4c17-af23-0b4694293918","resolution":{"observed_at":"2026-05-11T09:16:04.419338Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2604.13602","last_updated":"2026-04-15T08:11:34Z","snapshot_observed_at":"2026-07-06T23:01:32.542040Z","submitted_at":"2026-04-15T08:11:34Z","title":"Reward Hacking in the Era of Large Models: Mechanisms, Emergent Misalignment, Challenges","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-05-10T13:58:53.430492Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2604.13602"},"observation_digest":"sha256:212a776125df4384015df8aec9a10b6e470a22e50eb828ebe39ac9c344fc8b48","observation_id":"ee674794-cfa2-4c6b-abdd-e3641071a541","resolution":{"observed_at":"2026-05-10T14:00:28.159008Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2604.14265","last_updated":"2026-04-15T17:12:56Z","snapshot_observed_at":"2026-07-06T23:02:05.116649Z","submitted_at":"2026-04-15T17:12:56Z","title":"Reinforcement Learning via Value Gradient Flow","version":1},"reference_index":66,"source":"arxiv_source","source_observed_at":"2026-05-10T13:18:16.532434Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2604.14265"},"observation_digest":"sha256:bfddc919bc2aa062fa243abd59071feb83e724784961a9b817d50696b6e9742b","observation_id":"d3ebbd09-47a7-412e-97bf-eac652a28814","resolution":{"observed_at":"2026-05-10T13:20:25.604855Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2604.19544","last_updated":"2026-04-21T15:02:50Z","snapshot_observed_at":"2026-07-06T23:06:12.542013Z","submitted_at":"2026-04-21T15:02:50Z","title":"DT2IT-MRM: Debiased Preference Construction and Iterative Training for Multimodal Reward Modeling","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-05-10T01:45:30.001398Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2604.19544"},"observation_digest":"sha256:fe6eab31f298c9835bbf8f9adfe452827ca619ff9165c19e73587bc46623efd7","observation_id":"8bdc2aa2-39d8-4cc2-a6a1-fa90e214838c","resolution":{"observed_at":"2026-05-11T13:26:03.869033Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2605.02626","last_updated":"2026-05-04T14:15:24Z","snapshot_observed_at":"2026-07-30T07:44:04.614445Z","submitted_at":"2026-05-04T14:15:24Z","title":"Gradient-Gated DPO: Stabilizing Preference Optimization in Language Models","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-05-08T18:35:13.659698Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2605.02626"},"observation_digest":"sha256:2136f331c2fe93c4483b673ec4c6c2f2f0d3bc94859ac1593143518ff8e7a138","observation_id":"3a406e49-0558-4235-92b8-ab3799e9c7d8","resolution":{"observed_at":"2026-05-09T06:20:41.718303Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2605.09119","last_updated":"2026-05-09T19:07:35Z","snapshot_observed_at":"2026-07-06T23:21:16.454273Z","submitted_at":"2026-05-09T19:07:35Z","title":"Personalized Alignment Revisited: The Necessity and Sufficiency of User Diversity","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-05-12T03:16:09.213612Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2605.09119"},"observation_digest":"sha256:c5e07a8cefce7226ab5a542860f2387149a2f9e67f7bab96603bfcc3b32b5f5a","observation_id":"9dcd4027-37e4-456c-b1e5-32c00cd4bbd3","resolution":{"observed_at":"2026-05-12T03:16:18.568531Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2605.11974","last_updated":"2026-05-12T11:31:18Z","snapshot_observed_at":"2026-07-06T23:23:44.276236Z","submitted_at":"2026-05-12T11:31:18Z","title":"Towards Order Fairness: Mitigating LLMs Order Sensitivity through Dual Group Advantage Optimization","version":1},"reference_index":47,"source":"arxiv_source","source_observed_at":"2026-05-13T07:32:58.404947Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2605.11974"},"observation_digest":"sha256:3084f804e4d2c58f4f2046b2a025a30c19d86a4106f1a84c80afe9a66afb0d97","observation_id":"9e9cfa9b-6513-47ce-a9ae-ac8768437ab2","resolution":{"observed_at":"2026-05-13T07:37:29.945673Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2605.15012","last_updated":"2026-05-14T16:12:30Z","snapshot_observed_at":"2026-07-06T23:26:23.179482Z","submitted_at":"2026-05-14T16:12:30Z","title":"Boosting Reinforcement Learning with Verifiable Rewards via Randomly Selected Few-Shot Guidance","version":1},"reference_index":79,"source":"pdf_text","source_observed_at":"2026-05-15T03:18:26.590871Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2605.15012"},"observation_digest":"sha256:9976061ce6027f07e16d373996aec7b3007888866ec4519978bab15812615a1c","observation_id":"90b60aec-fea5-48c8-a609-4e07d988c717","resolution":{"observed_at":"2026-05-15T03:19:43.017450Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2606.05054","last_updated":"2026-06-03T16:12:30Z","snapshot_observed_at":"2026-07-06T23:45:06.925235Z","submitted_at":"2026-06-03T16:12:30Z","title":"Boosting Self-Consistency with Ranking","version":1},"reference_index":185,"source":"arxiv_source","source_observed_at":"2026-06-28T06:49:58.051659Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2606.05054"},"observation_digest":"sha256:5dbe88eef038544ae1345544d6fd7f222b6e153ebae3e6c0463c355b194cf637","observation_id":"8fe3cacf-d709-4e12-97df-291176f3fe31","resolution":{"observed_at":"2026-06-28T06:51:44.296018Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2606.09043","last_updated":"2026-06-08T05:24:15Z","snapshot_observed_at":"2026-08-02T05:26:25.857871Z","submitted_at":"2026-06-08T05:24:15Z","title":"DynaCF: Mitigating Shortcut Learning in Reward Models via Dynamic Counterfactual Sensitivity","version":1},"reference_index":40,"source":"arxiv_source","source_observed_at":"2026-06-27T17:26:17.072017Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2606.09043"},"observation_digest":"sha256:c53caef785f90f68299fcfd2a937e3005efb974983f75ebfe759f6b8c39f3843","observation_id":"7b24150e-32d1-49a0-aa08-65e809dabaa6","resolution":{"observed_at":"2026-06-27T17:31:06.957727Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2606.09711","last_updated":"2026-06-08T16:32:54Z","snapshot_observed_at":"2026-07-06T23:49:03.237958Z","submitted_at":"2026-06-08T16:32:54Z","title":"Proxy Reward Internalization and Mechanistic Exploitation: A Learned Precursor to Reward Hacking and Its Generalization","version":1},"reference_index":115,"source":"arxiv_source","source_observed_at":"2026-06-27T16:26:34.918099Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2606.09711"},"observation_digest":"sha256:18ca5dd8f762f15caf81c1dd99a896c2fab42292de21df4a760ab47c4abf0901","observation_id":"bcf61e05-b703-4838-96f2-ca6ee50f3ef2","resolution":{"observed_at":"2026-07-03T01:37:30.515245Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2606.10528","last_updated":"2026-06-09T07:57:50Z","snapshot_observed_at":"2026-07-06T23:49:41.940954Z","submitted_at":"2026-06-09T07:57:50Z","title":"Representation-Aware Advantage Estimation: Your Reward Model Provides More Than A Scalar Output","version":1},"reference_index":37,"source":"arxiv_source","source_observed_at":"2026-06-27T13:39:17.701196Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2606.10528"},"observation_digest":"sha256:d2141ffd265fe3e6440ba97f0d33b78d545fc6027ba8ac0db2b596f42191358d","observation_id":"24da175c-92c6-47ff-9ec0-a4ccd107e1a7","resolution":{"observed_at":"2026-07-03T04:47:38.282385Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2606.13209","last_updated":"2026-06-11T11:19:03Z","snapshot_observed_at":"2026-08-07T06:04:00.877585Z","submitted_at":"2026-06-11T11:19:03Z","title":"Understanding helpfulness and harmless tension in reward models","version":1},"reference_index":39,"source":"arxiv_source","source_observed_at":"2026-06-27T07:06:23.680572Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2606.13209"},"observation_digest":"sha256:2d270c3a7233b8f180adcbd75b7127aeec30a284d94074e50d5451d235cd3fd1","observation_id":"84b2b752-56a8-4a70-ac76-0f528db5dc7a","resolution":{"observed_at":"2026-07-03T14:18:22.991762Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2606.21943","last_updated":"2026-06-20T08:20:41Z","snapshot_observed_at":"2026-07-06T23:56:54.959593Z","submitted_at":"2026-06-20T08:20:41Z","title":"Modularized Reinforcement Learning on LLMs: From MDP Creation to Exploration and Learning","version":1},"reference_index":210,"source":"pdf_text","source_observed_at":"2026-06-26T12:15:08.304150Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2606.21943"},"observation_digest":"sha256:ff8c2796051dace0b61ee2673cfaf8331b3bd43420ef76a2d3c7b68bab089d0e","observation_id":"33796558-f5e7-46a1-87c9-9ca4b3415e13","resolution":{"observed_at":"2026-07-04T07:59:40.105153Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":"2401.06080","doi":"10.48550/arxiv.2401.06080","metadata_source":"arxiv_reference","pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Secrets of RLHF in large language models part II: Reward modeling","venue":"arXiv (Cornell University)","work_id":"f37659bb-882a-46af-89a3-bf53a8c0969c","year":2024},"citing_paper":{"arxiv_id":"2606.28707","last_updated":"2026-06-27T03:25:53Z","snapshot_observed_at":"2026-07-07T00:02:47.924620Z","submitted_at":"2026-06-27T03:25:53Z","title":"BV-Blend: Uncertainty-Weighted Historical Baselines for Stable Critic-Free RL with Verifiable Rewards","version":1},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-06-30T10:07:39.554999Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2606.28707"},"observation_digest":"sha256:a22847c6b39bda1eed88ebbe81771a8a72f1fb009e952f83314a7c2797ddd2fd","observation_id":"90eb243c-e305-431b-99ab-b8b2084915af","resolution":{"observed_at":"2026-06-30T12:44:40.135832Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-07-14T15:37:01.391649Z","title":"Secrets of rlhf in large language models part ii: Reward modeling","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.09796","last_updated":"2026-07-20T03:41:16Z","snapshot_observed_at":"2026-08-08T03:21:32.168317Z","submitted_at":"2026-07-09T09:20:25Z","title":"Metadata-Free Meta-Reweighted Direct Preference Optimization under Noisy Preference Labels","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-07-14T15:37:01.391649Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2607.09796"},"observation_digest":"sha256:0525599fe7f9b77cc6eaeb5660a537392f5e06b5b80d814042c6d06f9b27e7cb","observation_id":"e39d2557-3329-4f19-bc1b-9b02bb77a51b","resolution":{"observed_at":"2026-07-14T15:37:01.391649Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-02T08:00:53.827430Z","title":"Secrets of rlhf in large language models part ii: Reward modeling,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.09796","last_updated":"2026-07-20T03:41:16Z","snapshot_observed_at":"2026-08-08T03:21:32.168317Z","submitted_at":"2026-07-09T09:20:25Z","title":"Metadata-Free Meta-Reweighted Direct Preference Optimization under Noisy Preference Labels","version":2},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-02T08:00:53.827430Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2607.09796"},"observation_digest":"sha256:cb21ce151690e90bc936e1556b667a31201a356af2b73b2ccbbfd109cc991724","observation_id":"e433b4e7-168e-4078-ad68-e57a0bc95423","resolution":{"observed_at":"2026-08-02T08:00:53.827430Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.06080","snapshot_observed_at":"2026-08-01T03:14:12.745471Z","title":"arXiv:2401.06080","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.26094","last_updated":"2026-07-28T02:09:26Z","snapshot_observed_at":"2026-08-05T20:24:58.318060Z","submitted_at":"2026-07-28T02:09:26Z","title":"Meta-Learned Reward Shaping for Reinforcement Learning from Human Feedback","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-01T03:14:12.745471Z"},"links":{"cited_paper":"/paper/2401.06080","citing_paper":"/paper/2607.26094"},"observation_digest":"sha256:2f6bd10d9408a20258f830640dd5b5b1509af2a386bd77c36b1e893454aa8b73","observation_id":"36eec2bb-fe64-48d6-9715-b767de896496","resolution":{"observed_at":"2026-08-01T03:14:12.745471Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2401.06080/citation-record","integrity":"/paper/2401.06080/integrity","json":"/paper/2401.06080/citation-record.json","paper":"/paper/2401.06080"},"outbound":[],"paper":{"arxiv_id":"2401.06080","last_updated":"2024-01-12T09:46:10Z","latest_version":2,"primary_category":"cs.AI","snapshot_observed_at":"2026-07-06T17:14:24.413664Z","submitted_at":"2024-01-11T17:56:59Z","title":"Secrets of RLHF in Large Language Models Part II: Reward Modeling"},"reference_resolution":{"displayed":0,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":0,"verified_exact":0,"verified_fuzzy":0},"total_outbound_references":0},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"thesis":"As of 9 August 2026, this Paper Citation Record lists 0 of 0 outbound references and 34 inbound Pith citation observations for arXiv:2401.06080."}