{"as_of":"2026-08-17T01:25:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:db049389fd1fbed364ddeed354cdce7c248698f6fec2867e2bc4912880ed48b4","coverage":[{"denominator":36,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":36,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-15T19:38:51.086103Z","state":"measured"},{"denominator":42,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":42,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-16T06:30:59.297886+00:00","state":"measured"},{"denominator":6,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":6,"source":"paper_references, paper_reference_links","source_observed_at":"2026-06-27T18:08:19.797150Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-07-04T13:29:50.941267Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"cited_work":{"arxiv_id":"2506.15421","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.15421","snapshot_observed_at":"2026-07-04T13:29:50.941267Z","title":"Reward models in deep reinforcement learning: A survey","venue":null,"work_id":"06baa68d-907b-4b0e-8ebe-783f606e1cbf","year":2025},"citing_paper":{"arxiv_id":"2604.15757","last_updated":"2026-04-17T07:01:24Z","snapshot_observed_at":"2026-08-15T00:44:09.007448Z","submitted_at":"2026-04-17T07:01:24Z","title":"Multi-objective Reinforcement Learning With Augmented States Requires Rewards After Deployment","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-05-10T09:38:05.280629Z"},"links":{"cited_paper":"/paper/2506.15421","citing_paper":"/paper/2604.15757"},"observation_digest":"sha256:b6f5946c1411b6e9c52cc292b28b56662187a804f51f02335122a9d4d0d5d149","observation_id":"9c30b6bc-215c-4a92-8b91-d9a93b451a39","resolution":{"observed_at":"2026-05-10T09:38:41.945064Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"cited_work":{"arxiv_id":"2506.15421","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.15421","snapshot_observed_at":"2026-07-04T13:29:50.941267Z","title":"Reward models in deep reinforcement learning: A survey","venue":null,"work_id":"06baa68d-907b-4b0e-8ebe-783f606e1cbf","year":2025},"citing_paper":{"arxiv_id":"2604.20627","last_updated":"2026-04-22T14:41:56Z","snapshot_observed_at":"2026-08-14T15:20:54.355766Z","submitted_at":"2026-04-22T14:41:56Z","title":"Occupancy Reward Shaping: Improving Credit Assignment for Offline Goal-Conditioned Reinforcement Learning","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-05-10T00:19:48.053466Z"},"links":{"cited_paper":"/paper/2506.15421","citing_paper":"/paper/2604.20627"},"observation_digest":"sha256:83c3154aa6aa5e9249064fd03499adaab3e886089d8fadca80ae6fcb4cf80d17","observation_id":"e53582ca-beba-4ed5-90ca-36d1efeb6910","resolution":{"observed_at":"2026-05-10T00:24:47.223690Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"cited_work":{"arxiv_id":"2506.15421","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.15421","snapshot_observed_at":"2026-07-04T13:29:50.941267Z","title":"Reward models in deep reinforcement learning: A survey","venue":null,"work_id":"06baa68d-907b-4b0e-8ebe-783f606e1cbf","year":2025},"citing_paper":{"arxiv_id":"2605.13276","last_updated":"2026-05-14T06:59:41Z","snapshot_observed_at":"2026-08-12T12:19:08.594635Z","submitted_at":"2026-05-13T09:54:31Z","title":"D-VLA: A High-Concurrency Distributed Asynchronous Reinforcement Learning Framework for Vision-Language-Action Models","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-05-14T19:45:26.458859Z"},"links":{"cited_paper":"/paper/2506.15421","citing_paper":"/paper/2605.13276"},"observation_digest":"sha256:9f37598b387fc89bda9d5d74e1060fe0fcef94542a382ca5a4683b2e3bfee390","observation_id":"79cea2e5-be26-43e8-9720-51304d27cddc","resolution":{"observed_at":"2026-05-14T19:47:53.425028Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"cited_work":{"arxiv_id":"2506.15421","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.15421","snapshot_observed_at":"2026-07-04T13:29:50.941267Z","title":"Reward models in deep reinforcement learning: A survey","venue":null,"work_id":"06baa68d-907b-4b0e-8ebe-783f606e1cbf","year":2025},"citing_paper":{"arxiv_id":"2605.13276","last_updated":"2026-05-14T06:59:41Z","snapshot_observed_at":"2026-08-12T12:19:08.594635Z","submitted_at":"2026-05-13T09:54:31Z","title":"D-VLA: A High-Concurrency Distributed Asynchronous Reinforcement Learning Framework for Vision-Language-Action Models","version":2},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-05-15T06:04:17.574622Z"},"links":{"cited_paper":"/paper/2506.15421","citing_paper":"/paper/2605.13276"},"observation_digest":"sha256:4f3e1f2bae62168021b0b94a6480f1fed7fa1e38df56321995ecd46179837b61","observation_id":"ee58a84b-58e0-4b21-80ad-9659fcf303f4","resolution":{"observed_at":"2026-05-15T06:05:06.483419Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"cited_work":{"arxiv_id":"2506.15421","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.15421","snapshot_observed_at":"2026-07-04T13:29:50.941267Z","title":"Reward models in deep reinforcement learning: A survey","venue":null,"work_id":"06baa68d-907b-4b0e-8ebe-783f606e1cbf","year":2025},"citing_paper":{"arxiv_id":"2606.09925","last_updated":"2026-06-07T12:24:18Z","snapshot_observed_at":"2026-08-02T22:00:53.121300Z","submitted_at":"2026-06-07T12:24:18Z","title":"AudioProcessBench: Benchmark for Identifying Process Errors in Audio-Grounded Reasoning","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-06-27T18:08:19.797150Z"},"links":{"cited_paper":"/paper/2506.15421","citing_paper":"/paper/2606.09925"},"observation_digest":"sha256:816d993ae333e82d61165c6902436edc6c76d23ea9448e8fffdfbb337933bf82","observation_id":"6c8a3fc2-9855-4232-b2f2-9bf323ba31e0","resolution":{"observed_at":"2026-07-02T23:37:27.332423Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"cited_work":{"arxiv_id":"2506.15421","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.15421","snapshot_observed_at":"2026-07-04T13:29:50.941267Z","title":"Reward models in deep reinforcement learning: A survey","venue":null,"work_id":"06baa68d-907b-4b0e-8ebe-783f606e1cbf","year":2025},"citing_paper":{"arxiv_id":"2606.26930","last_updated":"2026-06-25T12:05:40Z","snapshot_observed_at":"2026-08-16T22:08:02.834262Z","submitted_at":"2026-06-25T12:05:40Z","title":"PortraitGen: Exemplar-Driven GRPO with Dual-Reward Guidance for Photorealistic Portrait Generation","version":1},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-06-26T05:14:14.053344Z"},"links":{"cited_paper":"/paper/2506.15421","citing_paper":"/paper/2606.26930"},"observation_digest":"sha256:f3c732fffdd36e0f943f900346f33b89d7b9bf50341c223cee3dfc894b351712","observation_id":"ad523e39-1735-4f45-8802-1b24dff731b1","resolution":{"observed_at":"2026-07-04T13:29:50.942774Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2506.15421/citation-record","integrity":"/paper/2506.15421/integrity","json":"/paper/2506.15421/citation-record.json","paper":"/paper/2506.15421"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2312.09187","last_updated":"2024-07-12T21:14:32Z","snapshot_observed_at":"2026-08-16T14:34:40.156321Z","submitted_at":"2023-12-14T18:06:17Z","title":"Vision-Language Models as a Source of Rewards","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.09187","snapshot_observed_at":"2026-08-15T19:38:50.445678Z","title":"Vision-language models as a source of rewards.arXiv preprint arXiv:2312.09187,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.445678Z"},"links":{"cited_paper":"/paper/2312.09187","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:44eaac617e8a6b6f70d0eb102c1f8a950daabc2597bb310e19b9bd8c84fb25f1","observation_id":"49a2ba99-8f48-418d-a2b3-2d472e72612c","resolution":{"observed_at":"2026-08-15T19:38:50.445678Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1802.06070","last_updated":"2018-10-09T23:19:52Z","snapshot_observed_at":"2026-08-14T19:45:07.867570Z","submitted_at":"2018-02-16T18:57:57Z","title":"Diversity is All You Need: Learning Skills without a Reward Function","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1802.06070","snapshot_observed_at":"2026-08-15T19:38:50.461416Z","title":"Diversity is all you need: Learning skills without a re- ward function.arXiv preprint arXiv:1802.06070,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.461416Z"},"links":{"cited_paper":"/paper/1802.06070","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:c769bf4700e4428aa38a7a639763190d10e3f9e32c1e1e7e9356246050a06f3a","observation_id":"4b81b121-5b55-402d-b1e7-7a72d76bf2d1","resolution":{"observed_at":"2026-08-15T19:38:50.461416Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2006.13900","last_updated":"2021-03-17T21:54:55Z","snapshot_observed_at":"2026-08-16T13:18:39.507010Z","submitted_at":"2020-06-24T17:35:15Z","title":"Quantifying Differences in Reward Functions","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2006.13900","snapshot_observed_at":"2026-08-15T19:38:50.478051Z","title":"Quantifying differences in reward functions.arXiv preprint arXiv:2006.13900,","venue":null,"work_id":null,"year":2006},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.478051Z"},"links":{"cited_paper":"/paper/2006.13900","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:b936747e5c9a3a6542fa28829e4ea4bdd5b90b64e887717c277ac36ac1a8a09f","observation_id":"f002d8c3-1e2c-423d-8e3e-a11ad8568f40","resolution":{"observed_at":"2026-08-15T19:38:50.478051Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2203.13553","last_updated":"2022-03-25T10:19:35Z","snapshot_observed_at":"2026-08-16T17:35:48.500115Z","submitted_at":"2022-03-25T10:19:35Z","title":"Preprocessing Reward Functions for Interpretability","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2203.13553","snapshot_observed_at":"2026-08-15T19:38:50.544516Z","title":"Preprocessing reward functions for interpretability.arXiv preprint arXiv:2203.13553,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.544516Z"},"links":{"cited_paper":"/paper/2203.13553","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:a8e954db73c12de0f85cb772b8e7266041a40facf4da02ed2cca613b6b87ff5b","observation_id":"0bff0728-f114-4b9d-8edc-c33ad3690e8d","resolution":{"observed_at":"2026-08-15T19:38:50.544516Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2010.03691","last_updated":"2020-12-03T01:34:00Z","snapshot_observed_at":"2026-08-16T19:15:04.857580Z","submitted_at":"2020-10-07T23:38:47Z","title":"Regularized Inverse Reinforcement Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2010.03691","snapshot_observed_at":"2026-08-15T19:38:50.578945Z","title":"Regularized inverse rein- forcement learning.arXiv preprint arXiv:2010.03691,","venue":null,"work_id":null,"year":2010},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.578945Z"},"links":{"cited_paper":"/paper/2010.03691","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:baff6c608a72c0e71027fe38f7ad25e96e639d1c98e9d56a9348e667d67e59f0","observation_id":"f4a188ef-28ba-45b9-b6d5-9f3237367050","resolution":{"observed_at":"2026-08-15T19:38:50.578945Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:38:50.627819Z","title":"A survey of reinforcement learning from human feedback.arXiv preprint arXiv:2312.14925, 10,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.627819Z"},"links":{"citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:b596b72d8fc0326914f753fbb1f85259caa3b9d2b7098f0b8bb1d5c527723dd0","observation_id":"9270cc22-8d05-44ab-bba5-0b97572f10d2","resolution":{"observed_at":"2026-08-15T19:38:50.627819Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.00957","last_updated":"2023-03-02T04:24:29Z","snapshot_observed_at":"2026-08-16T15:51:31.299086Z","submitted_at":"2023-03-02T04:24:29Z","title":"Preference Transformer: Modeling Human Preferences using Transformers for RL","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.00957","snapshot_observed_at":"2026-08-15T19:38:50.639257Z","title":"Preference transformer: Modeling human preferences using transformers for rl.arXiv preprint arXiv:2303.00957,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.639257Z"},"links":{"cited_paper":"/paper/2303.00957","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:bbe15f0302b96ec92be9aecf841c2590e7f8683099e240d3d249d9c827b692d3","observation_id":"d6570f42-3b80-4095-b1cc-01223c13b89b","resolution":{"observed_at":"2026-08-15T19:38:50.639257Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:38:52.179406Z","title":"Empowerment: A universal agent-centric measure of control","venue":null,"work_id":"f854c006-2c05-4f3c-8bc4-e14e270231a4","year":2005},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.650866Z"},"links":{"citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:c47ee7882126d3100d064f7569ef903852ba635f9908ced0410cb30af256ebc0","observation_id":"dc08f9b9-f160-4338-a95e-482f090b333c","resolution":{"observed_at":"2026-08-15T19:38:52.185603Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2201.08299","last_updated":"2022-09-02T10:46:40Z","snapshot_observed_at":"2026-08-16T17:27:02.445517Z","submitted_at":"2022-01-20T17:06:42Z","title":"Goal-Conditioned Reinforcement Learning: Problems and Solutions","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2201.08299","snapshot_observed_at":"2026-08-15T19:38:50.662293Z","title":"Goal- conditioned reinforcement learning: Problems and solutions","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.662293Z"},"links":{"cited_paper":"/paper/2201.08299","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:5f5187d8ca26366f85e45fa5c53543ac058799ea0b478de3710e309a4be1499a","observation_id":"fb6b9eb2-7d20-4713-b7cb-c33da20962a7","resolution":{"observed_at":"2026-08-15T19:38:50.662293Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.12843","last_updated":"2024-11-19T20:17:04Z","snapshot_observed_at":"2026-08-17T01:00:48.700288Z","submitted_at":"2024-11-19T20:17:04Z","title":"Reward Modeling with Ordinal Feedback: Wisdom of the Crowd","version":1},"cited_work":{"arxiv_id":"2411.12843","doi":null,"metadata_source":"pith","pith_arxiv_id":"2411.12843","snapshot_observed_at":"2026-08-15T19:38:51.596823Z","title":"Reward Modeling with Ordinal Feedback: Wisdom of the Crowd","venue":"cs.LG","work_id":"5c54f993-3f8b-4aa0-a8df-40b2ecedb49e","year":2024},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.668075Z"},"links":{"cited_paper":"/paper/2411.12843","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:b7d24fed0b30bb9d2d8b992ed28c2ef9b014630f4de13c622cb6d6de511a1685","observation_id":"5144f0ef-b109-4920-a17b-a2d97aab60bc","resolution":{"observed_at":"2026-08-15T19:38:51.656840Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.08967","last_updated":"2024-12-13T04:44:11Z","snapshot_observed_at":"2026-08-16T14:26:54.639216Z","submitted_at":"2024-01-17T04:43:21Z","title":"ReFT: Reasoning with Reinforced Fine-Tuning","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.08967","snapshot_observed_at":"2026-08-15T19:38:50.679115Z","title":"Reft: Reasoning with reinforced fine-tuning","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.679115Z"},"links":{"cited_paper":"/paper/2401.08967","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:98476f9cf6b781b1a784aa19ab117198363ee2c367fb83fc1b89c9b414300e5c","observation_id":"a6529e8c-1975-4c4e-86d9-64d2cd39f9a1","resolution":{"observed_at":"2026-08-15T19:38:50.679115Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.13350","last_updated":"2024-01-19T17:33:36Z","snapshot_observed_at":"2026-08-16T16:13:43.873702Z","submitted_at":"2022-11-23T23:31:14Z","title":"Choreographer: Learning and Adapting Skills in Imagination","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.13350","snapshot_observed_at":"2026-08-15T19:38:50.698388Z","title":"Choreographer: Learning and adapting skills in imagination.arXiv preprint arXiv:2211.13350,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.698388Z"},"links":{"cited_paper":"/paper/2211.13350","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:7adb82c1831f17d2d9070b26b8510116c0eaaa855bb3926c5ae294814a411d89","observation_id":"4795901f-7eab-45bf-ab6e-6c5ea4aec581","resolution":{"observed_at":"2026-08-15T19:38:50.698388Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.02623","last_updated":"2025-01-16T08:18:01Z","snapshot_observed_at":"2026-08-16T13:02:59.359890Z","submitted_at":"2024-11-04T21:31:04Z","title":"Learning to Assist Humans without Inferring Rewards","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.02623","snapshot_observed_at":"2026-08-15T19:38:50.792909Z","title":"Learning to assist humans without inferring re- wards.arXiv preprint arXiv:2411.02623,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.792909Z"},"links":{"cited_paper":"/paper/2411.02623","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:7090859165d29807b6cc19e417b678636e399b868ac719dca5f3c92f245ae865","observation_id":"61063ea3-cb44-4b2e-9e8c-769bdd690a8a","resolution":{"observed_at":"2026-08-15T19:38:50.792909Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.08887","last_updated":"2024-03-10T04:30:17Z","snapshot_observed_at":"2026-08-16T14:52:27.542277Z","submitted_at":"2023-10-13T06:43:11Z","title":"METRA: Scalable Unsupervised RL with Metric-Aware Abstraction","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.08887","snapshot_observed_at":"2026-08-15T19:38:50.841430Z","title":"Metra: Scalable unsupervised rl with metric-aware abstraction.arXiv preprint arXiv:2310.08887,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.841430Z"},"links":{"cited_paper":"/paper/2310.08887","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:de1539e1ae509f72c2c910ea3039fd4da8361fe00c47e61b275df2f5159cd3ef","observation_id":"598ec36a-b1d1-4312-86f5-6eeb6144bcfb","resolution":{"observed_at":"2026-08-15T19:38:50.841430Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.12921","last_updated":"2024-03-14T12:16:00Z","snapshot_observed_at":"2026-08-16T14:50:37.995504Z","submitted_at":"2023-10-19T17:17:06Z","title":"Vision-Language Models are Zero-Shot Reward Models for Reinforcement Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.12921","snapshot_observed_at":"2026-08-15T19:38:50.847290Z","title":"Vision-language models are zero-shot reward models for reinforcement learning.arXiv preprint arXiv:2310.12921,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.847290Z"},"links":{"cited_paper":"/paper/2310.12921","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:c92abb785bda9fa489f086d9967c49ec296f2bf7f93173d7de9eef0421a3d123","observation_id":"b9c43ce1-7ec8-434d-99cf-8d878f4a37cb","resolution":{"observed_at":"2026-08-15T19:38:50.847290Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.15257","last_updated":"2024-12-12T15:29:41Z","snapshot_observed_at":"2026-08-16T14:57:22.692732Z","submitted_at":"2023-09-26T20:31:19Z","title":"STARC: A General Framework For Quantifying Differences Between Reward Functions","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.15257","snapshot_observed_at":"2026-08-15T19:38:50.852504Z","title":"Starc: A general frame- work for quantifying differences between reward functions.arXiv preprint arXiv:2309.15257,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.852504Z"},"links":{"cited_paper":"/paper/2309.15257","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:7efaf424e7fedeb3813f13484e38b911dd4c8f54de09f2ccb75613cc3e24e2f3","observation_id":"26e7ff2b-2682-45f3-8248-ac6aaa0779a6","resolution":{"observed_at":"2026-08-15T19:38:50.852504Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2307.09288","last_updated":"2023-07-19T17:08:59Z","snapshot_observed_at":"2026-08-07T12:56:43.323460Z","submitted_at":"2023-07-18T14:31:57Z","title":"Llama 2: Open Foundation and Fine-Tuned Chat Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.09288","snapshot_observed_at":"2026-08-15T19:38:50.858219Z","title":"Llama 2: Open foundation and fine-tuned chat models.arXiv preprint arXiv:2307.09288,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.858219Z"},"links":{"cited_paper":"/paper/2307.09288","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:dbe0b364ba061e4d26e84cbd4117283488ee03f51d51100479dabe7025ec8c2d","observation_id":"fd93ee8b-7900-4168-838c-4bfeb2a8f248","resolution":{"observed_at":"2026-08-15T19:38:50.858219Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.17032","last_updated":"2025-11-02T13:42:19Z","snapshot_observed_at":"2026-08-13T22:24:37.672685Z","submitted_at":"2024-07-24T06:35:05Z","title":"Gymnasium: A Standard Interface for Reinforcement Learning Environments","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.17032","snapshot_observed_at":"2026-08-15T19:38:50.863987Z","title":"Gymnasium: A standard interface for reinforcement learning environments","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.863987Z"},"links":{"cited_paper":"/paper/2407.17032","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:11f2834c3fea0df334d50ab5fc9720ac8d2846e3279ecc26fc1bcaa7ee522b2e","observation_id":"b80eeca1-34af-449e-9d5f-de88fe30a17a","resolution":{"observed_at":"2026-08-15T19:38:50.863987Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.08828","last_updated":"2024-04-12T21:59:42Z","snapshot_observed_at":"2026-08-16T14:01:18.453354Z","submitted_at":"2024-04-12T21:59:42Z","title":"Hindsight PRIORs for Reward Learning from Human Preferences","version":1},"cited_work":{"arxiv_id":"2404.08828","doi":null,"metadata_source":"pith","pith_arxiv_id":"2404.08828","snapshot_observed_at":"2026-08-15T19:38:51.348238Z","title":"Hindsight PRIORs for Reward Learning from Human Preferences","venue":"cs.LG","work_id":"2d23c5cf-cbda-4fcc-a15f-27ca45429ede","year":2024},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.869505Z"},"links":{"cited_paper":"/paper/2404.08828","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:edff94dbfc1983b72b7b3f02ed8a31fb01d496d482ccb15bac49433f40fe3471","observation_id":"199389b0-881b-4c00-9e30-6e75e105ec4a","resolution":{"observed_at":"2026-08-15T19:38:51.407764Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2309.11489","last_updated":"2024-05-25T06:42:10Z","snapshot_observed_at":"2026-08-16T14:59:02.373142Z","submitted_at":"2023-09-20T17:39:13Z","title":"Text2Reward: Reward Shaping with Language Models for Reinforcement Learning","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.11489","snapshot_observed_at":"2026-08-15T19:38:51.030036Z","title":"Text2reward: Auto- mated dense reward function generation for reinforcement learn- ing.arXiv preprint arXiv:2309.11489,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:51.030036Z"},"links":{"cited_paper":"/paper/2309.11489","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:3735774ca20d8e38033bc01e254ea67aee90d7374d9b6bf4b05237b27c1d7787","observation_id":"bea44e5a-ebf5-4d1a-a72c-bffd2ba218a8","resolution":{"observed_at":"2026-08-15T19:38:51.030036Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.08152","last_updated":"2024-08-15T13:40:03Z","snapshot_observed_at":"2026-08-16T13:26:15.351308Z","submitted_at":"2024-08-15T13:40:03Z","title":"DeepSeek-Prover-V1.5: Harnessing Proof Assistant Feedback for Reinforcement Learning and Monte-Carlo Tree Search","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.08152","snapshot_observed_at":"2026-08-15T19:38:51.075704Z","title":"Deepseek-prover-v1","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:51.075704Z"},"links":{"cited_paper":"/paper/2408.08152","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:bc4e6ecb14787b87063a399263f774abe3dcbf5862c2f1e5da9f6c5119ec2784","observation_id":"81ce2191-54d4-409a-8c40-53578b8eaa83","resolution":{"observed_at":"2026-08-15T19:38:51.075704Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.15240","last_updated":"2025-02-22T10:21:46Z","snapshot_observed_at":"2026-08-16T13:23:25.049278Z","submitted_at":"2024-08-27T17:57:45Z","title":"Generative Verifiers: Reward Modeling as Next-Token Prediction","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.15240","snapshot_observed_at":"2026-08-15T19:38:51.080835Z","title":"Generative verifiers: Reward modeling as next-token prediction.arXiv preprint arXiv:2408.15240,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:51.080835Z"},"links":{"cited_paper":"/paper/2408.15240","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:fce5bc2003d8510360c3029b5507ac02740598030b9b3b48b2e7d258274444ad","observation_id":"9dbf7c04-eef8-4c1e-83be-43a1cee1455f","resolution":{"observed_at":"2026-08-15T19:38:51.080835Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T19:38:52.161008Z","title":"Maximum entropy inverse reinforcement learning","venue":null,"work_id":"9f2d9279-a176-4eb5-8617-0e1ad0c7443e","year":2008},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:51.086103Z"},"links":{"citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:c17b3c886063548d2e73893fe5c16201cb0c9887cd1f1bc7b05561fbc7af0d53","observation_id":"82014311-ddbe-4e41-8db7-7b212b576b57","resolution":{"observed_at":"2026-08-15T19:38:52.167651Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1907.08225","last_updated":"2020-02-14T10:16:54Z","snapshot_observed_at":"2026-08-14T16:04:34.670855Z","submitted_at":"2019-07-18T18:07:47Z","title":"Dynamical Distance Learning for Semi-Supervised and Unsupervised Skill Discovery","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1907.08225","snapshot_observed_at":"2026-08-15T19:38:50.489300Z","title":"Dynamical distance learning for semi-supervised and unsupervised skill discovery.arXiv preprint arXiv:1907.08225,","venue":null,"work_id":null,"year":1907},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":1950,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.489300Z"},"links":{"cited_paper":"/paper/1907.08225","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:244ed455953888bbebbdc91a4a1472ff2aa13e9c60e5e9abbb9d0353831755b5","observation_id":"b88bbc5f-199d-4b32-961a-8cb31b19f2c7","resolution":{"observed_at":"2026-08-15T19:38:50.489300Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1810.12894","last_updated":"2018-10-30T17:44:42Z","snapshot_observed_at":"2026-08-14T18:06:54.993797Z","submitted_at":"2018-10-30T17:44:42Z","title":"Exploration by Random Network Distillation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1810.12894","snapshot_observed_at":"2026-08-15T19:38:50.450710Z","title":"Exploration by random network distillation.arXiv preprint arXiv:1810.12894,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":1952,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.450710Z"},"links":{"cited_paper":"/paper/1810.12894","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:e5652f80718593a425af4b0a92287c37e918f71e2720e3645e0ec6d398e79442","observation_id":"e4c7d4f9-2d29-479c-9d14-f6b860535c79","resolution":{"observed_at":"2026-08-15T19:38:50.450710Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.11602","last_updated":"2022-11-21T16:00:31Z","snapshot_observed_at":"2026-08-16T17:36:31.228792Z","submitted_at":"2022-11-21T16:00:31Z","title":"Improving Multimodal Interactive Agents with Reinforcement Learning from Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.11602","snapshot_observed_at":"2026-08-15T19:38:50.410219Z","title":"Improving multimodal interactive agents with reinforcement learning from human feedback.arXiv preprint arXiv:2211.11602,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":2004,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.410219Z"},"links":{"cited_paper":"/paper/2211.11602","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:53f99b35198af59ba3f535c38bb4fcc7aafe09a36f44f0d30449cf85f2ca7576","observation_id":"ebb72092-549a-4dec-84b7-b23e74c8330b","resolution":{"observed_at":"2026-08-15T19:38:50.410219Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2206.02231","last_updated":"2023-09-06T21:13:28Z","snapshot_observed_at":"2026-08-16T17:35:08.472912Z","submitted_at":"2022-06-05T17:58:02Z","title":"Models of human preference for learning reward functions","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2206.02231","snapshot_observed_at":"2026-08-15T19:38:50.656340Z","title":"Models of hu- man preference for learning reward functions.arXiv preprint arXiv:2206.02231,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":2005,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.656340Z"},"links":{"cited_paper":"/paper/2206.02231","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:b74ad30a4d4d6a0769715f3644560853f807f71ca0dd454f42112145f5143003","observation_id":"baff5cc1-6d0e-4bf8-adc7-3bcd9acfebef","resolution":{"observed_at":"2026-08-15T19:38:50.656340Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.12948","last_updated":"2026-01-04T03:57:36Z","snapshot_observed_at":"2026-08-15T12:33:55.451951Z","submitted_at":"2025-01-22T15:19:35Z","title":"DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.12948","snapshot_observed_at":"2026-08-15T19:38:50.483634Z","title":"Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning.arXiv preprint arXiv:2501.12948,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":2011,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.483634Z"},"links":{"cited_paper":"/paper/2501.12948","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:04df13f23de2bd8e48fe443126d0c95acc89e04ed5ad91c876f01dbeea239344","observation_id":"d8b3e62f-02a8-49eb-b66c-a8dc62d1b0ee","resolution":{"observed_at":"2026-08-15T19:38:50.483634Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1710.11248","last_updated":"2018-08-13T18:33:24Z","snapshot_observed_at":"2026-08-14T20:18:41.596978Z","submitted_at":"2017-10-30T21:22:28Z","title":"Learning Robust Rewards with Adversarial Inverse Reinforcement Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1710.11248","snapshot_observed_at":"2026-08-15T19:38:50.473011Z","title":"Learning robust rewards with adversarial inverse reinforcement learning.arXiv preprint arXiv:1710.11248,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":2016,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.473011Z"},"links":{"cited_paper":"/paper/1710.11248","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:c1f0c80de541e63cd2760501612b7c3483c0f36707b7a5ae205399157398c1c8","observation_id":"875992f6-9b86-4dba-a3c3-852304b4c120","resolution":{"observed_at":"2026-08-15T19:38:50.473011Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.12773","last_updated":"2023-10-19T14:22:03Z","snapshot_observed_at":"2026-08-14T19:10:00.850271Z","submitted_at":"2023-10-19T14:22:03Z","title":"Safe RLHF: Safe Reinforcement Learning from Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.12773","snapshot_observed_at":"2026-08-15T19:38:50.456084Z","title":"Safe rlhf: Safe re- inforcement learning from human feedback.arXiv preprint arXiv:2310.12773,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":2017,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.456084Z"},"links":{"cited_paper":"/paper/2310.12773","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:3bd9cc2d4a8e6e906839027f9f8b1f175dd9cd93d1937620f778c78472fc9627","observation_id":"d5b5aa4f-bf1a-44fe-8b45-54f78a626e19","resolution":{"observed_at":"2026-08-15T19:38:50.456084Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2212.08073","last_updated":"2022-12-15T06:19:23Z","snapshot_observed_at":"2026-08-16T03:49:00.703994Z","submitted_at":"2022-12-15T06:19:23Z","title":"Constitutional AI: Harmlessness from AI Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2212.08073","snapshot_observed_at":"2026-08-15T19:38:50.439486Z","title":"Constitutional ai: Harm- lessness from ai feedback.arXiv preprint arXiv:2212.08073,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":2020,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.439486Z"},"links":{"cited_paper":"/paper/2212.08073","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:1b696932ef89ce4953ab167224fa34347ee23a17ffd9c9bec75be8ca758c8bfc","observation_id":"53580f90-e529-44f7-a411-577256bbb502","resolution":{"observed_at":"2026-08-15T19:38:50.439486Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2002.06038","last_updated":"2020-02-14T13:57:22Z","snapshot_observed_at":"2026-08-16T00:01:17.496734Z","submitted_at":"2020-02-14T13:57:22Z","title":"Never Give Up: Learning Directed Exploration Strategies","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2002.06038","snapshot_observed_at":"2026-08-15T19:38:50.433306Z","title":"Never give up: Learning directed exploration strategies.arXiv preprint arXiv:2002.06038,","venue":null,"work_id":null,"year":2002},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":2021,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.433306Z"},"links":{"cited_paper":"/paper/2002.06038","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:adfcc1c3b2704f52fa5bea50b04b714fc96cd16e189f8bc08684106db8fa1617","observation_id":"64e3677c-cd60-4840-8c23-e9dacf260f04","resolution":{"observed_at":"2026-08-15T19:38:50.433306Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1611.03852","last_updated":"2016-11-25T08:09:55Z","snapshot_observed_at":"2026-08-14T21:30:48.662143Z","submitted_at":"2016-11-11T20:53:45Z","title":"A Connection between Generative Adversarial Networks, Inverse Reinforcement Learning, and Energy-Based Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1611.03852","snapshot_observed_at":"2026-08-15T19:38:50.467651Z","title":"A connection between generative adversarial networks, inverse reinforcement learning, and energy-based models.arXiv preprint arXiv:1611.03852,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":2022,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.467651Z"},"links":{"cited_paper":"/paper/1611.03852","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:bb11e9af96a1a3c645eb687568a84056280839ced9eb3c2575ff16b8db6b93cb","observation_id":"1b420d97-7414-4ab5-a1da-b8a05c7f42bc","resolution":{"observed_at":"2026-08-15T19:38:50.467651Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.00166","last_updated":"2023-09-29T22:10:01Z","snapshot_observed_at":"2026-08-16T14:56:21.668991Z","submitted_at":"2023-09-29T22:10:01Z","title":"Motif: Intrinsic Motivation from Artificial Intelligence Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.00166","snapshot_observed_at":"2026-08-15T19:38:50.645727Z","title":"Motif: Intrinsic motivation from artificial intelli- gence feedback.arXiv preprint arXiv:2310.00166,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":2023,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.645727Z"},"links":{"cited_paper":"/paper/2310.00166","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:3703ac7a32a4c46ed00f8a11bc0fd5298b6acbb12b57c13f5e2a2b3beb725f9f","observation_id":"1be62d4a-6672-4734-b03a-985e95e42ca8","resolution":{"observed_at":"2026-08-15T19:38:50.645727Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.01878","last_updated":"2025-01-24T19:13:34Z","snapshot_observed_at":"2026-08-16T14:22:00.944616Z","submitted_at":"2024-02-02T20:08:10Z","title":"LiPO: Listwise Preference Optimization through Learning-to-Rank","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.01878","snapshot_observed_at":"2026-08-15T19:38:50.674235Z","title":"Lipo: Listwise preference optimiza- tion through learning-to-rank.arXiv preprint arXiv:2402.01878,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.674235Z"},"links":{"cited_paper":"/paper/2402.01878","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:c149f91fc328811720fd4f10135d0d646488e9f2a501008269f4948a580f49aa","observation_id":"2fe16626-45d2-4e09-a520-a493d26b41fe","resolution":{"observed_at":"2026-08-15T19:38:50.674235Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2201.10081","last_updated":"2022-01-25T03:48:00Z","snapshot_observed_at":"2026-08-16T17:26:13.400175Z","submitted_at":"2022-01-25T03:48:00Z","title":"Dynamics-Aware Comparison of Learned Reward Functions","version":1},"cited_work":{"arxiv_id":"2201.10081","doi":null,"metadata_source":"pith","pith_arxiv_id":"2201.10081","snapshot_observed_at":"2026-08-15T19:38:51.197671Z","title":"Dynamics-Aware Comparison of Learned Reward Functions","venue":"cs.LG","work_id":"0a3dc8b9-dd10-4ad3-a3e0-5975cae5195a","year":2022},"citing_paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey","version":1},"reference_index":2025,"source":"pdf_text","source_observed_at":"2026-08-15T19:38:50.959942Z"},"links":{"cited_paper":"/paper/2201.10081","citing_paper":"/paper/2506.15421"},"observation_digest":"sha256:52c42b7e4d7fb467f21520636f7340a90bd6b7e8486b1f35f90794c5574edde9","observation_id":"5f0cf442-4de2-42dd-a4a1-b515064d66f3","resolution":{"observed_at":"2026-08-15T19:38:51.286313Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-16T06:30:59.297886+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2506.15421","last_updated":"2025-06-18T12:46:39Z","latest_version":1,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-16T17:35:22.013765Z","submitted_at":"2025-06-18T12:46:39Z","title":"Reward Models in Deep Reinforcement Learning: A Survey"},"reference_resolution":{"displayed":36,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":31,"verified_exact":3,"verified_fuzzy":2},"total_outbound_references":36},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-16T06:30:59.297886+00:00","source":"crossref"},{"observed_at":"2026-08-16T06:30:54.164669+00:00","source":"retraction_watch"}],"thesis":"As of 17 August 2026, this Paper Citation Record lists 36 of 36 outbound references and 6 inbound Pith citation observations for arXiv:2506.15421."}