{"as_of":"2026-08-19T18:57:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:3f546bac15a872fda8028e07b1e0158f8b45746fe0e89f2fb798355073549f13","coverage":[{"denominator":58,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":58,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-15T21:10:57.673384Z","state":"measured"},{"denominator":61,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":61,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-19T06:32:44.657259+00:00","state":"measured"},{"denominator":3,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":3,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-03T15:55:45.589124Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-07-04T07:59:40.634737Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"cited_work":{"arxiv_id":"2505.11081","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2505.11081","snapshot_observed_at":"2026-07-04T07:59:40.634737Z","title":"D., Tarassov, E., Pietquin, O., Richemond, P","venue":null,"work_id":"0c6e680e-f6a0-45dc-8a32-ea62c9b81b62","year":2025},"citing_paper":{"arxiv_id":"2512.15605","last_updated":"2026-05-25T15:54:35Z","snapshot_observed_at":"2026-08-16T11:27:16.123116Z","submitted_at":"2025-12-17T17:14:26Z","title":"Autoregressive Language Models are Secretly Energy-Based Models: Insights into the Lookahead Capabilities of Next-Token Prediction","version":3},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-05-16T21:28:20.658539Z"},"links":{"cited_paper":"/paper/2505.11081","citing_paper":"/paper/2512.15605"},"observation_digest":"sha256:1e5e729b02f830aae27091fd9490d9b4b78ea6df4ed180aff683fa5cbd333f6f","observation_id":"93209482-94e8-4983-a010-6f5035bca00a","resolution":{"observed_at":"2026-05-16T21:28:33.501043Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2505.11081","snapshot_observed_at":"2026-08-03T15:55:45.589124Z","title":"D., Tarassov, E., Pietquin, O., Richemond, P","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2512.15605","last_updated":"2026-05-25T15:54:35Z","snapshot_observed_at":"2026-08-16T11:27:16.123116Z","submitted_at":"2025-12-17T17:14:26Z","title":"Autoregressive Language Models are Secretly Energy-Based Models: Insights into the Lookahead Capabilities of Next-Token Prediction","version":4},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-03T15:55:45.589124Z"},"links":{"cited_paper":"/paper/2505.11081","citing_paper":"/paper/2512.15605"},"observation_digest":"sha256:330493222c4a2b1514141e93ea9cc3948866b790b05d9119b391a26d26382c30","observation_id":"97d5248d-0f2e-4647-ad0c-b1edf88dc07f","resolution":{"observed_at":"2026-08-03T15:55:45.589124Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"cited_work":{"arxiv_id":"2505.11081","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2505.11081","snapshot_observed_at":"2026-07-04T07:59:40.634737Z","title":"D., Tarassov, E., Pietquin, O., Richemond, P","venue":null,"work_id":"0c6e680e-f6a0-45dc-8a32-ea62c9b81b62","year":2025},"citing_paper":{"arxiv_id":"2606.21943","last_updated":"2026-06-20T08:20:41Z","snapshot_observed_at":"2026-08-15T21:55:25.625601Z","submitted_at":"2026-06-20T08:20:41Z","title":"Modularized Reinforcement Learning on LLMs: From MDP Creation to Exploration and Learning","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-06-26T12:15:08.304150Z"},"links":{"cited_paper":"/paper/2505.11081","citing_paper":"/paper/2606.21943"},"observation_digest":"sha256:c8a39133da84a7d78925875066c3818c2d8bc0af8d8df26d8a09e04e336dbbf9","observation_id":"b304f723-78b5-4404-8aac-290f8627d5f8","resolution":{"observed_at":"2026-07-04T07:59:40.636098Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2505.11081/citation-record","integrity":"/paper/2505.11081/integrity","json":"/paper/2505.11081/citation-record.json","paper":"/paper/2505.11081"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.455667Z","title":"A learning algorithm for boltzmann machines.Cognitive science, 9(1):147–169, 1985","venue":null,"work_id":null,"year":1985},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.455667Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:a9f52afcc1925fd7710927a430bb5663d57202c048e077e6da95c781f25e0633","observation_id":"96bc527e-1924-4ed5-899f-78b34614b53e","resolution":{"observed_at":"2026-08-15T21:10:57.455667Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.14740","last_updated":"2024-02-26T18:26:25Z","snapshot_observed_at":"2026-08-09T14:30:33.899591Z","submitted_at":"2024-02-22T17:52:34Z","title":"Back to Basics: Revisiting REINFORCE Style Optimization for Learning from Human Feedback in LLMs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.14740","snapshot_observed_at":"2026-08-15T21:10:57.460970Z","title":"Back to basics: Revisiting reinforce style optimization for learning from human feedback in llms.arXiv preprint arXiv:2402.14740, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.460970Z"},"links":{"cited_paper":"/paper/2402.14740","citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:99bd8912ccf40e4793a7da3d38434a6cfe0bd006b83bd448fad0fcbf9accdc96","observation_id":"022836f6-d4e3-417e-bedf-3e606660425c","resolution":{"observed_at":"2026-08-15T21:10:57.460970Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.295557Z","title":"A general theoretical paradigm to understand learning from human preferences","venue":null,"work_id":"043430ff-ad3f-4a2a-bfcb-2cd3e84274c7","year":2024},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.465428Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:392c0c6899490e0860fdec71ff9cd3da6dc841aa9ffcf588857768f392153569","observation_id":"b78145be-ec0b-4216-ad7d-683137e28b51","resolution":{"observed_at":"2026-08-15T21:10:58.300968Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2204.05862","last_updated":"2022-04-12T15:02:38Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-04-12T15:02:38Z","title":"Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2204.05862","snapshot_observed_at":"2026-08-15T21:10:57.469525Z","title":"Training a helpful and harmless assistant with reinforcement learning from human feedback.arXiv preprint arXiv:2204.05862, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.469525Z"},"links":{"cited_paper":"/paper/2204.05862","citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:fb7a481e8586a05b269c2b9752daa7cdf9a34cf82948029721e049727f026030","observation_id":"3055a8d9-9ab5-4f18-a28d-9fe7d258d290","resolution":{"observed_at":"2026-08-15T21:10:57.469525Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.473289Z","title":"Residual algorithms: Reinforcement learning with function approximation","venue":null,"work_id":null,"year":1995},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.473289Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:c1fbb93ddbb2f5d4a0edccb58694ecdfc06ec4c7e9dc393d94114ddbf409d751","observation_id":"fb6a178d-74de-4e94-9dd6-e05e1e01a526","resolution":{"observed_at":"2026-08-15T21:10:57.473289Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.277691Z","title":"Linear least-squares algorithms for temporal difference learning","venue":null,"work_id":"4558d74d-c3a5-4718-807d-032c40a56b9d","year":1996},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.477125Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:094c535eed57183666b2281426c8c2d76962f4683c9c4b478856bafee72c63a5","observation_id":"bbbc3035-ec37-4dd1-8425-63e714c4e2a6","resolution":{"observed_at":"2026-08-15T21:10:58.281123Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.481630Z","title":"Deep reinforcement learning from human preferences.Advances in neural information processing systems, 30, 2017","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.481630Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:076b79cfa2e90bd63c2c9a8f4dfbd794a7d1ce4e609071b149c2b1efb99b11db","observation_id":"a1108bc0-169c-41a0-854e-0abcb65ef2ba","resolution":{"observed_at":"2026-08-15T21:10:57.481630Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2504.00698","last_updated":"2025-04-14T12:37:51Z","snapshot_observed_at":"2026-08-17T05:47:31.276123Z","submitted_at":"2025-04-01T12:08:07Z","title":"Command A: An Enterprise-Ready Large Language Model","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2504.00698","snapshot_observed_at":"2026-08-15T21:10:57.485815Z","title":"Command a: An enterprise-ready large language model.arXiv preprint arXiv:2504.00698, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.485815Z"},"links":{"cited_paper":"/paper/2504.00698","citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:97a52c643f8bf9079fc8a84a2d70cc46ac4ebcb83b12b4c2bf63e48a4cdfe3b2","observation_id":"baf27859-184c-430f-851d-a484ccfc99ed","resolution":{"observed_at":"2026-08-15T21:10:57.485815Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.489789Z","title":"Ultrafeedback: Boosting language models with high-quality feedback","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.489789Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:96aa3ba44e93b99fd5f658cd0a9f8cf762c576bb65d493a87a049cf52f85a3c5","observation_id":"82fcb595-98d5-40a0-bef5-ac256bebfadd","resolution":{"observed_at":"2026-08-15T21:10:57.489789Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.252855Z","title":"Off-policy actor-critic","venue":null,"work_id":"701bfc0d-08e1-4a58-848a-02656b05677f","year":2012},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.493246Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:bfe329f272fa0bfb0c3b7ffe7084c0deacad15fc6c89bdf84a92875219329ba3","observation_id":"39953d5b-35fd-4628-af56-85097a7b5ccb","resolution":{"observed_at":"2026-08-15T21:10:58.257084Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.01306","last_updated":"2024-11-19T18:12:45Z","snapshot_observed_at":"2026-08-17T15:29:47.883677Z","submitted_at":"2024-02-02T10:53:36Z","title":"KTO: Model Alignment as Prospect Theoretic Optimization","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.01306","snapshot_observed_at":"2026-08-15T21:10:57.496903Z","title":"Kto: Model alignment as prospect theoretic optimization.arXiv preprint arXiv:2402.01306, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.496903Z"},"links":{"cited_paper":"/paper/2402.01306","citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:f6d00e54be955c8854e43428b10880969f1cbc2d4fe5e0f96ca577879a76a723","observation_id":"1b13d439-6bce-4319-af00-4e9ba47dddf9","resolution":{"observed_at":"2026-08-15T21:10:57.496903Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.240398Z","title":"Contrastive policy gradient: Aligning LLMs on sequence-level scores in a supervised-friendly fashion","venue":null,"work_id":"4d82b0f8-8c25-4e91-80bb-3b5d0cdf8a06","year":2024},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.501190Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:36683af06ff98360be97474272b39affb1d7269f4eff78305a348763d86f5876","observation_id":"23227b70-862b-4e9c-a2cc-d948c9e3472e","resolution":{"observed_at":"2026-08-15T21:10:58.244639Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.504944Z","title":"Addressing function approximation error in actor-critic methods","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.504944Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:ba6f5f8479d57937e0c4126c6e9c677087a44124da2bffd1acf839b76c04a2ef","observation_id":"c1a4af39-ee1d-404d-ba1b-9a38153ca2c2","resolution":{"observed_at":"2026-08-15T21:10:57.504944Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.221413Z","title":"Is the bellman residual a bad proxy?Advances in Neural Information Processing Systems, 30, 2017","venue":null,"work_id":"c2d83547-d6a4-4108-8f9c-41df4d03f86c","year":2017},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.508495Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:fdc11efcc87434d1c59fb331ed75bbd49278d4e7f59e988d4dfa91010482bf48","observation_id":"03cc769b-3d6b-4332-be6e-40c227e85f9e","resolution":{"observed_at":"2026-08-15T21:10:58.225965Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.209325Z","title":"A theory of regularized markov decision processes","venue":null,"work_id":"8f931d7b-284b-4d59-be32-31a63266b7c9","year":2019},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.511648Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:9b0cf7910e542e9b3545d45677cb00619a41a826e74e44c16c631c199bd19e66","observation_id":"a9ab6012-21e4-402a-945d-2ee1c13f9281","resolution":{"observed_at":"2026-08-15T21:10:58.213433Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.19188","last_updated":"2024-06-27T14:07:38Z","snapshot_observed_at":"2026-08-19T04:42:51.609754Z","submitted_at":"2024-06-27T14:07:38Z","title":"Averaging log-likelihoods in direct alignment","version":1},"cited_work":{"arxiv_id":"2406.19188","doi":null,"metadata_source":"pith","pith_arxiv_id":"2406.19188","snapshot_observed_at":"2026-08-15T21:10:57.837237Z","title":"Averaging log-likelihoods in direct alignment","venue":"cs.LG","work_id":"e6bf71d2-f203-47ec-9438-d7300eab62f9","year":2024},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.515108Z"},"links":{"cited_paper":"/paper/2406.19188","citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:6798e16944f15e5311de7936e4e3af4fb76445e6f9b2c1559646dc81f71264e4","observation_id":"926f8381-b10b-47f2-b092-ae0fc4c8be2c","resolution":{"observed_at":"2026-08-15T21:10:57.843680Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.197981Z","title":"Efficient (soft) q-learning for text generation with limited good data","venue":null,"work_id":"6a792bba-2265-4f80-91d2-1fe6bf43619b","year":2022},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.518966Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:2082811e591083802e06852c5582414abc96d5ecc30b51e1074a25c46b9fc1cf","observation_id":"7841342c-d83f-4e51-8577-1a5e7aff03a6","resolution":{"observed_at":"2026-08-15T21:10:58.202042Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.185998Z","title":"Rainbow: Combining improvements in deep reinforcement learning","venue":null,"work_id":"2a824422-c645-4cd3-a56b-2b4faa4e162b","year":2018},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.522717Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:0d6a87a9759ba5fe102f39672bff3ae5a47ec1c3ebd0a8c22fd729e1385eaf59","observation_id":"c8d8304b-f413-4196-b281-102636062b26","resolution":{"observed_at":"2026-08-15T21:10:58.190200Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1904.09751","last_updated":"2020-02-14T21:56:30Z","snapshot_observed_at":"2026-07-06T07:47:32.745963Z","submitted_at":"2019-04-22T07:17:18Z","title":"The Curious Case of Neural Text Degeneration","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1904.09751","snapshot_observed_at":"2026-08-15T21:10:57.525931Z","title":"The curious case of neural text degeneration.arXiv preprint arXiv:1904.09751, 2019","venue":null,"work_id":null,"year":1904},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.525931Z"},"links":{"cited_paper":"/paper/1904.09751","citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:b865c437cecaeaa7c41ba9f38cc894e14274b9655e8c5e094a93f5d05fb09a74","observation_id":"8f9ae8ed-8cc0-4579-98b0-39027f78708e","resolution":{"observed_at":"2026-08-15T21:10:57.525931Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.05193","last_updated":"2024-11-27T00:05:44Z","snapshot_observed_at":"2026-08-16T13:01:55.976106Z","submitted_at":"2024-11-07T21:36:52Z","title":"Q-SFT: Q-Learning for Language Models via Supervised Fine-Tuning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.05193","snapshot_observed_at":"2026-08-15T21:10:57.529911Z","title":"Q-sft: Q-learning for language models via supervised fine-tuning.arXiv preprint arXiv:2411.05193, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.529911Z"},"links":{"cited_paper":"/paper/2411.05193","citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:1fdb61a4a7f32e26a52939b07210b791b189260773a209a52f10ac3090766a99","observation_id":"a5e731ea-e33b-4d1a-b2ac-aa81f4b0a581","resolution":{"observed_at":"2026-08-15T21:10:57.529911Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.09302","last_updated":"2025-02-11T01:46:35Z","snapshot_observed_at":"2026-08-18T16:37:15.346928Z","submitted_at":"2024-10-11T23:29:20Z","title":"Enhancing Multi-Step Reasoning Abilities of Language Models through Direct Q-Function Optimization","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.09302","snapshot_observed_at":"2026-08-15T21:10:57.533374Z","title":"Enhancing multi-step reasoning abilities of language models through direct q-function optimization.arXiv preprint arXiv:2410.09302, 2024","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.533374Z"},"links":{"cited_paper":"/paper/2410.09302","citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:53db5a0ac468d8e0367646ecc749927a8ad91f38d640de17a6ba66a81a6b5865","observation_id":"7e8b128c-b30f-4025-8bfc-aca1ec19eb6b","resolution":{"observed_at":"2026-08-15T21:10:57.533374Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.173921Z","title":"Buy 4 reinforce samples, get a baseline for free! In Deep Reinforcement Learning Meets Structured Prediction (ICLR Workshop), 2019","venue":null,"work_id":"12fbd91e-eafa-48c9-adbf-efca6be4707c","year":2019},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.537151Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:5199fed86eaf3387e4704886692a14e3ed148475478240e01d0ca52373ab5f72","observation_id":"b0f8d21c-88ef-48f9-8f51-a387710ca392","resolution":{"observed_at":"2026-08-15T21:10:58.178590Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.161299Z","title":"Offline reinforcement learning with implicit q-learning","venue":null,"work_id":"ffaebac4-a40e-46a5-ba9a-f83aae439f83","year":2022},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.540652Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:7f8a060b46da9eadbc6d172ffe0c5888703f6143b7e9b6bf9497ff90f9fdb37c","observation_id":"18b626a5-f6de-4ed2-a6b5-75372c95bf39","resolution":{"observed_at":"2026-08-15T21:10:58.165611Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.544320Z","title":"Coderl: Mastering code generation through pretrained models and deep reinforcement learning.Advances in Neural Information Processing Systems, 35:21314–21328, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.544320Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:904888d4ad10793270c7ead0f2b3e49328bbfe0e41cbe60cf5826817a3f5ad52","observation_id":"d01fb1bf-eab7-4558-9fa1-8110c78f4171","resolution":{"observed_at":"2026-08-15T21:10:57.544320Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.14734","last_updated":"2024-11-01T20:05:19Z","snapshot_observed_at":"2026-08-16T13:50:07.192537Z","submitted_at":"2024-05-23T16:01:46Z","title":"SimPO: Simple Preference Optimization with a Reference-Free Reward","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.14734","snapshot_observed_at":"2026-08-15T21:10:57.547830Z","title":"Simpo: Simple preference optimization with a reference-free reward.arXiv preprint arXiv:2405.14734, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.547830Z"},"links":{"cited_paper":"/paper/2405.14734","citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:67156b9f8481d8ea4621bc54098501c9a5abbb20b79ecf8fadc601b8440db1dc","observation_id":"ee8ee31d-38fc-4f4e-a2a0-dde7bd755e87","resolution":{"observed_at":"2026-08-15T21:10:57.547830Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.551783Z","title":"Human-level control through deep reinforcement learning.nature, 518(7540):529–533, 2015","venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.551783Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:8df5ad91a3336ffb5ddd077246f337d0c275f1dd1db069406f13a14189b12810","observation_id":"19866da0-4419-4c82-9b34-aacf251dae46","resolution":{"observed_at":"2026-08-15T21:10:57.551783Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.131740Z","title":"Safe and efficient off-policy reinforcement learning.Advances in neural information processing systems, 29, 2016","venue":null,"work_id":"271637c3-7847-4dfb-ba1a-fbec8293301b","year":2016},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.555210Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:44a7e0bce98a8d5accb652c098527d84d1b5f00d810e3b35c84c27d9dae20c1c","observation_id":"5d67656b-899e-45f6-8477-1b55cc5f9f80","resolution":{"observed_at":"2026-08-15T21:10:58.137184Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.119745Z","title":"Bridging the gap between value and policy based reinforcement learning.Advances in neural information processing systems, 30, 2017","venue":null,"work_id":"d9b5c51a-8c39-403f-a75d-b317a5787bd4","year":2017},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.559081Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:85bd6b568a4c74702aa30eadf72d2a85f7998db622ea2d14a372ac2931d242fc","observation_id":"d7aed369-8a7f-4d2d-b39c-59cbe960d7f5","resolution":{"observed_at":"2026-08-15T21:10:58.124138Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.107838Z","title":"Policy invariance under reward transformations: Theory and application to reward shaping","venue":null,"work_id":"079a7fe1-c4ef-4f16-9860-d3d08a78093d","year":1999},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.562577Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:ac0d5351bc41d472dc2f687b0aa7b428abf1e1bbbf7552ae92a74c61b1b7a311","observation_id":"e2f0cecf-46b6-4c59-85dc-37550bbca7e5","resolution":{"observed_at":"2026-08-15T21:10:58.111655Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.566791Z","title":"Training language models to follow instructions with human feedback.Advances in neural information processing systems, 35:27730–27744, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.566791Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:1b49dea577c7055164c153dcbf2299ab3abbfcddc2c5d7cc6312a78ee083b5bc","observation_id":"7b96bb31-8a46-446e-a0ae-24b5043a707b","resolution":{"observed_at":"2026-08-15T21:10:57.566791Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.571105Z","title":"Gorilla: Large language model connected with massive apis.Advances in Neural Information Processing Systems, 37:126544–126565, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.571105Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:49ab0101358f5b333fa4b233fb0565847bafffb4163721f828dd8c346b854775","observation_id":"b4789803-cb77-42f6-b5a6-86680bec27ff","resolution":{"observed_at":"2026-08-15T21:10:57.571105Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.082850Z","title":"Direct preference optimization: Your language model is secretly a reward model.Advances in Neural Information Processing Systems, 36, 2023","venue":null,"work_id":"43c5f26b-3aed-4de0-93ac-a4d664109e71","year":2023},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.574896Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:534777346f9ea60662928dcd9c7ec621452a8d81b399dd27bb6e45d7ac33de42","observation_id":"8c7dfce9-c30a-4e81-9231-aef6ca36efa0","resolution":{"observed_at":"2026-08-15T21:10:58.086669Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2404.12358","last_updated":"2024-08-12T21:13:35Z","snapshot_observed_at":"2026-08-16T13:59:41.097676Z","submitted_at":"2024-04-18T17:37:02Z","title":"From $r$ to $Q^*$: Your Language Model is Secretly a Q-Function","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.12358","snapshot_observed_at":"2026-08-15T21:10:57.579584Z","title":"From r to q∗: Your language model is secretly a q-function.arXiv preprint arXiv:2404.12358, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.579584Z"},"links":{"cited_paper":"/paper/2404.12358","citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:ac780310c795688e7a3a7610bf2af1b7ca6590c5c724b5cdc1e5c33e5a0131e2","observation_id":"a54fcf35-98e9-4c86-a287-e57a05b36e2d","resolution":{"observed_at":"2026-08-15T21:10:57.579584Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.19107","last_updated":"2024-05-29T14:11:29Z","snapshot_observed_at":"2026-08-16T13:48:11.416703Z","submitted_at":"2024-05-29T14:11:29Z","title":"Offline Regularised Reinforcement Learning for Large Language Models Alignment","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.19107","snapshot_observed_at":"2026-08-15T21:10:57.583738Z","title":"Offline regularised reinforcement learning for large language models alignment.arXiv preprint arXiv:2405.19107, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.583738Z"},"links":{"cited_paper":"/paper/2405.19107","citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:7e15b1a0fbec46c2d74f980e5ce6389a33f48698e1ed765e4595904af979833e","observation_id":"fcc0837a-f55f-43ba-b31e-9718bc8d6d7d","resolution":{"observed_at":"2026-08-15T21:10:57.583738Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.071531Z","title":"Factually consistent summarization via reinforcement learning with textual entailment feedback","venue":null,"work_id":"615cd1e4-b04d-42ae-8aed-28665c3e7e5b","year":2023},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.587091Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:3c60610ca16753e1392354081546695a6376f8bec77c1a3ebb9ed64a847f9ab2","observation_id":"0fdd4b4d-02c6-4e73-b278-efb41499f604","resolution":{"observed_at":"2026-08-15T21:10:58.075496Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.059283Z","title":"Approx- imate modified policy iteration and its application to the game of tetris.J","venue":null,"work_id":"9955fd46-2d97-4744-8cab-90c0492bcbf6","year":2015},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.590663Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:74368a0a374cb13fde7a23b149e1634f66bdbd0d9ac2dc850a71cebe619784b5","observation_id":"bcb36d6c-bbef-405f-b719-f0586d4e4ccf","resolution":{"observed_at":"2026-08-15T21:10:58.063154Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1707.06347","last_updated":"2017-08-28T09:20:06Z","snapshot_observed_at":"2026-08-15T20:26:32.102285Z","submitted_at":"2017-07-20T02:32:33Z","title":"Proximal Policy Optimization Algorithms","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1707.06347","snapshot_observed_at":"2026-08-15T21:10:57.594034Z","title":"Proximal policy optimization algorithms.arXiv preprint arXiv:1707.06347, 2017","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.594034Z"},"links":{"cited_paper":"/paper/1707.06347","citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:2f024ca2bcb3fbb9d772ea01b63f0cd48f7d642d43accff617c4e2d32ade3b51","observation_id":"213c66aa-c1af-481d-9019-40c64fbe60cf","resolution":{"observed_at":"2026-08-15T21:10:57.594034Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.14655","last_updated":"2024-12-02T12:37:46Z","snapshot_observed_at":"2026-08-17T05:06:22.790424Z","submitted_at":"2024-05-23T14:53:54Z","title":"Multi-turn Reinforcement Learning from Preference Human Feedback","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.14655","snapshot_observed_at":"2026-08-15T21:10:57.597496Z","title":"Multi-turn reinforcement learning from preference human feedback.arXiv preprint arXiv:2405.14655, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.597496Z"},"links":{"cited_paper":"/paper/2405.14655","citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:2963841b9f27a36344cf83a1bd24809f15c2cf5292dc736c066a074bcc18d296","observation_id":"d85991ec-24c4-476d-b750-246c340cbfee","resolution":{"observed_at":"2026-08-15T21:10:57.597496Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.045815Z","title":"Offline RL for natural language generation with implicit language q learning","venue":null,"work_id":"17f4f357-8074-44ec-bce9-80ac5e358564","year":2023},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.601286Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:afbbce5c5ce22814f7142c46f92a0fc10c7f376f256f415e35de70c46b5c9a18","observation_id":"9cb689e5-5810-4219-a172-3681e5b10f94","resolution":{"observed_at":"2026-08-15T21:10:58.051009Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.033649Z","title":"Generalized preference optimization: A unified approach to offline alignment","venue":null,"work_id":"d53cf789-9a18-412b-bf74-885c346b3907","year":2024},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.604883Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:15d215e9ea8e9a04154b5ed8a11c28bc8fbe378b8a56df5ff5e6ac257be244f6","observation_id":"3b0adb19-4dbd-4282-aa01-6293c0b7be72","resolution":{"observed_at":"2026-08-15T21:10:58.037824Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2503.19612","last_updated":"2025-03-28T18:02:54Z","snapshot_observed_at":"2026-08-18T14:05:17.731016Z","submitted_at":"2025-03-25T12:52:38Z","title":"RL-finetuning LLMs from on- and off-policy data with a single algorithm","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.19612","snapshot_observed_at":"2026-08-15T21:10:57.608629Z","title":"Rl-finetuning llms from on-and off-policy data with a single algorithm.arXiv preprint arXiv:2503.19612, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.608629Z"},"links":{"cited_paper":"/paper/2503.19612","citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:1d22297d6e1703f4cad8bdeecc4038c65efac2e7e3b064469c952f4481ecb3b5","observation_id":"d4e4e0e5-f2b5-4ddd-bbf6-eee1db7c913d","resolution":{"observed_at":"2026-08-15T21:10:57.608629Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.022677Z","title":"Leverage the average: an analysis of kl regularization in reinforcement learning.Advances in Neural Information Processing Systems, 33:12163–12174, 2020","venue":null,"work_id":"53cf4396-997d-4a95-abd7-f7919d1c56b7","year":2020},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.612498Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:aa7f841d89f44cbfd5c0412b1f41353bba6a1769303811e8f0dac30db27e6bf0","observation_id":"fb2927fd-5aa3-4949-8f0f-b7a9482c7e55","resolution":{"observed_at":"2026-08-15T21:10:58.026707Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:58.011799Z","title":"Munchausen reinforcement learning.Advances in Neural Information Processing Systems, 33:4235–4246, 2020","venue":null,"work_id":"4ec31dce-0d7c-41ee-ac81-d4215267ee0b","year":2020},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.615592Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:b50c195c1a83a78e4a8b7ce0c897f9b63c59106f936e6290b3a0b8358dfb3f0c","observation_id":"077620c5-8f80-437f-aeaf-373d6e7e9b1e","resolution":{"observed_at":"2026-08-15T21:10:58.015588Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.619137Z","title":"Dueling network architectures for deep reinforcement learning","venue":null,"work_id":null,"year":1995},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.619137Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:49ffb13d5c754321bea23c3e94bee7640611d3bf4f7769dfbe8b03d6359dd115","observation_id":"5d2d18e3-512f-4d79-8f00-ef19195b33f8","resolution":{"observed_at":"2026-08-15T21:10:57.619137Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.994186Z","title":"Potential-based shaping and q-value initialization are equivalent.Journal of Artificial Intelligence Research, 19:205–208, 2003","venue":null,"work_id":"e8ebc4d7-74bf-444e-8b75-2532eb408a55","year":2003},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.622900Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:29f22fe880a3f4184ec295276504f21fb4d2afffe36f9b152a77d9b4483683dd","observation_id":"a58f1446-8375-4031-87a3-1d57a67489d5","resolution":{"observed_at":"2026-08-15T21:10:57.997686Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.983883Z","title":"Function optimization using connectionist reinforcement learning algorithms.Connection Science, 3(3):241–268, 1991","venue":null,"work_id":"c582b057-b401-4480-a6ca-bc8753038500","year":1991},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.626223Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:98166cc6c2f5b9f205fc37e3b2707f41ebdd6e4c3b6c46d12300643f9ffdc742","observation_id":"5b979ec3-bb05-4a3c-a3e8-793c27b5aa84","resolution":{"observed_at":"2026-08-15T21:10:57.987572Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2409.02392","last_updated":"2025-02-27T22:10:16Z","snapshot_observed_at":"2026-08-16T13:21:33.623046Z","submitted_at":"2024-09-04T02:41:04Z","title":"Building Math Agents with Multi-Turn Iterative Preference Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.02392","snapshot_observed_at":"2026-08-15T21:10:57.629840Z","title":"Building math agents with multi-turn iterative preference learning.arXiv preprint arXiv:2409.02392, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.629840Z"},"links":{"cited_paper":"/paper/2409.02392","citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:7910f002c4f8fbbda14b7e3761352978331c7eb5009917e69be5953b64aa1248","observation_id":"a82447f1-a91f-42cb-aa73-3b7e5c467616","resolution":{"observed_at":"2026-08-15T21:10:57.629840Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.973052Z","title":"InThe Twelfth International Conference on Learning Representations, 2024","venue":null,"work_id":"e00aa1f2-24ca-4caf-8a38-d5ba077a5058","year":2024},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.633448Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:17359de0a8cae0474a14c805645d2b23dc04376cb38451d0db7217baf8013e7f","observation_id":"0a0bb9dc-e5d6-43fd-a9a8-d6514018fe9f","resolution":{"observed_at":"2026-08-15T21:10:57.976771Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.16793","last_updated":"2025-02-24T11:29:08Z","snapshot_observed_at":"2026-08-18T08:35:53.576897Z","submitted_at":"2024-06-24T16:56:41Z","title":"Adam-mini: Use Fewer Learning Rates To Gain More","version":7},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.16793","snapshot_observed_at":"2026-08-15T21:10:57.637034Z","title":"Adam-mini: Use fewer learning rates to gain more.arXiv preprint arXiv:2406.16793, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.637034Z"},"links":{"cited_paper":"/paper/2406.16793","citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:12c375c270c1c0d03d27617780af9c3a04bf2cc126b5e8a30db0b8cf5976a500","observation_id":"6372540e-0a55-4552-b6ba-acefa4250a03","resolution":{"observed_at":"2026-08-15T21:10:57.637034Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.10425","last_updated":"2023-05-17T17:57:10Z","snapshot_observed_at":"2026-08-18T20:14:23.882982Z","submitted_at":"2023-05-17T17:57:10Z","title":"SLiC-HF: Sequence Likelihood Calibration with Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.10425","snapshot_observed_at":"2026-08-15T21:10:57.640889Z","title":"Slic-hf: Sequence likelihood calibration with human feedback.arXiv preprint arXiv:2305.10425, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.640889Z"},"links":{"cited_paper":"/paper/2305.10425","citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:e212eee7840adb7724c30ce7daa2a7b44e0a7fe9b8d0c37eed4942c0704cb237","observation_id":"ebbe1492-80e1-46c9-90d7-5d7cc68d3708","resolution":{"observed_at":"2026-08-15T21:10:57.640889Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.19446","last_updated":"2024-02-29T18:45:56Z","snapshot_observed_at":"2026-08-19T03:42:50.568302Z","submitted_at":"2024-02-29T18:45:56Z","title":"ArCHer: Training Language Model Agents via Hierarchical Multi-Turn RL","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.19446","snapshot_observed_at":"2026-08-15T21:10:57.644647Z","title":"Archer: Training language model agents via hierarchical multi-turn rl.arXiv preprint arXiv:2402.19446, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.644647Z"},"links":{"cited_paper":"/paper/2402.19446","citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:0f76516c2388550cdacd5091d04e9f757c97ca77161779ab7e9141a889cae33e","observation_id":"e9c88e6f-9758-4e5a-ba60-8ab220d3031b","resolution":{"observed_at":"2026-08-15T21:10:57.644647Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.961967Z","title":"PhD thesis, Carnegie Mellon University, 2010","venue":null,"work_id":"7b42e557-51c5-4a1f-ae21-edf4069ed3b7","year":2010},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.648893Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:78386799ff30bc4ee3901b94e6df7a8dedc935bdd36474422de7c8df4c611723","observation_id":"0cc62e57-a085-422d-a0f6-ddefbeed1118","resolution":{"observed_at":"2026-08-15T21:10:57.966526Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.950900Z","title":null,"venue":null,"work_id":"71a81198-b41a-4309-9716-1e3be61f0863","year":null},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.652994Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:0edfef09c2addad8cca345deac15d216f6d4188e3095fe6c2aec12c45c4eab5b","observation_id":"940fde9c-4619-400c-a29f-6a51a3203b6f","resolution":{"observed_at":"2026-08-15T21:10:57.954395Z","resolver_source":"raw_fallback","status":"parse_uncertain"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.939140Z","title":"Initialization trick","venue":null,"work_id":"1742e1ae-0128-4b06-9380-6e20eb50a880","year":null},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.656959Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:e7cdf11a37713fda0a049cdb9214717ab46ac9ac63cc4357523013809eb67e83","observation_id":"ebcac538-19f2-45bf-bd52-700034ba3dd1","resolution":{"observed_at":"2026-08-15T21:10:57.942715Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.928391Z","title":"This contrasts with other RL-finetuning approaches, such as DRO [ 34] or CoPG [ 12], that involve a square term per sequence of the batch","venue":null,"work_id":"bfd9741f-78c5-4dab-b483-adc78bdb66c3","year":2025},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.660461Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:7e21f30a3ad08bbc1ebfb3979a9f862386e0730762074f91efd8c5cb1d468d11","observation_id":"683665a4-f333-4b52-a187-eda081d5cf04","resolution":{"observed_at":"2026-08-15T21:10:57.932010Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.917342Z","title":"These tools use machine learning algorithms to generate realistic voices and faces","venue":null,"work_id":"6daf259c-12bd-48a9-80b9-e1d06eebdf17","year":null},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.665274Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:3dcdd501822ca8da6a5b8c58ae93d643d2c1f0f8ac4a4f6ebc8d78b60c92bfed","observation_id":"638e2006-da73-4454-b1aa-9c417e26ecd1","resolution":{"observed_at":"2026-08-15T21:10:57.921229Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.906168Z","title":"The more reference audio you have, the better the result","venue":null,"work_id":"38a8dd1c-8743-494a-99c8-71ff14d66e23","year":null},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.669412Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:67782eecdda5c82304c1bb8140c8c8919e10a28976aacc872303861ac1795fe6","observation_id":"108adfb6-2a4c-4e93-9832-893323e94382","resolution":{"observed_at":"2026-08-15T21:10:57.910171Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T21:10:57.892861Z","title":"This process may take some time, depending on the complexity of the task and the 35","venue":null,"work_id":"4425c41a-8a2e-4922-9f2f-2f0f22492a07","year":null},"citing_paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs","version":1},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-15T21:10:57.673384Z"},"links":{"citing_paper":"/paper/2505.11081"},"observation_digest":"sha256:17fa7423d01c102f918a6fbadb186f9ae81fdef81ef53097976544ab08ceadd4","observation_id":"21c39961-af15-46bf-8967-c76c9f35c729","resolution":{"observed_at":"2026-08-15T21:10:57.897515Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2505.11081","last_updated":"2025-05-16T10:12:11Z","latest_version":1,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-17T20:05:05.299606Z","submitted_at":"2025-05-16T10:12:11Z","title":"ShiQ: Bringing back Bellman to LLMs"},"reference_resolution":{"displayed":58,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":1,"unresolved":27,"verified_exact":1,"verified_fuzzy":29},"total_outbound_references":58},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"thesis":"As of 19 August 2026, this Paper Citation Record lists 58 of 58 outbound references and 3 inbound Pith citation observations for arXiv:2505.11081."}