{"as_of":"2026-08-19T18:26:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:e4c229e433f3ab85cfdcfb031f018611e43822ea2e38085a122372231aa89eed","coverage":[{"denominator":27,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":27,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-15T15:02:45.665708Z","state":"measured"},{"denominator":27,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":27,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-19T06:32:44.657259+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2608.02951/citation-record","integrity":"/paper/2608.02951/integrity","json":"/paper/2608.02951/citation-record.json","paper":"/paper/2608.02951"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2204.05862","last_updated":"2022-04-12T15:02:38Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-04-12T15:02:38Z","title":"Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2204.05862","snapshot_observed_at":"2026-08-15T15:02:45.558196Z","title":"Training a helpful and harmless assistant with reinforcement learning from human feedback.arXiv preprint arXiv:2204.05862,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.558196Z"},"links":{"cited_paper":"/paper/2204.05862","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:b2288cb3f5cfca215e098337ac68298f4154cdb510cbc3d1856472682f8e253c","observation_id":"ee422279-496a-4840-9e7d-bfb307e0ec00","resolution":{"observed_at":"2026-08-15T15:02:45.558196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.13639","last_updated":"2024-04-30T14:36:26Z","snapshot_observed_at":"2026-08-16T17:38:24.988221Z","submitted_at":"2023-10-20T16:37:56Z","title":"Contrastive Preference Learning: Learning from Human Feedback without RL","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.13639","snapshot_observed_at":"2026-08-15T15:02:45.577850Z","title":"Contrastive preference learning: learning from human feedback without rl.arXiv preprint arXiv:2310.13639,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.577850Z"},"links":{"cited_paper":"/paper/2310.13639","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:84b935beef02fa7b56cd6e04f810d7c48902216dd045c00a2ecc282d6e0b0db4","observation_id":"d290f816-01cf-4562-8b67-462bbce1b352","resolution":{"observed_at":"2026-08-15T15:02:45.577850Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2307.15217","last_updated":"2023-09-11T17:25:24Z","snapshot_observed_at":"2026-08-17T11:20:55.974248Z","submitted_at":"2023-07-27T22:29:25Z","title":"Open Problems and Fundamental Limitations of Reinforcement Learning from Human Feedback","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.15217","snapshot_observed_at":"2026-08-15T15:02:45.582408Z","title":"Open problems and fundamental limitations of reinforcement learning from human feedback.arXiv preprint arXiv:2307.15217,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.582408Z"},"links":{"cited_paper":"/paper/2307.15217","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:ebb23b5b01ef73e2c3199c76223733c7f26aef2a7825c53a50c52be889cef39a","observation_id":"98b5c4aa-31eb-42e3-a4ba-65e39627fcad","resolution":{"observed_at":"2026-08-15T15:02:45.582408Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2009.11462","last_updated":"2020-09-25T20:22:26Z","snapshot_observed_at":"2026-08-18T05:13:24.803705Z","submitted_at":"2020-09-24T03:17:19Z","title":"RealToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.11462","snapshot_observed_at":"2026-08-15T15:02:45.609451Z","title":"Realtoxicityprompts: Evaluating neural toxic degeneration in language models.arXiv preprint arXiv:2009.11462,","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.609451Z"},"links":{"cited_paper":"/paper/2009.11462","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:d8e4f17a4c8eb64d25ddb352f26bc6688fda0d775841ce759fd3d6bb5f0aec99","observation_id":"3a480b73-c493-4684-92e3-dccca44d5ce0","resolution":{"observed_at":"2026-08-15T15:02:45.609451Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.01876","last_updated":"2025-06-17T17:54:41Z","snapshot_observed_at":"2026-08-17T22:25:12.546901Z","submitted_at":"2025-02-03T23:08:42Z","title":"Reinforcement Learning with Segment Feedback","version":2},"cited_work":{"arxiv_id":"2502.01876","doi":null,"metadata_source":"pith","pith_arxiv_id":"2502.01876","snapshot_observed_at":"2026-08-15T15:02:45.745385Z","title":"Reinforcement Learning with Segment Feedback","venue":"cs.LG","work_id":"866c8f08-455b-4b4d-a561-be944139224d","year":2025},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.617996Z"},"links":{"cited_paper":"/paper/2502.01876","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:164cc03ea2e365fcb82a3a752c7eeda747f3daac90e6426701145cdc9a870446","observation_id":"2f6a9377-259a-45e1-9173-21492ddfe58f","resolution":{"observed_at":"2026-08-15T15:02:45.752291Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1412.6980","last_updated":"2017-01-30T01:27:54Z","snapshot_observed_at":"2026-08-17T19:26:44.032537Z","submitted_at":"2014-12-22T13:54:29Z","title":"Adam: A Method for Stochastic Optimization","version":9},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1412.6980","snapshot_observed_at":"2026-08-15T15:02:45.622442Z","title":"Adam: A method for stochastic optimization.arXiv preprint arXiv:1412.6980,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.622442Z"},"links":{"cited_paper":"/paper/1412.6980","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:0bd1b4747cd489e8d7017426b750431395da5287d16f7266c11a38bde9d2f5a8","observation_id":"605b043e-97f6-4fc4-89d8-ad25e4b7c7df","resolution":{"observed_at":"2026-08-15T15:02:45.622442Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1711.05101","last_updated":"2019-01-04T21:01:49Z","snapshot_observed_at":"2026-08-14T20:13:52.872565Z","submitted_at":"2017-11-14T14:24:06Z","title":"Decoupled Weight Decay Regularization","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1711.05101","snapshot_observed_at":"2026-08-15T15:02:45.631715Z","title":"Decoupled weight decay regularization.arXiv preprint arXiv:1711.05101,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.631715Z"},"links":{"cited_paper":"/paper/1711.05101","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:49958c836bbfc6d82d875650f605d3c82d580b010febc9c5f7fcc311120e3e3a","observation_id":"08c4f306-f481-4970-b9c9-d8c977c836e5","resolution":{"observed_at":"2026-08-15T15:02:45.631715Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:02:46.081511Z","title":"Approximating kl divergence, 2020.URL http://joschu","venue":null,"work_id":"9219d6dc-8fe2-4c6f-9468-acbbb41bf6da","year":2020},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.636173Z"},"links":{"citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:e7d133fa328d0f231127ec99c76807c91ee894a1ad4b18836f0391f4dfe643b9","observation_id":"e28fc222-6012-45fa-b84a-03663fb3c60f","resolution":{"observed_at":"2026-08-15T15:02:46.087144Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:02:46.066136Z","title":"15 A.2 Comparison of Preference Models","venue":null,"work_id":"fefaa32e-bfb3-4243-afb2-0c6d068c0aa3","year":2011},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.641390Z"},"links":{"citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:53d78a1d2e3653725fb22ec4786e439afc04d6698b4c9569ac1e0d75a74d5310","observation_id":"4fb484bb-1eb7-4c64-a6d9-305513f72586","resolution":{"observed_at":"2026-08-15T15:02:46.071148Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:02:46.050405Z","title":"This makes them inapplicable to many modern RL problems","venue":null,"work_id":"99979dca-aa42-458c-91d9-4ae07314779c","year":2017},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.646471Z"},"links":{"citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:f91b8f8f1ab5d6c90ed68d64abd9010bca1b34504f7e862e8dfb8218f5ef1611","observation_id":"ba2a8584-11b2-422a-9c6f-84a761d8d5d0","resolution":{"observed_at":"2026-08-15T15:02:46.055188Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:02:46.035761Z","title":"Specifically, even if the partial returns of two segments are the same, 15 humans could still prefer one trajectory segment based on the value of the last state and action","venue":null,"work_id":"5370b888-8067-490b-9063-d02e46db776e","year":2022},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.651001Z"},"links":{"citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:45abae03d8bcf3a439025641f6129aecb48756c462c4449d50e5d5c44a932bac","observation_id":"13ac6ae2-d619-4207-9ba0-d10702023097","resolution":{"observed_at":"2026-08-15T15:02:46.040674Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:02:46.005825Z","title":"During training, policies were Gaussian with a parameter controlling the log standard deviation for each dimension in the action space","venue":null,"work_id":"cc1c36f4-24a2-42ee-b4dc-34f0e3914dcb","year":2022},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.660952Z"},"links":{"citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:14ff792c20778370036dcdf4f821e2dab84c8750ccb0b4f3f910a665f768e273","observation_id":"156a5eb1-7f77-42e1-b4bb-c7055b8afd38","resolution":{"observed_at":"2026-08-15T15:02:46.010880Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:02:45.991632Z","title":"The global gradient norm is clipped to1.0","venue":null,"work_id":"9da15089-a36e-4a49-8e6a-86037ce4d528","year":2020},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.665708Z"},"links":{"citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:66673f50652484e42475c5df104200617c34dc7be0b5d09600ff24d48f688476","observation_id":"ff72a96f-415a-4d8e-9695-9a1ab3b7e032","resolution":{"observed_at":"2026-08-15T15:02:45.996416Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T15:02:46.020817Z","title":"We observe that in Ant-v5 usingπθt´1 as the second policy out performs setting the second policy toπθref","venue":null,"work_id":"dc801843-3a74-4c18-8bd7-a9e03a56f720","year":2019},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":1000,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.655771Z"},"links":{"citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:fd3ab3eea7d365aefe5bdf3631b5580962f3690ea83ebc5ac49ffb3137c47d8d","observation_id":"fe301495-77e8-4b26-9e33-1856d6f1db27","resolution":{"observed_at":"2026-08-15T15:02:46.025703Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2206.02231","last_updated":"2023-09-06T21:13:28Z","snapshot_observed_at":"2026-08-16T17:35:08.472912Z","submitted_at":"2022-06-05T17:58:02Z","title":"Models of human preference for learning reward functions","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2206.02231","snapshot_observed_at":"2026-08-15T15:02:45.573168Z","title":"12 Qining Zhang and Lei Ying","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":1952,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.573168Z"},"links":{"cited_paper":"/paper/2206.02231","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:ce90ec2434b02f47adfceb5c0a56c56fcaed4bb8db570fc1005620e4dacb9f44","observation_id":"34a6412f-ff93-4201-886c-6617e19fa7e0","resolution":{"observed_at":"2026-08-15T15:02:45.573168Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.18629","last_updated":"2024-06-26T17:43:06Z","snapshot_observed_at":"2026-08-06T00:24:52.274888Z","submitted_at":"2024-06-26T17:43:06Z","title":"Step-DPO: Step-wise Preference Optimization for Long-chain Reasoning of LLMs","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.18629","snapshot_observed_at":"2026-08-15T15:02:45.591812Z","title":"Step-dpo: Step-wise preference optimization for long-chain reasoning of llms.arXiv preprint arXiv:2406.18629,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2012,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.591812Z"},"links":{"cited_paper":"/paper/2406.18629","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:1da7fad89cb73d8ec699bc654c4d4c1ba9edb499b1f3e9fd9f5deacd7a299667","observation_id":"14ed6ea2-811c-45d0-84b0-51b5553374de","resolution":{"observed_at":"2026-08-15T15:02:45.591812Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1707.06347","last_updated":"2017-08-28T09:20:06Z","snapshot_observed_at":"2026-08-15T20:26:32.102285Z","submitted_at":"2017-07-20T02:32:33Z","title":"Proximal Policy Optimization Algorithms","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1707.06347","snapshot_observed_at":"2026-08-15T15:02:45.548076Z","title":"Proximal policy optimization algorithms.arXiv preprint arXiv:1707.06347,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2013,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.548076Z"},"links":{"cited_paper":"/paper/1707.06347","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:c000929c6d44fec8b6589c821e2ce2c9e71c65314aff70b4137926c325e4c588","observation_id":"1ff964b8-3fa5-467a-8dfa-88ee91dd45af","resolution":{"observed_at":"2026-08-15T15:02:45.548076Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2208.07587","last_updated":"2022-08-25T06:47:48Z","snapshot_observed_at":"2026-08-16T16:38:58.777792Z","submitted_at":"2022-08-16T08:00:43Z","title":"Making Reinforcement Learning Work on Swimmer","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2208.07587","snapshot_observed_at":"2026-08-15T15:02:45.626921Z","title":"Making reinforcement learning work on swimmer.arXiv preprint arXiv:2208.07587,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2014,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.626921Z"},"links":{"cited_paper":"/paper/2208.07587","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:c29a54c983983f706a4522648f70fdee0c708bdf6f3ecf1c1cc49591530d92e4","observation_id":"52bf253f-7a60-4fdd-acb2-db84ff3810f2","resolution":{"observed_at":"2026-08-15T15:02:45.626921Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2106.05091","last_updated":"2021-06-09T14:10:50Z","snapshot_observed_at":"2026-08-18T07:40:57.268095Z","submitted_at":"2021-06-09T14:10:50Z","title":"PEBBLE: Feedback-Efficient Interactive Reinforcement Learning via Relabeling Experience and Unsupervised Pre-training","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2106.05091","snapshot_observed_at":"2026-08-15T15:02:45.601173Z","title":"Pebble: Feedback-efficient interactive reinforcement learning via relabeling experience and unsupervised pre-training.arXiv preprint arXiv:2106.05091,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2017,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.601173Z"},"links":{"cited_paper":"/paper/2106.05091","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:589f9cf821e4bfe6654a6c3369c35572fc18fc46feeef1e09a6f5f44f562b489","observation_id":"526c189c-caec-415d-a723-96ba1e20aceb","resolution":{"observed_at":"2026-08-15T15:02:45.601173Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2302.12192","last_updated":"2023-02-23T17:34:53Z","snapshot_observed_at":"2026-08-19T01:47:45.494420Z","submitted_at":"2023-02-23T17:34:53Z","title":"Aligning Text-to-Image Models using Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2302.12192","snapshot_observed_at":"2026-08-15T15:02:45.567940Z","title":"Aligning text-to-image models using human feedback.arXiv preprint arXiv:2302.12192, 2023b","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2018,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.567940Z"},"links":{"cited_paper":"/paper/2302.12192","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:9e266abbf9443b904b0cb01f17460f026b696f4fab4ffb4897c717ef01555ad4","observation_id":"179e76b8-8da8-4c1f-8fe0-de3fa777a8c2","resolution":{"observed_at":"2026-08-15T15:02:45.567940Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1312.5602","last_updated":"2013-12-19T16:00:08Z","snapshot_observed_at":"2026-08-17T17:19:33.060917Z","submitted_at":"2013-12-19T16:00:08Z","title":"Playing Atari with Deep Reinforcement Learning","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1312.5602","snapshot_observed_at":"2026-08-15T15:02:45.542769Z","title":"Playing atari with deep reinforcement learning.arXiv preprint arXiv:1312.5602,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2019,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.542769Z"},"links":{"cited_paper":"/paper/1312.5602","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:5318bb2459540478125836ea135c04b169ae1d2b3474e99061eafb2ba8ed8bb6","observation_id":"b0d7fa27-dac7-474f-9532-0b7e02d9daa7","resolution":{"observed_at":"2026-08-15T15:02:45.542769Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.00957","last_updated":"2023-03-02T04:24:29Z","snapshot_observed_at":"2026-08-16T15:51:31.299086Z","submitted_at":"2023-03-02T04:24:29Z","title":"Preference Transformer: Modeling Human Preferences using Transformers for RL","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.00957","snapshot_observed_at":"2026-08-15T15:02:45.613860Z","title":"Preference transformer: Modeling human preferences using transformers for rl.arXiv preprint arXiv:2303.00957,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2020,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.613860Z"},"links":{"cited_paper":"/paper/2303.00957","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:6fe958d824549c6dcce994ba4e1cf655166b081274c7d770fe5dbb32ca581508","observation_id":"38abb7ee-237b-4218-993a-661c3d4087de","resolution":{"observed_at":"2026-08-15T15:02:45.613860Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.17032","last_updated":"2025-11-02T13:42:19Z","snapshot_observed_at":"2026-08-13T22:24:37.672685Z","submitted_at":"2024-07-24T06:35:05Z","title":"Gymnasium: A Standard Interface for Reinforcement Learning Environments","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.17032","snapshot_observed_at":"2026-08-15T15:02:45.605223Z","title":"Gymnasium: A standard interface for reinforcement learning environments.arXiv preprint arXiv:2407.17032,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2021,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.605223Z"},"links":{"cited_paper":"/paper/2407.17032","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:62026dc2485933fe74844bb71626734b9ddb4b7b5a0d19397824121ddac7091e","observation_id":"b85b8801-d6b7-4c14-a2b0-d595bc8b65a0","resolution":{"observed_at":"2026-08-15T15:02:45.605223Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.00267","last_updated":"2024-09-03T14:01:54Z","snapshot_observed_at":"2026-08-15T10:07:24.540946Z","submitted_at":"2023-09-01T05:53:33Z","title":"RLAIF vs. RLHF: Scaling Reinforcement Learning from Human Feedback with AI Feedback","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.00267","snapshot_observed_at":"2026-08-15T15:02:45.552796Z","title":"Rlaif vs","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2022,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.552796Z"},"links":{"cited_paper":"/paper/2309.00267","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:48b2fa2effa45e46f2c56c278e80ea45143262a3e7d0b69c0d9fef2e92c4d27d","observation_id":"ea2ce2c8-9538-4056-8515-e1ea757041a9","resolution":{"observed_at":"2026-08-15T15:02:45.552796Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.03300","last_updated":"2024-04-27T15:25:53Z","snapshot_observed_at":"2026-08-06T14:58:42.911363Z","submitted_at":"2024-02-05T18:55:32Z","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.03300","snapshot_observed_at":"2026-08-15T15:02:45.563041Z","title":"Deepseekmath: Pushing the limits of mathematical reasoning in open language models","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2023,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.563041Z"},"links":{"cited_paper":"/paper/2402.03300","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:6486fbb919f9200b4b3bda68b6d9a7911ae4b6462fc3c46215bda19ea867ef6c","observation_id":"227b8e35-4610-4640-bc74-c6d145f81fd4","resolution":{"observed_at":"2026-08-15T15:02:45.563041Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2506.03066","last_updated":"2026-07-17T19:03:49Z","snapshot_observed_at":"2026-08-19T03:42:01.548140Z","submitted_at":"2025-06-03T16:42:39Z","title":"Sign-SZPO: Provable Preference-based Reinforcement Learning with an Unknown Link Function","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2506.03066","snapshot_observed_at":"2026-08-15T15:02:45.587036Z","title":"Provable reinforcement learning from human feedback with an unknown link function","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.587036Z"},"links":{"cited_paper":"/paper/2506.03066","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:21f5b118b441acd1e8b3348776793c0cb6e9836acecbe41ce0478ab1741de9c1","observation_id":"f7f7b12a-67c5-4df6-b868-14f03998d776","resolution":{"observed_at":"2026-08-15T15:02:45.587036Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.04792","last_updated":"2024-02-29T20:59:17Z","snapshot_observed_at":"2026-08-16T14:20:38.839188Z","submitted_at":"2024-02-07T12:31:13Z","title":"Direct Language Model Alignment from Online AI Feedback","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.04792","snapshot_observed_at":"2026-08-15T15:02:45.596461Z","title":"Shangmin Guo, Biao Zhang, Tianlin Liu, Tianqi Liu, Misha Khalman, Felipe Llinares, Alexandre Rame, Thomas Mesnard, Yao Zhao, Bilal Piot, et al","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling","version":1},"reference_index":2026,"source":"pdf_text","source_observed_at":"2026-08-15T15:02:45.596461Z"},"links":{"cited_paper":"/paper/2402.04792","citing_paper":"/paper/2608.02951"},"observation_digest":"sha256:5f11a49b66ec41bf9b8fb6ec20e06244b2e315724d8afcae1d75d746ade70b31","observation_id":"5d978a87-4205-43fd-bd9a-0b116899fbf5","resolution":{"observed_at":"2026-08-15T15:02:45.596461Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2608.02951","last_updated":"2026-08-03T23:28:06Z","latest_version":1,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-19T00:16:24.852363Z","submitted_at":"2026-08-03T23:28:06Z","title":"SP3O: Reinforcement Learning from Segment Preferences without Reward Modeling"},"reference_resolution":{"displayed":27,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":19,"verified_exact":1,"verified_fuzzy":7},"total_outbound_references":27},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"thesis":"As of 19 August 2026, this Paper Citation Record lists 27 of 27 outbound references and 0 inbound Pith citation observations for arXiv:2608.02951."}