{"as_of":"2026-08-11T14:29:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:38cff260fa85245ec25ae6a154ee5c40d29901fe9e41cb79bd3b9df944f10b0d","coverage":[{"denominator":80,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":80,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-10T16:58:02.706784Z","state":"measured"},{"denominator":86,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":86,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-11T06:34:44.6726+00:00","state":"measured"},{"denominator":6,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":6,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-04T22:59:14.448812Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-07-03T04:17:37.298243Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.12735","snapshot_observed_at":"2026-08-04T22:59:14.448812Z","title":"Online preference alignment for language models via count-based exploration.arXiv preprint arXiv:2501.12735,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.06941","last_updated":"2025-09-08T17:52:56Z","snapshot_observed_at":"2026-08-07T12:11:53.268632Z","submitted_at":"2025-09-08T17:52:56Z","title":"Outcome-based Exploration for LLM Reasoning","version":1},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-04T22:59:14.448812Z"},"links":{"cited_paper":"/paper/2501.12735","citing_paper":"/paper/2509.06941"},"observation_digest":"sha256:d8ee9b5a2cdd79b34c5a1cc82353c0f523024072bc3d3e504ee4d67d2755c6b5","observation_id":"a61d8773-626d-46fe-b3bf-f7fa28628346","resolution":{"observed_at":"2026-08-04T22:59:14.448812Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.12735","snapshot_observed_at":"2026-08-04T18:53:02.606163Z","title":"Online prefer- ence alignment for language models via count-based exploration.arXiv preprint arXiv:2501.12735,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.09675","last_updated":"2025-09-11T17:59:17Z","snapshot_observed_at":"2026-08-07T12:25:43.109797Z","submitted_at":"2025-09-11T17:59:17Z","title":"CDE: Curiosity-Driven Exploration for Efficient Reinforcement Learning in Large Language Models","version":1},"reference_index":2021,"source":"pdf_text","source_observed_at":"2026-08-04T18:53:02.606163Z"},"links":{"cited_paper":"/paper/2501.12735","citing_paper":"/paper/2509.09675"},"observation_digest":"sha256:b08abf68cfdcc3d2de5e4c73a44834c4286a27af6287ff39c73aaed4f23861a1","observation_id":"f0ebcc62-be89-4ba5-b101-e6d25f04a06e","resolution":{"observed_at":"2026-08-04T18:53:02.606163Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.12735","snapshot_observed_at":"2026-08-04T10:09:01.222940Z","title":"Online preference alignment for language models via count-based exploration.arXiv preprint arXiv:2501.12735,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2510.11686","last_updated":"2026-07-15T17:43:29Z","snapshot_observed_at":"2026-08-08T16:17:52.499482Z","submitted_at":"2025-10-13T17:49:05Z","title":"Representation-Based Exploration for Language Models: From Test-Time to Post-Training","version":2},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-04T10:09:01.222940Z"},"links":{"cited_paper":"/paper/2501.12735","citing_paper":"/paper/2510.11686"},"observation_digest":"sha256:00fd0e1c604b68572bc703243df3b31686110174886d305f7d984e1442c07e47","observation_id":"b02aeed5-5e6d-4ec3-a9fd-2532a5403d52","resolution":{"observed_at":"2026-08-04T10:09:01.222940Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"cited_work":{"arxiv_id":"2501.12735","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2501.12735","snapshot_observed_at":"2026-07-03T04:17:37.298243Z","title":"arXiv preprint arXiv:2501.12735 , year=","venue":null,"work_id":"ed782cfe-3240-4098-b5b8-77340b192a69","year":2025},"citing_paper":{"arxiv_id":"2605.31455","last_updated":"2026-05-29T15:49:13Z","snapshot_observed_at":"2026-08-06T06:42:59.945235Z","submitted_at":"2026-05-29T15:49:13Z","title":"DRIFT: Decoupled Rollouts and Importance-Weighted Fine-Tuning for Efficient Multi-Turn Optimization","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-06-28T23:16:49.358792Z"},"links":{"cited_paper":"/paper/2501.12735","citing_paper":"/paper/2605.31455"},"observation_digest":"sha256:5b14e7e78deb9cd1f6184b27e6507b41e0f6771bb04596571270be05c81527bd","observation_id":"82d9f26c-c454-4aa4-90cd-cef9da877dcd","resolution":{"observed_at":"2026-06-28T23:22:47.450020Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"cited_work":{"arxiv_id":"2501.12735","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2501.12735","snapshot_observed_at":"2026-07-03T04:17:37.298243Z","title":"arXiv preprint arXiv:2501.12735 , year=","venue":null,"work_id":"ed782cfe-3240-4098-b5b8-77340b192a69","year":2025},"citing_paper":{"arxiv_id":"2606.06080","last_updated":"2026-06-04T12:16:39Z","snapshot_observed_at":"2026-07-06T23:45:56.266116Z","submitted_at":"2026-06-04T12:16:39Z","title":"On Advantage Estimates for Max@K Policy Gradients","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-06-28T02:21:57.143016Z"},"links":{"cited_paper":"/paper/2501.12735","citing_paper":"/paper/2606.06080"},"observation_digest":"sha256:93b96c3bf00934e0c0772d5822f0ed0a6e7f9951206fa338cee4e31c675d3d3c","observation_id":"7e4c3140-62f9-4b08-b63f-b3e808a90c04","resolution":{"observed_at":"2026-07-02T12:06:56.459065Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"cited_work":{"arxiv_id":"2501.12735","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2501.12735","snapshot_observed_at":"2026-07-03T04:17:37.298243Z","title":"arXiv preprint arXiv:2501.12735 , year=","venue":null,"work_id":"ed782cfe-3240-4098-b5b8-77340b192a69","year":2025},"citing_paper":{"arxiv_id":"2606.10768","last_updated":"2026-06-09T12:21:27Z","snapshot_observed_at":"2026-08-08T12:03:53.959178Z","submitted_at":"2026-06-09T12:21:27Z","title":"N-GRPO: Embedding-Level Neighbor Mixing for Enhanced Policy Optimization","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-06-27T14:02:05.833651Z"},"links":{"cited_paper":"/paper/2501.12735","citing_paper":"/paper/2606.10768"},"observation_digest":"sha256:379a3ea0ab8c75fd4bbbd0635afc60099ba19620ba87dce8cdc6491bbd330105","observation_id":"6ae9a656-4640-495b-853d-d45c2a6cb363","resolution":{"observed_at":"2026-07-03T04:17:37.299709Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2501.12735/citation-record","integrity":"/paper/2501.12735/integrity","json":"/paper/2501.12735/citation-record.json","paper":"/paper/2501.12735"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.282596Z","title":"write newline","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.282596Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:b48609d703f82b76701c1fc007e487072dfc6a23613c04d5ea554ec3dcf238b1","observation_id":"ef53fa48-6b45-4b67-8dd4-5df4ab33c1c1","resolution":{"observed_at":"2026-08-10T16:58:02.282596Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.289381Z","title":"Improved algorithms for linear stochastic bandits","venue":null,"work_id":null,"year":2011},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.289381Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:3b8544d7814223fc50a13b179219d1a4def5bafbc9293215c2c2ebd0254d6770","observation_id":"fa65da28-31df-4ceb-8871-a66f902761bb","resolution":{"observed_at":"2026-08-10T16:58:02.289381Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.295415Z","title":"Reinforcement learning: Theory and algorithms","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.295415Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:2bb9640218ee7299589c8c5cb167b2012f192caa6eca9ed4af9e481de178c030","observation_id":"37ebaddc-f5eb-4bb8-b750-5cc9186a9199","resolution":{"observed_at":"2026-08-10T16:58:02.295415Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.14740","last_updated":"2024-02-26T18:26:25Z","snapshot_observed_at":"2026-08-09T14:30:33.899591Z","submitted_at":"2024-02-22T17:52:34Z","title":"Back to Basics: Revisiting REINFORCE Style Optimization for Learning from Human Feedback in LLMs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.14740","snapshot_observed_at":"2026-08-10T16:58:02.300277Z","title":"Back to basics: Revisiting reinforce style optimization for learning from human feedback in llms","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.300277Z"},"links":{"cited_paper":"/paper/2402.14740","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:a609737be88582e5898cda532a0a4df629ee8981eae5ed33e43598753d03dbde","observation_id":"335e907c-2771-49c7-9467-6a51634de33f","resolution":{"observed_at":"2026-08-10T16:58:02.300277Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.306276Z","title":"A general theoretical paradigm to understand learning from human preferences","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.306276Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:db703dc805b97993af8dbbddeebf7bdbdfec660f167b6a20700072322af7c3db","observation_id":"a205591e-4273-4ad1-9286-0445c7a496d4","resolution":{"observed_at":"2026-08-10T16:58:02.306276Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:03.896772Z","title":"Dynamic bottleneck for robust self-supervised exploration","venue":null,"work_id":"c28c5091-50ce-4d5f-bc05-6b89bf749e92","year":2021},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.311797Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:edbef3d1c93e090b411251eb5512808a50f0e301cc3c613dae54e5d3be4f9f2d","observation_id":"3db2ce02-fa1f-4a4d-9889-dd930552cb92","resolution":{"observed_at":"2026-08-10T16:58:03.901418Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:03.880587Z","title":"Principled exploration via optimistic bootstrapping and backward induction","venue":null,"work_id":"c751e69a-3d38-4771-84c1-de033b729510","year":2021},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.317754Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:7a760a482f8aa07cdef3aae03dd3fb2645228bd084576d2deef00d7151ea51e1","observation_id":"61b7b2fc-bf1c-45b0-83c0-bb113d54659a","resolution":{"observed_at":"2026-08-10T16:58:03.885692Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:03.864667Z","title":"Pessimistic bootstrapping for uncertainty-driven offline reinforcement learning","venue":null,"work_id":"47163cfb-7506-4930-a027-9b1cbe20640c","year":2022},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.323338Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:53d49de0bd348047f60d3c89d0f55558e82168b874ff88547ebe8067bfc8b4ba","observation_id":"3f01d01d-c28a-46fb-acec-1ec4fedcad04","resolution":{"observed_at":"2026-08-10T16:58:03.869749Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:03.848329Z","title":"Pessimistic value iteration for multi-task data sharing in offline reinforcement learning","venue":null,"work_id":"6056a134-fce2-4c9d-af07-7d968e148cfc","year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.328525Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:39a67d73cf61969587b5ceb2f18d50301ead0304cc2632d1adabb8f617dd239d","observation_id":"5663f3e6-4208-4386-9523-0f3419e42898","resolution":{"observed_at":"2026-08-10T16:58:03.853497Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2204.05862","last_updated":"2022-04-12T15:02:38Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-04-12T15:02:38Z","title":"Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2204.05862","snapshot_observed_at":"2026-08-10T16:58:02.333468Z","title":"Training a helpful and harmless assistant with reinforcement learning from human feedback","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.333468Z"},"links":{"cited_paper":"/paper/2204.05862","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:28b0d1bffb5b309bb1b31c486fcbad133b5786318c6762033465bda4815cf538","observation_id":"1b9ff02c-0539-4d21-b69d-ec8d6295a0cf","resolution":{"observed_at":"2026-08-10T16:58:02.333468Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.338317Z","title":"Unifying count-based exploration and intrinsic motivation","venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.338317Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:9ad17db4e0879c01e1909d02a6784f1a1b1909cacc7ae5cb7ba75b84fcf62f84","observation_id":"22139940-ed4e-454f-88a4-306882a4fed1","resolution":{"observed_at":"2026-08-10T16:58:02.338317Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.343378Z","title":"Rank analysis of incomplete block designs: I","venue":null,"work_id":null,"year":1952},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.343378Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:9881a18aed8d31019e4f9a237f6bb69094434bdaa13b34a6fd4f07dc922bffcd","observation_id":"0298d3f8-8a7d-45ee-889c-57af64d913f9","resolution":{"observed_at":"2026-08-10T16:58:02.343378Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1810.12894","last_updated":"2018-10-30T17:44:42Z","snapshot_observed_at":"2026-07-06T07:11:32.319931Z","submitted_at":"2018-10-30T17:44:42Z","title":"Exploration by Random Network Distillation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1810.12894","snapshot_observed_at":"2026-08-10T16:58:02.349211Z","title":"Exploration by random network distillation","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.349211Z"},"links":{"cited_paper":"/paper/1810.12894","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:208ada01b63a81aa50c75c0114a5a19a5c54e48698af393b82adde2ae04b881c","observation_id":"91614892-6328-480f-83ad-5b5073a4cfc9","resolution":{"observed_at":"2026-08-10T16:58:02.349211Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.19320","last_updated":"2025-02-19T00:51:58Z","snapshot_observed_at":"2026-07-06T18:22:10.094519Z","submitted_at":"2024-05-29T17:51:42Z","title":"Value-Incentivized Preference Optimization: A Unified Approach to Online and Offline RLHF","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.19320","snapshot_observed_at":"2026-08-10T16:58:02.354557Z","title":"Value-incentivized preference optimization: A unified approach to online and offline rlhf","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.354557Z"},"links":{"cited_paper":"/paper/2405.19320","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:2da25d1a48454b3e6ecc9baf36fe3cd962942066cee0fe9327165912f4edc112","observation_id":"5daa008d-20d7-4214-a794-a7d172dd9392","resolution":{"observed_at":"2026-08-10T16:58:02.354557Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.359974Z","title":"Deep reinforcement learning from human preferences","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.359974Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:0d8664630806c07e6cb3650ede3f6a77322e9eecef34302dcba0678c544a542a","observation_id":"ae3edc00-5f1d-441e-8640-e3d5dd19f24c","resolution":{"observed_at":"2026-08-10T16:58:02.359974Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1803.05457","last_updated":"2018-03-14T18:04:21Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2018-03-14T18:04:21Z","title":"Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1803.05457","snapshot_observed_at":"2026-08-10T16:58:02.365180Z","title":"Think you have solved question answering? try arc, the ai2 reasoning challenge","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.365180Z"},"links":{"cited_paper":"/paper/1803.05457","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:cefe7c0e3391decbf10ce8c65d8477d106e168a471d41250a696daef409a4bbf","observation_id":"a83b5501-fa90-47cc-8437-fe75775a3151","resolution":{"observed_at":"2026-08-10T16:58:02.365180Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2110.14168","last_updated":"2021-11-18T00:23:45Z","snapshot_observed_at":"2026-08-07T01:45:38.840969Z","submitted_at":"2021-10-27T04:49:45Z","title":"Training Verifiers to Solve Math Word Problems","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2110.14168","snapshot_observed_at":"2026-08-10T16:58:02.370586Z","title":"Training verifiers to solve math word problems","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.370586Z"},"links":{"cited_paper":"/paper/2110.14168","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:6037a7780204ba0e62a69c93638ab004da2de7795acf857ffc6d613198044481","observation_id":"73039118-e167-4d2e-9a6a-31bfef77f552","resolution":{"observed_at":"2026-08-10T16:58:02.370586Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.01377","last_updated":"2024-07-16T03:24:39Z","snapshot_observed_at":"2026-08-02T07:46:40.319683Z","submitted_at":"2023-10-02T17:40:01Z","title":"UltraFeedback: Boosting Language Models with Scaled AI Feedback","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.01377","snapshot_observed_at":"2026-08-10T16:58:02.376355Z","title":"Ultrafeedback: Boosting language models with high-quality feedback","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.376355Z"},"links":{"cited_paper":"/paper/2310.01377","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:5c9c48ebd7b2f40eb9122265fc24db11b4d9aa937d39245f830814f8d1768322","observation_id":"96c70b83-bb3e-4d7d-880a-138e758e79d4","resolution":{"observed_at":"2026-08-10T16:58:02.376355Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.07863","last_updated":"2024-11-12T11:18:43Z","snapshot_observed_at":"2026-08-07T08:32:00.916309Z","submitted_at":"2024-05-13T15:50:39Z","title":"RLHF Workflow: From Reward Modeling to Online RLHF","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.07863","snapshot_observed_at":"2026-08-10T16:58:02.381265Z","title":"Rlhf workflow: From reward modeling to online rlhf","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.381265Z"},"links":{"cited_paper":"/paper/2405.07863","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:53dce39d518b1f337b666aea05f973011dd35bf29135be5e4e568134ec8001bc","observation_id":"c77a1b4f-c896-445a-8633-bd811307e2fa","resolution":{"observed_at":"2026-08-10T16:58:02.381265Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.21783","last_updated":"2024-11-23T23:27:33Z","snapshot_observed_at":"2026-08-10T16:40:37.411115Z","submitted_at":"2024-07-31T17:54:27Z","title":"The Llama 3 Herd of Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.21783","snapshot_observed_at":"2026-08-10T16:58:02.386536Z","title":"The llama 3 herd of models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.386536Z"},"links":{"cited_paper":"/paper/2407.21783","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:c586e100e5c2b3d6357e3883e5ab67e2fa60a3ae61d085f6a7bc631c2362105d","observation_id":"0353558b-556c-4724-afc8-e0a726a9b717","resolution":{"observed_at":"2026-08-10T16:58:02.386536Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.04475","last_updated":"2025-03-10T09:27:03Z","snapshot_observed_at":"2026-07-06T17:56:23.317089Z","submitted_at":"2024-04-06T02:29:02Z","title":"Length-Controlled AlpacaEval: A Simple Way to Debias Automatic Evaluators","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.04475","snapshot_observed_at":"2026-08-10T16:58:02.393058Z","title":"Length-controlled alpacaeval: A simple way to debias automatic evaluators","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.393058Z"},"links":{"cited_paper":"/paper/2404.04475","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:ce2aaa5446820fbfcc138c6394dbc25f644ced36d1a813fbe187e75c9be6be11","observation_id":"5a1c866b-ff66-4914-a6a9-2bf484d45a3b","resolution":{"observed_at":"2026-08-10T16:58:02.393058Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.00396","last_updated":"2024-06-04T18:35:09Z","snapshot_observed_at":"2026-08-10T06:44:57.475366Z","submitted_at":"2024-02-01T07:32:24Z","title":"Efficient Exploration for LLMs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.00396","snapshot_observed_at":"2026-08-10T16:58:02.400792Z","title":"Efficient exploration for llms","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.400792Z"},"links":{"cited_paper":"/paper/2402.00396","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:0083a0cfb1e3ba87f55a12e38a0c4c5f8a5c0a9411e6c3b1a4762b672d3f26f1","observation_id":"cf3c38bb-a4c6-4643-9338-a732163ca67a","resolution":{"observed_at":"2026-08-10T16:58:02.400792Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:03.797197Z","title":"Kto: Model alignment as prospect theoretic optimization","venue":null,"work_id":"b39a6f2c-f426-4c33-879f-32c1f5226338","year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.406030Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:85dc0bc1804b4e1fac92641e6cc904c8c987637071352a4e5dc1f6cd53ee7cc8","observation_id":"90b6bc5f-ed61-4879-a5b2-7aa9de428e58","resolution":{"observed_at":"2026-08-10T16:58:03.802452Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.01306","last_updated":"2024-11-19T18:12:45Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-02-02T10:53:36Z","title":"KTO: Model Alignment as Prospect Theoretic Optimization","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.01306","snapshot_observed_at":"2026-08-10T16:58:02.411074Z","title":"Kto: Model alignment as prospect theoretic optimization","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.411074Z"},"links":{"cited_paper":"/paper/2402.01306","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:da0fa464913b3dd3a79011a73d311f4f3df55045bcafec7a3387f72793878e5f","observation_id":"bf25acec-14a5-45e5-96e7-ffe138407b89","resolution":{"observed_at":"2026-08-10T16:58:02.411074Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.417149Z","title":"Scaling laws for reward model overoptimization","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.417149Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:6824f59238e04d8856e0a77b9d7c1e30774325fa61373a48ce94969cc5e93fb6","observation_id":"70831cce-6fb2-4675-b1f0-a9132a2859b4","resolution":{"observed_at":"2026-08-10T16:58:02.417149Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.422098Z","title":"A framework for few-shot language model evaluation, 07 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.422098Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:a582bf4383bfae3d1687dc21a2e9fccdb17c539a2d05f273334e89fe7a80591c","observation_id":"c4d4090a-b281-4761-b526-efe9d7227328","resolution":{"observed_at":"2026-08-10T16:58:02.422098Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.04792","last_updated":"2024-02-29T20:59:17Z","snapshot_observed_at":"2026-08-10T08:55:18.452315Z","submitted_at":"2024-02-07T12:31:13Z","title":"Direct Language Model Alignment from Online AI Feedback","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.04792","snapshot_observed_at":"2026-08-10T16:58:02.426898Z","title":"Direct language model alignment from online ai feedback","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.426898Z"},"links":{"cited_paper":"/paper/2402.04792","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:4a29f88dd9d8dc289a927a464da70c184912987c8dd6a3b786a6bad685141e40","observation_id":"bb455c0a-59fe-442c-8611-7e6a6b9419db","resolution":{"observed_at":"2026-08-10T16:58:02.426898Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:03.761180Z","title":"Exploration in deep reinforcement learning: From single-agent to multiagent domain","venue":null,"work_id":"c2435431-3b97-4040-b9f4-7e6142c86225","year":2023},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.431968Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:b4f2211054dc87c2bc72e1871987e261e0da5843901ce2950b90b4d649427e55","observation_id":"f617c9d4-9cf5-475a-98cc-be6e812296e7","resolution":{"observed_at":"2026-08-10T16:58:03.765809Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:03.743611Z","title":"VIME: variational information maximizing exploration","venue":null,"work_id":"c6dd9316-4668-4de2-9786-faced7d064c8","year":2016},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.436780Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:31c34e476bbd453478afd12fc9bef8985fd6848a3acfef763256b403028deab4","observation_id":"77bf73e0-b47e-4b04-abc8-3d37966d4aab","resolution":{"observed_at":"2026-08-10T16:58:03.749626Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.441734Z","title":"Lo RA : Low-rank adaptation of large language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":30,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.441734Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:f4147db02e0cc100c95841da65ba1e94f87ab3c966d02d75f463b4ba4a1d1e44","observation_id":"f3290652-12c8-4769-b09c-31b9aadc1879","resolution":{"observed_at":"2026-08-10T16:58:02.441734Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:03.716569Z","title":"Unpacking dpo and ppo: Disentangling best practices for learning from preference feedback","venue":null,"work_id":"67e319f0-d69c-4b8d-b69d-37185eee6204","year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.446933Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:2db93af044ad64582e1863b469037f1cb6b40e5f54a764658ce5a6adb27283ea","observation_id":"b2f05f73-cd26-4f7a-a356-b5d5ecccb49e","resolution":{"observed_at":"2026-08-10T16:58:03.721788Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2306.02561","last_updated":"2023-06-30T21:39:54Z","snapshot_observed_at":"2026-08-10T19:14:51.255301Z","submitted_at":"2023-06-05T03:32:26Z","title":"LLM-Blender: Ensembling Large Language Models with Pairwise Ranking and Generative Fusion","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.02561","snapshot_observed_at":"2026-08-10T16:58:02.451981Z","title":"Llm-blender: Ensembling large language models with pairwise ranking and generative fusion","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.451981Z"},"links":{"cited_paper":"/paper/2306.02561","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:2630d6982ca3097cc3fac3e190d1badb231ffebbaceb1e4b4fe1b9ac8938acb4","observation_id":"c7e4530f-8b0b-4cb2-83e8-46d35f5a7a31","resolution":{"observed_at":"2026-08-10T16:58:02.451981Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:03.700426Z","title":"Provably efficient reinforcement learning with linear function approximation","venue":null,"work_id":"9dd3e9cc-c0c1-42bc-bb02-883628fd1430","year":2020},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.457552Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:d08df1cd6a9d6a8637088cf54abd54eaac3e88bca20bcd8778798ba66fd8a41a","observation_id":"afc77d64-92f3-4511-810d-a71c1d8ed02d","resolution":{"observed_at":"2026-08-10T16:58:03.705640Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:03.683218Z","title":"Kearns and Satinder P","venue":null,"work_id":"726269b1-9a7b-40bf-9c84-9183a886fadd","year":2002},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.463450Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:3d7b90b7816fa0908db7d977f6fd4f772acf87b4eac18dabc8f41b0ff7f193f0","observation_id":"1f1c2440-5c48-4ebb-8250-3e39785ed575","resolution":{"observed_at":"2026-08-10T16:58:03.688239Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.13787","last_updated":"2024-06-08T16:40:12Z","snapshot_observed_at":"2026-08-02T18:11:57.036767Z","submitted_at":"2024-03-20T17:49:54Z","title":"RewardBench: Evaluating Reward Models for Language Modeling","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.13787","snapshot_observed_at":"2026-08-10T16:58:02.469030Z","title":"Rewardbench: Evaluating reward models for language modeling","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.469030Z"},"links":{"cited_paper":"/paper/2403.13787","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:9b04eb2d2c353798af834d56a1cc06a01309df6fb38bbb61904280a0be06614e","observation_id":"de4c7455-0eea-4f23-876b-64f9db55da2b","resolution":{"observed_at":"2026-08-10T16:58:02.469030Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.475812Z","title":"Bandit algorithms","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":36,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.475812Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:5c3030ff5355c43da61aa0fff84acc9d8e466eb1b53a5f858c5afb4b0626486b","observation_id":"8b3b8d5d-bf58-40fc-ae56-2429ed449777","resolution":{"observed_at":"2026-08-10T16:58:02.475812Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:03.656898Z","title":"A mechanistic understanding of alignment algorithms: A case study on dpo and toxicity","venue":null,"work_id":"34d171fc-71fa-40b1-80ce-ce9b7d0a90a6","year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":37,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.481760Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:aec067c9abfb949ae09838db07a50ea8b871c056d6fc70808db992e6ec8c1cdf","observation_id":"dee82772-2af4-4232-8af8-0e9167470206","resolution":{"observed_at":"2026-08-10T16:58:03.662098Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.11253","last_updated":"2024-06-25T13:39:52Z","snapshot_observed_at":"2026-08-01T08:51:52.481299Z","submitted_at":"2024-02-17T11:25:26Z","title":"Aligning Large Language Models by On-Policy Self-Judgment","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.11253","snapshot_observed_at":"2026-08-10T16:58:02.487409Z","title":"Aligning large language models by on-policy self-judgment","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":38,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.487409Z"},"links":{"cited_paper":"/paper/2402.11253","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:43e663859177a80dc3ca060182626eed5b8fb00bc663a06c56665fa884e84bc7","observation_id":"0fcf4a93-0e7a-4cd1-9a7b-4f9298893c9b","resolution":{"observed_at":"2026-08-10T16:58:02.487409Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2109.07958","last_updated":"2022-05-08T02:43:02Z","snapshot_observed_at":"2026-08-03T16:26:48.747700Z","submitted_at":"2021-09-08T17:15:27Z","title":"TruthfulQA: Measuring How Models Mimic Human Falsehoods","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2109.07958","snapshot_observed_at":"2026-08-10T16:58:02.492624Z","title":"Truthfulqa: Measuring how models mimic human falsehoods","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":39,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.492624Z"},"links":{"cited_paper":"/paper/2109.07958","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:b12cda37d20e287f8111c7c843e026838d9f44d1cdf36fec38b3fd7c3ade29a9","observation_id":"2708c191-c90d-48ba-9d50-7948c84594fb","resolution":{"observed_at":"2026-08-10T16:58:02.492624Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.498083Z","title":"Flipping coins to estimate pseudocounts for exploration in reinforcement learning","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":40,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.498083Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:e87d70d004c22f9ef54fb0fc1a3329bf42a5151211143e9fabd9f27ef41e626e","observation_id":"3cdb3c77-9ce4-4617-987a-4ab43bd70dce","resolution":{"observed_at":"2026-08-10T16:58:02.498083Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:03.629204Z","title":"The sample complexity of exploration in the multi-armed bandit problem","venue":null,"work_id":"7705a535-e5e2-4b69-aa61-a6bfdcaf45fd","year":2004},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":41,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.502922Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:c2e70cd1fa54b8c4a1940fcb8bcdd105b1fed711ce25bd3c08f0837be309c028","observation_id":"45194ecf-3380-4c65-8ebc-83eff819febe","resolution":{"observed_at":"2026-08-10T16:58:03.634569Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2405.14734","last_updated":"2024-11-01T20:05:19Z","snapshot_observed_at":"2026-08-03T02:02:15.325394Z","submitted_at":"2024-05-23T16:01:46Z","title":"SimPO: Simple Preference Optimization with a Reference-Free Reward","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.14734","snapshot_observed_at":"2026-08-10T16:58:02.508364Z","title":"Simpo: Simple preference optimization with a reference-free reward","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":42,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.508364Z"},"links":{"cited_paper":"/paper/2405.14734","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:5115646a775df71c3ac92475accdcb2f792f7349d6931fc58dfc680416a1229f","observation_id":"0aa02464-c6fb-408b-9cb4-382731f3dc7e","resolution":{"observed_at":"2026-08-10T16:58:02.508364Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.513411Z","title":"Introducing meta llama 3: The most capable openly available llm to date","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":43,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.513411Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:19b7b6d320b83ff455effac57880f5dedf1e3e0756f0664e5bb3dfddb2c30615","observation_id":"91a4e0ba-81d1-414d-8fee-113b0b2aebab","resolution":{"observed_at":"2026-08-10T16:58:02.513411Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1809.02789","last_updated":"2018-09-08T11:47:16Z","snapshot_observed_at":"2026-08-08T02:19:52.596062Z","submitted_at":"2018-09-08T11:47:16Z","title":"Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1809.02789","snapshot_observed_at":"2026-08-10T16:58:02.518948Z","title":"Can a suit of armor conduct electricity? a new dataset for open book question answering","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":44,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.518948Z"},"links":{"cited_paper":"/paper/1809.02789","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:964e5ef18c2bd37a50ba3a545564fcfb2b2e90aa888c338d8d4ea804f7a97c6c","observation_id":"983d111d-4900-4e47-bba8-dc096b0a7fbe","resolution":{"observed_at":"2026-08-10T16:58:02.518948Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.00886","last_updated":"2024-06-11T16:25:52Z","snapshot_observed_at":"2026-07-06T16:55:53.666800Z","submitted_at":"2023-12-01T19:26:23Z","title":"Nash Learning from Human Feedback","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.00886","snapshot_observed_at":"2026-08-10T16:58:02.524062Z","title":"Nash learning from human feedback","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":45,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.524062Z"},"links":{"cited_paper":"/paper/2312.00886","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:1084418c85f76ba157e0938e546baf0560bcce9a93ddfad844b9fb23c946c9df","observation_id":"3dadfa13-9fd9-4bbf-846e-7ebd4a259c5d","resolution":{"observed_at":"2026-08-10T16:58:02.524062Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.528746Z","title":"Count-based exploration with neural density models","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":46,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.528746Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:6726520a71e26cfe5639128e6550cfa18c7f5e9bfd2b85c2cd67abf619f45d35","observation_id":"bde05da0-a5d9-4f72-8cf7-223df294b920","resolution":{"observed_at":"2026-08-10T16:58:02.528746Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.533763Z","title":"Training language models to follow instructions with human feedback","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":47,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.533763Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:b672fd56b34163e5bc96ae8509cbab65a65498079702f027f9e408f422254b42","observation_id":"b0cc0e5f-ebaf-4182-b347-dbc033d14cd2","resolution":{"observed_at":"2026-08-10T16:58:02.533763Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.06281","last_updated":"2024-01-03T12:20:35Z","snapshot_observed_at":"2026-08-06T08:22:38.908366Z","submitted_at":"2023-12-11T10:35:32Z","title":"EQ-Bench: An Emotional Intelligence Benchmark for Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.06281","snapshot_observed_at":"2026-08-10T16:58:02.539280Z","title":"Eq-bench: An emotional intelligence benchmark for large language models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":48,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.539280Z"},"links":{"cited_paper":"/paper/2312.06281","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:80dbc7e0c408aab0e26e096084181b9bc8d19a5f23e80ef5e6928fdfd75af5b5","observation_id":"b8995c09-eae0-4c6b-813d-d75ac4a71731","resolution":{"observed_at":"2026-08-10T16:58:02.539280Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.545044Z","title":"Curiosity-driven exploration by self-supervised prediction","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":49,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.545044Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:1e18fb19df1a4cb7d8cd58af25f16c775e362b0e801a108c6f69ba85d67c089c","observation_id":"925ec6a9-62f9-4792-bce0-e83dcb298f1f","resolution":{"observed_at":"2026-08-10T16:58:02.545044Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:03.567122Z","title":"Contrastive ucb: Provably efficient contrastive self-supervised learning in online reinforcement learning","venue":null,"work_id":"723b64f7-3533-4663-86ee-ba622abbd91f","year":null},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":50,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.550680Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:7fcf6f1340d8adafa4d5264080bb5a18e393b3668dd1b5d2ab78c5dc606a6caa","observation_id":"4860ad6f-df33-4618-ad38-765b80d8bf23","resolution":{"observed_at":"2026-08-10T16:58:03.572349Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.556057Z","title":"Direct preference optimization: Your language model is secretly a reward model","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":51,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.556057Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:d1232dc1f5e38034d51e5df5b8da592ddc85a80fdb6ee4d6a3491b88e860861e","observation_id":"3f86d261-1d25-49dd-b357-a0df6ebbcbdf","resolution":{"observed_at":"2026-08-10T16:58:02.556057Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.02900","last_updated":"2024-11-05T01:44:14Z","snapshot_observed_at":"2026-07-06T18:25:35.827334Z","submitted_at":"2024-06-05T03:41:37Z","title":"Scaling Laws for Reward Model Overoptimization in Direct Alignment Algorithms","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.02900","snapshot_observed_at":"2026-08-10T16:58:02.560951Z","title":"Scaling laws for reward model overoptimization in direct alignment algorithms","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":52,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.560951Z"},"links":{"cited_paper":"/paper/2406.02900","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:6193368733789cea14dc838bab5b4c671bc3750d7b0eab836af04a78e4c77475","observation_id":"dc65dccf-c32c-4e79-840d-5f81612232a5","resolution":{"observed_at":"2026-08-10T16:58:02.560951Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.12358","last_updated":"2024-08-12T21:13:35Z","snapshot_observed_at":"2026-07-06T18:02:19.428801Z","submitted_at":"2024-04-18T17:37:02Z","title":"From $r$ to $Q^*$: Your Language Model is Secretly a Q-Function","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.12358","snapshot_observed_at":"2026-08-10T16:58:02.566480Z","title":"From r to q*: Your language model is secretly a q-function","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":53,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.566480Z"},"links":{"cited_paper":"/paper/2404.12358","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:49ac766c2e55d9253516dde1653e2547e1131eb2352432afb2ccaa2c761acc66","observation_id":"fd5effcb-c5e7-486e-9db5-38f54e5ee978","resolution":{"observed_at":"2026-08-10T16:58:02.566480Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:03.538429Z","title":"Optimistic exploration even with a pessimistic initialisation","venue":null,"work_id":"d036ef18-83f0-4366-bb9c-ade6c23b23de","year":2020},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":54,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.572108Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:d12fa858e5256fb0d5d53b467e558840a3309165f06de7a7462b4f864cbb9ad6","observation_id":"0a117eff-091f-4b7d-a6bc-4bb6fe7826a2","resolution":{"observed_at":"2026-08-10T16:58:03.543956Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1707.06347","last_updated":"2017-08-28T09:20:06Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2017-07-20T02:32:33Z","title":"Proximal Policy Optimization Algorithms","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1707.06347","snapshot_observed_at":"2026-08-10T16:58:02.578564Z","title":"Proximal policy optimization algorithms","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":55,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.578564Z"},"links":{"cited_paper":"/paper/1707.06347","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:80eb4e21522a78d7b6e09bbe609d64b942154f639bf400fda9b2daab99076911","observation_id":"f3b8509e-4cee-4633-a644-45c1cc7f43ba","resolution":{"observed_at":"2026-08-10T16:58:02.578564Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.03716","last_updated":"2024-07-10T23:15:49Z","snapshot_observed_at":"2026-08-03T14:09:43.107865Z","submitted_at":"2023-10-05T17:38:28Z","title":"A Long Way to Go: Investigating Length Correlations in RLHF","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.03716","snapshot_observed_at":"2026-08-10T16:58:02.583705Z","title":"A long way to go: Investigating length correlations in rlhf","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":56,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.583705Z"},"links":{"cited_paper":"/paper/2310.03716","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:25b3a3bebb00e1e907cbe1a9e72227c72d7ba0b3525cd1cb3d4a66b764dd0b0a","observation_id":"4974abf7-e451-4b7c-b3a2-52f85b3ce142","resolution":{"observed_at":"2026-08-10T16:58:02.583705Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.01511","last_updated":"2024-08-07T03:46:30Z","snapshot_observed_at":"2026-08-09T21:37:23.855026Z","submitted_at":"2024-05-02T17:44:41Z","title":"D2PO: Discriminator-Guided DPO with Response Evaluation Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.01511","snapshot_observed_at":"2026-08-10T16:58:02.589047Z","title":"D2po: Discriminator-guided dpo with response evaluation models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":57,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.589047Z"},"links":{"cited_paper":"/paper/2405.01511","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:08fb3053206a2d96c4ee455a04bc970bd3e9bc52925fe7783cfc22be4b22d30e","observation_id":"83b75950-a1ae-4a1c-81e5-8a156403d141","resolution":{"observed_at":"2026-08-10T16:58:02.589047Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:03.521600Z","title":"An analysis of model-based interval estimation for markov decision processes","venue":null,"work_id":"909d52ac-c7e0-4198-8cf7-81eb1bd0987d","year":2008},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":58,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.594628Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:9bc62ef9a2bbacd94ee8039d10780cfbd546c818977d372943a05f9fcab26fbf","observation_id":"3eab34b1-641e-424b-a777-9405db12d272","resolution":{"observed_at":"2026-08-10T16:58:03.527064Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.04056","last_updated":"2024-06-13T14:45:36Z","snapshot_observed_at":"2026-08-09T09:34:04.204036Z","submitted_at":"2024-01-08T17:55:02Z","title":"A Minimaximalist Approach to Reinforcement Learning from Human Feedback","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.04056","snapshot_observed_at":"2026-08-10T16:58:02.599465Z","title":"A minimaximalist approach to reinforcement learning from human feedback","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":59,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.599465Z"},"links":{"cited_paper":"/paper/2401.04056","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:d379ee23031977f77022d4cf3dd1710a0fa04e1ef421ac15d546db7ab2b75347","observation_id":"fd4f4167-2e88-4f61-850d-ab1fbbdf79d0","resolution":{"observed_at":"2026-08-10T16:58:02.599465Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.604270Z","title":"\\# exploration: A study of count-based exploration for deep reinforcement learning","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":60,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.604270Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:5f6b3915e0f4ee29f433fded3f01406f1c91c0ed760803234545a978766bee16","observation_id":"f8fd4d4e-54de-4202-9681-2119cceb56c2","resolution":{"observed_at":"2026-08-10T16:58:02.604270Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.05749","last_updated":"2024-05-28T23:25:15Z","snapshot_observed_at":"2026-08-10T05:17:53.947932Z","submitted_at":"2024-02-08T15:33:09Z","title":"Generalized Preference Optimization: A Unified Approach to Offline Alignment","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.05749","snapshot_observed_at":"2026-08-10T16:58:02.609795Z","title":"Generalized preference optimization: A unified approach to offline alignment","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":61,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.609795Z"},"links":{"cited_paper":"/paper/2402.05749","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:e4e52aaef5bf8a44cb91b3bc032ca4e5cfc5fe8af3281f91d790b692e7525213","observation_id":"d5d1e5d9-8723-4de7-97a1-bf97586244a1","resolution":{"observed_at":"2026-08-10T16:58:02.609795Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2307.09288","last_updated":"2023-07-19T17:08:59Z","snapshot_observed_at":"2026-08-07T12:56:43.323460Z","submitted_at":"2023-07-18T14:31:57Z","title":"Llama 2: Open Foundation and Fine-Tuned Chat Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.09288","snapshot_observed_at":"2026-08-10T16:58:02.614756Z","title":"Llama 2: Open foundation and fine-tuned chat models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":62,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.614756Z"},"links":{"cited_paper":"/paper/2307.09288","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:af8996a76edfa40a7e5da3487f9d813f6caf3823788cc41daace6e2fe8283bf9","observation_id":"28198650-2280-43b9-b673-bd3300888985","resolution":{"observed_at":"2026-08-10T16:58:02.614756Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.16944","last_updated":"2023-10-25T19:25:16Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-10-25T19:25:16Z","title":"Zephyr: Direct Distillation of LM Alignment","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.16944","snapshot_observed_at":"2026-08-10T16:58:02.620383Z","title":"Zephyr: Direct distillation of lm alignment","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":63,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.620383Z"},"links":{"cited_paper":"/paper/2310.16944","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:773411878efd911943712def14d839304e558c1218babec3608913fa35ffd74b","observation_id":"11f1b9f5-4ff3-400a-9676-bd2544fa9720","resolution":{"observed_at":"2026-08-10T16:58:02.620383Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.21046","last_updated":"2024-05-31T17:39:06Z","snapshot_observed_at":"2026-07-31T03:29:04.443362Z","submitted_at":"2024-05-31T17:39:06Z","title":"Exploratory Preference Optimization: Harnessing Implicit Q*-Approximation for Sample-Efficient RLHF","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.21046","snapshot_observed_at":"2026-08-10T16:58:02.626291Z","title":"Exploratory preference optimization: Harnessing implicit q*-approximation for sample-efficient rlhf","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":64,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.626291Z"},"links":{"cited_paper":"/paper/2405.21046","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:e865d7241abb1447a5d23f9f2f8ab2330f2a3e1eb5df481f519d7f3d401fa61e","observation_id":"90197c55-9f72-42fe-a34a-f099727d8e81","resolution":{"observed_at":"2026-08-10T16:58:02.626291Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.631290Z","title":"Iterative preference learning from human feedback: Bridging theory and practice for rlhf under kl-constraint","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":65,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.631290Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:1d48061be65c060c934aee877b9ccc5eed0df65ac28b11133ffc48539a1f36a1","observation_id":"ff9d91df-bbb1-4686-bece-ff3fa3c263c4","resolution":{"observed_at":"2026-08-10T16:58:02.631290Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:03.483523Z","title":"Is dpo superior to ppo for llm alignment? a comprehensive study","venue":null,"work_id":"27d604d6-0dd6-4abe-b238-b696b5d19db7","year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":66,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.636237Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:5dc12f3acf0a4b7142b4a3bf9feddc24df814d844e8dd8d5246761d02f3e484b","observation_id":"b68010ec-8d77-469d-8b74-cc3561b50960","resolution":{"observed_at":"2026-08-10T16:58:03.488698Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.640741Z","title":"Rorl: Robust offline reinforcement learning via conservative smoothing","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":67,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.640741Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:4c9cbd49e67d3034fc177ac1c2fb1f4f4baf8d6b014a299a21e7e874c20a6b35","observation_id":"a5c652fc-42fe-4a2b-8e5a-f41cf3cf9f65","resolution":{"observed_at":"2026-08-10T16:58:02.640741Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.04652","last_updated":"2025-01-21T10:12:05Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-03-07T16:52:49Z","title":"Yi: Open Foundation Models by 01.AI","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.04652","snapshot_observed_at":"2026-08-10T16:58:02.645376Z","title":"Yi: Open foundation models by 01","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":68,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.645376Z"},"links":{"cited_paper":"/paper/2403.04652","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:7712b532fae6feb4a6c15e7c1dba2692fa08dd46d898f5d6533d588ef424facb","observation_id":"64c656ab-1840-42e8-b787-1f0d1cb7924d","resolution":{"observed_at":"2026-08-10T16:58:02.645376Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.04920","last_updated":"2024-10-10T10:05:43Z","snapshot_observed_at":"2026-08-02T15:09:55.584633Z","submitted_at":"2024-04-07T11:20:32Z","title":"Regularized Conditional Diffusion Model for Multi-Task Preference Alignment","version":2},"cited_work":{"arxiv_id":"2404.04920","doi":null,"metadata_source":"pith","pith_arxiv_id":"2404.04920","snapshot_observed_at":"2026-08-10T16:58:02.857375Z","title":"Regularized Conditional Diffusion Model for Multi-Task Preference Alignment","venue":"cs.LG","work_id":"ea6f31a7-84e9-4772-9332-581320fdd929","year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":69,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.649973Z"},"links":{"cited_paper":"/paper/2404.04920","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:d76a5dbd744fd76f6bf0312e2fc0bc9ed41fa5a0c71491605cb13f09fb6fa55f","observation_id":"da437e9d-fa08-415d-8c39-920c81aaccf4","resolution":{"observed_at":"2026-08-10T16:58:02.864613Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.10020","last_updated":"2025-03-28T00:06:51Z","snapshot_observed_at":"2026-08-07T08:02:34.857823Z","submitted_at":"2024-01-18T14:43:47Z","title":"Self-Rewarding Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.10020","snapshot_observed_at":"2026-08-10T16:58:02.655481Z","title":"Self-rewarding language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":70,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.655481Z"},"links":{"cited_paper":"/paper/2401.10020","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:bc3921734fe9aa8aa609692155eb094ebc4d871a80a5314b27895e8b764c224b","observation_id":"16e79d25-7dbb-489f-ba33-e2376f4b90c8","resolution":{"observed_at":"2026-08-10T16:58:02.655481Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.13586","last_updated":"2025-03-03T14:24:45Z","snapshot_observed_at":"2026-07-06T19:35:21.868885Z","submitted_at":"2024-10-17T14:21:32Z","title":"Preference Aligned Diffusion Planner for Quadrupedal Locomotion Control","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.13586","snapshot_observed_at":"2026-08-10T16:58:02.660431Z","title":"Preference aligned diffusion planner for quadrupedal locomotion control","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":71,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.660431Z"},"links":{"cited_paper":"/paper/2410.13586","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:84d23c5d10b0b0aad0c641ee57b38c71fe24b232ffe108f3e9d34ceb073afa78","observation_id":"158014e7-edc5-41f4-84c0-7e149477fe99","resolution":{"observed_at":"2026-08-10T16:58:02.660431Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1905.07830","last_updated":"2019-05-19T23:57:23Z","snapshot_observed_at":"2026-08-10T16:18:23.994244Z","submitted_at":"2019-05-19T23:57:23Z","title":"HellaSwag: Can a Machine Really Finish Your Sentence?","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1905.07830","snapshot_observed_at":"2026-08-10T16:58:02.666433Z","title":"Hellaswag: Can a machine really finish your sentence? arXiv preprint arXiv:1905.07830, 2019","venue":null,"work_id":null,"year":1905},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":72,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.666433Z"},"links":{"cited_paper":"/paper/1905.07830","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:5fb38a37ffe2326f1d3d1da663b8a933015b4eb35976c4189b9927c2e3161205","observation_id":"e93129e2-bb24-4e68-b060-75ec78a33bfd","resolution":{"observed_at":"2026-08-10T16:58:02.666433Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.19332","last_updated":"2024-11-05T07:21:18Z","snapshot_observed_at":"2026-07-06T18:22:10.094519Z","submitted_at":"2024-05-29T17:59:07Z","title":"Self-Exploring Language Models: Active Preference Elicitation for Online Alignment","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.19332","snapshot_observed_at":"2026-08-10T16:58:02.671392Z","title":"Self-exploring language models: Active preference elicitation for online alignment","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":73,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.671392Z"},"links":{"cited_paper":"/paper/2405.19332","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:1ff869f7e9b1b62a5bbbe1184e3e5b9a130b1292a141c95f86dc27390b332d7f","observation_id":"8589d06f-41f9-4b44-9136-39958e26ad64","resolution":{"observed_at":"2026-08-10T16:58:02.671392Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.10425","last_updated":"2023-05-17T17:57:10Z","snapshot_observed_at":"2026-08-10T10:05:24.083432Z","submitted_at":"2023-05-17T17:57:10Z","title":"SLiC-HF: Sequence Likelihood Calibration with Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.10425","snapshot_observed_at":"2026-08-10T16:58:02.677210Z","title":"Slic-hf: Sequence likelihood calibration with human feedback","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":74,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.677210Z"},"links":{"cited_paper":"/paper/2305.10425","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:cb489ca1a885a24f01505d977c08f23e1562bcf8169c0feb7c9cd44eef9b70bd","observation_id":"ccd42246-84e3-40b2-ba18-505b2e9a2926","resolution":{"observed_at":"2026-08-10T16:58:02.677210Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.682027Z","title":"Judging llm-as-a-judge with mt-bench and chatbot arena","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":75,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.682027Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:7f40ea49025931d2cf1582777995f182f9076c53246f89af158795b1384f5aa2","observation_id":"219b48b0-ef9c-4a92-966c-d08c93b6e255","resolution":{"observed_at":"2026-08-10T16:58:02.682027Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:03.447417Z","title":"Principled reinforcement learning with human feedback from pairwise or k-wise comparisons","venue":null,"work_id":"e26586a4-b667-4f09-84fa-2ca6866f1ba3","year":2023},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":76,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.686792Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:8a365953349856b95a49d51a964a32e39d0bacc0a9f72efbcea1c91f9e7e5938","observation_id":"f1210001-b081-4da7-b0eb-90c876f76849","resolution":{"observed_at":"2026-08-10T16:58:03.452536Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1909.08593","last_updated":"2020-01-08T23:02:36Z","snapshot_observed_at":"2026-08-11T04:34:07.318549Z","submitted_at":"2019-09-18T17:33:39Z","title":"Fine-Tuning Language Models from Human Preferences","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1909.08593","snapshot_observed_at":"2026-08-10T16:58:02.691569Z","title":"Fine-tuning language models from human preferences","venue":null,"work_id":null,"year":1909},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":77,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.691569Z"},"links":{"cited_paper":"/paper/1909.08593","citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:4e841fc9254db000aa0babee3d42eaa012f9cbe217e65a596885812f651758cc","observation_id":"7c8604ef-bd9c-4599-9b57-b61fe2bbadc3","resolution":{"observed_at":"2026-08-10T16:58:02.691569Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.696219Z","title":"@esa (Ref","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":78,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.696219Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:91bbc3d64f8d3371271212359e7f17dfda15452ab4c16a0ea10fda57e4d1854f","observation_id":"335f4395-a3a5-471e-99aa-b46c6a07584b","resolution":{"observed_at":"2026-08-10T16:58:02.696219Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:02.701890Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":79,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.701890Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:9199a2d2e07e9e1dd264cc632ed756f94c26b1afcd7b7016fef36e87901903bb","observation_id":"3f144e43-f8d6-4621-b292-9fe9213e1530","resolution":{"observed_at":"2026-08-10T16:58:02.701890Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T16:58:03.411740Z","title":"a&!Ï \"l=-BpEMUs J5ū ?bvtCy O (^rڜH - Y`J* aH/'V t@Ys ;ӓj(u B FBa 竑 6 ^mN OB Y>X 5 >D Q=h .+' A Ի, _|k P(qd/T) nV P C/ۿA+ڽ.W b< ydQx> gQ`U Ԡ < a","venue":null,"work_id":"4cd7e4e9-7ac1-41b0-8756-063f661aa991","year":2000},"citing_paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration","version":3},"reference_index":80,"source":"arxiv_source","source_observed_at":"2026-08-10T16:58:02.706784Z"},"links":{"citing_paper":"/paper/2501.12735"},"observation_digest":"sha256:932603cc8142925843755070c040182575f668e762f8a4f6ff93f9bed3e206a2","observation_id":"34a2b941-c2a4-40be-922c-02facace7c73","resolution":{"observed_at":"2026-08-10T16:58:03.416315Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2501.12735","last_updated":"2025-02-07T02:13:27Z","latest_version":3,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-10T16:48:54.578052Z","submitted_at":"2025-01-22T09:12:09Z","title":"Online Preference Alignment for Language Models via Count-based Exploration"},"reference_resolution":{"displayed":80,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":61,"verified_exact":1,"verified_fuzzy":18},"total_outbound_references":80},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"thesis":"As of 11 August 2026, this Paper Citation Record lists 80 of 80 outbound references and 6 inbound Pith citation observations for arXiv:2501.12735."}