{"as_of":"2026-08-15T11:30:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:8e4fcf39ee681af871f509826fab58c750751e6fd4241a62b4ad45b4995fdc65","coverage":[{"denominator":78,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":78,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-06T14:45:42.780856Z","state":"measured"},{"denominator":79,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":79,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-15T06:32:42.880941+00:00","state":"measured"},{"denominator":1,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":1,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-03T17:21:27.531024Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2507.18182","snapshot_observed_at":"2026-08-03T17:21:27.531024Z","title":"CoRR , volume =","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2512.11001","last_updated":"2026-07-07T14:01:07Z","snapshot_observed_at":"2026-08-12T13:49:43.584085Z","submitted_at":"2025-12-10T20:16:20Z","title":"Rethinking Query Optimization for Multi-Agent Systems [Vision]","version":2},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-03T17:21:27.531024Z"},"links":{"cited_paper":"/paper/2507.18182","citing_paper":"/paper/2512.11001"},"observation_digest":"sha256:af771fb407dd241c9e6187b7dff15a6e56c562f86380867fbb8aad1673235172","observation_id":"0c7b0f32-2814-431a-9a24-54afd8f992c0","resolution":{"observed_at":"2026-08-03T17:21:27.531024Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2507.18182/citation-record","integrity":"/paper/2507.18182/integrity","json":"/paper/2507.18182/citation-record.json","paper":"/paper/2507.18182"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-07T07:30:12.213965Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-08-06T14:45:42.561961Z","title":"Gpt-4 technical report","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.561961Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:98efb4908d5c08fb2d13583ac2514bac2272c03a748521642851325461b00145","observation_id":"5216c661-9e79-4deb-9ee1-06cda3c739d5","resolution":{"observed_at":"2026-08-06T14:45:42.561961Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.12712","last_updated":"2023-04-13T20:41:31Z","snapshot_observed_at":"2026-08-03T04:49:15.195814Z","submitted_at":"2023-03-22T16:51:28Z","title":"Sparks of Artificial General Intelligence: Early experiments with GPT-4","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.12712","snapshot_observed_at":"2026-08-06T14:45:42.566066Z","title":"Sparks of artificial general intelligence: Early experiments with gpt-4","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.566066Z"},"links":{"cited_paper":"/paper/2303.12712","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:08e06d9eae147c24992177578617528e10c88021e28d3b812f7edf9c4758dfd6","observation_id":"cb2282c2-f384-4bd5-b127-3f706c3f7a62","resolution":{"observed_at":"2026-08-06T14:45:42.566066Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2302.13971","last_updated":"2023-02-27T17:11:15Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-02-27T17:11:15Z","title":"LLaMA: Open and Efficient Foundation Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2302.13971","snapshot_observed_at":"2026-08-06T14:45:42.569568Z","title":"Llama: Open and efficient foundation language models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.569568Z"},"links":{"cited_paper":"/paper/2302.13971","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:ea0399ed616ef8cc91e6bd794b604bc7eae1c4f134d732ce8c4558e809e7cc3d","observation_id":"2760630c-638e-44ae-86a4-a0c55b30d2d3","resolution":{"observed_at":"2026-08-06T14:45:42.569568Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.11805","last_updated":"2025-05-09T21:04:06Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-12-19T02:39:27Z","title":"Gemini: A Family of Highly Capable Multimodal Models","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.11805","snapshot_observed_at":"2026-08-06T14:45:42.572487Z","title":"Gemini: a family of highly capable multimodal models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.572487Z"},"links":{"cited_paper":"/paper/2312.11805","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:42341a1977357a3186341b6f79c608a2df96b7f3853a53658f8346d46c33c691","observation_id":"f9293f9f-fa3a-444a-b726-edeaf7c1a832","resolution":{"observed_at":"2026-08-06T14:45:42.572487Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.618524Z","title":"The economic potential of generative ai: The next productivity frontier, 2023","venue":null,"work_id":"26cec241-0f45-4c87-a546-19389588577e","year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.575869Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:93db60776a354542f7810442efd2051fd3873f4f544c0b6f4cbe32e10affdf6c","observation_id":"218de190-04dc-4364-95d0-850a2e06143d","resolution":{"observed_at":"2026-08-06T14:45:43.621636Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.609493Z","title":"Pwc is accelerating adoption of ai with chatgpt enterprise in us and uk and with clients, 2024","venue":null,"work_id":"b9cc843b-c0dd-4e0e-ae33-52e533f19f01","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.579151Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:5d953fc4ca66e29c675f41b980785c9c5cb5737f1154b915dcfa2fb268b03d30","observation_id":"a77f83e9-b076-47b2-bab1-54b48e044733","resolution":{"observed_at":"2026-08-06T14:45:43.612570Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2404.01869","last_updated":"2024-08-06T11:58:53Z","snapshot_observed_at":"2026-08-14T23:20:45.882944Z","submitted_at":"2024-04-02T11:46:31Z","title":"Beyond Accuracy: Evaluating the Reasoning Behavior of Large Language Models -- A Survey","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.01869","snapshot_observed_at":"2026-08-06T14:45:42.582894Z","title":"Beyond accuracy: evaluating the reasoning behavior of large language models–a survey","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.582894Z"},"links":{"cited_paper":"/paper/2404.01869","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:7ef86bc6c1a55baa988bb817a9b5bbf50e917b1cf73f2a4082165aee1fc08238","observation_id":"8d2d8d73-ee8e-4180-82a9-32fcbcc1dac6","resolution":{"observed_at":"2026-08-06T14:45:42.582894Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.599983Z","title":"Shortcut learning of large language models in natural language understanding","venue":null,"work_id":"44f72b25-531b-433d-9d88-6a5261b5a888","year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.585882Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:02761c6377ac93d246431cc0d93cbec78be2b02c62bbda23396ff25529851b77","observation_id":"58eb326e-4fdb-4ed6-9560-5333f4283e41","resolution":{"observed_at":"2026-08-06T14:45:43.603251Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2405.03205","last_updated":"2025-05-30T20:24:51Z","snapshot_observed_at":"2026-08-13T00:14:16.157621Z","submitted_at":"2024-05-06T07:10:09Z","title":"Anchored Answers: Unravelling Positional Bias in GPT-2's Multiple-Choice Questions","version":3},"cited_work":{"arxiv_id":"2405.03205","doi":null,"metadata_source":"pith","pith_arxiv_id":"2405.03205","snapshot_observed_at":"2026-08-06T14:45:43.141576Z","title":"Anchored Answers: Unravelling Positional Bias in GPT-2's Multiple-Choice Questions","venue":"cs.CL","work_id":"8e2c79e6-a649-4b6c-a46d-005320e120a0","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.591793Z"},"links":{"cited_paper":"/paper/2405.03205","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:85719dbab02af3ef237f7e5fda1de650bd2d91cc0b71da26ca2021fa6060654e","observation_id":"f349f8bc-052c-40fb-9d71-1c872ffe3904","resolution":{"observed_at":"2026-08-06T14:45:43.145432Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.15393","last_updated":"2024-10-20T13:47:39Z","snapshot_observed_at":"2026-08-14T17:33:38.973788Z","submitted_at":"2024-10-20T13:47:39Z","title":"CalibraEval: Calibrating Prediction Distribution to Mitigate Selection Bias in LLMs-as-Judges","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.15393","snapshot_observed_at":"2026-08-06T14:45:42.595463Z","title":"Calibraeval: Calibrating prediction distribution to mitigate selection bias in llms-as-judges","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.595463Z"},"links":{"cited_paper":"/paper/2410.15393","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:c671b129276feb696b5d2532e620c52f4129f2c8ca8868a72cba247a2ee1c3f6","observation_id":"48e8a152-485c-49e4-9a07-f5963bc2dd57","resolution":{"observed_at":"2026-08-06T14:45:42.595463Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.08382","last_updated":"2024-08-20T08:07:49Z","snapshot_observed_at":"2026-08-13T00:32:02.037290Z","submitted_at":"2024-04-12T10:36:15Z","title":"Look at the Text: Instruction-Tuned Language Models are More Robust Multiple Choice Selectors than You Think","version":2},"cited_work":{"arxiv_id":"2404.08382","doi":null,"metadata_source":"pith","pith_arxiv_id":"2404.08382","snapshot_observed_at":"2026-08-06T14:45:43.121164Z","title":"Look at the Text: Instruction-Tuned Language Models are More Robust Multiple Choice Selectors than You Think","venue":"cs.CL","work_id":"f7af48c2-fb1d-4fff-939c-198e46fe32d9","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.598610Z"},"links":{"cited_paper":"/paper/2404.08382","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:84d61bf4acda458150de6cacddf6fed5a1e675bc1da0282f477fa7bae9cae955","observation_id":"804d4f28-d333-42c6-9101-1f053eb2d790","resolution":{"observed_at":"2026-08-06T14:45:43.124439Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.601429Z","title":"Language models are few-shot learners","venue":null,"work_id":null,"year":1901},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.601429Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:fe14eecbc22d6f82136370d6f6e1622cc6795ad421cdb4021d889d4149e452c2","observation_id":"e2ddc13a-a790-4983-8e6b-0df7fcbf2a43","resolution":{"observed_at":"2026-08-06T14:45:42.601429Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.604125Z","title":"Exploring the limits of transfer learning with a unified text-to-text transformer","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.604125Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:493b54e4c8fa617d77681e7dee616623ac112076c3950210b3c00a954c0a1aa1","observation_id":"0cc113ea-39ac-4f19-a6ef-57a2f6c3590c","resolution":{"observed_at":"2026-08-06T14:45:42.604125Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2009.03300","last_updated":"2021-01-12T18:57:11Z","snapshot_observed_at":"2026-08-13T20:44:28.824685Z","submitted_at":"2020-09-07T17:59:25Z","title":"Measuring Massive Multitask Language Understanding","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.03300","snapshot_observed_at":"2026-08-06T14:45:42.607027Z","title":"Measuring massive multitask language understanding","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.607027Z"},"links":{"cited_paper":"/paper/2009.03300","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:7a6c0dffb87556cd1c4bac0248eefea000d7ca3deead26ccd018ca2901ba8887","observation_id":"d5ea368c-46eb-4423-8f61-3f20b0bb8b88","resolution":{"observed_at":"2026-08-06T14:45:42.607027Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.578988Z","title":"Commonsenseqa: A question answering challenge targeting commonsense knowledge","venue":null,"work_id":"a3a89ee0-eae5-42f1-b6a6-910030603c84","year":2019},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.609512Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:dfddadf27cb7db670ff7024ffabfacbf9f409a786e13c4f9e475131c11ee661c","observation_id":"792dc029-15b8-4a49-bf2f-6bb7d7223cfd","resolution":{"observed_at":"2026-08-06T14:45:43.581828Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1803.05457","last_updated":"2018-03-14T18:04:21Z","snapshot_observed_at":"2026-08-14T19:36:07.505691Z","submitted_at":"2018-03-14T18:04:21Z","title":"Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1803.05457","snapshot_observed_at":"2026-08-06T14:45:42.612530Z","title":"Think you have solved question answering? try arc, the ai2 reasoning challenge","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.612530Z"},"links":{"cited_paper":"/paper/1803.05457","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:8fcdc8ba673db991d6c12bbf2c462f3ea78c983c627a6d267cc67f8703624287","observation_id":"ca26ebfc-5895-458f-80aa-cdd3cebe1982","resolution":{"observed_at":"2026-08-06T14:45:42.612530Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.615299Z","title":"Can a suit of armor conduct electricity? a new dataset for open book question answering","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.615299Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:013522e6d3b8e2cafffe5d5538bb645f178124e17cb69f14bf1276cc64d8a0d1","observation_id":"ad484efe-39c1-4c9f-9489-a11f91e62622","resolution":{"observed_at":"2026-08-06T14:45:42.615299Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.564500Z","title":"Beyond the imitation game: Quantifying and extrapolating the capabilities of language models","venue":null,"work_id":"538e3469-9e3a-4b00-9f56-93b604fbd42b","year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.618154Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:8516563a644bd3a4f61851ed7fa0b826a9751256d20ca269ce143a251a49d92a","observation_id":"bd3fae3e-6895-46f2-8f8a-9d697081e08e","resolution":{"observed_at":"2026-08-06T14:45:43.567456Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-06T14:45:42.620508Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.620508Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:8b1411a0e1fba627ec97119fc9a82c82fd044cec9a017d9d4cbff06d61b18942","observation_id":"b91a51a6-a339-4d1e-b70a-3fb91eafeb9b","resolution":{"observed_at":"2026-08-06T14:45:42.620508Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.555280Z","title":"M3exam: A multilingual, multimodal, multilevel benchmark for examining large language models","venue":null,"work_id":"b8e6f6a4-e43b-49b6-9d87-b40b0f23ed6f","year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.623159Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:60ee92ec2d6a8d75031f1ad5b0fc9cb08a4093b01af6a42fbe542ec566cd5a15","observation_id":"a999ef10-10e4-4015-be27-90b254a96c08","resolution":{"observed_at":"2026-08-06T14:45:43.558566Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.546941Z","title":"Agieval: A human-centric benchmark for evaluating foundation models","venue":null,"work_id":"c49e0475-f072-4e9b-b7e2-5d746de2ad93","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.625905Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:10e1418c54537c6cf9e58459ff4ad97ebb4e9a4fb7f7f96012194eeaae01bf37","observation_id":"8123bda6-f4b8-46fc-8f07-d6a3cbaaecc7","resolution":{"observed_at":"2026-08-06T14:45:43.549898Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.628168Z","title":"Medmcqa: A large-scale multi-subject multi-choice dataset for medical domain question answering","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.628168Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:9ad039640d39849f2e59e7a498b4d3669b60f07050a49e99dc0bda65ee3665fa","observation_id":"d3e3c2aa-2ceb-4a65-8b2d-724c83c9f536","resolution":{"observed_at":"2026-08-06T14:45:42.628168Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.630597Z","title":"Crowdsourcing multiple choice science questions","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.630597Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:69e9a0786a616cf5d48531f575cef3e9d2cc3b6e7561466976ff2e05bb5832da","observation_id":"df6b1510-bd09-462b-9813-4e35ba20437d","resolution":{"observed_at":"2026-08-06T14:45:42.630597Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.526391Z","title":"From live data to high-quality benchmarks: The arena-hard pipeline","venue":null,"work_id":"d2cc26dd-a3df-4180-9301-9b6feeb22bdb","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.632984Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:16a3f443532e0c7eefcf42bd953009eb07ad2feeb44ab2e09e61be89b4882e15","observation_id":"0d09c583-8dd4-40cc-b490-2e0d5ca64a9d","resolution":{"observed_at":"2026-08-06T14:45:43.529542Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.635387Z","title":"Chatbot arena: An open platform for evaluating llms by human preference","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.635387Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:35037defde1f3ad2cd424a71e5d71a2ace3697690ccdae6956efaabfc95145f0","observation_id":"e3db5bdc-e636-4f69-a407-95adab2f9f80","resolution":{"observed_at":"2026-08-06T14:45:42.635387Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.511515Z","title":"Large language models are not fair evaluators","venue":null,"work_id":"f01a0b4d-339b-43d5-a8d5-2487e630526a","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.638295Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:0b4f016f9589cf809bb5d7c1f21f92ab74ec85eb5f7fdaf5a69671514519292b","observation_id":"9f7a62ba-9fde-4987-ae00-0777e92a8a65","resolution":{"observed_at":"2026-08-06T14:45:43.514598Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.502649Z","title":"Apbench and benchmarking large language model performance in fundamental astrodynamics problems for space engineering","venue":null,"work_id":"30fa044b-d9f8-4706-b9de-a997593d90b1","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.640769Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:ee203c9bae6d6e26d32d1c885ca1de49db991a2b847b3ddebe998c8cb30ca46c","observation_id":"ab2c8cd0-06b0-4f5d-9b3f-6be3e265e722","resolution":{"observed_at":"2026-08-06T14:45:43.505622Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.491806Z","title":"Where is the answer? an empirical study of positional bias for parametric knowledge extraction in language model","venue":null,"work_id":"77e371de-f3b0-4f55-a699-30a6aa37ee82","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.643189Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:9c93b73ec932d726a945fd86849263c110738c2af8ba01189d7c66d027204ddf","observation_id":"ecf966ea-e5af-4d75-a462-0c8c733f6360","resolution":{"observed_at":"2026-08-06T14:45:43.495745Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.482212Z","title":"Option symbol matters: Investigating and mitigating multiple-choice option symbol bias of large language models","venue":null,"work_id":"7180469c-5f47-4a21-8d0e-a84e0922d60f","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.645709Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:20d55869ed4bb0f2456d2c15a86c5eeb31ddd2a9c35b72a0044d2ecc157857bb","observation_id":"64cad35f-936a-4df2-9b8a-c5f5aff12226","resolution":{"observed_at":"2026-08-06T14:45:43.485373Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.473409Z","title":"Large language models sensitivity to the order of options in multiple- choice questions","venue":null,"work_id":"2f0fce3e-c072-4e6c-ae37-0b260076ee19","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.648245Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:98a16a026bd45e8b4b7b81a848e562bc6308558846771e3f10d25e587586be16","observation_id":"0627c88b-3464-416e-bfce-57c47b93af5b","resolution":{"observed_at":"2026-08-06T14:45:43.476599Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2309.03882","last_updated":"2024-02-22T01:40:35Z","snapshot_observed_at":"2026-08-15T09:16:07.428258Z","submitted_at":"2023-09-07T17:44:56Z","title":"Large Language Models Are Not Robust Multiple Choice Selectors","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.03882","snapshot_observed_at":"2026-08-06T14:45:42.650856Z","title":"Large language models are not robust multiple choice selectors","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.650856Z"},"links":{"cited_paper":"/paper/2309.03882","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:2eb85429414081e0571114ff43d72d5fd3448d290da233a277f4335ae7f9f089","observation_id":"23bf2d76-da00-464e-8b44-1cf3feee1a49","resolution":{"observed_at":"2026-08-06T14:45:42.650856Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.464099Z","title":"Fool your (vision and) language model with embarrassingly simple permutations","venue":null,"work_id":"f526553c-6805-41ed-b934-7fb550133fb5","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.653438Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:3aab434ef9a45fc5c89ef2560439d3a3fc592f213a608461a74f1e025466aa26","observation_id":"fee25609-a4fd-4c9e-b357-493252f824f9","resolution":{"observed_at":"2026-08-06T14:45:43.467133Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.454963Z","title":"Teacher-student training for debiasing: General permutation debiasing for large language models","venue":null,"work_id":"a7826d21-d4cc-46c1-98c0-2920a71344d3","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.656108Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:c9b5f42cfaa809f6176e589852293126c11cb27743e349a7aba1361949394f9f","observation_id":"4fd8a892-a18c-4df6-acf8-52a6a94ba69c","resolution":{"observed_at":"2026-08-06T14:45:43.458257Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2409.18857","last_updated":"2025-05-17T04:21:30Z","snapshot_observed_at":"2026-08-14T23:21:15.425666Z","submitted_at":"2024-09-27T15:53:54Z","title":"Mitigating Selection Bias with Node Pruning and Auxiliary Options","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.18857","snapshot_observed_at":"2026-08-06T14:45:42.658846Z","title":"Mitigating selection bias with node pruning and auxiliary options","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.658846Z"},"links":{"cited_paper":"/paper/2409.18857","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:ae54b7dca4a745c5a6ddead3e2d2f0600e30faf4cbe6987d3df9a5c2a4d7eac1","observation_id":"49d0fb8b-749a-4c49-a354-e988cd73e886","resolution":{"observed_at":"2026-08-06T14:45:42.658846Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.445455Z","title":"Unveiling selection biases: Exploring order and token sensitivity in large language models","venue":null,"work_id":"a7abbaa0-c4bc-41fa-9085-c9a87c8aa3a0","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.661589Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:c1e94b33eb69dd4e726fc1f0a3bd7a1f83c9ef2d533ab15ee6781e4b7b9bac69","observation_id":"27c90e41-76f3-4519-8ba4-c8865cdb35ec","resolution":{"observed_at":"2026-08-06T14:45:43.448857Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.664558Z","title":"Chain-of-thought prompting elicits reasoning in large language models.Advances in neural information processing systems, 35:24824–24837, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.664558Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:f0b17a6c56ec46c7afd2ff93d0523adfe9c53e98a12eda6d428419021f9ddf0f","observation_id":"c5073e1d-af59-4fab-a0aa-60dbc4c9be1b","resolution":{"observed_at":"2026-08-06T14:45:42.664558Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.667059Z","title":"Large language models are zero-shot reasoners","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.667059Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:39cb751826a6aeb1fba351b2aee877397ee0b74f85eba61d8014ec95af3e22bc","observation_id":"c5c97aeb-d96c-41b5-b53c-15482b0c79ff","resolution":{"observed_at":"2026-08-06T14:45:42.667059Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2203.11171","last_updated":"2023-03-07T17:57:37Z","snapshot_observed_at":"2026-07-06T12:50:22.773056Z","submitted_at":"2022-03-21T17:48:52Z","title":"Self-Consistency Improves Chain of Thought Reasoning in Language Models","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2203.11171","snapshot_observed_at":"2026-08-06T14:45:42.669715Z","title":"Self-consistency improves chain of thought reasoning in language models.arXiv preprint arXiv:2203.11171, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.669715Z"},"links":{"cited_paper":"/paper/2203.11171","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:6fcb464ab7a5d78ec71e068182a51dc8890ea23c09e28744b87af3ebcc7c03f4","observation_id":"7ed7787b-ed82-49ae-9e66-846a8d6348ed","resolution":{"observed_at":"2026-08-06T14:45:42.669715Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.424270Z","title":"Star: Self-taught reasoner bootstrapping reasoning with reasoning","venue":null,"work_id":"09ea7ace-09d5-4008-a05a-da4bc56b5b43","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.672549Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:25c4a5175e6d0bbba53e261154748ccf849ba762c1ae07fc01885b5ecb4d980c","observation_id":"5bbae5f3-6a6e-4388-ae67-6029fe8ad69a","resolution":{"observed_at":"2026-08-06T14:45:43.427766Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2205.10625","last_updated":"2023-04-16T22:08:08Z","snapshot_observed_at":"2026-08-06T09:00:42.886249Z","submitted_at":"2022-05-21T15:34:53Z","title":"Least-to-Most Prompting Enables Complex Reasoning in Large Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2205.10625","snapshot_observed_at":"2026-08-06T14:45:42.676271Z","title":"Least-to-most prompting enables complex reasoning in large language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.676271Z"},"links":{"cited_paper":"/paper/2205.10625","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:6ad1881a2a6b9bca5ce5856397d0561bae283145ce5cd1038d1ed154112ea901","observation_id":"e06a5064-bf7a-4a76-b3ef-2bab70a4aafa","resolution":{"observed_at":"2026-08-06T14:45:42.676271Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.680112Z","title":"React: Synergizing reasoning and acting in language models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.680112Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:a2b138fd1d3fef9a89ea7dcac59dffaf150e8504876d5fae5ba4353b54a0fcc1","observation_id":"7d3b7e95-806d-40bb-8b0a-78b2baf192ec","resolution":{"observed_at":"2026-08-06T14:45:42.680112Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.409524Z","title":"Debiasing in-context learning by instructing llms how to follow demonstrations","venue":null,"work_id":"0b3d6a8a-b63f-4e9f-8b97-efa13030a9c5","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.683542Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:faeaef1c54fb1f7e2262b1b8728bc69a32b91b32b565ee0ec971699a0632d18f","observation_id":"9d7f7b60-f742-4ed4-8262-538aa0aa21a9","resolution":{"observed_at":"2026-08-06T14:45:43.412777Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.12588","last_updated":"2023-10-23T01:27:38Z","snapshot_observed_at":"2026-08-02T13:06:11.850456Z","submitted_at":"2022-11-22T21:06:00Z","title":"Program of Thoughts Prompting: Disentangling Computation from Reasoning for Numerical Reasoning Tasks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.12588","snapshot_observed_at":"2026-08-06T14:45:42.686346Z","title":"Program of thoughts prompting: Disentangling computation from reasoning for numerical reasoning tasks","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.686346Z"},"links":{"cited_paper":"/paper/2211.12588","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:01ab1bfd31513a2cf0b5c5655760d3ed42e8864c9f9caf092e6c3d4cb848a346","observation_id":"568a757c-58f2-439f-9301-512f20c93783","resolution":{"observed_at":"2026-08-06T14:45:42.686346Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.400365Z","title":"Prompt sketching for large language models","venue":null,"work_id":"c882cf97-8b8d-4d70-98fc-07d04088d229","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.689179Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:69e488db4e7a157b9d8de13d1530dc38227e97eafa2acae2dcb01c47a96e9585","observation_id":"46c97963-d8cb-490f-b595-6a688a93b16c","resolution":{"observed_at":"2026-08-06T14:45:43.403188Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.691864Z","title":"Self-refine: Iterative refinement with self-feedback","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.691864Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:32819bfed242d44fdb8d37b550f586d006d8f74e3ff98758c8b029010e52958e","observation_id":"25ae7ea6-21e6-4729-a55d-f3fa6c05b86a","resolution":{"observed_at":"2026-08-06T14:45:42.691864Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2212.08073","last_updated":"2022-12-15T06:19:23Z","snapshot_observed_at":"2026-08-02T04:53:58.766070Z","submitted_at":"2022-12-15T06:19:23Z","title":"Constitutional AI: Harmlessness from AI Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2212.08073","snapshot_observed_at":"2026-08-06T14:45:42.694535Z","title":"Constitutional ai: Harmlessness from ai feedback","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.694535Z"},"links":{"cited_paper":"/paper/2212.08073","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:1fde49d6227842b0da00b7082f9b3aa97cd07f2b777acc8d86a688b366d2b964","observation_id":"88a31bde-6223-435b-90a9-6c6349be6b86","resolution":{"observed_at":"2026-08-06T14:45:42.694535Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.386360Z","title":"Neurologic decoding:(un) supervised neural text generation with predicate logic constraints","venue":null,"work_id":"17436e4e-b7ac-4482-ba86-bc4535076acc","year":2021},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.697124Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:95edc1deafbdaa4edd798eacef6de00b36b6b18a0188576c49af3194f437a286","observation_id":"74233023-f874-45d8-8741-9eebd5e2d606","resolution":{"observed_at":"2026-08-06T14:45:43.389274Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.376968Z","title":"Calibration of pre-trained transformers","venue":null,"work_id":"4eb64bff-b1ab-46f6-853c-19564353a8dc","year":2020},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.699431Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:0d2ebc900ac99ad92ad5b55fa889729419775c7a74421db1ddbd823157b0834b","observation_id":"fe1876f5-78d7-4d4c-b641-4d40858ffdf3","resolution":{"observed_at":"2026-08-06T14:45:43.379919Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.701730Z","title":"Calibrate before use: Improving few-shot performance of language models","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.701730Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:51dc184962a9d8df936661a77bef62e58c495d76273d9c09854ef977a285ad26","observation_id":"5ceeb87a-9260-4404-b871-4ed2589f7ac2","resolution":{"observed_at":"2026-08-06T14:45:42.701730Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.362088Z","title":"Calibrating language models with adaptive temperature scaling","venue":null,"work_id":"dc539f5f-9bcb-4f21-a55a-795a4af0b706","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.704480Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:bf120f6dbcc298f8a807b020b7a5c0b461083f002828a8be00947ce366664334","observation_id":"5f4750a8-0d4e-46bd-9a14-32a3ef571d81","resolution":{"observed_at":"2026-08-06T14:45:43.365364Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.353566Z","title":"Calibrating large language models with sample consistency","venue":null,"work_id":"6372a039-fe73-42c4-8ec0-bf2f1b4f88a4","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.707052Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:b70f28f2b665d62bddfcb007d38e5105093df3adeed7fbca51284a52ccf6793a","observation_id":"291e6d06-dc20-49f7-91ee-40bff5409dc0","resolution":{"observed_at":"2026-08-06T14:45:43.356435Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.344466Z","title":"Benchmarking uncertainty quantification methods for large language models with lm-polygraph","venue":null,"work_id":"3f9d4d4a-aeaa-41e9-b271-fa18dc13bd6b","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.709284Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:87ffda0ec428b8906cb9933788bf2eb5078acec6cb5228ab8b3e0936e8c0651d","observation_id":"f28128f1-d3a7-4447-b8ad-b0a4c509c41b","resolution":{"observed_at":"2026-08-06T14:45:43.347645Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.335449Z","title":"Thermometer: towards universal calibration for large language models","venue":null,"work_id":"6c2a6462-0269-44b0-8cb6-b6011eddebd3","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.711759Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:54b74e153672df97f795dfe156293958df1c9457ac3093522b72a9033461574c","observation_id":"c7ad5102-cabb-474c-b834-4c0c79213041","resolution":{"observed_at":"2026-08-06T14:45:43.338319Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.326679Z","title":"Monte carlo temperature: a robust sampling strategy for llm’s uncertainty quantification methods","venue":null,"work_id":"995fc3c7-98ac-4dae-865e-fdd67f5bd0a8","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.714104Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:9382734b3a0132e7175d470917147531d336b6041638b8f53ade15d4cc1f93b3","observation_id":"981f2f23-405a-4692-a00e-7f33bd4543e3","resolution":{"observed_at":"2026-08-06T14:45:43.329636Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.716462Z","title":"Charm: Calibrating reward models with chatbot arena scores","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.716462Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:107eda83f11f6124f4c0141d348a7d4130127f8fc55eeb90836b88e441344a66","observation_id":"30a565e2-2bc1-44e8-947c-a349b0dbe6b3","resolution":{"observed_at":"2026-08-06T14:45:42.716462Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.719088Z","title":"Restoring calibration for aligned large language models: A calibration-aware fine-tuning approach","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.719088Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:d5cdc30a4c39ad6887bfa0387d278f56ee16eac94190993504530ce8983ed663","observation_id":"8941c8ba-16f9-438b-9dcb-e1edbbcd7d68","resolution":{"observed_at":"2026-08-06T14:45:42.719088Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.317814Z","title":"Uncertainty estimation in large language models to support biodiversity conservation","venue":null,"work_id":"9033ea43-e6ba-4f73-a35b-fc6af24e1820","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.721589Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:051ab34a3185ec46878e85f0b02b1a0671642ae93c87faa3d7cfb1f2aafd29da","observation_id":"54a0abea-7d07-4838-bc15-dd125385f050","resolution":{"observed_at":"2026-08-06T14:45:43.321254Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2302.02083","last_updated":"2024-11-04T19:51:53Z","snapshot_observed_at":"2026-08-14T23:20:10.618972Z","submitted_at":"2023-02-04T03:50:01Z","title":"Evaluating Large Language Models in Theory of Mind Tasks","version":7},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2302.02083","snapshot_observed_at":"2026-08-06T14:45:42.724363Z","title":"Theory of mind may have spontaneously emerged in large language models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":59,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.724363Z"},"links":{"cited_paper":"/paper/2302.02083","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:5968b140dbaeb53710fc57e02d60a74e64299ced7b170e529d5f9bfb81c36295","observation_id":"c4da330a-b34e-4de7-b732-7b850c1a77d2","resolution":{"observed_at":"2026-08-06T14:45:42.724363Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.309088Z","title":"Neural theory-of-mind? on the limits of social intelligence in large lms","venue":null,"work_id":"e168591c-c5af-4d54-8403-56f8ec19a568","year":2022},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.727107Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:c395bb31f35afbb7339e03502bcdf8f9957c6c027d0b3b7caaa15d4945ee09a3","observation_id":"45e80a02-291f-45f8-af0f-f452ba01f3b9","resolution":{"observed_at":"2026-08-06T14:45:43.311892Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.300905Z","title":"Social iqa: Commonsense reasoning about social interactions","venue":null,"work_id":"d44cf160-8666-45a0-a063-a612a33dd624","year":2019},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":61,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.729494Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:a270374a09f8a2e79c1b21331a8a167b9c9012cd5962e70384993a57eef4f78c","observation_id":"3732c923-91aa-4618-9630-d810928a1e8b","resolution":{"observed_at":"2026-08-06T14:45:43.303710Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.291488Z","title":"Large language models are not strong abstract reasoners","venue":null,"work_id":"85db9d18-8640-4fe2-929b-59a6b63bfe9e","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":62,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.732181Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:cf4ec115f63290675bb7ee22b434d72b07b43c8222e8928469418a1bf4068291","observation_id":"f6ae2707-9ad2-4811-ade7-46f523ba93a3","resolution":{"observed_at":"2026-08-06T14:45:43.295120Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.283071Z","title":"Coglm: Tracking cognitive development of large language models","venue":null,"work_id":"da25771b-e8e2-42a5-b73d-369faad7fbb2","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":63,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.734531Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:b3aee1670538580ac77c1939a33dfe50756bd346c3ad6cfc09e8ea25f7bf28d1","observation_id":"0c5f58cd-35f4-43ac-a61b-3680e204b522","resolution":{"observed_at":"2026-08-06T14:45:43.285900Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.275035Z","title":"V-alphasocial: Benchmark and self-reflective chain-of- thought generation for visual social commonsense reasoning","venue":null,"work_id":"0201843c-0e81-429f-bd89-0ea966e6cef2","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":64,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.736943Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:f90db3810e4edd32f66a4975ea47d595b09d61b6cd6db7472a224dfb56b445ce","observation_id":"51baff84-0779-4c6e-9fb3-16938bc28486","resolution":{"observed_at":"2026-08-06T14:45:43.277771Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:42.739479Z","title":"Mind2web: Towards a generalist agent for the web","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":65,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.739479Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:d4d27188649891cd3438a4dc1319bd1898792ecf68143f7379c6da8dd41569a5","observation_id":"627a74e2-98d7-42b2-929e-0df52d2e2100","resolution":{"observed_at":"2026-08-06T14:45:42.739479Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2506.21506","last_updated":"2025-07-03T15:47:40Z","snapshot_observed_at":"2026-08-15T08:08:37.615147Z","submitted_at":"2025-06-26T17:32:50Z","title":"Mind2Web 2: Evaluating Agentic Search with Agent-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2506.21506","snapshot_observed_at":"2026-08-06T14:45:42.742049Z","title":"Mind2web 2: Evaluating agentic search with agent-as-a-judge","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":66,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.742049Z"},"links":{"cited_paper":"/paper/2506.21506","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:6324726f9a820fd9f30433530c3038e1ba37690b2b703ea1e711332ae4740282","observation_id":"7592be3c-8ba4-4e30-8d6d-1e4ef8e92ddf","resolution":{"observed_at":"2026-08-06T14:45:42.742049Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2011.04006","last_updated":"2020-11-08T15:53:56Z","snapshot_observed_at":"2026-08-09T15:12:21.086848Z","submitted_at":"2020-11-08T15:53:56Z","title":"Long Range Arena: A Benchmark for Efficient Transformers","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2011.04006","snapshot_observed_at":"2026-08-06T14:45:42.745219Z","title":"Long range arena: A benchmark for efficient transformers","venue":null,"work_id":null,"year":2011},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":67,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.745219Z"},"links":{"cited_paper":"/paper/2011.04006","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:37ec806698b6fa8b9aaeb4b2598dbc922932da5c9782e3c8bf23d964496781ad","observation_id":"7bed5f91-00c3-4549-b38e-a7350617a961","resolution":{"observed_at":"2026-08-06T14:45:42.745219Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.03358","last_updated":"2025-06-09T14:31:04Z","snapshot_observed_at":"2026-08-15T09:49:12.117405Z","submitted_at":"2025-02-05T16:53:45Z","title":"Minerva: A Programmable Memory Test Benchmark for Language Models","version":2},"cited_work":{"arxiv_id":"2502.03358","doi":null,"metadata_source":"pith","pith_arxiv_id":"2502.03358","snapshot_observed_at":"2026-08-06T14:45:42.844290Z","title":"Minerva: A Programmable Memory Test Benchmark for Language Models","venue":"cs.CL","work_id":"3f8ea708-e16b-48f0-9731-2e1b0891df17","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":68,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.748430Z"},"links":{"cited_paper":"/paper/2502.03358","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:f1e691aaab1011f5b03530f1d253412c97b1e1621ab096e28ccc0b7cdd6d4285","observation_id":"91e16ef8-01f0-4903-8d7c-ca63f47db4b5","resolution":{"observed_at":"2026-08-06T14:45:42.847863Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.261227Z","title":"L-eval: Instituting standardized evaluation for long context language models","venue":null,"work_id":"be5060db-c12c-4b48-8175-3dede473bde0","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":69,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.751816Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:2701b0552800d656595cf4b7d1bfc5c34a2a98572f2b78aefe0679ca649ea04d","observation_id":"0b7abe6e-7add-4c05-9ba7-0a785830c544","resolution":{"observed_at":"2026-08-06T14:45:43.264284Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.01437","last_updated":"2024-07-12T17:20:34Z","snapshot_observed_at":"2026-08-14T23:21:56.102809Z","submitted_at":"2024-07-01T16:32:16Z","title":"Needle in the Haystack for Memory Based Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.01437","snapshot_observed_at":"2026-08-06T14:45:42.754635Z","title":"Needle in the haystack for memory based large language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":70,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.754635Z"},"links":{"cited_paper":"/paper/2407.01437","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:f87f699bde35606b9947e637a12b2dfabd6d9226e71c1cfef562b29fc89b24f4","observation_id":"af100c43-3815-4d90-a3df-597d7498c86d","resolution":{"observed_at":"2026-08-06T14:45:42.754635Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.04681","last_updated":"2024-08-08T04:55:03Z","snapshot_observed_at":"2026-08-12T23:06:49.868443Z","submitted_at":"2024-08-08T04:55:03Z","title":"Conversational AI Powered by Large Language Models Amplifies False Memories in Witness Interviews","version":1},"cited_work":{"arxiv_id":"2408.04681","doi":null,"metadata_source":"pith","pith_arxiv_id":"2408.04681","snapshot_observed_at":"2026-08-06T14:45:42.820633Z","title":"Conversational AI Powered by Large Language Models Amplifies False Memories in Witness Interviews","venue":"cs.CL","work_id":"93769f36-8b0c-4227-a8e7-0d85d32a7df5","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":71,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.757628Z"},"links":{"cited_paper":"/paper/2408.04681","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:2b092e36cc566d16352caa8c47bd310c7d4a998e869eba8dc9292c8cd5b8fc7d","observation_id":"a3c961f2-fb53-4177-96d8-a6b899f08052","resolution":{"observed_at":"2026-08-06T14:45:42.825972Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.252766Z","title":"Sentence-bert: Sentence embeddings using siamese bert-networks","venue":null,"work_id":"ec79078e-0857-4260-944f-df0816aa2b56","year":2019},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":72,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.760596Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:a6329e4125af8810a0ed3e2867456595483ab606fc657aa664efab4a0088a07d","observation_id":"15d3363f-d81e-4c91-ab3b-69513f717179","resolution":{"observed_at":"2026-08-06T14:45:43.255832Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.243283Z","title":"Introduction to information retrieval , volume 39","venue":null,"work_id":"7ed06abb-43ee-447f-9ac5-7664f39ee98c","year":2008},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":73,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.763686Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:e7fb483106019c9c3e264cc41afb8ea1ac73eb7e4e296a0847a91ab8c77f5a93","observation_id":"e5677550-9e97-4b0e-9fda-11a30c4b5b32","resolution":{"observed_at":"2026-08-06T14:45:43.246950Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.234678Z","title":"The claude 3 model family: Opus, sonnet, haiku","venue":null,"work_id":"0bc329c8-d67c-4a4a-9911-2eb66e9732a7","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":74,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.766616Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:bfcbb5491535a4b0650fcbdf30814b3775038d8f6c52741c1c1e7348fdf9d782","observation_id":"c7c6cf1c-fa05-4c7a-9208-b05183f4260b","resolution":{"observed_at":"2026-08-06T14:45:43.237288Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.226348Z","title":"Model card addendum: Claude 3.5 haiku and sonnet","venue":null,"work_id":"85d7a1e4-8284-499d-af7b-65a3d75b3e52","year":2025},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":75,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.769395Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:7d977fcbfe4e51a0ecbafb5ea853b104bd3823074953bf80c84f6c77e52ec894","observation_id":"e6138f8e-1207-4806-a43c-7e73eb63aefa","resolution":{"observed_at":"2026-08-06T14:45:43.229332Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.05530","last_updated":"2024-12-16T17:39:39Z","snapshot_observed_at":"2026-08-14T18:15:53.516440Z","submitted_at":"2024-03-08T18:54:20Z","title":"Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.05530","snapshot_observed_at":"2026-08-06T14:45:42.772324Z","title":"Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":76,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.772324Z"},"links":{"cited_paper":"/paper/2403.05530","citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:a5e064d3247c54c8d91444f7b64e43edc004a2ab723555b5c0a0e9170cd319db","observation_id":"8918cd5d-be6f-4bd8-b8f9-282fccf6cdc2","resolution":{"observed_at":"2026-08-06T14:45:42.772324Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.218646Z","title":"Introducing meta llama 3","venue":null,"work_id":"23b059f6-b0d9-4fdb-a5d4-aa9c5f99aa8e","year":2024},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":77,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.775275Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:1422c62d60dcf2005a0b7ef16dbf51c49d72e287692090997476dcd2fe2f65f2","observation_id":"e4eee62f-382c-4feb-ad6a-30862ead2565","resolution":{"observed_at":"2026-08-06T14:45:43.221259Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.210394Z","title":"The serial position effect of free recall","venue":null,"work_id":"0ca13c23-e523-42be-8605-c2a74ce291f0","year":1962},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":78,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.778207Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:fa644936afc45d084dcdf1e5d447efbc8b4ed5cbcf4af8c6717779f2921ca9b1","observation_id":"de7165fe-97ad-4746-84c9-0e7cb86218a3","resolution":{"observed_at":"2026-08-06T14:45:43.213126Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T14:45:43.201035Z","title":"luck-free","venue":null,"work_id":"6b354254-27b0-4920-aa2b-25962c330bd9","year":1966},"citing_paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models","version":2},"reference_index":79,"source":"pdf_text","source_observed_at":"2026-08-06T14:45:42.780856Z"},"links":{"citing_paper":"/paper/2507.18182"},"observation_digest":"sha256:b0e1dc4181eff1d10eb16b71504b9deea5c2304383ad3ea29f3398d82c669014","observation_id":"1e0237f2-0b8c-4141-b590-1b0b2faa909c","resolution":{"observed_at":"2026-08-06T14:45:43.203784Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2507.18182","last_updated":"2025-08-04T15:53:52Z","latest_version":2,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-15T09:16:42.772104Z","submitted_at":"2025-07-24T08:28:17Z","title":"SCOPE: Stochastic and Counterbiased Option Placement for Evaluating Large Language Models"},"reference_resolution":{"displayed":78,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":34,"verified_exact":4,"verified_fuzzy":40},"total_outbound_references":78},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"thesis":"As of 15 August 2026, this Paper Citation Record lists 78 of 78 outbound references and 1 inbound Pith citation observation for arXiv:2507.18182."}