{"as_of":"2026-08-10T06:18:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:85f1f386a04d5f538aac1392f69fd2eaeb1b06d0d79a14341091583cdc44933a","coverage":[{"denominator":37,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":37,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-07T12:35:29.070019Z","state":"measured"},{"denominator":37,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":37,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-09T06:31:02.800959+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2505.24263/citation-record","integrity":"/paper/2505.24263/integrity","json":"/paper/2505.24263/citation-record.json","paper":"/paper/2505.24263"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:25.126963Z","title":"URL: \" 'urlintro :=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:25.126963Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:5b1e01eb8519f34b78be45a98d60d51df254c04b11f6b707268b7fc26709e30d","observation_id":"48403ae3-c436-4c4d-8b1b-b0882126a52f","resolution":{"observed_at":"2026-08-07T12:35:25.126963Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:25.285963Z","title":"write newline","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:25.285963Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:8209dc2184d73b7c3d2b0aea8ba44009492f5a70d41fd8948ba14fb24424561b","observation_id":"552d78f0-3bb1-4696-8fe9-f0b5a18c4c1a","resolution":{"observed_at":"2026-08-07T12:35:25.285963Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:25.410371Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:25.410371Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:37a3cdf156b65e4b68747b67671b5df1914c1103235c4472ba1a6b2e06676da9","observation_id":"381ca8e0-ebc8-45b7-beb2-c466d9774410","resolution":{"observed_at":"2026-08-07T12:35:25.410371Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:30.129677Z","title":null,"venue":null,"work_id":"395edd94-e160-4ca8-9358-5fbc14c79fd1","year":2020},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:25.533380Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:7907fe6639801740a27cd73bf1e73962c94ec6bb5f1be2ab190c44d322fb1918","observation_id":"5d579d1d-a646-4908-bd15-bd401705277f","resolution":{"observed_at":"2026-08-07T12:35:30.205471Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2005.14165","last_updated":"2020-07-22T19:47:17Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2020-05-28T17:29:03Z","title":"Language Models are Few-Shot Learners","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2005.14165","snapshot_observed_at":"2026-08-07T12:35:25.691779Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:25.691779Z"},"links":{"cited_paper":"/paper/2005.14165","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:6aff98f865aa31bb3fdb02d3ac82eae155e5692362854a7879db9900e16db611","observation_id":"ac6a04c4-0f96-44ff-a4d0-5d758d64bf20","resolution":{"observed_at":"2026-08-07T12:35:25.691779Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2202.07646","last_updated":"2023-03-06T06:28:18Z","snapshot_observed_at":"2026-08-03T19:51:05.627063Z","submitted_at":"2022-02-15T18:48:31Z","title":"Quantifying Memorization Across Neural Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2202.07646","snapshot_observed_at":"2026-08-07T12:35:25.837301Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:25.837301Z"},"links":{"cited_paper":"/paper/2202.07646","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:f5cf2eaf2e78fea399b9c603fb714b30adb39c03cd1400dbdc4f24a2876ad5dc","observation_id":"7d2be1ac-92ae-4e22-9908-3792e513e538","resolution":{"observed_at":"2026-08-07T12:35:25.837301Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:25.945153Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:25.945153Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:b4ea5f25483220953c48e0919f3839a3dfc0e9e5b977926bcdb28f095e2fad39","observation_id":"1faed743-79ba-435e-a9d9-a5ba3cb31840","resolution":{"observed_at":"2026-08-07T12:35:25.945153Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2110.14168","last_updated":"2021-11-18T00:23:45Z","snapshot_observed_at":"2026-08-07T01:45:38.840969Z","submitted_at":"2021-10-27T04:49:45Z","title":"Training Verifiers to Solve Math Word Problems","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2110.14168","snapshot_observed_at":"2026-08-07T12:35:26.047017Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.047017Z"},"links":{"cited_paper":"/paper/2110.14168","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:230791720766f5a3379cb3299004fdd71d0be7cd4216318ab3db192025a2f1f7","observation_id":"6fb46aeb-d547-4ee8-8969-fc01656f5fca","resolution":{"observed_at":"2026-08-07T12:35:26.047017Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.18966","last_updated":"2025-05-09T23:13:37Z","snapshot_observed_at":"2026-07-06T19:39:15.025945Z","submitted_at":"2024-10-24T17:58:22Z","title":"Does Data Contamination Detection Work (Well) for LLMs? A Survey and Evaluation on Detection Assumptions","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.18966","snapshot_observed_at":"2026-08-07T12:35:26.123753Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.123753Z"},"links":{"cited_paper":"/paper/2410.18966","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:4423201e39a0bfaed3b3b31d2e47da799a8fc18119af092e4dacda2b23fc1aff","observation_id":"d447cf9b-c72d-4059-a0f0-3344f8a8dc83","resolution":{"observed_at":"2026-08-07T12:35:26.123753Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.04127","last_updated":"2025-01-10T14:31:21Z","snapshot_observed_at":"2026-08-09T20:09:37.487347Z","submitted_at":"2024-06-06T14:49:06Z","title":"Are We Done with MMLU?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.04127","snapshot_observed_at":"2026-08-07T12:35:26.208319Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.208319Z"},"links":{"cited_paper":"/paper/2406.04127","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:a3400163950933912514726de3230456220d151e5fd22f60c3cb636c2ecabadf","observation_id":"9b4366e9-05f8-49cd-a94a-145685687d6d","resolution":{"observed_at":"2026-08-07T12:35:26.208319Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.08493","last_updated":"2024-02-21T22:02:26Z","snapshot_observed_at":"2026-08-06T07:48:14.479004Z","submitted_at":"2023-08-16T16:48:57Z","title":"Time Travel in LLMs: Tracing Data Contamination in Large Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.08493","snapshot_observed_at":"2026-08-07T12:35:26.309692Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.309692Z"},"links":{"cited_paper":"/paper/2308.08493","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:aa1c4a7eab752cfad4d77709002678506c3d78084a6cf608b80e9655c26db0f1","observation_id":"65d8c614-fee6-4973-98cc-704b3e9632b4","resolution":{"observed_at":"2026-08-07T12:35:26.309692Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.21783","last_updated":"2024-11-23T23:27:33Z","snapshot_observed_at":"2026-07-06T18:55:11.576666Z","submitted_at":"2024-07-31T17:54:27Z","title":"The Llama 3 Herd of Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.21783","snapshot_observed_at":"2026-08-07T12:35:26.414783Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.414783Z"},"links":{"cited_paper":"/paper/2407.21783","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:389148121c14d7948c9292ed1d606852b1403fcc9337a3f4ef4dd115f3ef52c9","observation_id":"4f148745-1121-4a19-8c0b-296d37a97674","resolution":{"observed_at":"2026-08-07T12:35:26.414783Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:26.515744Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.515744Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:d0be9d8f58fed0201672ef10cd53b2e666ea309f9eefeedb26eb77a95b9c6802","observation_id":"f015e85b-9744-4278-af9d-5a08ebfe5715","resolution":{"observed_at":"2026-08-07T12:35:26.515744Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2009.03300","last_updated":"2021-01-12T18:57:11Z","snapshot_observed_at":"2026-08-09T10:28:06.906299Z","submitted_at":"2020-09-07T17:59:25Z","title":"Measuring Massive Multitask Language Understanding","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.03300","snapshot_observed_at":"2026-08-07T12:35:26.585502Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.585502Z"},"links":{"cited_paper":"/paper/2009.03300","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:4531f39cfbd4254d4ffe938eb919cb3ce0fb3f2418374714142a60cc05643dad","observation_id":"f872dafe-c61b-4cd0-b962-c71c5ba91f46","resolution":{"observed_at":"2026-08-07T12:35:26.585502Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:29.959788Z","title":null,"venue":null,"work_id":"b957eb66-990d-45ca-a47c-09218f629875","year":2021},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.693692Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:e03bb7e924f924909890479b8f81f504762c71a4e9636c831a7d562ecba640a9","observation_id":"9fa753c5-b81e-466b-9385-c969a3c49ecf","resolution":{"observed_at":"2026-08-07T12:35:30.011952Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2106.09685","last_updated":"2021-10-16T18:40:34Z","snapshot_observed_at":"2026-08-07T07:43:16.294957Z","submitted_at":"2021-06-17T17:37:18Z","title":"LoRA: Low-Rank Adaptation of Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2106.09685","snapshot_observed_at":"2026-08-07T12:35:26.812818Z","title":"Hu, Yelong Shen, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, and Weizhu Chen","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.812818Z"},"links":{"cited_paper":"/paper/2106.09685","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:c59db28848be12fd26a82b26139dbd5e0b8a0e1ec5c0f59f74a0703e8f24acf6","observation_id":"f23ba1cc-33d7-4198-8548-4725c998a439","resolution":{"observed_at":"2026-08-07T12:35:26.812818Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.07853","last_updated":"2022-02-03T04:29:23Z","snapshot_observed_at":"2026-08-06T13:05:17.451329Z","submitted_at":"2021-03-14T06:10:47Z","title":"Membership Inference Attacks on Machine Learning: A Survey","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.07853","snapshot_observed_at":"2026-08-07T12:35:26.911572Z","title":"Yu, and Xuyun Zhang","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.911572Z"},"links":{"cited_paper":"/paper/2103.07853","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:eb4e97a5d68a5c96c117b94bc8ceb75c0b56931c38c614edd753806f018a2698","observation_id":"088ddac3-4d2c-4796-97d3-8db2db9abe7b","resolution":{"observed_at":"2026-08-07T12:35:26.911572Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.10677","last_updated":"2023-09-27T01:15:49Z","snapshot_observed_at":"2026-07-06T16:20:36.866744Z","submitted_at":"2023-09-19T15:02:58Z","title":"Estimating Contamination via Perplexity: Quantifying Memorisation in Language Model Evaluation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.10677","snapshot_observed_at":"2026-08-07T12:35:26.977795Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:26.977795Z"},"links":{"cited_paper":"/paper/2309.10677","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:119526328a74358580c476c75974b5d213aa93028cec96b6615c8fff17a7f007","observation_id":"3a55477a-f0cc-4719-8e26-4ed2979c2896","resolution":{"observed_at":"2026-08-07T12:35:26.977795Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:27.076291Z","title":null,"venue":null,"work_id":null,"year":2004},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:27.076291Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:85421e13663ef20aed5f0d42c1c47572bd3d82ac27832658370a1e35ec7fe602","observation_id":"c0af4880-be2f-4f7f-b237-dd4aed103c85","resolution":{"observed_at":"2026-08-07T12:35:27.076291Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:29.730038Z","title":"Lin, Jacob Hilton, and Owain Evans","venue":null,"work_id":"34a24a37-5e90-4ed9-8580-aedf62e6b92e","year":2021},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:27.190607Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:38d4efddd0b3d4f2b057c1f8a6c6fb6a5da2cf17da184791a3c9046db3d356b5","observation_id":"aca29502-3b06-4411-898b-37b2eb2c6e40","resolution":{"observed_at":"2026-08-07T12:35:29.821277Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.19437","last_updated":"2025-02-18T17:26:38Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-12-27T04:03:16Z","title":"DeepSeek-V3 Technical Report","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.19437","snapshot_observed_at":"2026-08-07T12:35:27.321284Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:27.321284Z"},"links":{"cited_paper":"/paper/2412.19437","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:f860528aad2d7b66e816dc236ae92865c8c4b1ec082d664818dd3a960f218333","observation_id":"e29cbdeb-17ed-44ca-be0f-87b2bd22759a","resolution":{"observed_at":"2026-08-07T12:35:27.321284Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:27.401062Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:27.401062Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:f85d16204c82fcf33753956497cecd1457bf32009bdf5944e123ffc2189646b9","observation_id":"537db04c-fc95-4991-b246-5bc604ff91bc","resolution":{"observed_at":"2026-08-07T12:35:27.401062Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2409.01790","last_updated":"2025-02-28T02:40:58Z","snapshot_observed_at":"2026-08-07T05:21:59.292674Z","submitted_at":"2024-09-03T11:09:44Z","title":"Training on the Benchmark Is Not All You Need","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.01790","snapshot_observed_at":"2026-08-07T12:35:27.501910Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:27.501910Z"},"links":{"cited_paper":"/paper/2409.01790","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:fcef29dd7c027d48dab7f0d4dc10ad896ebc053e5ccf5a2a9fedcd312319af17","observation_id":"b4c6546f-481d-4c0f-a8a6-1c9b078a59b3","resolution":{"observed_at":"2026-08-07T12:35:27.501910Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-07T07:30:12.213965Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-08-07T12:35:27.657313Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:27.657313Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:02ddfe100bdcf56d5bfff7007941ffa712c6fbdc2aa2df6a5ba20569d2bffbe2","observation_id":"022bc4e4-6bca-4107-a61d-e4d2afd41e0d","resolution":{"observed_at":"2026-08-07T12:35:27.657313Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T12:35:27.763705Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:27.763705Z"},"links":{"citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:33ad29600695973c3f7ea746e6af451ab6b674decf7775567dc2ac9a0632b61b","observation_id":"85948a63-eefd-4c53-8708-a7d5933f2fbb","resolution":{"observed_at":"2026-08-07T12:35:27.763705Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.15115","last_updated":"2025-01-03T02:18:21Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-12-19T17:56:09Z","title":"Qwen2.5 Technical Report","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.15115","snapshot_observed_at":"2026-08-07T12:35:27.876148Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:27.876148Z"},"links":{"cited_paper":"/paper/2412.15115","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:b36a7be89095705ad45aedf191df112036f297d282b2ccb89b74b1d44a7f8cb7","observation_id":"457c0c24-c0e0-4a64-bfc4-7e22fc3024c4","resolution":{"observed_at":"2026-08-07T12:35:27.876148Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2210.12353","last_updated":"2023-03-17T00:52:56Z","snapshot_observed_at":"2026-08-09T19:18:49.025361Z","submitted_at":"2022-10-22T05:04:54Z","title":"Leveraging Large Language Models for Multiple Choice Question Answering","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2210.12353","snapshot_observed_at":"2026-08-07T12:35:27.989088Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:27.989088Z"},"links":{"cited_paper":"/paper/2210.12353","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:229ebd57d376f22cc9826743b58d3a984d04f4dbb0196b999077f3ec37b2ca2a","observation_id":"c8d45bf4-cb03-4b7e-b8af-fae0dddd9161","resolution":{"observed_at":"2026-08-07T12:35:27.989088Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2409.09927","last_updated":"2024-12-08T19:15:26Z","snapshot_observed_at":"2026-08-03T19:23:12.561275Z","submitted_at":"2024-09-16T02:04:33Z","title":"Towards Data Contamination Detection for Modern Large Language Models: Limitations, Inconsistencies, and Oracle Challenges","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2409.09927","snapshot_observed_at":"2026-08-07T12:35:28.145036Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:28.145036Z"},"links":{"cited_paper":"/paper/2409.09927","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:ed419c97f48a3cc2e53ad1847315871e2027e75caf7d311f80baa1b3892a2fed","observation_id":"a1645894-ad5c-4459-87c3-38c1f24c1685","resolution":{"observed_at":"2026-08-07T12:35:28.145036Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1904.09728","last_updated":"2019-09-09T17:29:55Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2019-04-22T05:36:37Z","title":"SocialIQA: Commonsense Reasoning about Social Interactions","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1904.09728","snapshot_observed_at":"2026-08-07T12:35:28.259215Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:28.259215Z"},"links":{"cited_paper":"/paper/1904.09728","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:cb1524719c4bc8c3297ddf5347c8cdfc43e9baa00899d010ea1f9150075791e5","observation_id":"d4a2cc5c-414e-4f72-93f1-84b947cfecfd","resolution":{"observed_at":"2026-08-07T12:35:28.259215Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.11805","last_updated":"2025-05-09T21:04:06Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-12-19T02:39:27Z","title":"Gemini: A Family of Highly Capable Multimodal Models","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.11805","snapshot_observed_at":"2026-08-07T12:35:28.402129Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":30,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:28.402129Z"},"links":{"cited_paper":"/paper/2312.11805","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:73401030d5f56d3e33a6d615ea5e1ec7caf24cdf4e1026a38661d8a9a082b802","observation_id":"877f8036-0eb7-428d-b4b5-b2904ec2975f","resolution":{"observed_at":"2026-08-07T12:35:28.402129Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.08295","last_updated":"2024-04-16T12:52:47Z","snapshot_observed_at":"2026-08-03T03:29:01.959523Z","submitted_at":"2024-03-13T06:59:16Z","title":"Gemma: Open Models Based on Gemini Research and Technology","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.08295","snapshot_observed_at":"2026-08-07T12:35:28.488255Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:28.488255Z"},"links":{"cited_paper":"/paper/2403.08295","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:d03fbaafd9f82d70d953b6584898ba49745b4a3432487f8c767e7a47acfdadd7","observation_id":"17509d7d-145e-4f20-b4c0-cafb062a0914","resolution":{"observed_at":"2026-08-07T12:35:28.488255Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.00118","last_updated":"2024-10-02T15:22:49Z","snapshot_observed_at":"2026-08-02T16:20:09.773989Z","submitted_at":"2024-07-31T19:13:07Z","title":"Gemma 2: Improving Open Language Models at a Practical Size","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.00118","snapshot_observed_at":"2026-08-07T12:35:28.599455Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:28.599455Z"},"links":{"cited_paper":"/paper/2408.00118","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:cc7cfcb483bb327812dc34e78eb1296de9887431ee1ddb4c63473bcaef8423d1","observation_id":"c169a4fc-480e-431e-9945-25576704a8e1","resolution":{"observed_at":"2026-08-07T12:35:28.599455Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2302.13971","last_updated":"2023-02-27T17:11:15Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-02-27T17:11:15Z","title":"LLaMA: Open and Efficient Foundation Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2302.13971","snapshot_observed_at":"2026-08-07T12:35:28.705768Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:28.705768Z"},"links":{"cited_paper":"/paper/2302.13971","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:6febe35a69db8f085c9112d5312df613ad3558d00357826e10a3c3dbf7fd47ce","observation_id":"d931bec7-4dd3-40f8-b778-8aef1d04f872","resolution":{"observed_at":"2026-08-07T12:35:28.705768Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.01574","last_updated":"2024-11-06T02:54:00Z","snapshot_observed_at":"2026-08-06T00:29:17.674418Z","submitted_at":"2024-06-03T17:53:00Z","title":"MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.01574","snapshot_observed_at":"2026-08-07T12:35:28.812524Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:28.812524Z"},"links":{"cited_paper":"/paper/2406.01574","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:72d3002b0f25355da4c3614cc6da8d9584980c65e0da38cead2a1afdcb95c53a","observation_id":"7265672e-7d0f-4ee7-aab8-3d9c9f00974d","resolution":{"observed_at":"2026-08-07T12:35:28.812524Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1910.03771","last_updated":"2020-07-14T03:42:34Z","snapshot_observed_at":"2026-07-06T08:27:58.343233Z","submitted_at":"2019-10-09T03:23:22Z","title":"HuggingFace's Transformers: State-of-the-art Natural Language Processing","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1910.03771","snapshot_observed_at":"2026-08-07T12:35:28.896218Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:28.896218Z"},"links":{"cited_paper":"/paper/1910.03771","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:bf7e3d4a5fb943cac52373e441b46bab408c9cf2bcefa856e3bcbfb6b297cb35","observation_id":"cc895bca-1fb5-4f50-872f-432387d4c5bd","resolution":{"observed_at":"2026-08-07T12:35:28.896218Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.18824","last_updated":"2024-04-29T16:05:36Z","snapshot_observed_at":"2026-08-09T17:21:02.287748Z","submitted_at":"2024-04-29T16:05:36Z","title":"Benchmarking Benchmark Leakage in Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.18824","snapshot_observed_at":"2026-08-07T12:35:28.982799Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":36,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:28.982799Z"},"links":{"cited_paper":"/paper/2404.18824","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:da825585864ce67c7cc0b0d48436f1518cb446331771d3330f44471ed56eb04d","observation_id":"cbd67ca9-a106-43a1-a3b5-d6a351c50b4f","resolution":{"observed_at":"2026-08-07T12:35:28.982799Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1905.07830","last_updated":"2019-05-19T23:57:23Z","snapshot_observed_at":"2026-07-31T00:09:56.948833Z","submitted_at":"2019-05-19T23:57:23Z","title":"HellaSwag: Can a Machine Really Finish Your Sentence?","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1905.07830","snapshot_observed_at":"2026-08-07T12:35:29.070019Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation","version":1},"reference_index":37,"source":"arxiv_source","source_observed_at":"2026-08-07T12:35:29.070019Z"},"links":{"cited_paper":"/paper/1905.07830","citing_paper":"/paper/2505.24263"},"observation_digest":"sha256:060a71598938e64e7dd13a6be99aa7d3372d76d4d4437683102ca820a4b83bb4","observation_id":"dfe21523-ee1b-44fc-b0db-954c381c2c42","resolution":{"observed_at":"2026-08-07T12:35:29.070019Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2505.24263","last_updated":"2025-05-30T06:37:39Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-09T17:21:24.329215Z","submitted_at":"2025-05-30T06:37:39Z","title":"Simulating Training Data Leakage in Multiple-Choice Benchmarks for LLM Evaluation"},"reference_resolution":{"displayed":37,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":36,"verified_exact":0,"verified_fuzzy":1},"total_outbound_references":37},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"thesis":"As of 10 August 2026, this Paper Citation Record lists 37 of 37 outbound references and 0 inbound Pith citation observations for arXiv:2505.24263."}