{"as_of":"2026-08-10T06:19:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:60aa51206aa03b210d5b7056ab4569553917f6f771dc3db50e0c9808c44eade3","coverage":[{"denominator":51,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":51,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-05T10:31:02.319589Z","state":"measured"},{"denominator":59,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":59,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-09T06:31:02.800959+00:00","state":"measured"},{"denominator":8,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":8,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-02T13:46:08.844007Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-06-30T23:45:07.998622Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"cited_work":{"arxiv_id":"2509.04013","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2509.04013","snapshot_observed_at":"2026-06-30T23:45:07.998622Z","title":"On robustness and reliability of benchmark-based evaluation of llms","venue":null,"work_id":"b9028098-f624-4214-ba6e-82cd64cb2a14","year":2025},"citing_paper":{"arxiv_id":"2512.10687","last_updated":"2026-04-20T11:46:57Z","snapshot_observed_at":"2026-08-02T16:01:53.254884Z","submitted_at":"2025-12-11T14:34:40Z","title":"Safe for Whom? Rethinking How We Evaluate the Safety of LLMs for Real Users","version":2},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-05-16T23:22:42.431997Z"},"links":{"cited_paper":"/paper/2509.04013","citing_paper":"/paper/2512.10687"},"observation_digest":"sha256:a50b6053b98af65abccd60ea7b46454e752b823eec14cb0e9696822b7ff1e43f","observation_id":"cd860ed7-1571-4add-9e0d-9db5135b3653","resolution":{"observed_at":"2026-05-16T23:23:39.942350Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"cited_work":{"arxiv_id":"2509.04013","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2509.04013","snapshot_observed_at":"2026-06-30T23:45:07.998622Z","title":"On robustness and reliability of benchmark-based evaluation of llms","venue":null,"work_id":"b9028098-f624-4214-ba6e-82cd64cb2a14","year":2025},"citing_paper":{"arxiv_id":"2604.26500","last_updated":"2026-07-03T09:45:31Z","snapshot_observed_at":"2026-07-31T15:15:29.527162Z","submitted_at":"2026-04-29T10:03:12Z","title":"StarDrinks: An English and Korean Test Set for SLU Evaluation in a Drink Ordering Scenario","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-05-07T11:27:03.620795Z"},"links":{"cited_paper":"/paper/2509.04013","citing_paper":"/paper/2604.26500"},"observation_digest":"sha256:a775b1a7bfc6cb13ac0536013dbf8e88f335d5d8e9bdfb23c6aceff6e0ad3aa5","observation_id":"a7bd56d7-f80a-4bfa-be9e-c15e06bc3b09","resolution":{"observed_at":"2026-05-12T09:21:25.388199Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"cited_work":{"arxiv_id":"2509.04013","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2509.04013","snapshot_observed_at":"2026-06-30T23:45:07.998622Z","title":"On robustness and reliability of benchmark-based evaluation of llms","venue":null,"work_id":"b9028098-f624-4214-ba6e-82cd64cb2a14","year":2025},"citing_paper":{"arxiv_id":"2605.07053","last_updated":"2026-05-26T07:47:31Z","snapshot_observed_at":"2026-08-09T23:23:44.876652Z","submitted_at":"2026-05-08T00:02:39Z","title":"GSM-SEM: Benchmark and Framework for Generating Semantically Variant Augmentations","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-05-11T00:57:55.616036Z"},"links":{"cited_paper":"/paper/2509.04013","citing_paper":"/paper/2605.07053"},"observation_digest":"sha256:0bdfd1324b4999d417d716f9f4803c1ad8d4b38086632bb410a8a52796b81dbd","observation_id":"b9447640-1d57-4f04-bb6a-4e4ce0214ec9","resolution":{"observed_at":"2026-05-11T04:55:59.932740Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"cited_work":{"arxiv_id":"2509.04013","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2509.04013","snapshot_observed_at":"2026-06-30T23:45:07.998622Z","title":"On robustness and reliability of benchmark-based evaluation of llms","venue":null,"work_id":"b9028098-f624-4214-ba6e-82cd64cb2a14","year":2025},"citing_paper":{"arxiv_id":"2605.07053","last_updated":"2026-05-26T07:47:31Z","snapshot_observed_at":"2026-08-09T23:23:44.876652Z","submitted_at":"2026-05-08T00:02:39Z","title":"GSM-SEM: Benchmark and Framework for Generating Semantically Variant Augmentations","version":2},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-06-30T23:42:00.965554Z"},"links":{"cited_paper":"/paper/2509.04013","citing_paper":"/paper/2605.07053"},"observation_digest":"sha256:cfcabd538db90705a0e01bfbc9f0d7cfdc1bb941783d103dde36e57619c470c9","observation_id":"86bb03c9-9819-40e2-95cb-6390ff59ae8e","resolution":{"observed_at":"2026-06-30T23:45:08.000489Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"cited_work":{"arxiv_id":"2509.04013","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2509.04013","snapshot_observed_at":"2026-06-30T23:45:07.998622Z","title":"On robustness and reliability of benchmark-based evaluation of llms","venue":null,"work_id":"b9028098-f624-4214-ba6e-82cd64cb2a14","year":2025},"citing_paper":{"arxiv_id":"2605.19382","last_updated":"2026-05-19T05:28:54Z","snapshot_observed_at":"2026-07-06T23:30:06.876413Z","submitted_at":"2026-05-19T05:28:54Z","title":"PRISM: A Benchmark for Programmatic Spatial-Temporal Reasoning","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-05-20T05:52:46.575173Z"},"links":{"cited_paper":"/paper/2509.04013","citing_paper":"/paper/2605.19382"},"observation_digest":"sha256:d55380f8f81dee079338d7911d62f2358d7d543e2174aa2c6f0606ed43ffd8fc","observation_id":"8e54f074-d4a3-4308-8237-8717c9608525","resolution":{"observed_at":"2026-05-20T05:53:04.325279Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"cited_work":{"arxiv_id":"2509.04013","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2509.04013","snapshot_observed_at":"2026-06-30T23:45:07.998622Z","title":"On robustness and reliability of benchmark-based evaluation of llms","venue":null,"work_id":"b9028098-f624-4214-ba6e-82cd64cb2a14","year":2025},"citing_paper":{"arxiv_id":"2605.27209","last_updated":"2026-05-26T16:02:00Z","snapshot_observed_at":"2026-08-03T11:00:10.339186Z","submitted_at":"2026-05-26T16:02:00Z","title":"Learning to Act under Noise: Enhancing Agent Robustness via Noisy Environments","version":1},"reference_index":85,"source":"pdf_text","source_observed_at":"2026-06-29T16:51:36.524194Z"},"links":{"cited_paper":"/paper/2509.04013","citing_paper":"/paper/2605.27209"},"observation_digest":"sha256:5da94fa7e90bb1724d06cd6ca974f43789b039c9c2e519e12c41f35081447309","observation_id":"b0a32282-5a01-4201-ade7-b59c79931a23","resolution":{"observed_at":"2026-06-29T16:53:40.568253Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"cited_work":{"arxiv_id":"2509.04013","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2509.04013","snapshot_observed_at":"2026-06-30T23:45:07.998622Z","title":"On robustness and reliability of benchmark-based evaluation of llms","venue":null,"work_id":"b9028098-f624-4214-ba6e-82cd64cb2a14","year":2025},"citing_paper":{"arxiv_id":"2605.30738","last_updated":"2026-05-29T02:02:31Z","snapshot_observed_at":"2026-08-01T17:11:51.059319Z","submitted_at":"2026-05-29T02:02:31Z","title":"MAVEN: Improving Generalization in Agentic Tool Calling","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-06-28T22:42:11.459778Z"},"links":{"cited_paper":"/paper/2509.04013","citing_paper":"/paper/2605.30738"},"observation_digest":"sha256:6f021e30544fe32eeccbe01ab47bbdf018480a541b982d4b518ad41a12e650f5","observation_id":"cacb1980-62da-4e63-b519-bb6e4d88023a","resolution":{"observed_at":"2026-06-28T22:42:46.135066Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2509.04013","snapshot_observed_at":"2026-08-02T13:46:08.844007Z","title":"Lunardi, V","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-08T05:06:39.163957Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:08.844007Z"},"links":{"cited_paper":"/paper/2509.04013","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:904251f2cc5307d4e8568affccdbd8f56fc72e86dc2b7a3700ee69de39253637","observation_id":"b9caf502-53cd-465a-a0b2-1d1934c03339","resolution":{"observed_at":"2026-08-02T13:46:08.844007Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2509.04013/citation-record","integrity":"/paper/2509.04013/integrity","json":"/paper/2509.04013/citation-record.json","paper":"/paper/2509.04013"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2108.07732","last_updated":"2021-08-16T03:57:30Z","snapshot_observed_at":"2026-08-02T19:23:53.535075Z","submitted_at":"2021-08-16T03:57:30Z","title":"Program Synthesis with Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2108.07732","snapshot_observed_at":"2026-08-05T10:31:02.079925Z","title":"Austin et al","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.079925Z"},"links":{"cited_paper":"/paper/2108.07732","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:3a7f01a0e9d9d7e57c24531c3b3071774f3f27b0ae8a341990c00ed84106f476","observation_id":"128e5b78-77f7-44b3-8ba5-540d8af3ab63","resolution":{"observed_at":"2026-08-05T10:31:02.079925Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.14762","last_updated":"2024-11-05T16:40:21Z","snapshot_observed_at":"2026-07-06T17:34:07.737296Z","submitted_at":"2024-02-22T18:21:59Z","title":"MT-Bench-101: A Fine-Grained Benchmark for Evaluating Large Language Models in Multi-Turn Dialogues","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.14762","snapshot_observed_at":"2026-08-05T10:31:02.085778Z","title":"Bai et al","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.085778Z"},"links":{"cited_paper":"/paper/2402.14762","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:d5a5bb0b483000f7ad482fe015e239a832917f99803d67724b4b7542a3c4b544","observation_id":"032f5bc1-5851-425b-9f95-bae93273db73","resolution":{"observed_at":"2026-08-05T10:31:02.085778Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.276805Z","title":"Bailey, N","venue":null,"work_id":"4f35a8ad-fcd3-4067-91cb-cacab403ec62","year":2008},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.090860Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:b77efc8630f34585db117a710ea09616458287b84bc976ca795d2e07c0d380a8","observation_id":"5e28eac2-5474-4fa9-9b64-922e1d3182a5","resolution":{"observed_at":"2026-08-05T10:31:03.281185Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.262796Z","title":null,"venue":null,"work_id":"571c5bc0-1114-416c-b411-418b8d82174c","year":2020},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.095785Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:346468f78ec3c154ab461f1ccbbb2582766addb68a6b8773668d2c255ea96fcf","observation_id":"766cc208-33f1-4b9c-adb1-e312d203b2bd","resolution":{"observed_at":"2026-08-05T10:31:03.267003Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.248066Z","title":"Burnell et al","venue":null,"work_id":"fd6d0acf-4966-476f-9f5a-bcb7dd8bd3f8","year":2023},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.101619Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:3f9ef4216c303966ad45e75bff77784c1fddbab64c98f46dfc695e1f16eff703","observation_id":"a3b2f874-c6b9-4767-9bdd-7e6b6e164b3a","resolution":{"observed_at":"2026-08-05T10:31:03.252544Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.232995Z","title":"Carterette, J","venue":null,"work_id":"0ef1ebd9-366a-4d71-b9ba-b10704ac9a99","year":2006},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.106197Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:26f627ff42071bb5bd90d72ab3b5951cffa0f41adef2d19ab654f15565e1ff77","observation_id":"5c5a3774-d159-438d-8764-c88fab8a9d99","resolution":{"observed_at":"2026-08-05T10:31:03.237912Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.219213Z","title":"Carterette, A","venue":null,"work_id":"829cffc5-792a-4868-9f8c-33976d533609","year":2015},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.111083Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:1e3c0339c440d595f9d9e4f74f5ec6f645106ba83e28f1103c1192a85bc21e1a","observation_id":"555fd79b-d022-455b-9f0d-cdfb09ccc4a0","resolution":{"observed_at":"2026-08-05T10:31:03.223737Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-08T11:58:24.516369Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-05T10:31:02.115716Z","title":"Chen et al","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.115716Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:cb7cdc08a1dac9614a21181a5e5ab726cddad78002ae1faf572361e3e6ac4058","observation_id":"0952470b-d78b-4187-b02c-03e45a157225","resolution":{"observed_at":"2026-08-05T10:31:02.115716Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.204980Z","title":null,"venue":null,"work_id":"fe7ddb9b-5dab-4009-afe7-cbfad782dd1e","year":2018},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.120756Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:ce8617d4738a639c32a29bec78d1705ade30fe9f3c2a0af56ec54a219d3d457f","observation_id":"73abcfd4-d6c3-4275-beb0-e0e23c633649","resolution":{"observed_at":"2026-08-05T10:31:03.209510Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.189835Z","title":"Clark, K","venue":null,"work_id":"a95ab14e-884e-4d51-a24d-29af50e180c3","year":2019},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.125172Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:d37e7ee1df0d3149e525a66621bc435574cc85e3ee6a5390dfc70c8c48107d1f","observation_id":"aa1b7741-f64a-4045-92a0-ab886997637b","resolution":{"observed_at":"2026-08-05T10:31:03.194167Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1803.05457","last_updated":"2018-03-14T18:04:21Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2018-03-14T18:04:21Z","title":"Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1803.05457","snapshot_observed_at":"2026-08-05T10:31:02.129784Z","title":"Clark, I","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.129784Z"},"links":{"cited_paper":"/paper/1803.05457","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:7079f8946bf272909aff865d132641359aec01080c10720c1adac1c207bae37d","observation_id":"cad664e0-b2a2-46bc-b0f5-99639c46deb1","resolution":{"observed_at":"2026-08-05T10:31:02.129784Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2110.14168","last_updated":"2021-11-18T00:23:45Z","snapshot_observed_at":"2026-08-07T01:45:38.840969Z","submitted_at":"2021-10-27T04:49:45Z","title":"Training Verifiers to Solve Math Word Problems","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2110.14168","snapshot_observed_at":"2026-08-05T10:31:02.134983Z","title":"Cobbe, V","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.134983Z"},"links":{"cited_paper":"/paper/2110.14168","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:20f2c6d1fa54c58832eb7500747bc12d778b950fe6261cadcbffe9a7b544158e","observation_id":"97370bb6-f807-4ab7-86d1-458051a8a23f","resolution":{"observed_at":"2026-08-05T10:31:02.134983Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.175680Z","title":"Frohberg and F","venue":null,"work_id":"a16e988b-d0b5-40da-84fc-b7a9a4efe273","year":2022},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.140039Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:81abd32a121e1d4a917bd95d6612ed965988336fe7b4c5f2d505375baa6e3125","observation_id":"3e54834d-57eb-4a74-9042-3cf4dbc2279b","resolution":{"observed_at":"2026-08-05T10:31:03.180081Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.161913Z","title":"Guiver, S","venue":null,"work_id":"8fc0995c-a3ba-4d62-a31b-42703f699841","year":2009},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.144420Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:442cc019ce0513a4ecff1fadd98c729d65dd43672842518e5bc4fef54bab15d7","observation_id":"2b89a67e-665c-4c54-b7c8-78dc26687eee","resolution":{"observed_at":"2026-08-05T10:31:03.166258Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.147852Z","title":null,"venue":null,"work_id":"9e2bc9e6-e006-45d1-ae11-5b6f58a6bd6a","year":2011},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.149440Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:00b3b03af9f465f5e4ca625b376a4b702e9651a43f0d93897f6a8bd3475bf709","observation_id":"1ae1f1f2-014a-4992-96a9-b8f3e2c3a9ab","resolution":{"observed_at":"2026-08-05T10:31:03.152235Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.133358Z","title":"Hendrycks, C","venue":null,"work_id":"2520f4e3-30e7-41cc-8e4f-22baab2cba60","year":2021},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.153996Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:22ba6355d917fa3aec2e2c08ac535511a29d60183a781802ec78c7c9d3b1772a","observation_id":"094ceee2-e613-465d-9876-fff9a937eda6","resolution":{"observed_at":"2026-08-05T10:31:03.137847Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2103.03874","last_updated":"2021-11-08T21:30:18Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2021-03-05T18:59:39Z","title":"Measuring Mathematical Problem Solving With the MATH Dataset","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.03874","snapshot_observed_at":"2026-08-05T10:31:02.158497Z","title":"Hendrycks et al","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.158497Z"},"links":{"cited_paper":"/paper/2103.03874","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:e3764e6941f7dd1e22de3929be66b3a49a00e981f06162c53e8acdcc94e1a4fd","observation_id":"04a26570-1182-4b11-bdd9-7fe9326ba056","resolution":{"observed_at":"2026-08-05T10:31:02.158497Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.120027Z","title":"Kim et al","venue":null,"work_id":"b1e92372-6eee-49ba-96a5-2428841a9627","year":2023},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.163069Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:f22741d6f5e90b90ac242a2ecd4daa82296aa20adbfe3ceb3083e5f29b627e87","observation_id":"1701abe3-569f-4e95-ba65-170ea1c8c103","resolution":{"observed_at":"2026-08-05T10:31:03.124043Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.106132Z","title":"Kojima, S","venue":null,"work_id":"a5af94bb-a515-444d-bf3c-a70a7b7d7de0","year":2022},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.167804Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:c374d6af59ce205b0cdd805c3f474e2ee133087874c3a92f0e7a32b31196e397","observation_id":"f47fa918-c496-460b-bb29-1aa8ff18da21","resolution":{"observed_at":"2026-08-05T10:31:03.110964Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.091645Z","title":null,"venue":null,"work_id":"7f7f5d95-d679-407b-8bfe-81e1f56d8fef","year":2017},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.172823Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:8bbbd95ed95c34058714e6190b02abb26290478be22e9a5d8c0b09c9b4cc84e5","observation_id":"d0e77fad-3de6-4f99-b2c2-a5e45e072299","resolution":{"observed_at":"2026-08-05T10:31:03.096351Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.14215","last_updated":"2024-11-21T15:25:08Z","snapshot_observed_at":"2026-07-06T19:53:47.605701Z","submitted_at":"2024-11-21T15:25:08Z","title":"Evaluating the Robustness of Analogical Reasoning in Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.14215","snapshot_observed_at":"2026-08-05T10:31:02.177268Z","title":"Lewis and M","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.177268Z"},"links":{"cited_paper":"/paper/2411.14215","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:cc049313838ccc0771e01aec5e183eac777ea91ea242b0cd4822bac304bd774a","observation_id":"25a96312-2b0b-475c-91b2-649f60b911e9","resolution":{"observed_at":"2026-08-05T10:31:02.177268Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2102.04664","last_updated":"2021-03-16T08:28:37Z","snapshot_observed_at":"2026-07-06T10:39:42.676631Z","submitted_at":"2021-02-09T06:16:25Z","title":"CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2102.04664","snapshot_observed_at":"2026-08-05T10:31:02.182126Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.182126Z"},"links":{"cited_paper":"/paper/2102.04664","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:0fbc464b7b65d2f7eab28ced29b7508f75c429b62a36735004129518f9403fdf","observation_id":"887ea66b-3234-48da-a262-04da90ae8d1f","resolution":{"observed_at":"2026-08-05T10:31:02.182126Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.076677Z","title":"Lunardi, D","venue":null,"work_id":"66268753-c069-4deb-bc73-8505a049c38d","year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.186580Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:1283b657ec495fe4fa0443a772ed533ed28b9a5bf2a74959ed91d23905fd0748","observation_id":"c0f0c23a-571e-4cbd-904c-acbbbaa255fd","resolution":{"observed_at":"2026-08-05T10:31:03.081688Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.062203Z","title":"Mihaylov, P","venue":null,"work_id":"2fac6036-8c29-445d-8839-c3e3e1a1be7a","year":2018},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.191311Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:93e92556affb6647d954146f57f6322ff80d339cd06949ab1d0e0da57e29dd66","observation_id":"96ccef58-4bca-4a9b-808f-ac8f72575dd8","resolution":{"observed_at":"2026-08-05T10:31:03.066683Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.047930Z","title":"Mitchell","venue":null,"work_id":"efab1f26-62b0-468b-9cdc-0efecfe3a8a8","year":2023},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.196651Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:191eda92656fe5cc712262fc57cf4ec4784d85f12bc24c866158bb7c5258d859","observation_id":"b1d0c527-9588-48b4-8bb9-4dbb8515e46e","resolution":{"observed_at":"2026-08-05T10:31:03.052407Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.034236Z","title":"Mitchell","venue":null,"work_id":"3f26a3f2-0dcd-47d9-bc38-8113fb7a7681","year":2023},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.201503Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:ffe61f623e0c430538d4016c349305096a35de7e3361100e87adafcac322de9c","observation_id":"226d59b7-7fb1-4c96-9e39-54c25d67787e","resolution":{"observed_at":"2026-08-05T10:31:03.038500Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1611.09268","last_updated":"2018-10-31T14:46:47Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2016-11-28T18:14:11Z","title":"MS MARCO: A Human Generated MAchine Reading COmprehension Dataset","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1611.09268","snapshot_observed_at":"2026-08-05T10:31:02.206347Z","title":"Nguyen, M","venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.206347Z"},"links":{"cited_paper":"/paper/1611.09268","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:264583a322f5c6df7f3e81959be4aa83cca54069d48680c0f8d7b6b5aff027ea","observation_id":"33a6f282-cfff-4201-bc73-309166eb850f","resolution":{"observed_at":"2026-08-05T10:31:02.206347Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.019911Z","title":"Ouyang, J","venue":null,"work_id":"32606852-2b24-434d-9cdc-2ebeed621c90","year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.211401Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:db8e7c28eadab47b0da2b406d857e2fe3297fb0a78ab29744349c899324371bc","observation_id":"fb30fad6-2087-4804-86dd-a88a403aa5df","resolution":{"observed_at":"2026-08-05T10:31:03.024374Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2502.20937","last_updated":"2025-05-21T11:54:57Z","snapshot_observed_at":"2026-08-07T17:40:12.481199Z","submitted_at":"2025-02-28T10:46:56Z","title":"Variations in Relevance Judgments and the Shelf Life of Test Collections","version":2},"cited_work":{"arxiv_id":"2502.20937","doi":null,"metadata_source":"pith","pith_arxiv_id":"2502.20937","snapshot_observed_at":"2026-08-05T10:31:02.626069Z","title":"Variations in Relevance Judgments and the Shelf Life of Test Collections","venue":"cs.IR","work_id":"8c8c2bce-b9d1-4de7-a718-e99fffdcb68e","year":2025},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.215986Z"},"links":{"cited_paper":"/paper/2502.20937","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:cf3f401791e985fa88b13e4525643cc0c7f4809bdae7cb78ca993193ae8bbb86","observation_id":"cd6581ea-42a0-4bde-8aa1-16a34cf5048f","resolution":{"observed_at":"2026-08-05T10:31:02.633586Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:03.005865Z","title":null,"venue":null,"work_id":"a8bdfb1e-f112-4818-951f-c728f9a9489c","year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.220869Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:2cc44c75af3ddce997f54aea038181a91261bfd722fcdecb193b2d4f71c93430","observation_id":"1928cce6-36f7-44eb-9b8a-6650c08abc75","resolution":{"observed_at":"2026-08-05T10:31:03.010228Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.989717Z","title":"Reuel-Lamparth, A","venue":null,"work_id":"357d1370-36aa-4fbd-bed6-381cd81a2660","year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.225854Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:799a21bff8423b8997e07adfe25aabbed276fabe3d1d5cb360420ba02e4fc2a1","observation_id":"4a6e8c2e-a0c5-4393-8137-486c69e9eee8","resolution":{"observed_at":"2026-08-05T10:31:02.994446Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.975132Z","title":"Sakaguchi, R","venue":null,"work_id":"3a62f652-271c-4467-adb5-a289fbf9bf80","year":2021},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.230285Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:7a7a804ce6f39dd67e0293a7feb42279a067949db3f53edbe7f6273df8b5a15c","observation_id":"9de30eba-ef39-4bd6-85ed-2ca5ebc5ff69","resolution":{"observed_at":"2026-08-05T10:31:02.979800Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.234717Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.234717Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:bb72fd40507eee579f9c087333811ce692d4838a34d34a5751187e95e0f5129a","observation_id":"8644bea2-f5d8-45ff-8543-8e3c73b0cf41","resolution":{"observed_at":"2026-08-05T10:31:02.234717Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.960501Z","title":"Sanderson","venue":null,"work_id":"0dee9963-5423-47ea-914e-ca0e4b1234ff","year":2010},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.238982Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:910631cab435c14875a6943fcc54d4d93f5c9058ddebcf16b080ae8285c1df21","observation_id":"f473b916-536e-47ec-a003-aa68c32eebdf","resolution":{"observed_at":"2026-08-05T10:31:02.964910Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.945740Z","title":"Sclar, Y","venue":null,"work_id":"27637637-9eed-4e8d-be88-d501d2ba2bf5","year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.243182Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:65cfefa6d7102c6986e49ee16e43d078a095358afca3a36e6fc8fba7677f3781","observation_id":"c31df391-90e6-4fb8-a8b9-f824cca61ea2","resolution":{"observed_at":"2026-08-05T10:31:02.950379Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.930829Z","title":"Sparck Jones and C","venue":null,"work_id":"0c61af6e-73e5-4f8e-b2fe-b7b8ce6e0232","year":1976},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.248104Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:2993aa9695e3fa26296f81425cc7b51b59dbfe966469e9626a89f15504d2f1d4","observation_id":"b87919ff-4415-45b9-9be5-d6d8977c26cc","resolution":{"observed_at":"2026-08-05T10:31:02.935585Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.253189Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.253189Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:ed43e965e351218280480aaf62864cbdf4c370a7c9037feb463c145505c8fa6a","observation_id":"b27103e1-f069-4ad6-83b9-9fbcef21a8a4","resolution":{"observed_at":"2026-08-05T10:31:02.253189Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.915009Z","title":null,"venue":null,"work_id":"e137a6e8-817d-4dc5-ad4b-0bc983f1a0c6","year":1998},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.257873Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:46c9f212ad2c30065dc1027c899378d7a9b87cbc6bd21d0a59d04f7faf2086af","observation_id":"49213a02-9fdd-4668-a7b4-068e583792e8","resolution":{"observed_at":"2026-08-05T10:31:02.920333Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.898800Z","title":null,"venue":null,"work_id":"0dfdc5cb-ebc9-4b6a-9f10-4f0eb1036252","year":2001},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.262534Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:2fe8c600de11b0570af3cfdc652e5eef7702e82fe0bc8ed3a505bc452cc287ab","observation_id":"3053029f-53cd-4dbf-b20d-ef2e25be86d0","resolution":{"observed_at":"2026-08-05T10:31:02.903240Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.883984Z","title":null,"venue":null,"work_id":"88aad451-0fc0-4507-9822-407de688a67d","year":2001},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.266985Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:99e6cfdc75514bbc9d0205b5021acb29eece09ee18b71ef420a951652fc3d11a","observation_id":"0ccf09db-f904-40b5-b722-508a892310dc","resolution":{"observed_at":"2026-08-05T10:31:02.888353Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2109.01652","last_updated":"2022-02-08T20:26:45Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2021-09-03T17:55:52Z","title":"Finetuned Language Models Are Zero-Shot Learners","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2109.01652","snapshot_observed_at":"2026-08-05T10:31:02.271888Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.271888Z"},"links":{"cited_paper":"/paper/2109.01652","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:842fff0fe38c8fd834dc3d3393acf407025ee487c7b04b5694b899c6bdf5b088","observation_id":"1d473cba-0ea7-4dcb-88e8-b00755e160ae","resolution":{"observed_at":"2026-08-05T10:31:02.271888Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.867837Z","title":null,"venue":null,"work_id":"05630510-2bd8-4770-ad01-71b9b778c190","year":2022},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.276957Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:2cb4397e915cdcd7aff85a820f6a9ef4642e4e8e6bb28811f54023e4700643e5","observation_id":"d2b7c9c3-2c39-49ea-bc90-c7c1ebba685c","resolution":{"observed_at":"2026-08-05T10:31:02.873133Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2206.07682","last_updated":"2022-10-26T05:06:24Z","snapshot_observed_at":"2026-08-02T15:56:35.249569Z","submitted_at":"2022-06-15T17:32:01Z","title":"Emergent Abilities of Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2206.07682","snapshot_observed_at":"2026-08-05T10:31:02.282896Z","title":"Wei et al","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.282896Z"},"links":{"cited_paper":"/paper/2206.07682","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:c108b870ba7c39394a58056438a6596fec695133c400daf5ff13d801e13031f3","observation_id":"5a247aea-7324-4277-b3d6-86a6d3e39e04","resolution":{"observed_at":"2026-08-05T10:31:02.282896Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.852513Z","title":"Welbl, N","venue":null,"work_id":"003ee5d0-e6ee-4d6f-bfae-7789a886b504","year":2017},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.287920Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:9c696a1f4f8b865764e459043341a966d243aba12918386bb76bac32ea0556cc","observation_id":"826811b5-17ba-4d78-a2d8-1eebc9fcec3b","resolution":{"observed_at":"2026-08-05T10:31:02.856887Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.837564Z","title":null,"venue":null,"work_id":"95ad9ef2-9522-462e-bbf9-2189a79c8389","year":2023},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.292423Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:170c233057b7b39c0c104b430bc28b1f3f1f4696097e8139aebe4183e06feb1c","observation_id":"bd8639d5-3661-49c9-9ae8-aeddd79afaae","resolution":{"observed_at":"2026-08-05T10:31:02.842118Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.821602Z","title":"Zellers, A","venue":null,"work_id":"6532334b-a8cd-4e0b-aca0-ff5ef003ac58","year":2019},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.296921Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:bb014ecc9d475dad642de065572827c7c6de0482d03f99cad59898b1ea321b5f","observation_id":"e282fa3b-bb90-4bd8-b2d2-6c7cae47a607","resolution":{"observed_at":"2026-08-05T10:31:02.826115Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.805697Z","title":null,"venue":null,"work_id":"101dfbfc-013a-4a83-8573-c4b49efed8e1","year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.301739Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:5b0915a52607349a573fc4d0d7c0e3528fcf922e73a0da9d8e70e7f9c3e30039","observation_id":"bc8a2d74-d951-4f55-ac91-916e6b2b04d7","resolution":{"observed_at":"2026-08-05T10:31:02.811178Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T10:31:02.790712Z","title":"Zheng et al","venue":null,"work_id":"5ba21bb9-a179-4553-babc-2d96a413bf5f","year":2023},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.306286Z"},"links":{"citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:fa9547dc20b1c1d2182f7d3b59beac932370a19a06be71e03f3d2e9a9a804f7d","observation_id":"4c487dba-c445-4e33-8390-0e3b677644ce","resolution":{"observed_at":"2026-08-05T10:31:02.795648Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2104.05938","last_updated":"2021-04-13T05:00:35Z","snapshot_observed_at":"2026-08-08T14:49:34.085953Z","submitted_at":"2021-04-13T05:00:35Z","title":"QMSum: A New Benchmark for Query-based Multi-domain Meeting Summarization","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2104.05938","snapshot_observed_at":"2026-08-05T10:31:02.310499Z","title":"Zhong et al","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.310499Z"},"links":{"cited_paper":"/paper/2104.05938","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:45dff9fcc7c7d85918edcf9c3e7d156212d2ab44509e556960b0cf14a9255838","observation_id":"01dc6a9e-3429-4a80-ae43-94fbd1f422be","resolution":{"observed_at":"2026-08-05T10:31:02.310499Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.17631","last_updated":"2025-03-01T17:06:43Z","snapshot_observed_at":"2026-08-06T12:42:08.815092Z","submitted_at":"2023-10-26T17:48:58Z","title":"JudgeLM: Fine-tuned Large Language Models are Scalable Judges","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.17631","snapshot_observed_at":"2026-08-05T10:31:02.314947Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.314947Z"},"links":{"cited_paper":"/paper/2310.17631","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:8a7fd734922f0cc94e3f97da979574b0e26a1953b1c9b48809cc8662e7c86926","observation_id":"86795f54-c045-4a17-81a9-74e30fcc2ffc","resolution":{"observed_at":"2026-08-05T10:31:02.314947Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.15877","last_updated":"2025-04-01T08:36:44Z","snapshot_observed_at":"2026-07-31T19:00:59.311189Z","submitted_at":"2024-06-22T15:52:04Z","title":"BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.15877","snapshot_observed_at":"2026-08-05T10:31:02.319589Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-05T10:31:02.319589Z"},"links":{"cited_paper":"/paper/2406.15877","citing_paper":"/paper/2509.04013"},"observation_digest":"sha256:8d5b7bf757b1d9303f281345182a029767060f5f3f6bb280829c2537625665cc","observation_id":"61763fde-4f72-4bea-a209-5e3cf9f54dc0","resolution":{"observed_at":"2026-08-05T10:31:02.319589Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-09T16:56:43.263828Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs"},"reference_resolution":{"displayed":51,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":27,"verified_exact":1,"verified_fuzzy":23},"total_outbound_references":51},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"thesis":"As of 10 August 2026, this Paper Citation Record lists 51 of 51 outbound references and 8 inbound Pith citation observations for arXiv:2509.04013."}