{"as_of":"2026-08-11T19:38:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:b5dcd468420c9fe17d10ad18a58fc101e196a93122f119677f166ed9f4b16c97","coverage":[{"denominator":13,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":13,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-06T21:45:02.174033Z","state":"measured"},{"denominator":13,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":13,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-11T06:34:44.6726+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2506.23735/citation-record","integrity":"/paper/2506.23735/integrity","json":"/paper/2506.23735/citation-record.json","paper":"/paper/2506.23735"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"1803.05457","last_updated":"2018-03-14T18:04:21Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2018-03-14T18:04:21Z","title":"Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1803.05457","snapshot_observed_at":"2026-08-06T21:45:01.136247Z","title":"Think you have solved question answering? try arc, the ai2 reasoning challenge, 2018","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2506.23735","last_updated":"2025-06-30T11:18:56Z","snapshot_observed_at":"2026-08-07T23:55:30.572332Z","submitted_at":"2025-06-30T11:18:56Z","title":"AutoEvoEval: An Automated Framework for Evolving Close-Ended LLM Evaluation Data","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-06T21:45:01.136247Z"},"links":{"cited_paper":"/paper/1803.05457","citing_paper":"/paper/2506.23735"},"observation_digest":"sha256:c8e80c5ee62c4c4a38d4cd0d658d667164b77873ec3e356593358309b397dec0","observation_id":"b59611fa-8696-4bb1-9e6f-0df2cd2a98a9","resolution":{"observed_at":"2026-08-06T21:45:01.136247Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T21:45:02.745704Z","title":"Deepseek-v3: Scaling open-source language models with mixture of experts","venue":null,"work_id":"f50e58ba-98ec-416d-b44a-1b8de3de02bf","year":2024},"citing_paper":{"arxiv_id":"2506.23735","last_updated":"2025-06-30T11:18:56Z","snapshot_observed_at":"2026-08-07T23:55:30.572332Z","submitted_at":"2025-06-30T11:18:56Z","title":"AutoEvoEval: An Automated Framework for Evolving Close-Ended LLM Evaluation Data","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-06T21:45:01.188367Z"},"links":{"citing_paper":"/paper/2506.23735"},"observation_digest":"sha256:b956d8bdb5ba85031e5cda59b3da5ce860028592ae852a8c7e36ec351d9c1646","observation_id":"025d6689-70c1-4393-902d-71418d1ccd7c","resolution":{"observed_at":"2026-08-06T21:45:02.796646Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T21:45:01.255575Z","title":"Black-box generation of adversarial text sequences to evade deep learning classifiers","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2506.23735","last_updated":"2025-06-30T11:18:56Z","snapshot_observed_at":"2026-08-07T23:55:30.572332Z","submitted_at":"2025-06-30T11:18:56Z","title":"AutoEvoEval: An Automated Framework for Evolving Close-Ended LLM Evaluation Data","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-06T21:45:01.255575Z"},"links":{"citing_paper":"/paper/2506.23735"},"observation_digest":"sha256:3d8063c4a2f284299e1a1c846b0721755819679a9e55fcfbbe94012e71b30a07","observation_id":"2a67eaf4-1d63-4ccc-9c49-ca13d65b5496","resolution":{"observed_at":"2026-08-06T21:45:01.255575Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2009.03300","last_updated":"2021-01-12T18:57:11Z","snapshot_observed_at":"2026-08-10T12:35:09.020030Z","submitted_at":"2020-09-07T17:59:25Z","title":"Measuring Massive Multitask Language Understanding","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.03300","snapshot_observed_at":"2026-08-06T21:45:01.381819Z","title":"Measuring massive multitask language understanding, 2021","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2506.23735","last_updated":"2025-06-30T11:18:56Z","snapshot_observed_at":"2026-08-07T23:55:30.572332Z","submitted_at":"2025-06-30T11:18:56Z","title":"AutoEvoEval: An Automated Framework for Evolving Close-Ended LLM Evaluation Data","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-06T21:45:01.381819Z"},"links":{"cited_paper":"/paper/2009.03300","citing_paper":"/paper/2506.23735"},"observation_digest":"sha256:e77417a88e3af38984d95e92f43331477149741711f33ad5ba7105ccd1a8377e","observation_id":"e6813975-75c2-4d2f-ad10-1df82cbb37cd","resolution":{"observed_at":"2026-08-06T21:45:01.381819Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.08322","last_updated":"2023-11-06T13:24:16Z","snapshot_observed_at":"2026-08-08T19:03:09.038168Z","submitted_at":"2023-05-15T03:20:19Z","title":"C-Eval: A Multi-Level Multi-Discipline Chinese Evaluation Suite for Foundation Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.08322","snapshot_observed_at":"2026-08-06T21:45:01.465673Z","title":"C-eval: A multi-level multi-discipline chinese evaluation suite for foundation models, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.23735","last_updated":"2025-06-30T11:18:56Z","snapshot_observed_at":"2026-08-07T23:55:30.572332Z","submitted_at":"2025-06-30T11:18:56Z","title":"AutoEvoEval: An Automated Framework for Evolving Close-Ended LLM Evaluation Data","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-06T21:45:01.465673Z"},"links":{"cited_paper":"/paper/2305.08322","citing_paper":"/paper/2506.23735"},"observation_digest":"sha256:2cd7b435ec7e7c645470a56a43ca2be12fcd039fcec720ba7830dae56a95c537","observation_id":"e14cfab9-b745-4349-ba58-6caa471c5fef","resolution":{"observed_at":"2026-08-06T21:45:01.465673Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T21:45:01.552009Z","title":"Adversarial text generation by search and learning","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.23735","last_updated":"2025-06-30T11:18:56Z","snapshot_observed_at":"2026-08-07T23:55:30.572332Z","submitted_at":"2025-06-30T11:18:56Z","title":"AutoEvoEval: An Automated Framework for Evolving Close-Ended LLM Evaluation Data","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-06T21:45:01.552009Z"},"links":{"citing_paper":"/paper/2506.23735"},"observation_digest":"sha256:e1755103a3909bcb15fa5496409402f8d63ed801f87875bb2e70d2f70edbbc26","observation_id":"ea7df36b-156b-492d-ab37-f536b1963162","resolution":{"observed_at":"2026-08-06T21:45:01.552009Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.04369","last_updated":"2023-09-08T15:00:41Z","snapshot_observed_at":"2026-08-08T00:46:30.288581Z","submitted_at":"2023-09-08T15:00:41Z","title":"Beyond Static Datasets: A Deep Interaction Approach to LLM Evaluation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.04369","snapshot_observed_at":"2026-08-06T21:45:01.648196Z","title":"Beyond static datasets: A deep interaction approach to llm evaluation","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.23735","last_updated":"2025-06-30T11:18:56Z","snapshot_observed_at":"2026-08-07T23:55:30.572332Z","submitted_at":"2025-06-30T11:18:56Z","title":"AutoEvoEval: An Automated Framework for Evolving Close-Ended LLM Evaluation Data","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-06T21:45:01.648196Z"},"links":{"cited_paper":"/paper/2309.04369","citing_paper":"/paper/2506.23735"},"observation_digest":"sha256:0f4123fa44d033d8366c6d110eae78d3d8e1d3f90cfc44ed81ca7786ceb0f60e","observation_id":"baaf731e-2d06-4501-9f47-67da1aceedf5","resolution":{"observed_at":"2026-08-06T21:45:01.648196Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.19740","last_updated":"2024-10-18T06:57:08Z","snapshot_observed_at":"2026-08-08T23:35:33.282872Z","submitted_at":"2024-05-30T06:38:32Z","title":"PertEval: Unveiling Real Knowledge Capacity of LLMs with Knowledge-Invariant Perturbations","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.19740","snapshot_observed_at":"2026-08-06T21:45:01.719577Z","title":"Perteval: Unveiling real knowledge capacity of llms with knowledge-invariant perturbations, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.23735","last_updated":"2025-06-30T11:18:56Z","snapshot_observed_at":"2026-08-07T23:55:30.572332Z","submitted_at":"2025-06-30T11:18:56Z","title":"AutoEvoEval: An Automated Framework for Evolving Close-Ended LLM Evaluation Data","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-06T21:45:01.719577Z"},"links":{"cited_paper":"/paper/2405.19740","citing_paper":"/paper/2506.23735"},"observation_digest":"sha256:2db9667b3d4194ce4d532be16741aa09406db7b9671a2067d1fabb6c71448cb5","observation_id":"53df4759-7c2d-4fb4-9e65-93c92e196499","resolution":{"observed_at":"2026-08-06T21:45:01.719577Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2105.11136","last_updated":"2021-05-25T08:24:19Z","snapshot_observed_at":"2026-07-06T11:12:04.788175Z","submitted_at":"2021-05-24T07:35:56Z","title":"Using Adversarial Attacks to Reveal the Statistical Bias in Machine Reading Comprehension Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2105.11136","snapshot_observed_at":"2026-08-06T21:45:01.788221Z","title":"Using adversarial attacks to reveal the statistical bias in machine reading comprehension models, 2021","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2506.23735","last_updated":"2025-06-30T11:18:56Z","snapshot_observed_at":"2026-08-07T23:55:30.572332Z","submitted_at":"2025-06-30T11:18:56Z","title":"AutoEvoEval: An Automated Framework for Evolving Close-Ended LLM Evaluation Data","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-06T21:45:01.788221Z"},"links":{"cited_paper":"/paper/2105.11136","citing_paper":"/paper/2506.23735"},"observation_digest":"sha256:c9ac398167a3a538f2ece19f7252ae07e8129d81f316526243e77771800ff2d2","observation_id":"ca5d8135-ce8e-42d7-bc25-ef1220d1d270","resolution":{"observed_at":"2026-08-06T21:45:01.788221Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2209.09513","last_updated":"2022-10-17T07:46:47Z","snapshot_observed_at":"2026-08-02T03:10:19.073297Z","submitted_at":"2022-09-20T07:04:24Z","title":"Learn to Explain: Multimodal Reasoning via Thought Chains for Science Question Answering","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2209.09513","snapshot_observed_at":"2026-08-06T21:45:01.892149Z","title":"Learn to explain: Multimodal reasoning via thought chains for science question answering, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2506.23735","last_updated":"2025-06-30T11:18:56Z","snapshot_observed_at":"2026-08-07T23:55:30.572332Z","submitted_at":"2025-06-30T11:18:56Z","title":"AutoEvoEval: An Automated Framework for Evolving Close-Ended LLM Evaluation Data","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-06T21:45:01.892149Z"},"links":{"cited_paper":"/paper/2209.09513","citing_paper":"/paper/2506.23735"},"observation_digest":"sha256:26d27ec4d4f5cafa492b26b787661085a11378b339942ec1f8ce60dcad9588fd","observation_id":"0b0ab9bf-24d6-45b8-b827-73b7a419d91d","resolution":{"observed_at":"2026-08-06T21:45:01.892149Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2203.14371","last_updated":"2022-03-27T18:59:16Z","snapshot_observed_at":"2026-08-10T19:09:39.267277Z","submitted_at":"2022-03-27T18:59:16Z","title":"MedMCQA : A Large-scale Multi-Subject Multi-Choice Dataset for Medical domain Question Answering","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2203.14371","snapshot_observed_at":"2026-08-06T21:45:01.978036Z","title":"Medmcqa : A large-scale multi-subject multi-choice dataset for medical domain question answering, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2506.23735","last_updated":"2025-06-30T11:18:56Z","snapshot_observed_at":"2026-08-07T23:55:30.572332Z","submitted_at":"2025-06-30T11:18:56Z","title":"AutoEvoEval: An Automated Framework for Evolving Close-Ended LLM Evaluation Data","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-06T21:45:01.978036Z"},"links":{"cited_paper":"/paper/2203.14371","citing_paper":"/paper/2506.23735"},"observation_digest":"sha256:ae85e70821df73febd5d8878052a9ceee7c13619ca9e1a28014fe129bbe99188","observation_id":"6fab41cb-ecc3-4953-aca6-697cfc9ef00b","resolution":{"observed_at":"2026-08-06T21:45:01.978036Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T21:45:02.588060Z","title":"Alcuna: Large language models meet new knowledge","venue":null,"work_id":"522a1f11-3bcf-4c02-bc03-7fdb10870e12","year":2023},"citing_paper":{"arxiv_id":"2506.23735","last_updated":"2025-06-30T11:18:56Z","snapshot_observed_at":"2026-08-07T23:55:30.572332Z","submitted_at":"2025-06-30T11:18:56Z","title":"AutoEvoEval: An Automated Framework for Evolving Close-Ended LLM Evaluation Data","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-06T21:45:02.073822Z"},"links":{"citing_paper":"/paper/2506.23735"},"observation_digest":"sha256:d7143c8b4f32e3c32180052cc0e658769c696835f5bda476b332f5bbdc5596b5","observation_id":"6c0eae8f-4d19-48d9-ac76-bfaf9f4e6033","resolution":{"observed_at":"2026-08-06T21:45:02.656878Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2310.14820","last_updated":"2023-10-23T11:40:05Z","snapshot_observed_at":"2026-07-06T16:37:07.197221Z","submitted_at":"2023-10-23T11:40:05Z","title":"ALCUNA: Large Language Models Meet New Knowledge","version":1},"cited_work":{"arxiv_id":"2310.14820","doi":null,"metadata_source":"pith","pith_arxiv_id":"2310.14820","snapshot_observed_at":"2026-08-06T21:45:02.258572Z","title":"ALCUNA: Large Language Models Meet New Knowledge","venue":"cs.CL","work_id":"48103ad1-1fd7-47d7-869c-f58f31365346","year":2023},"citing_paper":{"arxiv_id":"2506.23735","last_updated":"2025-06-30T11:18:56Z","snapshot_observed_at":"2026-08-07T23:55:30.572332Z","submitted_at":"2025-06-30T11:18:56Z","title":"AutoEvoEval: An Automated Framework for Evolving Close-Ended LLM Evaluation Data","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-06T21:45:02.174033Z"},"links":{"cited_paper":"/paper/2310.14820","citing_paper":"/paper/2506.23735"},"observation_digest":"sha256:d30ee42adedb7cd41d96fb6159d35ae6cde1d2cdf7c2da2cec56fe50a790cb56","observation_id":"bfd065d6-b13c-4793-8dbe-5967b7222084","resolution":{"observed_at":"2026-08-06T21:45:02.341402Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2506.23735","last_updated":"2025-06-30T11:18:56Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-07T23:55:30.572332Z","submitted_at":"2025-06-30T11:18:56Z","title":"AutoEvoEval: An Automated Framework for Evolving Close-Ended LLM Evaluation Data"},"reference_resolution":{"displayed":13,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":10,"verified_exact":1,"verified_fuzzy":2},"total_outbound_references":13},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"thesis":"As of 11 August 2026, this Paper Citation Record lists 13 of 13 outbound references and 0 inbound Pith citation observations for arXiv:2506.23735."}