{"as_of":"2026-08-08T19:03:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:a23537271b5dbd78edf4aa71dc9a7145cd251dd2fe7ec5c82b8b0846ca50a4ed","coverage":[{"denominator":104,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":100,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-06T23:25:52.911449Z","state":"measured"},{"denominator":101,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":101,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-08T06:32:00.761636+00:00","state":"measured"},{"denominator":1,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":1,"source":"paper_references, paper_reference_links","source_observed_at":"2026-05-15T04:54:26.888562Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-05-15T04:55:03.356720Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"cited_work":{"arxiv_id":"2506.18213","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2506.18213","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":null,"venue":null,"work_id":"55c951d9-d1bd-4dc4-9b8e-6a0dcdebc0e2","year":2025},"citing_paper":{"arxiv_id":"2605.14164","last_updated":"2026-05-13T22:39:10Z","snapshot_observed_at":"2026-07-06T23:25:39.415637Z","submitted_at":"2026-05-13T22:39:10Z","title":"Unsteady Metrics and Benchmarking Cultures of AI Model Builders","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-05-15T04:54:26.888562Z"},"links":{"cited_paper":"/paper/2506.18213","citing_paper":"/paper/2605.14164"},"observation_digest":"sha256:3916f6d1d96fc889770bd1b51414677bdeecc4c87cb7b22a5910bf91dcb1b67d","observation_id":"452b6faa-c3ac-43c7-8e0f-e9fd5cbe5e8c","resolution":{"observed_at":"2026-05-15T04:55:03.360332Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2506.18213/citation-record","integrity":"/paper/2506.18213/integrity","json":"/paper/2506.18213/citation-record.json","paper":"/paper/2506.18213"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:45.551846Z","title":"Early insights from developing question-answer evaluations for frontier AI , 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:45.551846Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:9b6f1ac613085fc93f5185ca4ecc6d0c5dd069fdfce5795b5e0e1cdcd3cd0380","observation_id":"56a46f12-71ee-4976-a6fb-c9092c2692c2","resolution":{"observed_at":"2026-08-06T23:25:45.551846Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:45.636681Z","title":"Benchmarking foundation models with language-model-as-an-examiner","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:45.636681Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:458cb6e6e23fd7318a7061fe4a1a9fc9a6be5dcdf0decf816156ae2958635062","observation_id":"0ac43f7a-de55-441e-8b8a-f3db4df5f146","resolution":{"observed_at":"2026-08-06T23:25:45.636681Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.12820","last_updated":"2024-11-19T19:13:56Z","snapshot_observed_at":"2026-08-07T10:40:57.223673Z","submitted_at":"2024-11-19T19:13:56Z","title":"Declare and Justify: Explicit assumptions in AI evaluations are necessary for effective regulation","version":1},"cited_work":{"arxiv_id":"2411.12820","doi":null,"metadata_source":"pith","pith_arxiv_id":"2411.12820","snapshot_observed_at":"2026-08-06T23:25:55.029019Z","title":"Declare and Justify: Explicit assumptions in AI evaluations are necessary for effective regulation","venue":"cs.AI","work_id":"04dcc4a1-9693-4b21-967b-7d7cd43475e9","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:45.709164Z"},"links":{"cited_paper":"/paper/2411.12820","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:19ad43bef649acb8d3877eb8633ede89e9e30f7dc3ac50acf6870cc80ea33b29","observation_id":"2aec37df-5ee1-4c78-bc3d-08e2ba2adcac","resolution":{"observed_at":"2026-08-06T23:25:55.126842Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:45.801503Z","title":"A quantitative study of nlp approaches to question difficulty estimation","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:45.801503Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:4f6ca2ca9b45525442e72c5acba2f76cd5e40f5c922b6b056e6185722c856143","observation_id":"a571063c-b4e5-48bf-9264-cd696e6d2955","resolution":{"observed_at":"2026-08-06T23:25:45.801503Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.12349","last_updated":"2024-04-18T17:26:01Z","snapshot_observed_at":"2026-07-06T18:02:19.428801Z","submitted_at":"2024-04-18T17:26:01Z","title":"Evaluating AI for Law: Bridging the Gap with Open-Source Solutions","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.12349","snapshot_observed_at":"2026-08-06T23:25:45.871791Z","title":"Evaluating ai for law: Bridging the gap with open-source solutions","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:45.871791Z"},"links":{"cited_paper":"/paper/2404.12349","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:9a0f03341aa0485714c2ee973c84b688602db80510ace47e95527aad2dce6ceb","observation_id":"b9b95a89-88bd-4c01-9327-54fbe29ccf2a","resolution":{"observed_at":"2026-08-06T23:25:45.871791Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:45.907864Z","title":"F., Ammanamanchi, P","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:45.907864Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:5c3c00fb1fcec3d1a01e60b3941e34dcac92b215b87edfd51af1d904080946cb","observation_id":"82e98749-5e33-43ee-bfb2-f52008d9eecf","resolution":{"observed_at":"2026-08-06T23:25:45.907864Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:45.928285Z","title":"R., Steunebrink, B","venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:45.928285Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:f88fa3ac14661d798400cf24c076856bb166b3add4b3fa2d4a6d06fe7fae4524","observation_id":"5ac09d81-d2ad-44df-b2e6-9c2291cff01f","resolution":{"observed_at":"2026-08-06T23:25:45.928285Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:46.009313Z","title":"T., Li, Y., Lundberg, S., et al","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:46.009313Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:5898be64d588b61ab8ad3bc943413700e2dea703d7af8c03ec7e82a2a8661033","observation_id":"5a6b3a82-9480-45a9-a5f2-8775b2a04c42","resolution":{"observed_at":"2026-08-06T23:25:46.009313Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.09221","last_updated":"2024-07-12T12:37:13Z","snapshot_observed_at":"2026-08-02T16:37:16.273986Z","submitted_at":"2024-07-12T12:37:13Z","title":"Evaluating AI Evaluation: Perils and Prospects","version":1},"cited_work":{"arxiv_id":"2407.09221","doi":null,"metadata_source":"pith","pith_arxiv_id":"2407.09221","snapshot_observed_at":"2026-08-06T23:25:54.866279Z","title":"Evaluating AI Evaluation: Perils and Prospects","venue":"cs.AI","work_id":"624c4eb0-84b1-417c-95a3-c8bdca1ad58e","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:46.120217Z"},"links":{"cited_paper":"/paper/2407.09221","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:a436ba9cb9ee6d9f7826a648d51a1d9ca1df92dc86b913899135086cd0f185ae","observation_id":"33e05663-89b0-4d63-afd2-e5be8e5a1bde","resolution":{"observed_at":"2026-08-06T23:25:54.924747Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2502.15620","last_updated":"2025-06-06T13:52:52Z","snapshot_observed_at":"2026-08-07T17:57:55.738681Z","submitted_at":"2025-02-21T17:44:05Z","title":"Paradigms of AI Evaluation: Mapping Goals, Methodologies and Culture","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.15620","snapshot_observed_at":"2026-08-06T23:25:46.193957Z","title":"Paradigms of ai evaluation: Mapping goals, methodologies and culture","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:46.193957Z"},"links":{"cited_paper":"/paper/2502.15620","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:ddd3568730f996e73aa324276902534a8e2df7af08d926a92a10ab8d579898f4","observation_id":"e55a7c09-46e6-4749-9b66-984a37501275","resolution":{"observed_at":"2026-08-06T23:25:46.193957Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.10711","last_updated":"2026-07-06T06:02:32Z","snapshot_observed_at":"2026-08-06T04:23:10.320556Z","submitted_at":"2025-01-18T09:51:57Z","title":"Code Benchmarks Should Prioritize Rigor, Reliability, and Reproducibility","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.10711","snapshot_observed_at":"2026-08-06T23:25:46.325101Z","title":"R., and Cheung, S.-C","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:46.325101Z"},"links":{"cited_paper":"/paper/2501.10711","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:5adc940408b566ba8020ab0149c790eeeee727ab7ae015227adbddb0681371fd","observation_id":"3973a4fa-89d9-446d-a73e-f298d673980b","resolution":{"observed_at":"2026-08-06T23:25:46.325101Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:46.408648Z","title":"L., Bucknall, B., Haupt, A., Wei, K., Scheurer, J., Hobbhahn, M., et al","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:46.408648Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:596a668272a000564f1be17191b5b91067d7d00ef5f88c610e38a55c17b2c6b3","observation_id":"59c604f2-32fe-4fd8-8786-6a2409fa4af1","resolution":{"observed_at":"2026-08-06T23:25:46.408648Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.09177","last_updated":"2024-10-02T10:43:07Z","snapshot_observed_at":"2026-08-01T16:03:38.510223Z","submitted_at":"2024-02-14T13:45:19Z","title":"Leveraging the Context through Multi-Round Interactions for Jailbreaking Attacks","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.09177","snapshot_observed_at":"2026-08-06T23:25:46.504076Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:46.504076Z"},"links":{"cited_paper":"/paper/2402.09177","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:f3967abfc8e6d44a3083d3481f125cf7cd7c65d68b4b02b8bc5eeae2c08781d0","observation_id":"072e7f91-91c7-413b-861d-bd6feb78d302","resolution":{"observed_at":"2026-08-06T23:25:46.504076Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:46.548943Z","title":"N., Li, T., Li, D., Zhu, B., Zhang, H., Jordan, M., Gonzalez, J","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:46.548943Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:02dfe4792d580c718c75ca63f4818667b139d642c873cdd6b2f85d21ef8715fb","observation_id":"b18bfa59-ee0d-4948-a88e-14d35f42f66e","resolution":{"observed_at":"2026-08-06T23:25:46.548943Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:46.659204Z","title":"On the limitations of reference-free evaluations of generated text","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:46.659204Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:405f758604af2884782916d725be76e11bed2756f4a9775e3eec77e778f1fec8","observation_id":"3807f2cd-3c9a-450e-8c1c-5b7852b3bea5","resolution":{"observed_at":"2026-08-06T23:25:46.659204Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:46.709874Z","title":"R., Guo, S., Valko, M., Lillicrap, T., Jimenez Rezende, D., Bengio, Y., Mozer, M","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:46.709874Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:1cdbec51360d68b5e5ff817497cc446bed6917d1c9f3acd76866712bd1adb843","observation_id":"adc3836f-5c87-4296-a62b-0f2741beb6ce","resolution":{"observed_at":"2026-08-06T23:25:46.709874Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:46.762683Z","title":"Generalization or memorization: Data contamination and trustworthy evaluation for large language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:46.762683Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:ca6f6ccec2b9b100716eaec02bd43f336891da8bfb3c2fafb83c666844432b7b","observation_id":"75646943-5fa9-44f0-8e46-1ff23591c1a8","resolution":{"observed_at":"2026-08-06T23:25:46.762683Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:46.873694Z","title":"W., Barocas, S., Atalla, C., Chouldechova, A., and Wallach, H","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:46.873694Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:5587f01d648004a3b7e53087702b8d90a0e433da652dafd1ff58adaf2292ce5d","observation_id":"eadbae44-69dc-4b5f-9fd4-33b86c0d57c3","resolution":{"observed_at":"2026-08-06T23:25:46.873694Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.06559","last_updated":"2025-05-25T23:36:47Z","snapshot_observed_at":"2026-08-08T15:02:02.036967Z","submitted_at":"2025-02-10T15:25:06Z","title":"Can We Trust AI Benchmarks? An Interdisciplinary Review of Current Issues in AI Evaluation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.06559","snapshot_observed_at":"2026-08-06T23:25:46.923418Z","title":"Can we trust ai benchmarks? an interdisciplinary review of current issues in ai evaluation","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:46.923418Z"},"links":{"cited_paper":"/paper/2502.06559","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:43b6c934b8f79deec7282ac7293f33795c1bde85bd74af3142216601f9087fa4","observation_id":"bd68057e-87f1-4259-aafc-0a5e2ca81e00","resolution":{"observed_at":"2026-08-06T23:25:46.923418Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:47.040113Z","title":"Second draft of the general purpose AI code of practice, April 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:47.040113Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:18ef2cceef0e9e8040bc27e716eefb02639c8eed6c67148fa10d176c0889632d","observation_id":"34127b5f-062a-4d44-87b3-254d0096ec13","resolution":{"observed_at":"2026-08-06T23:25:47.040113Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:47.131398Z","title":"Issue brief: Early best practices for frontier AI safety evaluations, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:47.131398Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:405a6eb0e9a80d2b23c794bd3a7ca7352a0ed0dc8b7836da85de33c07dafb5a1","observation_id":"b1fe028a-aa7d-46f4-9a9f-035d54b4ab7a","resolution":{"observed_at":"2026-08-06T23:25:47.131398Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:47.228222Z","title":"Llm-based nlg evaluation: Current status and challenges","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:47.228222Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:4bca061c435f6431cc73e20a44fad76ca6dace2a6a6343a8b6cbdadc68b78f43","observation_id":"3b78b36f-b168-48c2-8ac5-c73310969e8d","resolution":{"observed_at":"2026-08-06T23:25:47.228222Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:47.276468Z","title":"A case for better evaluation standards in nlg","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:47.276468Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:d04ded4fe9cbad0cb780ce4ee8eebbfe948fe344268b589cd8f2440703a94259","observation_id":"16e1fce5-805f-480c-b180-0b148e7989ba","resolution":{"observed_at":"2026-08-06T23:25:47.276468Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:47.313523Z","title":"Repairing the cracked foundation: A survey of obstacles in evaluation practices for generated text","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:47.313523Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:a3474b488bc52cb7c8ec97ace54fc255e7a4b7f6ee235966a1b3b34da634d956","observation_id":"c13ba138-d762-47b2-a3a6-d8d958f6c77b","resolution":{"observed_at":"2026-08-06T23:25:47.313523Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:47.379142Z","title":"Legalbench: A collaboratively built benchmark for measuring legal reasoning in large language models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:47.379142Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:c8fd1871c4734955036f5dac1e637ac520cfab0d16ea0544ae05c41a94b567bd","observation_id":"c51e64f6-6ba2-4066-945f-a0fbdd1330c7","resolution":{"observed_at":"2026-08-06T23:25:47.379142Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:47.435980Z","title":"R., Hullman, J., and Subramonyam, H","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:47.435980Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:e45be5a521506084247a3398c27cf055a8d8ba2864fad1bde663b79a811015ad","observation_id":"fe5c778a-c3f4-4eef-b481-74e220bd9707","resolution":{"observed_at":"2026-08-06T23:25:47.435980Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:47.478675Z","title":"Deception abilities emerged in large language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:47.478675Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:a9b47a8b15b5ec9dd8464d41ea54bc176713b5cc91d82129682ccb0b392dfa3f","observation_id":"9d66b178-9203-416a-b59f-cba00e3474cb","resolution":{"observed_at":"2026-08-06T23:25:47.478675Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.13988","last_updated":"2024-08-08T07:32:14Z","snapshot_observed_at":"2026-07-06T15:07:37.299096Z","submitted_at":"2023-03-24T13:24:41Z","title":"Machine Psychology","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.13988","snapshot_observed_at":"2026-08-06T23:25:47.543373Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:47.543373Z"},"links":{"cited_paper":"/paper/2303.13988","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:f9f53108e14bba3237b89428a17111d9ba2ba0882b5e9557a3795475310385e3","observation_id":"4ab00757-dd10-479e-bb7c-d091c9ea6e6e","resolution":{"observed_at":"2026-08-06T23:25:47.543373Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:47.585201Z","title":"a m \\\"a l \\","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:47.585201Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:8f7ddf5b362427659c93a56f08b72663f413d70a1672fa5cd1d894c3d257fe23","observation_id":"a97ec3b4-39a2-481f-b43a-458a2123d40c","resolution":{"observed_at":"2026-08-06T23:25:47.585201Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1086/732153","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:53.591234Z","title":"and Sharadin, N","venue":null,"work_id":"93abd661-4281-4435-b68d-ada35b44ebb2","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":30,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:47.621675Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:28026da5d7cca85c83b781b392e90854c9c15bdc35093be0a0c479050d466a83","observation_id":"9e54ed9c-31cb-4272-984b-aa9ada6b3ea8","resolution":{"observed_at":"2026-08-06T23:25:53.659452Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:47.677631Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:47.677631Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:497a79007bfbf46bc1908807912c72a2ee49a3059027332dd7dfaa7a993a3fca","observation_id":"ee802a70-7bb5-4a23-ba11-2a5ebf8ea74f","resolution":{"observed_at":"2026-08-06T23:25:47.677631Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:47.733085Z","title":"R., Srivastava, A., and Agrawal, P","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:47.733085Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:7e35fd49f3bc48ca217e91f943cf6b36d77dcf327f6cebbea110a87c26c771cf","observation_id":"710eef11-3f7c-4be0-82b0-5082899058cc","resolution":{"observed_at":"2026-08-06T23:25:47.733085Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.09135","last_updated":"2024-04-14T03:54:00Z","snapshot_observed_at":"2026-07-06T17:59:50.796592Z","submitted_at":"2024-04-14T03:54:00Z","title":"Unveiling LLM Evaluation Focused on Metrics: Challenges and Solutions","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.09135","snapshot_observed_at":"2026-08-06T23:25:47.781422Z","title":"and Zhou, X.-H","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:47.781422Z"},"links":{"cited_paper":"/paper/2404.09135","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:7ba9421cbeb73fb8b6f646248107887c0a89c75a095e85df1f67abbd9008b220","observation_id":"6dba6565-0142-4763-9a95-b2de90f8aebd","resolution":{"observed_at":"2026-08-06T23:25:47.781422Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.02839","last_updated":"2025-05-30T12:01:03Z","snapshot_observed_at":"2026-07-06T17:39:45.571775Z","submitted_at":"2024-03-05T10:20:52Z","title":"An Empirical Study of LLM-as-a-Judge for LLM Evaluation: Fine-tuned Judge Model is not a General Substitute for GPT-4","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.02839","snapshot_observed_at":"2026-08-06T23:25:47.829032Z","title":"On the limitations of fine-tuned judge models for llm evaluation","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:47.829032Z"},"links":{"cited_paper":"/paper/2403.02839","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:3400ebd4343d75529d7abb93ac8b1dce333ea3bed6df0def992790f0eb02737a","observation_id":"35245c9b-6ac1-4d25-abec-e0eb5ed53118","resolution":{"observed_at":"2026-08-06T23:25:47.829032Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:47.914166Z","title":"M ath P rompter: Mathematical reasoning using large language models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:47.914166Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:1f2ac9a1e60421f1454cdeeb62ffe7c1040b30f1fc610080c829fbf49866586e","observation_id":"aa3b399a-3ee0-4ed2-a902-c0981b5311d0","resolution":{"observed_at":"2026-08-06T23:25:47.914166Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.12011","last_updated":"2025-01-21T10:05:48Z","snapshot_observed_at":"2026-07-06T20:23:45.022640Z","submitted_at":"2025-01-21T10:05:48Z","title":"Reference-free Evaluation Metrics for Text Generation: A Survey","version":1},"cited_work":{"arxiv_id":"2501.12011","doi":null,"metadata_source":"pith","pith_arxiv_id":"2501.12011","snapshot_observed_at":"2026-08-06T23:25:54.655866Z","title":"Reference-free Evaluation Metrics for Text Generation: A Survey","venue":"cs.CL","work_id":"687a959b-7651-4233-95cd-51a719d3afee","year":2025},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":36,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:47.974612Z"},"links":{"cited_paper":"/paper/2501.12011","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:62930b3a717995be1299882e622953373a8f9073a9573c91294b2b4dc7323ab0","observation_id":"c1da4d38-4607-4083-9419-d342f740bf94","resolution":{"observed_at":"2026-08-06T23:25:54.728552Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2312.01276","last_updated":"2024-10-29T02:51:28Z","snapshot_observed_at":"2026-07-06T16:56:09.624313Z","submitted_at":"2023-12-03T04:28:19Z","title":"Toward best research practices in AI Psychology","version":2},"cited_work":{"arxiv_id":"2312.01276","doi":null,"metadata_source":"pith","pith_arxiv_id":"2312.01276","snapshot_observed_at":"2026-08-06T23:25:54.516563Z","title":"Toward best research practices in AI Psychology","venue":"cs.AI","work_id":"73b668de-a3f5-4efb-a0e4-030faa9fb04e","year":2023},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":37,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:48.034859Z"},"links":{"cited_paper":"/paper/2312.01276","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:e8198e11c469aa14ab534d59b35a029da83e8013f6f1c09bc78c9e6747c6188b","observation_id":"54925f13-7666-4392-99fa-5193e14fba91","resolution":{"observed_at":"2026-08-06T23:25:54.582700Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:48.087777Z","title":"Stop uploading test data in plain text: Practical strategies for mitigating data contamination by evaluation benchmarks","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":38,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:48.087777Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:953b997b61a8c63502330da84e374058363e59e551194edecfef58bf2a320d71","observation_id":"55acc792-899f-4c18-8a8f-b133f93409ff","resolution":{"observed_at":"2026-08-06T23:25:48.087777Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:48.125765Z","title":"Cladder: assessing causal reasoning in language models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":39,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:48.125765Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:7db295e4fd49c1f04e649d43ac6ee36d7fa901dda2608cc7cff02835e2c5d669","observation_id":"d45f20b3-e3e9-4e0c-ab3f-ce0a28917286","resolution":{"observed_at":"2026-08-06T23:25:48.125765Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:26:01.202800Z","title":"T., and Sch \\\"o lkopf, B","venue":null,"work_id":"8f39731e-3a3b-400c-8ba1-819d53f13e48","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":40,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:48.218739Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:2283ff507cdd69854ff277c5066a9fc4b1a20e3a9455aaa074c9c4e0ba4293c3","observation_id":"00a9df19-b712-4d06-904b-49ee352b09af","resolution":{"observed_at":"2026-08-06T23:26:01.298570Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:26:01.063119Z","title":"R., Rockt \\\"a schel, T., and Perez, E","venue":null,"work_id":"d81c4ff8-2c44-4ab0-9bfe-a69eda60b250","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":41,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:48.251258Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:fdfc38b0d09158a6b55547bcb0040b3c05f91ee16d7709a3ab5d182822310c30","observation_id":"d01022a7-f799-4646-a912-8e1f3a2d13f0","resolution":{"observed_at":"2026-08-06T23:26:01.127649Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:26:00.942627Z","title":"Causal reasoning and large language models: Opening a new frontier for causality","venue":null,"work_id":"bfd32c92-7198-4d37-ad2a-ae120d1de817","year":2023},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":42,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:48.331076Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:4797ad40b1248a5e2af94f1b0455f08d3e1771f7218c9e453ee9222071e5518e","observation_id":"a592d7bd-4f17-45c0-9458-f5c56aa4505e","resolution":{"observed_at":"2026-08-06T23:26:00.994216Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2505.21808","last_updated":"2025-05-27T22:26:51Z","snapshot_observed_at":"2026-08-07T13:20:32.802576Z","submitted_at":"2025-05-27T22:26:51Z","title":"AI Agent Governance: A Field Guide","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2505.21808","snapshot_observed_at":"2026-08-06T23:25:48.401240Z","title":"Ai agent governance: A field guide","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":43,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:48.401240Z"},"links":{"cited_paper":"/paper/2505.21808","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:16e30e364c7f2bc109157695b5ce183fe09469ec73207ace1de8759390484089","observation_id":"51d4cdcb-d43d-4b85-81ac-bf4ae402e3d6","resolution":{"observed_at":"2026-08-06T23:25:48.401240Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:26:00.811035Z","title":"Semantic uncertainty: Linguistic invariances for uncertainty estimation in natural language generation","venue":null,"work_id":"966af6f3-3f20-4ca6-ac5d-7194d703c574","year":2023},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":44,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:48.447578Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:74d374d3794d36ebdfcd0d6d98e137220bd5a71ae93f1b37ba0886e8d11fafee","observation_id":"bc670b74-da28-4b39-a8df-8559da724f6f","resolution":{"observed_at":"2026-08-06T23:26:00.883960Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:26:00.678834Z","title":"P., Wu, H., and Yu, H","venue":null,"work_id":"cc6aaab0-1eee-4eee-981d-a8c19802bdbb","year":2016},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":45,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:48.539508Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:91946e58c7d26d7aa566f39aed28263a984fcccb5354c0fab5f4d5786d4149bd","observation_id":"b0d73691-b2dd-42cd-820a-3f58668ca67f","resolution":{"observed_at":"2026-08-06T23:26:00.734787Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:26:00.511757Z","title":null,"venue":null,"work_id":"96b8ff7c-7589-4456-aeb8-b11339e1eb60","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":46,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:48.584691Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:72cc047ab1df9cc4b9bf30db7678a8edd43d5931d27053d00d8e7c060bfdef4f","observation_id":"5e4129ac-86c3-4faf-9444-68359e50e825","resolution":{"observed_at":"2026-08-06T23:26:00.584030Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:26:00.298470Z","title":"J., Kawaguchi, K., Gidel, G., Bengio, Y., Malkin, N., and Jain, M","venue":null,"work_id":"2ba490dc-a8c8-49ed-9f5f-0f0bd901838a","year":2025},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":47,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:48.664770Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:ba2697a33f9028e7b8edb446fde871b26b5765e407bb6c574b80d09047bc6eaf","observation_id":"ce71553f-637a-4f2d-ae50-a049c57634e1","resolution":{"observed_at":"2026-08-06T23:26:00.401461Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:26:00.082839Z","title":"Leveraging large language models for nlg evaluation: Advances and challenges","venue":null,"work_id":"24d9551d-2973-4302-923a-993120a43fd9","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":48,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:48.732715Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:8f1a8f0000771fb9110979a7b63110e4f578d54e522a9690ad698c3f7c7cea56","observation_id":"004986a1-1769-4978-b256-faebbe908244","resolution":{"observed_at":"2026-08-06T23:26:00.186452Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:59.916474Z","title":"D., Re, C., Acosta-Navas, D., Hudson, D","venue":null,"work_id":"265960d7-f78c-45a4-99ee-768379dfd0f0","year":2023},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":49,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:48.811403Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:09aa8d676bae5cb078a0bb0f1e38cb961de6edb63fd813da0b1a587ac0877c21","observation_id":"d6baae05-f4e8-4bc5-a9e8-b8c3293035cf","resolution":{"observed_at":"2026-08-06T23:25:59.995800Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2306.03100","last_updated":"2025-01-31T14:59:17Z","snapshot_observed_at":"2026-07-06T15:38:46.970127Z","submitted_at":"2023-06-01T00:01:43Z","title":"Rethinking Model Evaluation as Narrowing the Socio-Technical Gap","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.03100","snapshot_observed_at":"2026-08-06T23:25:48.867513Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":50,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:48.867513Z"},"links":{"cited_paper":"/paper/2306.03100","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:42f8e477eb76a607185e0f03042573e7ccbecf715ec13a2d49ef35b41c565c28","observation_id":"8cb3e554-1745-4e84-98c6-229d9424ab14","resolution":{"observed_at":"2026-08-06T23:25:48.867513Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:59.767685Z","title":"D., and Schmidt, L","venue":null,"work_id":"1abc3b7a-3b38-4b9d-9991-3c1c7371bcf1","year":2021},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":51,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:48.914946Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:5b3b67bb866a1cf122adbc56986492663447793d0f34d601fd6aa07d327c5ce8","observation_id":"b0b0b00a-9c26-45e8-8fab-8f335df450b4","resolution":{"observed_at":"2026-08-06T23:25:59.828688Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:59.577047Z","title":"Against the achilles' heel: A survey on red teaming for generative models","venue":null,"work_id":"daeff2ed-86f7-47de-9c32-d8e7f69a6202","year":2025},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":52,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:48.977933Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:556314557ddc2012948e2b3f74adc9d806066c744104d2e0480873ba4b60d64b","observation_id":"4a9429d4-ee19-4328-b541-4fb2a90973ca","resolution":{"observed_at":"2026-08-06T23:25:59.680442Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.18041","last_updated":"2024-02-28T04:35:51Z","snapshot_observed_at":"2026-07-06T17:36:36.437597Z","submitted_at":"2024-02-28T04:35:51Z","title":"Datasets for Large Language Models: A Comprehensive Survey","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.18041","snapshot_observed_at":"2026-08-06T23:25:49.043762Z","title":"Datasets for large language models: A comprehensive survey","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":53,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:49.043762Z"},"links":{"cited_paper":"/paper/2402.18041","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:f3293a3d51d10056302fac394d3a9f0ee3d62c327978b70e68bd2515ac175025","observation_id":"77577b03-da1b-45f3-806a-ebeccf02e3a6","resolution":{"observed_at":"2026-08-06T23:25:49.043762Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.09880","last_updated":"2024-10-14T02:11:29Z","snapshot_observed_at":"2026-08-05T15:21:23.913810Z","submitted_at":"2024-02-15T11:08:10Z","title":"Inadequacies of Large Language Model Benchmarks in the Era of Generative Artificial Intelligence","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.09880","snapshot_observed_at":"2026-08-06T23:25:49.091271Z","title":"R., Susnjak, T., Arachchilage, N., Liu, T., Watters, P., and Halgamuge, M","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":54,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:49.091271Z"},"links":{"cited_paper":"/paper/2402.09880","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:f88c76a04a6a83db43e8af988c70e1fe3259b849151d308cc85b25cbd09a5437","observation_id":"ac7fea2d-36bc-4c4a-96dd-4c3d6f2e3949","resolution":{"observed_at":"2026-08-06T23:25:49.091271Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1901.08644","last_updated":"2019-02-18T09:39:19Z","snapshot_observed_at":"2026-07-06T07:28:54.617767Z","submitted_at":"2019-01-24T21:11:59Z","title":"Ablation Studies in Artificial Neural Networks","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1901.08644","snapshot_observed_at":"2026-08-06T23:25:49.152147Z","title":"W., and Meisen, T","venue":null,"work_id":null,"year":1901},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":55,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:49.152147Z"},"links":{"cited_paper":"/paper/1901.08644","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:ba962c9d300e4715bb1c90ffb0f3fb89cb25f8f62d434a4c9fc882a163fb287a","observation_id":"963f8f01-a4d8-433f-b4dc-94a9d4fe8b73","resolution":{"observed_at":"2026-08-06T23:25:49.152147Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.00640","last_updated":"2024-11-01T14:57:16Z","snapshot_observed_at":"2026-08-07T10:39:44.274910Z","submitted_at":"2024-11-01T14:57:16Z","title":"Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.00640","snapshot_observed_at":"2026-08-06T23:25:49.236771Z","title":"Adding error bars to evals: A statistical approach to language model evaluations","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":56,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:49.236771Z"},"links":{"cited_paper":"/paper/2411.00640","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:dabd238928e2ee9e2de8c5150ca496f936359b63d982d58ffc0cae6015c96efd","observation_id":"6998e5e9-1914-483d-ae1e-92ed1e729faf","resolution":{"observed_at":"2026-08-06T23:25:49.236771Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:59.381713Z","title":"Auditing large language models: a three-layered approach","venue":null,"work_id":"78c3003d-bdd2-4909-848f-5a4da30a17fa","year":2023},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":57,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:49.298801Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:bcb9abcd1e7c89332eb8c3584cd3a5a513bede5c0c4ac7b723289f9d2f7ee204","observation_id":"4a602db4-2242-48e2-95bc-a683cd0dcb7b","resolution":{"observed_at":"2026-08-06T23:25:59.470894Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:59.233098Z","title":"Evaluating the performance of large language models via debates","venue":null,"work_id":"2861cb24-8d39-46f6-97bd-b7a0dd27b4fe","year":2025},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":58,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:49.379623Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:380afbcdc061ac077fa1c974585b31253fdbeb0c8a6aed18bd5b34357e46f206","observation_id":"a4c72738-21da-4f60-9f61-36aa311caba7","resolution":{"observed_at":"2026-08-06T23:25:59.307346Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:59.118140Z","title":"and Kapoor, S","venue":null,"work_id":"a703ba39-ccf1-4d4b-b00c-0fe389930f45","year":2023},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":59,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:49.434665Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:e4649c97bdfeef231657ac4981dd9007dd7a8b34c9f5c1436db1afafb47ec43a","observation_id":"4fd25a23-0496-49c1-83f2-fb582686609a","resolution":{"observed_at":"2026-08-06T23:25:59.162486Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:59.005600Z","title":"Oecd framework for the classification of ai systems","venue":null,"work_id":"4d83bb8d-4bca-4744-9fe8-7b84baaeee85","year":2022},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":60,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:49.515858Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:43a7d698c33daf8f59edb116701c575e7e63978ab440117e8a795192c5f44a46","observation_id":"e9c72da9-038e-43af-a136-36975fa07e9d","resolution":{"observed_at":"2026-08-06T23:25:59.068464Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:58.876259Z","title":"and Kang, E","venue":null,"work_id":"34c33eb6-02f4-4f01-9443-7de4023538f0","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":61,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:49.575099Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:a10a14b677f1b9ffa60d5477ced149aeb5810f3afa5dc22df3c3c2bbc1d7c1fb","observation_id":"c5ec3c96-8051-49f4-8dac-5ee39d976327","resolution":{"observed_at":"2026-08-06T23:25:58.945843Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:49.679281Z","title":"Llm evaluators recognize and favor their own generations","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":62,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:49.679281Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:9eb159b6b2e9fa574222588481787ec2e9b04a6584cd771bbbc764c1f915181c","observation_id":"adcbe876-acc1-4905-90ce-ce33c6e81ce5","resolution":{"observed_at":"2026-08-06T23:25:49.679281Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:58.702065Z","title":"T., and Soder, L","venue":null,"work_id":"71d64a1d-ca76-4033-a934-f744fe27dffd","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":63,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:49.735360Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:0bff5d2cd8877477a26e732e281295fc0137611af7462b91e53ed51eed20fc3e","observation_id":"fd39a5de-c451-41cb-81e9-2af393f33d57","resolution":{"observed_at":"2026-08-06T23:25:58.796939Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:58.554385Z","title":"Preliminary suggestions for rigorous gpai model evaluations","venue":null,"work_id":"c6f98a5c-5e95-45ba-bc5e-a394e68d03cb","year":2025},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":64,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:49.802752Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:63b7d07af09cedd59f2d408aa680e5091267613bb54563fdef4814b4a0ebd2a2","observation_id":"506118b0-2baf-4524-9b19-e0dbac91c2dd","resolution":{"observed_at":"2026-08-06T23:25:58.632455Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:49.881801Z","title":"Discovering language model behaviors with model-written evaluations","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":65,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:49.881801Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:5eeca86a6684f97b2a792cb1fd18881790c22cf37a9b46ffb5b8d3b79e0376af","observation_id":"c4123736-a938-4812-b18a-d77283e1e050","resolution":{"observed_at":"2026-08-06T23:25:49.881801Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.07458","last_updated":"2025-01-13T16:28:01Z","snapshot_observed_at":"2026-07-06T20:20:23.703146Z","submitted_at":"2025-01-13T16:28:01Z","title":"Understanding and Benchmarking Artificial Intelligence: OpenAI's o3 Is Not AGI","version":1},"cited_work":{"arxiv_id":"2501.07458","doi":null,"metadata_source":"pith","pith_arxiv_id":"2501.07458","snapshot_observed_at":"2026-08-06T23:25:54.322563Z","title":"Understanding and Benchmarking Artificial Intelligence: OpenAI's o3 Is Not AGI","venue":"cs.AI","work_id":"511d2f6e-6bdf-433d-a958-09a0096cbcfa","year":2025},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":66,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:49.962510Z"},"links":{"cited_paper":"/paper/2501.07458","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:35297136eeba184eb59a81db82a5c73ff6f1e66b7f4767412f657fcb0f1c08ea","observation_id":"e4697ef7-3a5d-4011-8d66-007b206b5763","resolution":{"observed_at":"2026-08-06T23:25:54.385746Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:58.434471Z","title":"The roots search tool: Data transparency for llms","venue":null,"work_id":"1ea2453a-b206-49db-ae94-73824f70113e","year":2023},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":67,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:50.099913Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:217bd8d7b7237f5128b37b930631108d5739a5ade8b7b06e77f83384715aa867","observation_id":"84ad029e-504d-43b8-b4b9-5782528e75fe","resolution":{"observed_at":"2026-08-06T23:25:58.501486Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:58.311582Z","title":"D., Denton, E., Bender, E","venue":null,"work_id":"15633591-0caa-482d-8786-a1e2878c2ac2","year":2021},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":68,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:50.202095Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:91e4fd25d3d6528b634845e5dfa46a961411090258976f09f1811026ed9d6ff6","observation_id":"bc042dc2-0ba5-4fe6-ae32-322039ba04fb","resolution":{"observed_at":"2026-08-06T23:25:58.364595Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:58.180437Z","title":"Large language model evaluation via multi ai agents: Preliminary results","venue":null,"work_id":"0545f93e-2894-4e6d-aa61-cdc2a18147f1","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":69,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:50.322225Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:e2d4c87eaada6747bf7a3754ff7ae5681bcf74de2860971bc1705dd881f013e6","observation_id":"26bacb55-2e61-4dc4-80f4-3ccf8a124bcc","resolution":{"observed_at":"2026-08-06T23:25:58.238597Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:58.027716Z","title":"A., Comanescu, R., Akbulut, C., Stepleton, T., Mateos-Garcia, J., Bergman, S., Kay, J., et al","venue":null,"work_id":"8a7f8fe7-a270-4b77-8683-ef2a92948292","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":70,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:50.433994Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:83ea9229059d876738ad01102a10be15ad0f6cac54187f6cd89562d14df59c0e","observation_id":"a6a02373-3a9c-496b-941d-1f75ed53b553","resolution":{"observed_at":"2026-08-06T23:25:58.105856Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:57.874849Z","title":"Betterbench: Assessing AI benchmarks, uncovering issues, and establishing best practices","venue":null,"work_id":"5504974d-407d-47fd-8a68-8dc46264a2af","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":71,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:50.530574Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:43be689973f80416fdf7dfa48d1d28965df241029d0427c986081be195cdcefb","observation_id":"e4c626e1-592a-4ede-a8c0-b2d74d041af9","resolution":{"observed_at":"2026-08-06T23:25:57.941298Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:57.705670Z","title":null,"venue":null,"work_id":"72312f44-c49d-421a-bbea-6e4bbdde5bb9","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":72,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:50.669286Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:8683aa304c40dbe1c8e2354bb5848c47f133c7f15b57c5b2c8a8c424e219039f","observation_id":"8911e041-adc9-4f40-ae33-939659ae9531","resolution":{"observed_at":"2026-08-06T23:25:57.803711Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.14981","last_updated":"2025-04-16T09:38:09Z","snapshot_observed_at":"2026-07-06T18:49:33.987133Z","submitted_at":"2024-07-20T21:13:56Z","title":"Open Problems in Technical AI Governance","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.14981","snapshot_observed_at":"2026-08-06T23:25:50.774056Z","title":"S., Rajkumar, N., Moës, N., Ladish, J., Bau, D., Bricman, P., Guha, N., Newman, J., Bengio, Y., South, T., Pentland, A., Koyejo, S., Kochenderfer, M","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":73,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:50.774056Z"},"links":{"cited_paper":"/paper/2407.14981","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:761b4cc595c0173607854275484794fdec67cb81e3673e37cb9832bf74d0f8f7","observation_id":"be10e650-3bda-45c4-81d5-af80f392c598","resolution":{"observed_at":"2026-08-06T23:25:50.774056Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:57.600284Z","title":"Better than random: reliable nlg human evaluation with constrained active sampling","venue":null,"work_id":"5d0b6e33-eea7-440b-8b4d-03f6d5078705","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":74,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:50.872859Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:8f917d3c34ef0a7dde640f8cb543118662919847c7b4dbb5e399bbf27b3eafd9","observation_id":"67ffcbf6-190b-4473-9af9-39fdcc71e123","resolution":{"observed_at":"2026-08-06T23:25:57.636905Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.07927","last_updated":"2025-03-16T06:23:34Z","snapshot_observed_at":"2026-08-05T17:55:26.008016Z","submitted_at":"2024-02-05T19:49:13Z","title":"A Systematic Survey of Prompt Engineering in Large Language Models: Techniques and Applications","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.07927","snapshot_observed_at":"2026-08-06T23:25:50.970160Z","title":"K., Saha, S., Jain, V., Mondal, S., and Chadha, A","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":75,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:50.970160Z"},"links":{"cited_paper":"/paper/2402.07927","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:2f9edd026ad0dbf837b5cbe03412552f813743cc8f7c106bbb00732dfb23304e","observation_id":"285b97cd-d038-4788-a98a-28ad185ed40d","resolution":{"observed_at":"2026-08-06T23:25:50.970160Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:57.455491Z","title":"L., and Agirre, E","venue":null,"work_id":"6bb7ff34-3cdf-4bb4-8d2c-be161a17f77a","year":2023},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":76,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:51.077904Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:2db1975d263e29b2bf404259436410286b3dee055345dd35127a3d9af143e23a","observation_id":"06f09258-be5a-4152-94b6-e678c1758d10","resolution":{"observed_at":"2026-08-06T23:25:57.523765Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:57.337797Z","title":"Targeting the benchmark: On methodology in current natural language processing research","venue":null,"work_id":"557f400f-7da1-4f70-a2e1-7f171d79f0db","year":2021},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":77,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:51.254431Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:76e349095481bd543219c3e1b7d43b7fd389543bcb53ddd0d373d952480d7fc3","observation_id":"15611d99-e4c8-47f7-9a13-65f0a6cd6eec","resolution":{"observed_at":"2026-08-06T23:25:57.394177Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.06608","last_updated":"2025-02-26T18:59:01Z","snapshot_observed_at":"2026-07-06T18:28:24.821865Z","submitted_at":"2024-06-06T18:10:11Z","title":"The Prompt Report: A Systematic Survey of Prompt Engineering Techniques","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.06608","snapshot_observed_at":"2026-08-06T23:25:51.375160Z","title":"S., Vidyadhara, S., Ki, D., Agrawal, S., Pham, C., Kroiz, G","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":78,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:51.375160Z"},"links":{"cited_paper":"/paper/2406.06608","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:f4aba098446d10676a313ed80b2e38f8a10490a295c1a50fad277082acddf68a","observation_id":"ab2b176f-b0c1-4383-8f99-317bb1790866","resolution":{"observed_at":"2026-08-06T23:25:51.375160Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:57.182612Z","title":"Quantifying language models' sensitivity to spurious features in prompt design or: How i learned to start worrying about prompt formatting","venue":null,"work_id":"cf58e2bb-faa6-4451-8024-bb7af50157a7","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":79,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:51.485501Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:a5a1dc8b0ecdc9b3a958403569c4524ac41d0841874331b449c8049ab4789e7a","observation_id":"8a29cb56-42af-44b9-bbe1-b5960eeb87d5","resolution":{"observed_at":"2026-08-06T23:25:57.223251Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2305.15324","last_updated":"2023-09-22T18:48:42Z","snapshot_observed_at":"2026-08-08T14:17:44.034837Z","submitted_at":"2023-05-24T16:38:43Z","title":"Model evaluation for extreme risks","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.15324","snapshot_observed_at":"2026-08-06T23:25:51.618603Z","title":"Model evaluation for extreme risks","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":80,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:51.618603Z"},"links":{"cited_paper":"/paper/2305.15324","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:4b55e59a8470f2615331ec2fc506a251f9614e1c1ac70c191a34013b48cbbea0","observation_id":"f7fa0bd3-765b-4d2d-828e-b39219f73bbf","resolution":{"observed_at":"2026-08-06T23:25:51.618603Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:57.054228Z","title":"CHOPS : CH at with customer profile systems for customer service with LLM s","venue":null,"work_id":"4f4ca999-6b54-4ae5-95e6-9bb1d23a73cc","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":81,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:51.763437Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:9bf3bd5ad44d7eda4e208ba551bbf1924bced7ed856205cce16d8e1d85848f9e","observation_id":"118d65c2-2e53-4ad1-b92e-aa2243f528a4","resolution":{"observed_at":"2026-08-06T23:25:57.126694Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2501.17399","last_updated":"2025-03-06T04:41:56Z","snapshot_observed_at":"2026-08-05T22:39:31.964846Z","submitted_at":"2025-01-29T03:29:24Z","title":"MultiChallenge: A Realistic Multi-Turn Conversation Evaluation Benchmark Challenging to Frontier LLMs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.17399","snapshot_observed_at":"2026-08-06T23:25:51.856959Z","title":"Multichallenge: A realistic multi-turn conversation evaluation benchmark challenging to frontier llms","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":82,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:51.856959Z"},"links":{"cited_paper":"/paper/2501.17399","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:45e4c4a6bc006b417f6fed83d55b50debc0710c1f42fc2f68b4ca2a6b8ad4079","observation_id":"c76d5d24-86a7-48e5-a1ca-4b014cc39af5","resolution":{"observed_at":"2026-08-06T23:25:51.856959Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:56.912096Z","title":"K., Grundy, E","venue":null,"work_id":"90b018e7-1782-4bc7-b464-917d46d24408","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":83,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:51.907553Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:129d8c4f61ca908518604ed0a0ebe90af500890eb5a54f022ecf7da1e89ea3aa","observation_id":"c1b18374-ce72-4fed-aaac-82f17e8759ca","resolution":{"observed_at":"2026-08-06T23:25:56.971466Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:56.775035Z","title":"A study of translation edit rate with targeted human annotation","venue":null,"work_id":"2c79f730-a7a1-474e-89d0-e150c9fdc479","year":2006},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":84,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:51.938242Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:5cf2805d688a5d9dc634cc3a44e6e8837d286408bdea5291a35dbb41e2bc2ebf","observation_id":"185e5305-7c7c-41c5-ace3-3f8266739e57","resolution":{"observed_at":"2026-08-06T23:25:56.857437Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2504.13839","last_updated":"2025-08-14T19:25:43Z","snapshot_observed_at":"2026-08-07T16:01:02.422821Z","submitted_at":"2025-04-18T17:59:59Z","title":"Audit Cards: Contextualizing AI Evaluations","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2504.13839","snapshot_observed_at":"2026-08-06T23:25:51.963205Z","title":"Audit cards: Contextualizing ai evaluations","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":85,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:51.963205Z"},"links":{"cited_paper":"/paper/2504.13839","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:9937228cb3717a93b294cd3b116b45ec463b23ee938e9d6621aadfafeaa91de9","observation_id":"2a49e061-705b-4e26-8d90-dee0a0afa5df","resolution":{"observed_at":"2026-08-06T23:25:51.963205Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.15250","last_updated":"2024-06-24T07:49:25Z","snapshot_observed_at":"2026-07-06T17:49:03.650707Z","submitted_at":"2024-03-22T14:47:35Z","title":"Comprehensive Reassessment of Large-Scale Evaluation Outcomes in LLMs: A Multifaceted Statistical Approach","version":2},"cited_work":{"arxiv_id":"2403.15250","doi":"10.48550/arxiv.2403.15250","metadata_source":"pith","pith_arxiv_id":"2403.15250","snapshot_observed_at":"2026-08-07T06:16:28.064256Z","title":"Comprehensive Reassessment of Large-Scale Evaluation Outcomes in LLMs: A Multifaceted Statistical Approach","venue":"cs.CL","work_id":"7ba08b26-046e-4482-96ba-07fb01572d2f","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":86,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:52.051239Z"},"links":{"cited_paper":"/paper/2403.15250","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:6310a45110e80c8900ce6b39d786730113bc97237aa20d7851b58fed8df04557","observation_id":"2ae1515b-33ea-46ea-ab73-2c5b181161eb","resolution":{"observed_at":"2026-08-06T23:25:53.477142Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"2506.08800","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:54.161970Z","title":"Measuring data science automation: A survey of evaluation tools for ai assistants and agents","venue":null,"work_id":"70901dac-e291-46be-9850-6dd8cc40c3ef","year":2025},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":87,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:52.117063Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:1ca492cd078d0a16d0eaf38b96269100bff5263b2b17a304ffb7ab76c3b76dc5","observation_id":"14000850-7019-4bca-9691-1f96adcad220","resolution":{"observed_at":"2026-08-06T23:25:54.212966Z","resolver_source":"raw_fallback","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:56.647081Z","title":null,"venue":null,"work_id":"303018ad-0596-4d77-9933-999714a26e82","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":88,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:52.187327Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:d20224dfa958601922f5de9351874353571a8f1c9b97e5f5a85704d0954c4c5d","observation_id":"be7fc519-6150-4e21-8327-ac6ee73cc049","resolution":{"observed_at":"2026-08-06T23:25:56.699837Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:56.522408Z","title":"Best practices for the human evaluation of automatically generated text","venue":null,"work_id":"072c570b-2d47-436b-9e05-54d4ca971dc3","year":2019},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":89,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:52.257240Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:7255e5bbe7da3fff68db499f48fad05cef57d951c997133d6e3e981b4f47669d","observation_id":"0fd337e5-aac8-4913-a99d-2c9805687893","resolution":{"observed_at":"2026-08-06T23:25:56.585602Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:56.366251Z","title":"M., Huang, W., Mungra, D., Yuanzhe Pang, R., Phang, J., Liu, H., Cho, K., and Bowman, S","venue":null,"work_id":"e286dfa2-d46e-41c9-9a17-012686429cdd","year":2021},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":90,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:52.284798Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:5765e3fe2064d8c6a886c568bf9666064dd20f44e1e1a4063280761b7c15b065","observation_id":"d86eae37-7fc0-4006-979d-4bc464c7168d","resolution":{"observed_at":"2026-08-06T23:25:56.435860Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:56.196052Z","title":"Mint: Evaluating llms in multi-turn interaction with tools and language feedback","venue":null,"work_id":"d1ceb2d9-6c27-4b49-b3e7-a24d2495f7f9","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":91,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:52.336057Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:b6456a806a61c7bfc8d98234fd644a0b2f4ba8431511f1971f183984d39bcdbe","observation_id":"c09b3736-65ef-49aa-8733-d6afc68e0bef","resolution":{"observed_at":"2026-08-06T23:25:56.273371Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2310.11986","last_updated":"2023-10-31T18:23:32Z","snapshot_observed_at":"2026-08-02T09:23:01.679296Z","submitted_at":"2023-10-18T14:13:58Z","title":"Sociotechnical Safety Evaluation of Generative AI Systems","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.11986","snapshot_observed_at":"2026-08-06T23:25:52.408995Z","title":"A., Mateos-Garcia, J., Bergman, S., Kay, J., Griffin, C., Bariach, B., et al","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":92,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:52.408995Z"},"links":{"cited_paper":"/paper/2310.11986","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:14af71c4ca28131f995147e233fe8992618c2e7217db9f0238928714d212c235","observation_id":"89be56e1-8322-4bbc-b116-dbb0f5f3bae1","resolution":{"observed_at":"2026-08-06T23:25:52.408995Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2503.05336","last_updated":"2025-03-13T00:09:08Z","snapshot_observed_at":"2026-08-07T17:22:23.185186Z","submitted_at":"2025-03-07T11:23:48Z","title":"Toward an Evaluation Science for Generative AI Systems","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.05336","snapshot_observed_at":"2026-08-06T23:25:52.454150Z","title":"D., Wallach, H., Mitchell, M., Wang, A., Salaudeen, O., Bommasani, R., Ganguli, D., Koyejo, S., and Isaac, W","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":93,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:52.454150Z"},"links":{"cited_paper":"/paper/2503.05336","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:66173ed992bbfc32ba0319677c6a236e18930d96f18c9423fc6330869d21ba40","observation_id":"7ea0c3d7-609e-468a-b979-7bd66ba0adf6","resolution":{"observed_at":"2026-08-06T23:25:52.454150Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:56.040049Z","title":"An ai system evaluation framework for advancing ai safety: Terminology, taxonomy, lifecycle mapping","venue":null,"work_id":"f3f0627e-fedb-45f6-84c9-c8024f73572d","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":94,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:52.511076Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:07baa84b46811e24b8691e57ca18c0a4ad051598ecbf105ff9cf9deb6ee27a19","observation_id":"2f2f12fe-501e-44e3-9f2d-6f33a124818d","resolution":{"observed_at":"2026-08-06T23:25:56.134856Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:55.869093Z","title":"A critical review of causal inference benchmarks for large language models","venue":null,"work_id":"c6445488-ba40-4a3f-b441-d31a0258ccdd","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":95,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:52.582138Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:0cc6fa45f47f444966b3f083bbda12b3401f7990de00947bb392c5e8a5c21e38","observation_id":"36c38833-b24c-4681-8198-f3d3749122ce","resolution":{"observed_at":"2026-08-06T23:25:55.938837Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:52.632974Z","title":"Evaluatology: The science and engineering of evaluation","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":96,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:52.632974Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:5bdf5daa3be8ce6ad95f67dfc2500a8c1e4ee984a1ed6c2401c4557abfca4dcc","observation_id":"1360cb66-5ad5-4bb0-8ac5-32e483f7ece0","resolution":{"observed_at":"2026-08-06T23:25:52.632974Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.08385","last_updated":"2025-07-22T23:36:25Z","snapshot_observed_at":"2026-07-06T19:31:31.809692Z","submitted_at":"2024-10-10T21:44:56Z","title":"Language model developers should report train-test overlap","version":2},"cited_work":{"arxiv_id":"2410.08385","doi":null,"metadata_source":"pith","pith_arxiv_id":"2410.08385","snapshot_observed_at":"2026-08-06T23:25:53.750928Z","title":"Language model developers should report train-test overlap","venue":"cs.LG","work_id":"00337004-df41-4c88-bf07-f0ee0f1feb64","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":97,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:52.690485Z"},"links":{"cited_paper":"/paper/2410.08385","citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:078a7f608efd0fa278d4d30b122787ffd1f93433bc680d9abed79c6dc6065562","observation_id":"b7d1310d-4790-46ef-997a-f330a0adec6a","resolution":{"observed_at":"2026-08-06T23:25:53.854157Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:55.716074Z","title":"Q., Shaw, R., Anthis, J","venue":null,"work_id":"b9fa7223-987d-4c78-b74d-86e6dcc52861","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":98,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:52.779117Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:70976a042d0a7a65ef1970b53024662228702aa5562060c01488781958d2ea04","observation_id":"00149752-744b-4812-a5f1-e70f9edab429","resolution":{"observed_at":"2026-08-06T23:25:55.777337Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:55.553262Z","title":"Pacost: Paired confidence significance testing for benchmark contamination detection in large language models","venue":null,"work_id":"5516a53b-50b4-404a-8fc3-5ddc5bfc6407","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":99,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:52.835619Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:815e22d3bdd45d1843f64c6da5b7c3e7c028829eecdfc75e2c6fb852f2647dac","observation_id":"ec954761-76ee-44a6-a158-ef68b503ea85","resolution":{"observed_at":"2026-08-06T23:25:55.639990Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T23:25:55.356931Z","title":"and Kanayet, F","venue":null,"work_id":"5b1846b5-d137-46c8-857c-77278ca67a88","year":2024},"citing_paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations","version":1},"reference_index":100,"source":"arxiv_source","source_observed_at":"2026-08-06T23:25:52.911449Z"},"links":{"citing_paper":"/paper/2506.18213"},"observation_digest":"sha256:931532f624c7b00b20c5f63d7ce1d262c246ba257225bd9acdc2f0f367fea1aa","observation_id":"739097ec-7486-4a77-88f7-9f66a3f77e30","resolution":{"observed_at":"2026-08-06T23:25:55.470953Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-08T06:32:00.761636+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2506.18213","last_updated":"2025-06-23T00:19:27Z","latest_version":1,"primary_category":"cs.AI","snapshot_observed_at":"2026-08-07T10:41:05.932475Z","submitted_at":"2025-06-23T00:19:27Z","title":"A Conceptual Framework for AI Capability Evaluations"},"reference_resolution":{"displayed":100,"state_counts":{"malformed_identifier":0,"metadata_mismatch":1,"parse_uncertain":0,"unresolved":54,"verified_exact":8,"verified_fuzzy":37},"total_outbound_references":104},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"thesis":"As of 8 August 2026, this Paper Citation Record lists 100 of 104 outbound references and 1 inbound Pith citation observation for arXiv:2506.18213."}