{"as_of":"2026-08-14T19:19:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:910d43d812650756bc59ae74ede6d58f40c8102c5b1ad87a964ebb898a8a1598","coverage":[{"denominator":111,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":100,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-07T14:26:35.172094Z","state":"measured"},{"denominator":105,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":105,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-14T06:32:32.682623+00:00","state":"measured"},{"denominator":5,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":5,"source":"paper_references, paper_reference_links","source_observed_at":"2026-07-12T05:51:08.101805Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-05-18T11:01:17.051563Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"cited_work":{"arxiv_id":"2505.18893","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2505.18893","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"arXiv:2505.18893 [cs.CY] https://arxiv.org/abs/2505.18893","venue":null,"work_id":"dd8979b5-087f-4fbc-9209-2345274614e0","year":2025},"citing_paper":{"arxiv_id":"2510.15297","last_updated":"2026-05-13T10:08:20Z","snapshot_observed_at":"2026-08-13T20:57:03.709500Z","submitted_at":"2025-10-17T04:07:29Z","title":"VERA-MH Concept Paper","version":4},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-05-18T07:00:15.411310Z"},"links":{"cited_paper":"/paper/2505.18893","citing_paper":"/paper/2510.15297"},"observation_digest":"sha256:03347505f785a1176ff81c120eda0726fe27cb61e25705b47d35d89ef22be0e0","observation_id":"81957865-3309-4ca0-b7f3-67330616efe2","resolution":{"observed_at":"2026-05-18T07:01:01.468094Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"cited_work":{"arxiv_id":"2505.18893","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2505.18893","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"arXiv:2505.18893 [cs.CY] https://arxiv.org/abs/2505.18893","venue":null,"work_id":"dd8979b5-087f-4fbc-9209-2345274614e0","year":2025},"citing_paper":{"arxiv_id":"2511.05501","last_updated":"2026-04-28T16:06:15Z","snapshot_observed_at":"2026-08-11T16:16:31.682685Z","submitted_at":"2025-09-30T21:36:23Z","title":"Towards Real-World Validity in Generative AI Benchmarks: Understanding and Designing Domain-Centered Evaluations for Journalism Practitioners","version":3},"reference_index":61,"source":"pdf_text","source_observed_at":"2026-05-18T10:59:16.139525Z"},"links":{"cited_paper":"/paper/2505.18893","citing_paper":"/paper/2511.05501"},"observation_digest":"sha256:2c1fe506454f17fd3c6111dddbe2d3e1b5a8b1e8aa3a683943b80b75bd6dcd3c","observation_id":"1a9ec25c-73eb-4398-9ea7-343f82de5a0c","resolution":{"observed_at":"2026-05-18T11:01:17.054618Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"cited_work":{"arxiv_id":"2505.18893","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2505.18893","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"arXiv:2505.18893 [cs.CY] https://arxiv.org/abs/2505.18893","venue":null,"work_id":"dd8979b5-087f-4fbc-9209-2345274614e0","year":2025},"citing_paper":{"arxiv_id":"2603.06811","last_updated":"2026-05-08T14:52:11Z","snapshot_observed_at":"2026-08-05T19:05:35.953649Z","submitted_at":"2026-03-06T19:17:50Z","title":"Making AI Evaluation Deployment Relevant Through Context Specification","version":3},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-05-15T14:46:09.944168Z"},"links":{"cited_paper":"/paper/2505.18893","citing_paper":"/paper/2603.06811"},"observation_digest":"sha256:863bbd790357f9863a193431c18e18ca69e52d9772c8da11a36a356a3327f45e","observation_id":"0a088f9a-f81a-47f6-9c1c-d282ffcf339a","resolution":{"observed_at":"2026-05-15T14:50:05.136829Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"cited_work":{"arxiv_id":"2505.18893","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2505.18893","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"arXiv:2505.18893 [cs.CY] https://arxiv.org/abs/2505.18893","venue":null,"work_id":"dd8979b5-087f-4fbc-9209-2345274614e0","year":2025},"citing_paper":{"arxiv_id":"2604.05151","last_updated":"2026-04-06T20:25:20Z","snapshot_observed_at":"2026-08-13T21:47:43.744832Z","submitted_at":"2026-04-06T20:25:20Z","title":"Context Collapse: Barriers to Adoption for Generative AI in Workplace Settings","version":1},"reference_index":108,"source":"pdf_text","source_observed_at":"2026-05-10T18:58:18.436603Z"},"links":{"cited_paper":"/paper/2505.18893","citing_paper":"/paper/2604.05151"},"observation_digest":"sha256:2e7675a29a73586cc254bf515c9c381a7f884c020d4920826f58c6b574bb39fa","observation_id":"4b82fe65-407b-4be2-be87-fa8e4fcb69dd","resolution":{"observed_at":"2026-05-10T23:35:52.592560Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2505.18893","snapshot_observed_at":"2026-07-12T05:51:08.101805Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.02955","last_updated":"2026-07-03T04:59:03Z","snapshot_observed_at":"2026-08-06T03:14:27.339658Z","submitted_at":"2026-07-03T04:59:03Z","title":"The Foreign Policy AI Evaluation Gap","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-07-12T05:51:08.101805Z"},"links":{"cited_paper":"/paper/2505.18893","citing_paper":"/paper/2607.02955"},"observation_digest":"sha256:7a1764004cad72c6feb95184aa6efa03f99f112f0b4e4f7c62290086b3ea89da","observation_id":"4fd5fcbf-fb47-42be-a0bc-346bc73b5029","resolution":{"observed_at":"2026-07-12T05:51:08.101805Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2505.18893/citation-record","integrity":"/paper/2505.18893/integrity","json":"/paper/2505.18893/citation-record.json","paper":"/paper/2505.18893"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2407.01294","last_updated":"2024-11-09T12:00:09Z","snapshot_observed_at":"2026-08-12T23:31:16.312926Z","submitted_at":"2024-07-01T13:47:53Z","title":"A Collaborative, Human-Centred Taxonomy of AI, Algorithmic, and Automation Harms","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.01294","snapshot_observed_at":"2026-08-07T14:26:34.857333Z","title":"A collaborative, human-centred taxonomy of ai, algorithmic, and automation harms","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.857333Z"},"links":{"cited_paper":"/paper/2407.01294","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:2eb155c6e2d2559ce1a6550b05eb178665adfc651d92112272e9c2040319efcb","observation_id":"b6498dfe-01a4-4634-8264-996152b6d865","resolution":{"observed_at":"2026-08-07T14:26:34.857333Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.861646Z","title":"The simple macroeconomics of ai","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.861646Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:151953b13b346d5cfbffa53bcb611c38a5f775ce4d3f9f6d5c1b5fda9d29ecbe","observation_id":"afb07c4a-ea11-47ed-8bbc-ef50e31253d9","resolution":{"observed_at":"2026-08-07T14:26:34.861646Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.865827Z","title":"Xue, Peter S","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.865827Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:30e691f5e46edbc868122400113abf7bbc554c2faff3b251f7341ff6387301fb","observation_id":"13aa2236-d300-49fa-99ff-cedd3cfd80e9","resolution":{"observed_at":"2026-08-07T14:26:34.865827Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1901.09749","last_updated":"2019-05-15T15:12:03Z","snapshot_observed_at":"2026-08-14T17:24:52.721128Z","submitted_at":"2019-01-28T15:47:07Z","title":"Fairwashing: the risk of rationalization","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1901.09749","snapshot_observed_at":"2026-08-07T14:26:34.869537Z","title":"Aïvodji et al","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.869537Z"},"links":{"cited_paper":"/paper/1901.09749","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:8d522f5db37ee5999123671de796fc81b1ff6bbc17ad864e900a9fe9ed39b0ea","observation_id":"7fd991af-7b41-4e67-9c06-d50e1bceecf0","resolution":{"observed_at":"2026-08-07T14:26:34.869537Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2204.05862","last_updated":"2022-04-12T15:02:38Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-04-12T15:02:38Z","title":"Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2204.05862","snapshot_observed_at":"2026-08-07T14:26:34.873016Z","title":"Bai et al","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.873016Z"},"links":{"cited_paper":"/paper/2204.05862","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:9073d7fd66a7d5fb517ec424bc5af51278a47f8791b7f31080303cfd2e4356fd","observation_id":"c75d47a0-c066-4280-aaae-84dfc118e1ab","resolution":{"observed_at":"2026-08-07T14:26:34.873016Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"2024.10232","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:36.343501Z","title":"Incentives and the replication crisis in social sciences: A critical review of open science practices","venue":null,"work_id":"1fb33d1c-1901-47ab-80dd-dc57992d5361","year":2025},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.877215Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:8f8e0cd8f046fce9bf1e8e827e8cb08a1a92125dfae15ce9a0872a06d94b65e9","observation_id":"0e5d9c44-e9f7-4219-9232-0e72a5efa586","resolution":{"observed_at":"2026-08-07T14:26:36.348633Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.880419Z","title":"On the dangers of stochastic parrots: Can language models be too big? In Proceedings of the 2021 ACM conference on fairness, accountability, and transparency, pages 610–623, 2021","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.880419Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:7098fa659c03cb4c784dc11f5f4942b0437b378f0691e8fbf437ee3aae89831a","observation_id":"d6d4a213-1e5c-4d61-aaef-8f1b1d76f227","resolution":{"observed_at":"2026-08-07T14:26:34.880419Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.883256Z","title":"Participatory ai for humani- tarian innovation: A briefing paper","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.883256Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:f6fcd1ef8e8a0761bc16b5e6ff6a4f96708410d2b1861799d8b61616c9a10426","observation_id":"de6df55b-695b-4d35-a3bb-34d9a51a4001","resolution":{"observed_at":"2026-08-07T14:26:34.883256Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.886559Z","title":"Bertrand and S","venue":null,"work_id":null,"year":2004},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.886559Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:93052ba56aa0de7ba95f45294bbf9577f04601f109a10314d02c46b949dce8db","observation_id":"9e69c03f-2cbb-4428-b26b-2f491535c4b3","resolution":{"observed_at":"2026-08-07T14:26:34.886559Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"2504.03359","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:36.204581Z","title":"A metrological framework for uncertainty evaluation in machine learning classification models, May 2025","venue":null,"work_id":"d1070c42-6721-4498-aaa2-1e3bf89baff7","year":2025},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.890340Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:4d3ade4a797b826268e10b949cbffa7bcbc03433f65f3b34e60be61269f03c1d","observation_id":"7c5fce2e-95ec-4374-8ce1-8140fa5b4cbd","resolution":{"observed_at":"2026-08-07T14:26:36.209532Z","resolver_source":"raw_fallback","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.896063Z","title":"Blaire et al","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.896063Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:bd12b19f7c5b141d0191284872939bd6bb978961b1c8a828760d432c3874740d","observation_id":"bbc6fd49-e9fc-40e6-b19c-f20c786f8bd7","resolution":{"observed_at":"2026-08-07T14:26:34.896063Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.899053Z","title":"Atlas of ai risks: Enhancing public understanding of ai risks","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.899053Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:ad6d50eb6d7ea8dbcc93165ca1eb11e4fefd2c1dd377a2c0be44c0f1b091b86a","observation_id":"ee319d7f-274d-44ef-90f0-d9866723a3a3","resolution":{"observed_at":"2026-08-07T14:26:34.899053Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.901755Z","title":"Bommasani, P","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.901755Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:0adb2c51afb4c9676220b39a996f4b49cf26d7461016d24c9c482c2c60877775","observation_id":"aedc55e4-e36c-4170-991f-bdfc8c010225","resolution":{"observed_at":"2026-08-07T14:26:34.901755Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.904300Z","title":"The impact of ai on the workforce: Tasks versus jobs? Economics Letters, 244:111971, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.904300Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:0be540525beac8d305c9c4c4820aeb8ad08e53780afc91b0aa75cd6d74e568ee","observation_id":"c75045d6-2669-42b7-afe8-ea8c61338dbd","resolution":{"observed_at":"2026-08-07T14:26:34.904300Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2011.13416","last_updated":"2020-12-10T08:51:12Z","snapshot_observed_at":"2026-07-06T10:18:08.698617Z","submitted_at":"2020-11-26T18:09:52Z","title":"Overcoming Failures of Imagination in AI Infused System Development and Deployment","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2011.13416","snapshot_observed_at":"2026-08-07T14:26:34.907267Z","title":"Overcoming failures of imagination in ai infused system development and deployment, 2020","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.907267Z"},"links":{"cited_paper":"/paper/2011.13416","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:f6480dcf322b6eb5d5ff239ddafa42603ac6292e92357390f1e982d29d90dd34","observation_id":"a4c7aa42-8f94-4c35-b56e-1b75abbdee98","resolution":{"observed_at":"2026-08-07T14:26:34.907267Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.910193Z","title":"Evaluating the replicability of social science experiments in nature and science between 2010 and 2015","venue":null,"work_id":null,"year":2010},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.910193Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:2309b6f47216f393b32e7cf01e7aebcabc91b9672be45b9f18a1dcf52210c6a6","observation_id":"f0be35d7-ac87-4eb3-becc-ddc532641f27","resolution":{"observed_at":"2026-08-07T14:26:34.910193Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.913162Z","title":"Carlini et al","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.913162Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:0e3bb7bc75144f31a4005f80e46fea8891b3a472c8113ed098799968592fe130","observation_id":"2d765c08-716b-4479-977c-fef9e604199a","resolution":{"observed_at":"2026-08-07T14:26:34.913162Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2202.07646","last_updated":"2023-03-06T06:28:18Z","snapshot_observed_at":"2026-08-03T19:51:05.627063Z","submitted_at":"2022-02-15T18:48:31Z","title":"Quantifying Memorization Across Neural Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2202.07646","snapshot_observed_at":"2026-08-07T14:26:34.915543Z","title":"Quantifying memorization across neural language models, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.915543Z"},"links":{"cited_paper":"/paper/2202.07646","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:cf2f374dd58a9c2a7621210dd5d031f9629244e76ee1b625014c2a52edfce6a4","observation_id":"6dc8f480-07f4-49d1-afcb-3cb4b57346d9","resolution":{"observed_at":"2026-08-07T14:26:34.915543Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.06634","last_updated":"2024-07-09T17:44:00Z","snapshot_observed_at":"2026-08-13T00:57:43.847856Z","submitted_at":"2024-03-11T11:46:12Z","title":"Stealing Part of a Production Language Model","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.06634","snapshot_observed_at":"2026-08-07T14:26:34.918711Z","title":"Feder Cooper, Katherine Lee, Matthew Jagielski, Milad Nasr, Arthur Conmy, Itay Yona, Eric Wallace, David Rolnick, and Florian Tramèr","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.918711Z"},"links":{"cited_paper":"/paper/2403.06634","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:16a8ad7e92745175cecb120f8b9c7ac2bb719608f9f5ddaec9144d203996fedf","observation_id":"76c89e6d-1f98-451e-ba59-bb9b2433b228","resolution":{"observed_at":"2026-08-07T14:26:34.918711Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2307.15217","last_updated":"2023-09-11T17:25:24Z","snapshot_observed_at":"2026-08-03T19:11:09.671782Z","submitted_at":"2023-07-27T22:29:25Z","title":"Open Problems and Fundamental Limitations of Reinforcement Learning from Human Feedback","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.15217","snapshot_observed_at":"2026-08-07T14:26:34.921482Z","title":"Michaud, Jacob Pfau, Dmitrii Krasheninnikov, Xin Chen, Lauro Langosco, Peter Hase, Erdem Bıyık, Anca Dragan, David Krueger, Dorsa Sadigh, and Dylan Hadfield-Menell","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.921482Z"},"links":{"cited_paper":"/paper/2307.15217","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:bf4f6d92e2dc2bc4c188d5084353380d1276b08d59f0f3470a68e511cc2ca850","observation_id":"c234d425-3000-4e85-8a67-7c03b38df923","resolution":{"observed_at":"2026-08-07T14:26:34.921482Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"gov/3618036","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.843730Z","title":null,"venue":null,"work_id":"d9e30207-b7da-4ae0-9377-927bd458c615","year":2022},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.924306Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:b94c3ce5412ea4c2aec2312eaac9a9bf25f30a5be26a2244bfdbefec1eef2337","observation_id":"b43a77c0-b060-4f31-b1de-fee303a5bda9","resolution":{"observed_at":"2026-08-07T14:26:35.865414Z","resolver_source":"raw_fallback","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.926568Z","title":"A survey on evaluation of large language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.926568Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:bbf8dedea942b8c45d9e792561bb857c8b064bbfb2e35ee6da62ca8bf1a85fc9","observation_id":"94969559-75ce-400e-9a19-bdd43a4e090d","resolution":{"observed_at":"2026-08-07T14:26:34.926568Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.08419","last_updated":"2024-07-18T18:24:57Z","snapshot_observed_at":"2026-08-12T12:29:45.113982Z","submitted_at":"2023-10-12T15:38:28Z","title":"Jailbreaking Black Box Large Language Models in Twenty Queries","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.08419","snapshot_observed_at":"2026-08-07T14:26:34.929840Z","title":"Pappas, and Eric Wong","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.929840Z"},"links":{"cited_paper":"/paper/2310.08419","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:b888d1a3f8d76aef4dae20273fc49ec21f9b498916c0a22d4454a824a7c153fd","observation_id":"91935f16-ca15-48b2-91f5-508b35d498fe","resolution":{"observed_at":"2026-08-07T14:26:34.929840Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.934020Z","title":"From representational harms to quality-of-service harms: A case study on llama 2 safety safeguards","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.934020Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:35c67b6906a9075d4f1adac8844099fae6a411db20e57476591d1d80558514ac","observation_id":"feac4b48-4dc3-46c2-83c5-59543cc190ee","resolution":{"observed_at":"2026-08-07T14:26:34.934020Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.01934","last_updated":"2024-12-02T19:50:00Z","snapshot_observed_at":"2026-08-12T13:25:20.838525Z","submitted_at":"2024-12-02T19:50:00Z","title":"A Shared Standard for Valid Measurement of Generative AI Systems' Capabilities, Risks, and Impacts","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.01934","snapshot_observed_at":"2026-08-07T14:26:34.937671Z","title":"Feder Cooper, Emily Corvi, P","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.937671Z"},"links":{"cited_paper":"/paper/2412.01934","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:5e31473562e29bf0b185d95672198b28133207aa89c180a0fc53fe6e278ac6a0","observation_id":"0cc1cb86-35dc-4ee9-892a-74586c65d2ac","resolution":{"observed_at":"2026-08-07T14:26:34.937671Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2011.03395","last_updated":"2020-11-24T19:16:02Z","snapshot_observed_at":"2026-08-07T12:11:27.017871Z","submitted_at":"2020-11-06T14:53:13Z","title":"Underspecification Presents Challenges for Credibility in Modern Machine Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2011.03395","snapshot_observed_at":"2026-08-07T14:26:34.941345Z","title":"D’Amour et al","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.941345Z"},"links":{"cited_paper":"/paper/2011.03395","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:258d5d9443eb535a6b0c0b23d719e5632316facc1d04f4041c57b32c4ead61c8","observation_id":"3dbf9447-7d0d-4ac6-90fd-f2d82493a473","resolution":{"observed_at":"2026-08-07T14:26:34.941345Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1007/s10551-013-1697-z","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:03:40.895421Z","title":null,"venue":"Journal of Business Ethics","work_id":"d0f5c0bc-667b-4687-be7a-b5f1b852b302","year":2014},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.944120Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:72b89f7aca05b5bde7ccde6cada24d1eb35e2e07fcebe2cad15eb344c0de348d","observation_id":"cf9b0241-1673-46db-96ed-9f827a69b119","resolution":{"observed_at":"2026-08-07T14:26:35.328171Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2106.11022","last_updated":"2021-06-10T09:49:34Z","snapshot_observed_at":"2026-08-13T19:05:56.465282Z","submitted_at":"2021-06-10T09:49:34Z","title":"Hard Choices in Artificial Intelligence","version":1},"cited_work":{"arxiv_id":"2106.11022","doi":"10.48550/arxiv.2106.11022","metadata_source":"pith","pith_arxiv_id":"2106.11022","snapshot_observed_at":"2026-08-07T18:16:16.850658Z","title":"Hard Choices in Artificial Intelligence","venue":"cs.CY","work_id":"e6d2099d-9686-4397-9807-3052071adb5e","year":2021},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.947324Z"},"links":{"cited_paper":"/paper/2106.11022","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:4623af6f545c1a5a53ec5d47828b948b79b62f3d5528ac617d672deb4c46fedb","observation_id":"d069ae07-c330-4943-a552-6cfd9f6509cd","resolution":{"observed_at":"2026-08-07T14:26:35.320244Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1909.03004","last_updated":"2019-09-06T16:40:42Z","snapshot_observed_at":"2026-07-06T08:19:45.897686Z","submitted_at":"2019-09-06T16:40:42Z","title":"Show Your Work: Improved Reporting of Experimental Results","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1909.03004","snapshot_observed_at":"2026-08-07T14:26:34.951230Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.951230Z"},"links":{"cited_paper":"/paper/1909.03004","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:697add73ffe7995de86f335ca37758252101d288b8c8af2a3a1c6566e96209b1","observation_id":"e215c4e5-f5c9-49f7-b5e3-d11823d243e2","resolution":{"observed_at":"2026-08-07T14:26:34.951230Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.954291Z","title":"Harmful speech detection by language models exhibits gender-queer dialect bias","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.954291Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:8c7c5fbcbeffac72ca17eef1228c4127894a2fb1bf60911790625c477d932b04","observation_id":"b94329f8-0d36-47c9-82e3-972b701a5976","resolution":{"observed_at":"2026-08-07T14:26:34.954291Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.07841","last_updated":"2024-09-16T13:18:23Z","snapshot_observed_at":"2026-08-14T13:12:42.596107Z","submitted_at":"2024-02-12T17:52:05Z","title":"Do Membership Inference Attacks Work on Large Language Models?","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.07841","snapshot_observed_at":"2026-08-07T14:26:34.957705Z","title":"Duan et al","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.957705Z"},"links":{"cited_paper":"/paper/2402.07841","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:a4edfd96c3144d0b7900da2b9b040f1e342dce2d9231698f410305904b322ead","observation_id":"2baf5d17-acee-49c1-b1c1-78de82ef832d","resolution":{"observed_at":"2026-08-07T14:26:34.957705Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2209.15259","last_updated":"2023-05-09T10:24:28Z","snapshot_observed_at":"2026-08-14T07:14:56.003140Z","submitted_at":"2022-09-30T06:36:49Z","title":"On the Impossible Safety of Large AI Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2209.15259","snapshot_observed_at":"2026-08-07T14:26:34.961038Z","title":"On the impossible safety of large ai models, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.961038Z"},"links":{"cited_paper":"/paper/2209.15259","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:d6322e5a0c8ee13a924e2abf2b03681380333e6b1eb8d2100c73c7f2362863b5","observation_id":"27a92922-9afe-4233-8bee-adfea636cfb1","resolution":{"observed_at":"2026-08-07T14:26:34.961038Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.965132Z","title":"Ethics Owners: A New Model of Organi- zational Responsibility in Data-Driven Technology Companies","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.965132Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:28e914323476980b21c2a83657fedc987729c3933a67b34b54f5511e48cba329","observation_id":"f81b78ef-f874-48e0-ac0e-dc952015b6c3","resolution":{"observed_at":"2026-08-07T14:26:34.965132Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.967839Z","title":"Can we trust ai benchmarks? an interdisciplinary review of current issues in ai evaluation, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.967839Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:585a81ef4fe19180a426381fbef2acb2859db29774ab58b84895b132145e915c","observation_id":"57bdd006-8a2b-4623-9240-18f9b683c32c","resolution":{"observed_at":"2026-08-07T14:26:34.967839Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1145/3686937","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"Code-ifying the law: How disciplinary divides afflict the development of legal software","venue":"Proceedings of the ACM on Human-Computer Interaction","work_id":"e1cb3664-ed93-48d7-a3d9-609c7e1d40ce","year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.970590Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:b444f4f2aca6602319ba72af1f7c72db4cfef5385c6319b44d6519de1116ba47","observation_id":"d9144f5c-a105-452e-b9b9-7fa940cfa70f","resolution":{"observed_at":"2026-08-07T14:26:35.309703Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1038/s44271-023-00003-2","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"The replication crisis has led to positive structural, procedural, and community changes","venue":"Communications Psychology","work_id":"d715cb26-8983-4d76-8f5e-af43bbb65f1b","year":2023},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.974416Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:8cafd7b02285f18509d81d480a3e6d12e19cb2d04863b97644dd200a6d1eb8b6","observation_id":"5c715418-5612-40a5-bc23-8f8a3049b866","resolution":{"observed_at":"2026-08-07T14:26:35.301963Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.977694Z","title":"Large ai models are cultural and social technologies","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.977694Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:6ab1efb6e02405bd4a620b3e57610fd0bc1bfcefc7a68e13f26bfa950e3cb541","observation_id":"1e77dd89-d8be-4a04-b85a-7049d0e3445e","resolution":{"observed_at":"2026-08-07T14:26:34.977694Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.980885Z","title":"Value sensitive design: Theory and methods","venue":null,"work_id":null,"year":2003},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.980885Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:b4bb10a8694bf370b883d093af8811e896a3a928074a878ed8500070f8009f55","observation_id":"912652e7-2c8f-45ad-af8a-ebe7d919bd4f","resolution":{"observed_at":"2026-08-07T14:26:34.980885Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1506.02142","last_updated":"2016-10-04T16:50:26Z","snapshot_observed_at":"2026-08-14T05:08:06.217969Z","submitted_at":"2015-06-06T12:30:43Z","title":"Dropout as a Bayesian Approximation: Representing Model Uncertainty in Deep Learning","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1506.02142","snapshot_observed_at":"2026-08-07T14:26:34.983661Z","title":"Dropout as a Bayesian Approximation: Representing Model Uncertainty in Deep Learning, October 2016","venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.983661Z"},"links":{"cited_paper":"/paper/1506.02142","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:8f63a0592bc1953deee0aa85eac1f29bcfc2ccc38c7de96f510f3eb7568f125f","observation_id":"922754ab-22d6-4c66-8236-2c38f5267b0e","resolution":{"observed_at":"2026-08-07T14:26:34.983661Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.987374Z","title":"Americans use ai in everyday prod- ucts without realizing it","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.987374Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:e8917301a56f7d624050dc452d12694deca682ee1e3ec148f23721ea26ef6ec4","observation_id":"216ad4eb-e6b6-419d-99f4-a0b1f140232a","resolution":{"observed_at":"2026-08-07T14:26:34.987374Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.990319Z","title":"Gao et al","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.990319Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:99c5bf094a314135652f6605f58a94a4f14e37a4b82d7bec631151c7d34b4b6b","observation_id":"02823ee1-8d02-44b8-850c-7dfb2eaf0e53","resolution":{"observed_at":"2026-08-07T14:26:34.990319Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2009.11462","last_updated":"2020-09-25T20:22:26Z","snapshot_observed_at":"2026-08-09T05:10:26.091710Z","submitted_at":"2020-09-24T03:17:19Z","title":"RealToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.11462","snapshot_observed_at":"2026-08-07T14:26:34.993546Z","title":"Real- toxicityprompts: Evaluating neural toxic degeneration in language models","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.993546Z"},"links":{"cited_paper":"/paper/2009.11462","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:2d84bc22d228844e0b72bc1b53ab5f7c1952b3ec0035382dd7763a042ff7caad","observation_id":"11d08791-fbb7-46e2-8098-3ec4fe98c7a1","resolution":{"observed_at":"2026-08-07T14:26:34.993546Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:34.996603Z","title":"Trust, attitudes and use of artificial intelligence: A global study 2025, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.996603Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:db99e10b11d830a375b35766f16ef1f29bc3b643f3729b42d44f2b0df88c9f17","observation_id":"3b750134-73a6-41bf-8529-f164a08264bb","resolution":{"observed_at":"2026-08-07T14:26:34.996603Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2302.12173","last_updated":"2023-05-05T14:26:17Z","snapshot_observed_at":"2026-08-12T14:45:22.117576Z","submitted_at":"2023-02-23T17:14:38Z","title":"Not what you've signed up for: Compromising Real-World LLM-Integrated Applications with Indirect Prompt Injection","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2302.12173","snapshot_observed_at":"2026-08-07T14:26:34.999548Z","title":"Greshake et al","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:34.999548Z"},"links":{"cited_paper":"/paper/2302.12173","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:fadfee44dc4756b45386165de05b7ec7c1d625b9369fd5dbae9175658253d655","observation_id":"21e2fd89-5fbc-4193-93a3-d61fdc487be4","resolution":{"observed_at":"2026-08-07T14:26:34.999548Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2503.04761","last_updated":"2025-02-11T00:46:43Z","snapshot_observed_at":"2026-08-08T13:35:20.441932Z","submitted_at":"2025-02-11T00:46:43Z","title":"Which Economic Tasks are Performed with AI? Evidence from Millions of Claude Conversations","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.04761","snapshot_observed_at":"2026-08-07T14:26:35.002831Z","title":"Troy, Dario Amodei, Jared Kaplan, Jack Clark, and Deep Ganguli","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.002831Z"},"links":{"cited_paper":"/paper/2503.04761","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:5b73bec6e89da94cdebb6c7c1ea47b2d1d87d59004e319ce166613abb48eb6e7","observation_id":"3967974e-db69-42e2-a3e9-3b0c560892d7","resolution":{"observed_at":"2026-08-07T14:26:35.002831Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.006652Z","title":"Hoffmann and H","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.006652Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:0d04a93c7bcc211ecf56ca0894754859b446323e15888333ed89e0cc8edb7120","observation_id":"f6658a23-0896-482b-8c79-329fa23fba89","resolution":{"observed_at":"2026-08-07T14:26:35.006652Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.01276","last_updated":"2024-10-29T02:51:28Z","snapshot_observed_at":"2026-08-13T16:05:35.611916Z","submitted_at":"2023-12-03T04:28:19Z","title":"Toward best research practices in AI Psychology","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.01276","snapshot_observed_at":"2026-08-07T14:26:35.009530Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.009530Z"},"links":{"cited_paper":"/paper/2312.01276","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:5bd077111b1e9c18d00ff85df3425bd6f6fd9b7be97cbf84ab5dd5c14717b209","observation_id":"c926737a-d81b-435e-bbcb-aad80a5548d2","resolution":{"observed_at":"2026-08-07T14:26:35.009530Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"9460.2021","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.731056Z","title":"Jansma, Anne M","venue":null,"work_id":"096637d5-ed9e-45af-bd34-f6c8dc473368","year":2021},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.012380Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:f465a16f5eb7e513cfad94de2eebe29d793f853b13d174695bb3e0caa18d0032","observation_id":"6dff38cc-eac9-4d50-bdad-b2c036b40754","resolution":{"observed_at":"2026-08-07T14:26:35.735095Z","resolver_source":"raw_fallback","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.015687Z","title":"Survey of hallucination in natural language generation","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.015687Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:a2fbb41b619c63d7bf7a51163804fcd5e8d7dd4a7caea16abf7b2b0e945cff29","observation_id":"7ed8dc78-8d6f-4246-9bf5-6470e13bef3f","resolution":{"observed_at":"2026-08-07T14:26:35.015687Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.018225Z","title":"Kapoor and A","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.018225Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:df49cc002db4b71eed0fb828e224586f8cdc5266095e34345bd4332c61a71272","observation_id":"d92c4d55-08cd-44fd-b9e2-78926ff72c47","resolution":{"observed_at":"2026-08-07T14:26:35.018225Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2207.07048","last_updated":"2022-07-14T16:44:59Z","snapshot_observed_at":"2026-08-14T14:36:33.379781Z","submitted_at":"2022-07-14T16:44:59Z","title":"Leakage and the Reproducibility Crisis in ML-based Science","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2207.07048","snapshot_observed_at":"2026-08-07T14:26:35.021540Z","title":"Leakage and the reproducibility crisis in ml-based science, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.021540Z"},"links":{"cited_paper":"/paper/2207.07048","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:5873ee2d8ab90dcfd4d9172f320704be946868e1fe64acc3d93b11902bc8d401","observation_id":"45e3f768-ff8e-4997-a239-db008ea79e4d","resolution":{"observed_at":"2026-08-07T14:26:35.021540Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.024268Z","title":"Dynabench: Rethinking benchmarking in NLP","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.024268Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:7e61a67e318704459bdc7bab14f9a5f3d831bcb7c699986e65a87d6016b55be3","observation_id":"38652cd6-1e13-4aee-85f1-c1a215f4f512","resolution":{"observed_at":"2026-08-07T14:26:35.024268Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.027858Z","title":"Audiocaps: Gener- ating captions for audios in the wild","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.027858Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:b0a5165f4f1cc85e2c285ff7b94508d8d46a9ba3b3b42c501cc8e3fa78eaff48","observation_id":"899bf88f-a594-43e8-9260-272c5c1f009e","resolution":{"observed_at":"2026-08-07T14:26:35.027858Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.13595","last_updated":"2023-11-28T18:16:11Z","snapshot_observed_at":"2026-08-13T05:44:32.405737Z","submitted_at":"2023-10-20T15:45:16Z","title":"The History and Risks of Reinforcement Learning and Human Feedback","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.13595","snapshot_observed_at":"2026-08-07T14:26:35.031480Z","title":"The history and risks of reinforcement learning and human feedback, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.031480Z"},"links":{"cited_paper":"/paper/2310.13595","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:f7fdf5385684906aee1d3d7c5a06649f1e47173e3ffb5f713bbc3bd5c44f7c66","observation_id":"7a40fea5-a5a8-448c-b5b5-a511fd157b94","resolution":{"observed_at":"2026-08-07T14:26:35.031480Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.10430","last_updated":"2024-06-14T22:22:17Z","snapshot_observed_at":"2026-08-12T23:42:20.175018Z","submitted_at":"2024-06-14T22:22:17Z","title":"Challenging the Machine: Contestability in Government AI Systems","version":1},"cited_work":{"arxiv_id":"2406.10430","doi":null,"metadata_source":"pith","pith_arxiv_id":"2406.10430","snapshot_observed_at":"2026-08-07T14:26:35.653468Z","title":"Challenging the Machine: Contestability in Government AI Systems","venue":"cs.CY","work_id":"0aeba562-92be-4de0-b9a0-df81f66fc5ce","year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.034525Z"},"links":{"cited_paper":"/paper/2406.10430","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:c4a63f1b1d0369c381db47423456e45fccbb1dcb59199f019dc08455dfc798ca","observation_id":"91a7ef01-7a82-4026-92ec-eb315053b58e","resolution":{"observed_at":"2026-08-07T14:26:35.657411Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1186/s40900-018-0136-9","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"Leask, Marlene Sandlund, Dawn A","venue":"Research Involvement and Engagement","work_id":"4313ea82-2aa6-4584-b3b7-daea2df233d4","year":2019},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.038251Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:ddca5bd1f870b1cacd5b409b85a47985f6123e5030b52d0958dd048cf808819a","observation_id":"64d8ebad-b86f-40e9-93af-4e72e8550326","resolution":{"observed_at":"2026-08-07T14:26:35.289958Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.041357Z","title":"Lenaerts-Bergmans","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.041357Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:5bee1e8bf3daf290d0f8a01ae032dd09ac8706be7b257c62c7485498699ee48e","observation_id":"f1be5504-2e3b-4263-849d-978860ae65e9","resolution":{"observed_at":"2026-08-07T14:26:35.041357Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.044083Z","title":"Ai sustainability in practice part one: Foundations for sustainable ai projects","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":59,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.044083Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:8b45df9625e3f56e8598dd0358341b7c71832f72ca6945a1add3e9b3a85ebbbe","observation_id":"1dc479c6-2bee-4ad7-adbd-e66ac3fdf5b9","resolution":{"observed_at":"2026-08-07T14:26:35.044083Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2312.16337","last_updated":"2023-12-26T21:17:46Z","snapshot_observed_at":"2026-08-14T13:06:26.250712Z","submitted_at":"2023-12-26T21:17:46Z","title":"Task Contamination: Language Models May Not Be Few-Shot Anymore","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.16337","snapshot_observed_at":"2026-08-07T14:26:35.047220Z","title":"Li and J","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.047220Z"},"links":{"cited_paper":"/paper/2312.16337","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:6769b944a8df3cfc0fc7c5db447cfcea90ba932f33062a41fce5c5387ffc07e9","observation_id":"ebe4ec8b-d75c-4c46-b21d-be97a5e8b2fb","resolution":{"observed_at":"2026-08-07T14:26:35.047220Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.15221","last_updated":"2024-09-04T00:58:59Z","snapshot_observed_at":"2026-08-12T22:56:25.482748Z","submitted_at":"2024-08-27T17:33:30Z","title":"LLM Defenses Are Not Robust to Multi-Turn Human Jailbreaks Yet","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.15221","snapshot_observed_at":"2026-08-07T14:26:35.050363Z","title":"LLM defenses are not robust to multi-turn human jailbreaks yet","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":61,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.050363Z"},"links":{"cited_paper":"/paper/2408.15221","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:0770638178ccb2e25dc8688558ee7c234454874945da4166a30217e699114084","observation_id":"025da3cb-08c6-465b-bc04-ee7fc410a902","resolution":{"observed_at":"2026-08-07T14:26:35.050363Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.03218","last_updated":"2024-05-15T19:16:09Z","snapshot_observed_at":"2026-08-09T09:20:14.433688Z","submitted_at":"2024-03-05T18:59:35Z","title":"The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning","version":7},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.03218","snapshot_observed_at":"2026-08-07T14:26:35.053073Z","title":"Li, Ann-Kathrin Dombrowski, Shashwat Goel, Long Phan, Gabriel Mukobi, Nathan Helm- Burger, Rassin Lababidi, Lennart Justen, Andrew B","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":62,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.053073Z"},"links":{"cited_paper":"/paper/2403.03218","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:1bce91c23d8ef65b832efabfd4f0e2d9f09e52708100e99fbb5e794f9ecd84e4","observation_id":"389ac5fe-c27c-46a5-89f0-ce659e480958","resolution":{"observed_at":"2026-08-07T14:26:35.053073Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.056436Z","title":"A survey of multimodel large language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":63,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.056436Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:a18d2179ef34ea9572103327570525c2e4ed84bd47b03860dbf49c2f5fa16d1b","observation_id":"80abc088-bfa2-4bad-af30-d22e37c4e5d1","resolution":{"observed_at":"2026-08-07T14:26:35.056436Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.059571Z","title":"Are we learn- ing yet? a meta review of evaluation failures across machine learning","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":64,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.059571Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:fbe871d5e053ee89a390fcc65ccd46c4d6844f99c0fda571f1ffe66ece3cf0e9","observation_id":"8321b673-345a-4700-8063-73ace51a30e3","resolution":{"observed_at":"2026-08-07T14:26:35.059571Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.2139/ssrn.5143605","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"Jaffe, Abhishek Nagaraj, Imke Reimers, Michael D","venue":"SSRN Electronic Journal","work_id":"2740bc97-23b3-402b-9708-e3fee75a32a8","year":2025},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":65,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.062561Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:e4f14bd26bf173d5e0ab3a4f8d81bb89830e4c41bd4804a1c6b61adb35aa3a86","observation_id":"71ae9f5c-30c4-4333-bb13-bb90ddc1d672","resolution":{"observed_at":"2026-08-07T14:26:35.276370Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.065115Z","title":"Madaio, Jingya Chen, Hanna Wallach, and Jennifer Wortman Vaughan","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":66,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.065115Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:c84bbcc707040546a99ec5e83a829df489d301c77335397879edb1c18b6dc4fe","observation_id":"1d8d6401-4b8e-4941-8a81-574bc731bfb5","resolution":{"observed_at":"2026-08-07T14:26:35.065115Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1177/20539517241290220","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"Martínez","venue":"Big Data & Society","work_id":"177db28e-6242-449e-b4e8-4ac9e4a0f819","year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":67,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.068275Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:6db2a7795058d8eac235fae067660af2162fcdb4bf67d34ea5ff439370488f58","observation_id":"b144b970-4bcf-458b-b05e-3de02192adfd","resolution":{"observed_at":"2026-08-07T14:26:35.263929Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.070857Z","title":"Co-creation for policy: Participatory methodologies to structure multi-stakeholder policymaking processes","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":68,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.070857Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:97df378cded8139ec8ca3d525f042577e7a97dc2242cccd3c6655cc1a629ee45","observation_id":"e3916f63-794f-41c3-8416-7a72083a1402","resolution":{"observed_at":"2026-08-07T14:26:35.070857Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.073654Z","title":"How the u.s","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":69,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.073654Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:949d7500da9ac63c1893b81a4171b6caecad7e38604c247a55469d49a9e67a28","observation_id":"d4b8510a-71f7-4262-867d-cd6958f7bc51","resolution":{"observed_at":"2026-08-07T14:26:35.073654Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.076211Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":70,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.076211Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:937903131d7ac8c743a3fe6c0694f3cebdbdbd49288d1d352ea270e4faf41126","observation_id":"a4196187-1308-4fdb-8173-321672f87d13","resolution":{"observed_at":"2026-08-07T14:26:35.076211Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.078657Z","title":"Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), Dublin, Ireland, May 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":71,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.078657Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:79296e4938d6423d764f1f6167b87f7c2777281da9822ac8560605007af468f4","observation_id":"297a28dc-07a4-41c4-8260-0beafa2de4b3","resolution":{"observed_at":"2026-08-07T14:26:35.078657Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.081533Z","title":"thick alignment","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":72,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.081533Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:95b70b2092e721c5ad533daa5180121d74bb09b0da09b3c3a23546234e5db0fb","observation_id":"384418a1-47a4-4377-82b2-4d29ddb2faad","resolution":{"observed_at":"2026-08-07T14:26:35.081533Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.083993Z","title":"Global law and policy tracker","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":73,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.083993Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:28318072e5426aed20978fd8dfcdcd7ccc678a97b68712629bb048b939dd2d6a","observation_id":"b0fb0488-7956-4493-b034-2062815e1e4b","resolution":{"observed_at":"2026-08-07T14:26:35.083993Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:36.448154Z","title":"Governing with artificial intelli- gence, June 2024","venue":null,"work_id":"570b0ab8-f3ac-4bed-8d80-ea606d2b2e29","year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":74,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.087421Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:1d4f3cc1d9cb459bee6384c000b635ad08fc8d037edd2510ed32375c3d13d468","observation_id":"2aa2749a-083c-4e8a-aecd-e8eee81c3ddf","resolution":{"observed_at":"2026-08-07T14:26:36.450268Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:36.441196Z","title":null,"venue":null,"work_id":"625fe543-108b-4129-a9b6-551a93a99dcc","year":2025},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":75,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.090372Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:b858e2a12e9051a9d9311f2211f477e54e1879e7f732a7f8f9ac3abdbe26efb8","observation_id":"6351948b-f2f7-4b81-a4ab-4334be4af9b2","resolution":{"observed_at":"2026-08-07T14:26:36.444429Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.10899","last_updated":"2023-12-18T23:38:05Z","snapshot_observed_at":"2026-08-13T04:59:02.365995Z","submitted_at":"2023-12-18T23:38:05Z","title":"Concrete Problems in AI Safety, Revisited","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.10899","snapshot_observed_at":"2026-08-07T14:26:35.092769Z","title":"Concrete problems in ai safety, revisited, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":76,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.092769Z"},"links":{"cited_paper":"/paper/2401.10899","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:3b2eb09b27a14512e0d2a4730eca985e16f3251a8bf26cd903c4e9b0a03947bd","observation_id":"5a5a82bc-8a7d-4109-b126-ca9d3ac66639","resolution":{"observed_at":"2026-08-07T14:26:35.092769Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:36.435515Z","title":"The fallacy of ai functionality","venue":null,"work_id":"b11c69b8-dcd5-4f9b-9783-ff94df943250","year":2022},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":77,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.095545Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:6ec91872137855dc0e6e51d4c37db0a8fe1ca120619ca71955f6b57c85bb3a61","observation_id":"8727bad5-1e4a-4d12-ad77-ec05658dcb6a","resolution":{"observed_at":"2026-08-07T14:26:36.437542Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1902.10811","last_updated":"2019-06-12T17:42:33Z","snapshot_observed_at":"2026-08-14T17:16:40.613707Z","submitted_at":"2019-02-13T20:35:44Z","title":"Do ImageNet Classifiers Generalize to ImageNet?","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1902.10811","snapshot_observed_at":"2026-08-07T14:26:35.097787Z","title":"Do imagenet classifiers generalize to imagenet?, 2019","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":78,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.097787Z"},"links":{"cited_paper":"/paper/1902.10811","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:39666bcb1190d8460179a018c08b99d939c0c3b1f41afc283a6f8756216eea01","observation_id":"02799c20-8c4a-4d18-91f6-cf96df412c46","resolution":{"observed_at":"2026-08-07T14:26:35.097787Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:36.429220Z","title":"Just what do you think you’re doing, dave?’ a checklist for responsible data use in nlp, 2021","venue":null,"work_id":"f1147e36-0016-43c8-9636-d90d0e3d50f8","year":2021},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":79,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.101462Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:ef422796218f26e664b1ce28fc773474ff87e940447400773acf91eb7b43811d","observation_id":"43776d69-a635-4a3f-ba4b-125f8b1983db","resolution":{"observed_at":"2026-08-07T14:26:36.431708Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.104955Z","title":"everyone wants to do the model work, not the data work","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":80,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.104955Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:001e09ca4eb093ec3164f0b62eec3c80664aebf9d3fca6575659898468cfe48d","observation_id":"1ecdc76c-e9db-4140-b20e-7139adcbf234","resolution":{"observed_at":"2026-08-07T14:26:35.104955Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.16822","last_updated":"2024-12-11T18:07:25Z","snapshot_observed_at":"2026-08-13T04:09:48.076759Z","submitted_at":"2024-02-26T18:47:27Z","title":"Rainbow Teaming: Open-Ended Generation of Diverse Adversarial Prompts","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.16822","snapshot_observed_at":"2026-08-07T14:26:35.108420Z","title":"Markosyan, Manish Bhatt, Yuning Mao, Minqi Jiang, Jack Parker-Holder, Jakob Foerster, Tim Rocktäschel, and Roberta Raileanu","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":81,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.108420Z"},"links":{"cited_paper":"/paper/2402.16822","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:e6b98f4cf9e86e6f0e981981ef3a872a674087fc03b604f25ed1e4def3c05590","observation_id":"c98d380a-778d-4c31-a646-cc36ab0d7ed6","resolution":{"observed_at":"2026-08-07T14:26:35.108420Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:36.422660Z","title":"Towards a standard for identifying and managing bias in artificial intelligence","venue":null,"work_id":"646013ed-3b2d-4161-8610-a679b03d822e","year":null},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":82,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.111174Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:050fcfbb7ca3b515892ff4d5c7312ade3aa972e45c5a4421bd862d96d7f33d2b","observation_id":"77e59401-666e-4cf9-a07c-51e4cd5cf6d9","resolution":{"observed_at":"2026-08-07T14:26:36.425294Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:36.416037Z","title":"The nist assessing risks and impacts of ai (aria) pilot evaluation plan","venue":null,"work_id":"bc34143d-ffda-428f-8bf0-56dffa30f798","year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":83,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.116595Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:123a97dc7d59edd36a80c963ade0a828b68c0443412f98f3407f7efd34e962c7","observation_id":"77bd4954-ec1b-43fa-822b-e3b8e23b9ea0","resolution":{"observed_at":"2026-08-07T14:26:36.418317Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:36.409222Z","title":"The assessing risks and impacts of ai (aria) program evaluation design document","venue":null,"work_id":"43ae8185-9d7b-490f-9c02-1e86350213af","year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":84,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.119294Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:423c577ab3f21247e18f393dba88f862dff5682799a924f23b29200b62835625","observation_id":"62a57b4b-68de-4eb0-890c-8997794ef9a3","resolution":{"observed_at":"2026-08-07T14:26:36.411404Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.121773Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":85,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.121773Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:91d9941cbf4d9feafec3e39898d5fb8f88f127b3fa10c1b9bf51f4f67f41c129","observation_id":"01c71d18-bb95-4afc-af30-579cb758e17d","resolution":{"observed_at":"2026-08-07T14:26:35.121773Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:36.403018Z","title":"Sociotech- nical harms of algorithmic systems: Scoping a taxonomy for harm reduction","venue":null,"work_id":"1848edf3-fa00-43e8-afc8-cd13518e0d08","year":2023},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":86,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.124830Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:ef4ea7cbc2fd23916c5c8f3821bf70551cae6c00b1f77794291a8bbe3c491261","observation_id":"600715f3-d7eb-4e86-8cf6-0fffa551734d","resolution":{"observed_at":"2026-08-07T14:26:36.405398Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:36.395627Z","title":"Taskbench: Benchmarking large language models for task automation","venue":null,"work_id":"e4fd6502-6152-4368-b9da-bc104d6e2814","year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":87,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.128077Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:6cb1fd61c1ea8f0b6b9bd3371b74a70eefca5437f2d24f061f6c5458f4752c3c","observation_id":"cac46a0c-4816-447f-98a6-528cff44561f","resolution":{"observed_at":"2026-08-07T14:26:36.398624Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2006.03463","last_updated":"2021-05-12T14:17:37Z","snapshot_observed_at":"2026-08-13T22:30:37.317307Z","submitted_at":"2020-06-05T14:10:09Z","title":"Sponge Examples: Energy-Latency Attacks on Neural Networks","version":2},"cited_work":{"arxiv_id":"2006.03463","doi":null,"metadata_source":"pith","pith_arxiv_id":"2006.03463","snapshot_observed_at":"2026-08-07T14:26:35.489693Z","title":"Sponge Examples: Energy-Latency Attacks on Neural Networks","venue":"cs.LG","work_id":"73091d1e-0852-4f08-a6da-39d285097609","year":2020},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":88,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.131240Z"},"links":{"cited_paper":"/paper/2006.03463","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:6a902a6bb62870f0bad96644047dff3fdf298c099d3466e8e8a766c016e2ec2e","observation_id":"750f819b-d2ca-4a8f-a004-eda5ac2c4dd0","resolution":{"observed_at":"2026-08-07T14:26:35.493129Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:36.389420Z","title":"The leaderboard illusion, 2025","venue":null,"work_id":"a7b526f7-3c40-471d-9842-3413d236e445","year":2025},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":89,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.133984Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:4b7c96df2b400f33d1bd0267a42802ccc8ac55497bd21cda477d2b0d1cc4c418","observation_id":"dfaf3510-8998-4495-8567-c8f1b0556c57","resolution":{"observed_at":"2026-08-07T14:26:36.391423Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.997991Z","title":"Participation is not a design fix for machine learning","venue":null,"work_id":"fe8670fb-e77d-455e-9c66-3b1f87207f89","year":2022},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":90,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.137449Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:6f8df370d3c4d9ab062ebcc75afb4beb6e260f2325d3f27eafa62bf0ba48f10f","observation_id":"b7fb97db-f9b2-4406-918e-49ef8d7cba90","resolution":{"observed_at":"2026-08-07T14:26:36.119406Z","resolver_source":"arxiv_id_nonexistent","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1007/s00146-021-01302-0","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-09T00:03:45.533247Z","title":"Slota, Kenneth R","venue":"AI & Society","work_id":"fb5edac7-a716-4d52-a646-1ba318ff7bb7","year":2023},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":91,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.140870Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:b511cdeba46031985cfbd77410f7c7a2853e3f0a991fded3af1314c83eb24a54","observation_id":"939774ff-aed7-4b9b-a9fe-e78202a7aabd","resolution":{"observed_at":"2026-08-07T14:26:35.253071Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.143758Z","title":"Smith, Saleema Amershi, Solon Barocas, Hanna Wallach, and Jennifer Wort- man Vaughan","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":92,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.143758Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:94285bca9ba68ddbf9807798a9d0e5878f9cba065cb685a1f3fca268a9381ea3","observation_id":"698e6155-3e17-4dba-bd06-041a6fad14b0","resolution":{"observed_at":"2026-08-07T14:26:35.143758Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2206.04615","last_updated":"2023-06-12T17:51:15Z","snapshot_observed_at":"2026-07-06T13:19:12.109592Z","submitted_at":"2022-06-09T17:05:34Z","title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2206.04615","snapshot_observed_at":"2026-08-07T14:26:35.146625Z","title":"Srivastava et al","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":93,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.146625Z"},"links":{"cited_paper":"/paper/2206.04615","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:31fa3f51a4f038db3a82e33305d2305ea331d7b8556e7dde7bc9ec00a85265ea","observation_id":"327a6d97-73b4-435a-a8ce-c9502763c6ea","resolution":{"observed_at":"2026-08-07T14:26:35.146625Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.07298","last_updated":"2024-05-06T15:52:03Z","snapshot_observed_at":"2026-08-13T05:52:22.536366Z","submitted_at":"2023-10-11T08:32:46Z","title":"Beyond Memorization: Violating Privacy Via Inference with Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.07298","snapshot_observed_at":"2026-08-07T14:26:35.149807Z","title":"Beyond memorization: Violating privacy via inference with large language models, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":94,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.149807Z"},"links":{"cited_paper":"/paper/2310.07298","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:4a370cc91615ee3be58f5ecc883e1bc224f26c6f4209e8138ecd2dde2b70a3cc","observation_id":"916cb51a-310c-4954-a515-4275a705dac3","resolution":{"observed_at":"2026-08-07T14:26:35.149807Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.152617Z","title":"Stadler, T","venue":null,"work_id":null,"year":2013},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":95,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.152617Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:ad8a1ea6733865dcf78c89a7707722c33e1820088d297b9a3deec723ded00fda","observation_id":"875059d9-86ee-4a16-9e5f-08c8a702a231","resolution":{"observed_at":"2026-08-07T14:26:35.152617Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.35650/aidp.4112.d.2025","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"The State of AI Governance Research: AI Safety and Reliability in Real World Commercial Deployment","venue":null,"work_id":"28b93e20-e670-4fa2-b4fd-e3738089f0fc","year":2025},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":96,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.155705Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:142690050c1548bfdf85fd567979a31fe686a770eceee1d022e9c5f7c83425de","observation_id":"acbb289e-5248-430f-b897-1ddec38e5994","resolution":{"observed_at":"2026-08-07T14:26:35.241286Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.13678","last_updated":"2024-12-18T10:05:43Z","snapshot_observed_at":"2026-08-13T14:24:47.389042Z","submitted_at":"2024-12-18T10:05:43Z","title":"Clio: Privacy-Preserving Insights into Real-World AI Use","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.13678","snapshot_observed_at":"2026-08-07T14:26:35.158699Z","title":"Sumers, Jared Mueller, William McEachen, Wes Mitchell, Shan Carter, Jack Clark, Jared Kaplan, and Deep Ganguli","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":97,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.158699Z"},"links":{"cited_paper":"/paper/2412.13678","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:cd7358e97a73738f03ca269009a4242eee5643eb566fe280221a0493a3eb8d6b","observation_id":"50853340-2cc3-44ab-99d2-a396c365cde0","resolution":{"observed_at":"2026-08-07T14:26:35.158699Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.6028/nist.ai.200-1","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T05:30:23.456663Z","title":"Theofanos, Y","venue":null,"work_id":"10935a1a-02b6-461b-a51c-afead997eabe","year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":98,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.161399Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:4ddf5c4f58842e06cf7f2583dfdaf9f0247d3df14de47d9706720ca812f49990","observation_id":"1b863056-85ca-4ad5-8a72-aa4dcb8dc7a6","resolution":{"observed_at":"2026-08-07T14:26:35.231664Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2002.08512","last_updated":"2020-02-20T00:56:11Z","snapshot_observed_at":"2026-08-13T23:28:26.539960Z","submitted_at":"2020-02-20T00:56:11Z","title":"The Problem with Metrics is a Fundamental Problem for AI","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2002.08512","snapshot_observed_at":"2026-08-07T14:26:35.164741Z","title":"Thomas and D","venue":null,"work_id":null,"year":2002},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":99,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.164741Z"},"links":{"cited_paper":"/paper/2002.08512","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:f0fe62f83bf01233292bf15d8186835f6523b89db1ad134073be95c4aa501494","observation_id":"2419bd08-0f63-4c67-9ce0-4d078fe84f63","resolution":{"observed_at":"2026-08-07T14:26:35.164741Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.11224","last_updated":"2024-05-08T15:50:31Z","snapshot_observed_at":"2026-08-13T00:28:26.173775Z","submitted_at":"2024-04-17T10:16:20Z","title":"Analytical results for uncertainty propagation through trained machine learning regression models","version":2},"cited_work":{"arxiv_id":"2404.11224","doi":null,"metadata_source":"pith","pith_arxiv_id":"2404.11224","snapshot_observed_at":"2026-08-07T14:26:35.391682Z","title":"Analytical results for uncertainty propagation through trained machine learning regression models","venue":"cs.LG","work_id":"f44a7df2-afc2-47f0-8925-f357fe427de2","year":2024},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":100,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.168688Z"},"links":{"cited_paper":"/paper/2404.11224","citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:60e58addc66dadcd9ff6c20cc6e22b4edf8eb5b91d99586961dcb0fc4066fcde","observation_id":"8eac081b-b223-465d-8eaf-9f6cbc90ee97","resolution":{"observed_at":"2026-08-07T14:26:35.394639Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T14:26:35.172094Z","title":null,"venue":null,"work_id":null,"year":1951},"citing_paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects","version":4},"reference_index":101,"source":"pdf_text","source_observed_at":"2026-08-07T14:26:35.172094Z"},"links":{"citing_paper":"/paper/2505.18893"},"observation_digest":"sha256:770180833cdaa87b156410b32aef13b1ac4ef3ef0d08da369ca849f543802c7e","observation_id":"3f77eb93-eaad-43e1-ba36-b1630bf5b225","resolution":{"observed_at":"2026-08-07T14:26:35.172094Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2505.18893","last_updated":"2025-05-30T14:09:51Z","latest_version":4,"primary_category":"cs.CY","snapshot_observed_at":"2026-08-14T16:53:14.677815Z","submitted_at":"2025-05-24T22:35:32Z","title":"Reality Check: A New Evaluation Ecosystem Is Necessary to Understand AI's Real World Effects"},"reference_resolution":{"displayed":100,"state_counts":{"malformed_identifier":0,"metadata_mismatch":2,"parse_uncertain":0,"unresolved":73,"verified_exact":16,"verified_fuzzy":9},"total_outbound_references":111},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"thesis":"As of 14 August 2026, this Paper Citation Record lists 100 of 111 outbound references and 5 inbound Pith citation observations for arXiv:2505.18893."}