{"as_of":"2026-08-09T23:30:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:d7b20413d2399e576eefefff085807e64fd36774dd341c87b8abf3a6279e480f","coverage":[{"denominator":0,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":29,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":29,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-09T06:31:02.800959+00:00","state":"measured"},{"denominator":29,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":29,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-09T04:45:34.602333Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-07-04T08:49:41.632097Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":"2103.14749","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-07-04T08:49:41.632097Z","title":"Confident learning: Estimating uncertainty in dataset labels.Journal of Artificial Intelligence Research, 70:1373–1411, 2021a","venue":null,"work_id":"328e364f-cefb-4793-8bdd-481ee68746ca","year":2023},"citing_paper":{"arxiv_id":"2405.00892","last_updated":"2026-04-30T18:09:59Z","snapshot_observed_at":"2026-07-06T18:08:30.875909Z","submitted_at":"2024-05-01T22:33:45Z","title":"Wake Vision: A Tailored Dataset and Benchmark Suite for TinyML Computer Vision Applications","version":6},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-05-24T01:06:48.298874Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2405.00892"},"observation_digest":"sha256:f587dbd5a3c4f1177aacc2ca554da20f5b4a1bbcb6de9d55ea21a93199911789","observation_id":"ae4bee83-4dc3-49df-9e32-52238d2e61be","resolution":{"observed_at":"2026-05-24T01:08:41.948583Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":"2103.14749","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-07-04T08:49:41.632097Z","title":"Confident learning: Estimating uncertainty in dataset labels.Journal of Artificial Intelligence Research, 70:1373–1411, 2021a","venue":null,"work_id":"328e364f-cefb-4793-8bdd-481ee68746ca","year":2023},"citing_paper":{"arxiv_id":"2411.04872","last_updated":"2025-12-23T02:23:47Z","snapshot_observed_at":"2026-08-04T15:54:46.196160Z","submitted_at":"2024-11-07T17:07:35Z","title":"FrontierMath: A Benchmark for Evaluating Advanced Mathematical Reasoning in AI","version":7},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-05-17T00:44:01.658214Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2411.04872"},"observation_digest":"sha256:eb8fdb33aa6fe1afaa3c2abc23d4314a4b8a3e5470e86649699c9dad5922d56a","observation_id":"cb113683-04c4-4100-9fa5-7dcd2199b93e","resolution":{"observed_at":"2026-05-17T00:44:01.690041Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-08-09T04:45:34.602333Z","title":"Pervasive label errors in test sets destabilize machine learning benchmarks","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2502.03461","last_updated":"2025-02-05T18:58:19Z","snapshot_observed_at":"2026-08-09T04:38:30.253150Z","submitted_at":"2025-02-05T18:58:19Z","title":"Do Large Language Model Benchmarks Test Reliability?","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-09T04:45:34.602333Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2502.03461"},"observation_digest":"sha256:96e9d50c1f361846ae791b621fcbc5b3e63c54b4f08c10086c5573a959190a2e","observation_id":"05fdc43a-0a68-41be-aaad-864ab2683443","resolution":{"observed_at":"2026-08-09T04:45:34.602333Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-08-08T14:44:41.854110Z","title":", author Athalye, A","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2502.06688","last_updated":"2025-05-22T09:22:23Z","snapshot_observed_at":"2026-08-08T14:37:43.230802Z","submitted_at":"2025-02-10T17:14:37Z","title":"Network Intrusion Datasets: A Survey, Limitations, and Recommendations","version":3},"reference_index":158,"source":"arxiv_source","source_observed_at":"2026-08-08T14:44:41.854110Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2502.06688"},"observation_digest":"sha256:6425435382394c453fa7c0d2d8d29bd8103b3f4583d682ddfe5236eba3295317","observation_id":"906ea888-465e-4af4-a526-fd00a2f4b67d","resolution":{"observed_at":"2026-08-08T14:44:41.854110Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-08-08T11:50:06.274969Z","title":"Pervasive label errors in test sets destabilize machine learning benchmarks,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2502.07721","last_updated":"2025-02-11T17:33:48Z","snapshot_observed_at":"2026-08-09T23:11:53.391895Z","submitted_at":"2025-02-11T17:33:48Z","title":"TMLC-Net: Transferable Meta Label Correction for Noisy Label Learning","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-08T11:50:06.274969Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2502.07721"},"observation_digest":"sha256:88c4722a2f7dcc0bb3491e16020340cf8b84338b08a20f3d10ba42d1c30fe211","observation_id":"8b001dc8-5ec3-459b-acf6-207481d05d37","resolution":{"observed_at":"2026-08-08T11:50:06.274969Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-08-07T20:55:05.895061Z","title":"G., Athalye, A., and Mueller, J","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2502.09696","last_updated":"2026-07-07T14:07:33Z","snapshot_observed_at":"2026-08-07T20:47:56.781130Z","submitted_at":"2025-02-13T18:59:11Z","title":"ZeroBench: An Impossible Visual Benchmark for Contemporary Large Multimodal Models","version":3},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-07T20:55:05.895061Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2502.09696"},"observation_digest":"sha256:602dc3591e386f9cd2e7f240fce9f07292c5f1746150eb63fc3ef91d0eefc1e7","observation_id":"62238fca-fcb3-4dbc-b782-a8949f7f8a78","resolution":{"observed_at":"2026-08-07T20:55:05.895061Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-08-07T15:29:34.923297Z","title":"Pervasive Label Errors in Test Sets Destabilize Benchmarks","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.14964","last_updated":"2025-05-20T22:57:35Z","snapshot_observed_at":"2026-08-07T15:24:13.333679Z","submitted_at":"2025-05-20T22:57:35Z","title":"The Achilles Heel of AI: Fundamentals of Risk-Aware Training Data for High-Consequence Models","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-07T15:29:34.923297Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2505.14964"},"observation_digest":"sha256:e79b3f503ae3cb104d3983d28945a6a36911345933cd2ce4560ff315464029c4","observation_id":"6564c559-b69d-4101-bf50-bc41bfde0b53","resolution":{"observed_at":"2026-08-07T15:29:34.923297Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-08-07T15:10:10.178439Z","title":"Pervasive label errors in test sets destabilize machine learning benchmarks.arXiv preprint arXiv:2103.14749, 2021","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.16149","last_updated":"2025-05-22T02:47:36Z","snapshot_observed_at":"2026-08-07T15:04:05.913617Z","submitted_at":"2025-05-22T02:47:36Z","title":"When VLMs Meet Image Classification: Test Sets Renovation via Missing Label Identification","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-07T15:10:10.178439Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2505.16149"},"observation_digest":"sha256:21d2ad7a21039adf38ae0e976161b01cbba4b80f8957a15500b9dd9d4c82d4ab","observation_id":"44cd219f-29c8-47d5-bc64-9c0ee8395bae","resolution":{"observed_at":"2026-08-07T15:10:10.178439Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-08-07T04:10:33.552063Z","title":"Pervasive label errors in test sets destabilize machine learning benchmarks","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2506.11615","last_updated":"2025-06-13T09:37:11Z","snapshot_observed_at":"2026-08-08T20:46:08.875943Z","submitted_at":"2025-06-13T09:37:11Z","title":"Machine Unlearning for Robust DNNs: Attribution-Guided Partitioning and Neuron Pruning in Noisy Environments","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-07T04:10:33.552063Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2506.11615"},"observation_digest":"sha256:1ca296b4d849af67b65afdc1b042de369a048d5a94b435808511d93658df53b7","observation_id":"c983470c-c349-46e1-9e9a-ff4b00faa454","resolution":{"observed_at":"2026-08-07T04:10:33.552063Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-08-07T00:00:21.325181Z","title":"G., Athalye, A., Mueller, J","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2506.15620","last_updated":"2025-06-18T16:51:26Z","snapshot_observed_at":"2026-08-09T15:31:20.285790Z","submitted_at":"2025-06-18T16:51:26Z","title":"GFLC: Graph-based Fairness-aware Label Correction for Fair Classification","version":1},"reference_index":9984,"source":"pdf_text","source_observed_at":"2026-08-07T00:00:21.325181Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2506.15620"},"observation_digest":"sha256:d6cad0df2e8a09fe9068904827ee21ec91e6f007a7dc1f8371ce32e44ac0c884","observation_id":"d581c8ac-9371-4066-90de-ec9b09c9c99f","resolution":{"observed_at":"2026-08-07T00:00:21.325181Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-08-06T17:40:19.810351Z","title":", author Athalye, A","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2507.10642","last_updated":"2025-07-14T16:37:20Z","snapshot_observed_at":"2026-08-09T22:35:20.064295Z","submitted_at":"2025-07-14T16:37:20Z","title":"First-of-its-kind AI model for bioacoustic detection using a lightweight associative memory Hopfield neural network","version":1},"reference_index":44,"source":"arxiv_source","source_observed_at":"2026-08-06T17:40:19.810351Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2507.10642"},"observation_digest":"sha256:b5a78cbd0dd28d128d0dcb83cb39662ee8b080407982c69eb5aec86bccf07e31","observation_id":"e61dbde6-c28c-4c7c-8d89-c116a8b5d06c","resolution":{"observed_at":"2026-08-06T17:40:19.810351Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-08-06T16:45:02.589586Z","title":"Confident learning: Estimating uncertainty in dataset labels","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2507.12750","last_updated":"2025-07-17T03:08:26Z","snapshot_observed_at":"2026-08-07T22:19:30.108386Z","submitted_at":"2025-07-17T03:08:26Z","title":"Multimodal-Guided Dynamic Dataset Pruning for Robust and Efficient Data-Centric Learning","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-06T16:45:02.589586Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2507.12750"},"observation_digest":"sha256:8e1da32e50e29365441ba9afb077fd3efdae2c630516d252c1dd7b94397c01ee","observation_id":"735d93c6-57d2-4076-b1c0-99856465a8ed","resolution":{"observed_at":"2026-08-06T16:45:02.589586Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-08-06T16:43:46.776213Z","title":"Pervasive label errors in test sets destabilize machine learning benchmarks,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2507.19511","last_updated":"2025-07-17T04:58:31Z","snapshot_observed_at":"2026-08-06T16:36:14.411067Z","submitted_at":"2025-07-17T04:58:31Z","title":"Advancing Mental Disorder Detection: A Comparative Evaluation of Transformer and LSTM Architectures on Social Media","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-06T16:43:46.776213Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2507.19511"},"observation_digest":"sha256:1021a1e1cde7537601346f3d11a5f308ff74454bd0876ee184e986b858a338af","observation_id":"4b3f88df-eb48-4c24-9b32-10475d3c1ecf","resolution":{"observed_at":"2026-08-06T16:43:46.776213Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-08-04T19:24:21.475663Z","title":"Northcutt, Anish Athalye, and Jonas Mueller","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2509.09311","last_updated":"2025-09-11T09:54:25Z","snapshot_observed_at":"2026-08-09T15:18:42.282208Z","submitted_at":"2025-09-11T09:54:25Z","title":"Image Recognition with Vision and Language Embeddings of VLMs","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-04T19:24:21.475663Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2509.09311"},"observation_digest":"sha256:a3be624c4c4fcb5bf45184e299bb32cbed7f92f8b2212f462bc881008580bb25","observation_id":"a1c85b76-e1e6-4e7f-b84f-3a570739fec6","resolution":{"observed_at":"2026-08-04T19:24:21.475663Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":"2103.14749","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-07-04T08:49:41.632097Z","title":"Confident learning: Estimating uncertainty in dataset labels.Journal of Artificial Intelligence Research, 70:1373–1411, 2021a","venue":null,"work_id":"328e364f-cefb-4793-8bdd-481ee68746ca","year":2023},"citing_paper":{"arxiv_id":"2511.02627","last_updated":"2026-06-11T15:36:40Z","snapshot_observed_at":"2026-08-08T22:40:29.987506Z","submitted_at":"2025-11-04T14:57:11Z","title":"DecompSR: A dataset for decomposed analyses of compositional multihop spatial reasoning","version":2},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-05-18T01:18:44.523602Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2511.02627"},"observation_digest":"sha256:631615279fbbc13f3fbd3363c3893e6b9c6b19ae5cadabdb0ff2a6688fea430a","observation_id":"cdd606df-3baf-4f78-9f09-0af27a748c93","resolution":{"observed_at":"2026-05-18T01:20:34.418769Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-08-04T00:11:51.702024Z","title":"Pervasive label errors in test sets destabilize machine learning benchmarks","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2511.02627","last_updated":"2026-06-11T15:36:40Z","snapshot_observed_at":"2026-08-08T22:40:29.987506Z","submitted_at":"2025-11-04T14:57:11Z","title":"DecompSR: A dataset for decomposed analyses of compositional multihop spatial reasoning","version":4},"reference_index":1999,"source":"pdf_text","source_observed_at":"2026-08-04T00:11:51.702024Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2511.02627"},"observation_digest":"sha256:ea2a058c8fd139fb98af252d0944852cf9d20129eea78c343a1d573aa0373297","observation_id":"538af51a-99cf-4371-af49-aea48be821c7","resolution":{"observed_at":"2026-08-04T00:11:51.702024Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-08-03T20:19:44.099592Z","title":"Pervasive label errors in test sets destabilize machine learning benchmarks,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2511.20586","last_updated":"2026-05-26T15:35:41Z","snapshot_observed_at":"2026-08-09T07:43:32.538358Z","submitted_at":"2025-11-25T18:15:36Z","title":"PaTAS: A Framework for Trust Propagation in Neural Networks Using Subjective Logic","version":4},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-03T20:19:44.099592Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2511.20586"},"observation_digest":"sha256:05ce9102c01bca08218fb31621a321ab82c4d6acef9ad5d4a1de2ff6654d7592","observation_id":"94067a78-7cdd-491b-b535-394938cf65af","resolution":{"observed_at":"2026-08-03T20:19:44.099592Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-08-03T06:59:18.572514Z","title":"G., Athalye, A., and Mueller, J","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2601.21564","last_updated":"2026-05-28T15:55:17Z","snapshot_observed_at":"2026-08-09T20:16:55.308363Z","submitted_at":"2026-01-29T11:28:02Z","title":"Representation Unlearning: Forgetting through Information Compression","version":2},"reference_index":2021,"source":"pdf_text","source_observed_at":"2026-08-03T06:59:18.572514Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2601.21564"},"observation_digest":"sha256:47cdb604c3a58c294f9bbb8fb1d4f2674c08e3d271ff43624c2ab8920909d804","observation_id":"7db6f62e-1de8-4d6d-b77a-8e2f1417505b","resolution":{"observed_at":"2026-08-03T06:59:18.572514Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-07-13T09:29:29.046109Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2604.05325","last_updated":"2026-07-02T05:47:57Z","snapshot_observed_at":"2026-07-13T09:29:28.756719Z","submitted_at":"2026-04-07T01:55:15Z","title":"Noise is not always detrimental: the capacity of quantum batteries is enhanced in black holes","version":2},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-07-13T09:29:29.046109Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2604.05325"},"observation_digest":"sha256:0fd584f89108664320a6fc2acb5d6ac721e86a83e17166d305d53154499277d0","observation_id":"120e7793-8c65-47a7-8bf5-281f9c6479f3","resolution":{"observed_at":"2026-07-13T09:29:29.046109Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":"2103.14749","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-07-04T08:49:41.632097Z","title":"Confident learning: Estimating uncertainty in dataset labels.Journal of Artificial Intelligence Research, 70:1373–1411, 2021a","venue":null,"work_id":"328e364f-cefb-4793-8bdd-481ee68746ca","year":2023},"citing_paper":{"arxiv_id":"2604.05329","last_updated":"2026-04-07T02:00:23Z","snapshot_observed_at":"2026-07-06T22:54:04.491386Z","submitted_at":"2026-04-07T02:00:23Z","title":"Semantic Trimming and Auxiliary Multi-step Prediction for Generative Recommendation","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-05-10T19:48:25.382343Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2604.05329"},"observation_digest":"sha256:e5d849f61c4ab02828cd131af2e23f6cda0a38a30666e2580446cd98314bdbd0","observation_id":"333fe779-501d-49ee-81fd-ba6230820886","resolution":{"observed_at":"2026-05-10T22:30:50.716942Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":"2103.14749","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-07-04T08:49:41.632097Z","title":"Confident learning: Estimating uncertainty in dataset labels.Journal of Artificial Intelligence Research, 70:1373–1411, 2021a","venue":null,"work_id":"328e364f-cefb-4793-8bdd-481ee68746ca","year":2023},"citing_paper":{"arxiv_id":"2604.17022","last_updated":"2026-04-30T07:31:04Z","snapshot_observed_at":"2026-07-06T23:04:14.688737Z","submitted_at":"2026-04-18T15:09:20Z","title":"Beyond Black-Box Labels: Interpretable Criteria for Diagnosing Subjective NLP Tasks","version":2},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-05-10T06:37:19.313543Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2604.17022"},"observation_digest":"sha256:af9d7d64053ab7fe288dfc98aeaed99b7a821f38770ab367f113953d2eed52c7","observation_id":"04d406b2-0b06-452c-a1cb-0acbbca4b45b","resolution":{"observed_at":"2026-05-10T06:41:36.875706Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":"2103.14749","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-07-04T08:49:41.632097Z","title":"Confident learning: Estimating uncertainty in dataset labels.Journal of Artificial Intelligence Research, 70:1373–1411, 2021a","venue":null,"work_id":"328e364f-cefb-4793-8bdd-481ee68746ca","year":2023},"citing_paper":{"arxiv_id":"2604.25923","last_updated":"2026-04-01T10:29:43Z","snapshot_observed_at":"2026-08-07T08:56:37.143577Z","submitted_at":"2026-04-01T10:29:43Z","title":"Evaluation Revisited: A Taxonomy of Evaluation Concerns in Natural Language Processing","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-05-13T22:52:17.310177Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2604.25923"},"observation_digest":"sha256:238315f97c01a6acef242872b89445cd47f59700a4116292a39f2f9e49ea67f8","observation_id":"a803ab18-8460-41a3-a06e-4bb99044d7c9","resolution":{"observed_at":"2026-05-13T22:53:23.358641Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":"2103.14749","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-07-04T08:49:41.632097Z","title":"Confident learning: Estimating uncertainty in dataset labels.Journal of Artificial Intelligence Research, 70:1373–1411, 2021a","venue":null,"work_id":"328e364f-cefb-4793-8bdd-481ee68746ca","year":2023},"citing_paper":{"arxiv_id":"2605.26973","last_updated":"2026-05-26T12:58:48Z","snapshot_observed_at":"2026-08-07T02:39:08.930568Z","submitted_at":"2026-05-26T12:58:48Z","title":"Signal-to-Noise Ratio and Sample Size Govern Representational Alignment in Neural Networks","version":1},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-06-29T15:46:31.066600Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2605.26973"},"observation_digest":"sha256:9b2e504ceb5759fde871ad762159c8ead249892f3613c052ed19edc36430973a","observation_id":"d085801d-28cc-4707-abe8-e8ab6d8f6c1c","resolution":{"observed_at":"2026-06-29T16:33:39.671679Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":"2103.14749","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-07-04T08:49:41.632097Z","title":"Confident learning: Estimating uncertainty in dataset labels.Journal of Artificial Intelligence Research, 70:1373–1411, 2021a","venue":null,"work_id":"328e364f-cefb-4793-8bdd-481ee68746ca","year":2023},"citing_paper":{"arxiv_id":"2605.29720","last_updated":"2026-05-28T10:15:46Z","snapshot_observed_at":"2026-07-06T23:39:05.858969Z","submitted_at":"2026-05-28T10:15:46Z","title":"Efficient, Validation-Free Intrinsic Quality Estimation for Large-Scale Face Recognition Datasets","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-06-29T08:34:18.117267Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2605.29720"},"observation_digest":"sha256:0b6b48282f9fb2f495362d18320028dd1eed5de7fecc2ea22bb114ca80f0319d","observation_id":"ce9465aa-659b-456a-a37c-3c98149b61e3","resolution":{"observed_at":"2026-06-29T08:53:16.473620Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":"2103.14749","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-07-04T08:49:41.632097Z","title":"Confident learning: Estimating uncertainty in dataset labels.Journal of Artificial Intelligence Research, 70:1373–1411, 2021a","venue":null,"work_id":"328e364f-cefb-4793-8bdd-481ee68746ca","year":2023},"citing_paper":{"arxiv_id":"2606.19186","last_updated":"2026-06-18T01:51:31Z","snapshot_observed_at":"2026-08-04T21:05:04.941577Z","submitted_at":"2026-06-17T15:27:15Z","title":"Learning to Annotate Delayed and False AEB Events: A Practical System for Extreme Class Imbalance and Asymmetric Label Noise","version":2},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-06-26T20:36:13.042553Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2606.19186"},"observation_digest":"sha256:3f0bf09fce2c5c0c4b82c258715edb13dd8aac0a578e69382b67ff6ff7bc3be5","observation_id":"b4ed5274-eeda-4b73-b64c-57ed3b2d434b","resolution":{"observed_at":"2026-07-04T01:09:19.667230Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":"2103.14749","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-07-04T08:49:41.632097Z","title":"Confident learning: Estimating uncertainty in dataset labels.Journal of Artificial Intelligence Research, 70:1373–1411, 2021a","venue":null,"work_id":"328e364f-cefb-4793-8bdd-481ee68746ca","year":2023},"citing_paper":{"arxiv_id":"2606.22437","last_updated":"2026-06-25T12:10:03Z","snapshot_observed_at":"2026-08-06T23:20:48.945969Z","submitted_at":"2026-06-21T10:57:43Z","title":"MMGist: A Comprehensive Multimodal Benchmark for 2027","version":2},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-06-26T11:05:14.573386Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2606.22437"},"observation_digest":"sha256:b07d0e566e0cf187f5e14527ae17cd0a5354f0b2ae280c9400016a04dbda9048","observation_id":"3796f0b2-e934-442e-a549-4c610d763e0f","resolution":{"observed_at":"2026-07-04T08:49:41.633870Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-07-14T06:14:56.071808Z","title":"Pervasive label errors in test sets destabilize machine learning benchmarks.arXiv preprint arXiv:2103.14749, 2021","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.11214","last_updated":"2026-07-13T08:07:45Z","snapshot_observed_at":"2026-08-08T02:10:41.465863Z","submitted_at":"2026-07-13T08:07:45Z","title":"A Novel Method to Evaluate Models on Unreliable, Noisy and Inconsistent Labels: Adaptive Resolution Label Aggregation (ARLA)","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-07-14T06:14:56.071808Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2607.11214"},"observation_digest":"sha256:0d3efaffb91252565b66ee4f1c9c96385e1d6f7a574997e65dfd882eb082afd7","observation_id":"c8e8a57a-9880-4be8-9c03-8b7aa0cb71ca","resolution":{"observed_at":"2026-07-14T06:14:56.071808Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-08-02T10:02:37.001079Z","title":"and Athalye, Anish and Mueller, Jonas , month = nov, year =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.16239","last_updated":"2026-06-26T00:31:00Z","snapshot_observed_at":"2026-08-09T20:03:40.575632Z","submitted_at":"2026-06-26T00:31:00Z","title":"BACON: Budgeted Human Calibration for Modeling and Evaluation with Multiple AI Judges","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-02T10:02:37.001079Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2607.16239"},"observation_digest":"sha256:70afde45e4fd6d4ba39c22b3145264f917b60dd5e03afcff0357d67a547b3f50","observation_id":"5d3000d0-13a7-4bfb-8455-5a40d5cdd5d4","resolution":{"observed_at":"2026-08-02T10:02:37.001079Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.14749","snapshot_observed_at":"2026-08-01T06:18:10.049087Z","title":"NeurIPS Datasets and Benchmarks Track , year =","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.21962","last_updated":"2026-07-24T04:19:45Z","snapshot_observed_at":"2026-08-07T16:22:09.041290Z","submitted_at":"2026-07-24T04:19:45Z","title":"Ground Truth First: A Longitudinal Evaluation Instrument for Agent Memory, and the Tenure Crossover in Memory-Architecture Rankings","version":1},"reference_index":107,"source":"arxiv_source","source_observed_at":"2026-08-01T06:18:10.049087Z"},"links":{"cited_paper":"/paper/2103.14749","citing_paper":"/paper/2607.21962"},"observation_digest":"sha256:8461d12f823704cc165438e0a3cbe42187f442956e969a1388e7fe2c6d0e3341","observation_id":"f2a424e3-bfad-492c-af98-d2b9780de8c4","resolution":{"observed_at":"2026-08-01T06:18:10.049087Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2103.14749/citation-record","integrity":"/paper/2103.14749/integrity","json":"/paper/2103.14749/citation-record.json","paper":"/paper/2103.14749"},"outbound":[],"paper":{"arxiv_id":"2103.14749","last_updated":"2021-11-07T13:04:04Z","latest_version":4,"primary_category":"stat.ML","snapshot_observed_at":"2026-08-04T22:08:50.379039Z","submitted_at":"2021-03-26T21:54:36Z","title":"Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks"},"reference_resolution":{"displayed":0,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":0,"verified_exact":0,"verified_fuzzy":0},"total_outbound_references":0},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"thesis":"As of 9 August 2026, this Paper Citation Record lists 0 of 0 outbound references and 29 inbound Pith citation observations for arXiv:2103.14749."}