{"as_of":"2026-08-13T12:57:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:c831dece009eae2eaa07b15d608bfdf3355998932909bd8977a7111113f1c826","coverage":[{"denominator":147,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":100,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-12T14:14:09.214957Z","state":"measured"},{"denominator":100,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":100,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-13T06:32:02.005865+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2411.15634/citation-record","integrity":"/paper/2411.15634/integrity","json":"/paper/2411.15634/citation-record.json","paper":"/paper/2411.15634"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.932253Z","title":"online\" 'onlinestring :=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.932253Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:17048ede8bbcd38737b332a55d2d44a179d5e435259d1946f28f69b4ca53d7ea","observation_id":"d3bbedf6-79da-44f1-a073-c8fb80e948f5","resolution":{"observed_at":"2026-08-12T14:14:08.932253Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.935276Z","title":"write newline","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.935276Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:8a3df2ed0b473dce58dd0ae98bacf20504bc543accddaa61eb22306619ce66e1","observation_id":"0b60fde8-63b8-4a72-ad12-297dd0acf522","resolution":{"observed_at":"2026-08-12T14:14:08.935276Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.48550/arxiv.2301.10684","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:18:38.108384Z","title":null,"venue":null,"work_id":"9e862592-e6b8-409f-afd7-57e957a14267","year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.937825Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:ff91096a753c2a09cc443ff0f1ecccbba79404d25b301bfbc0c86b0aa7513dc7","observation_id":"2d27847f-0aeb-4867-9b4c-2a60be69eb8f","resolution":{"observed_at":"2026-08-12T14:18:38.189288Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.940582Z","title":"Adams, Mark Wilson, and Wen-chung Wang","venue":null,"work_id":null,"year":1997},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.940582Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:f17c4c99389a3f104927462b8dbdb4e69346a295ca808f5cf6fa0e1ed1ad6f54","observation_id":"e96e75a5-d49d-46dd-b075-e2996a270651","resolution":{"observed_at":"2026-08-12T14:14:08.940582Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1810.03292","last_updated":"2020-11-06T13:40:14Z","snapshot_observed_at":"2026-07-06T07:06:35.439966Z","submitted_at":"2018-10-08T07:27:11Z","title":"Sanity Checks for Saliency Maps","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1810.03292","snapshot_observed_at":"2026-08-12T14:14:08.943711Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.943711Z"},"links":{"cited_paper":"/paper/1810.03292","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:8bf87ad2d7b5d0626865d28f425fa67c68c7badd1a6efca675d4d0f342f3619e","observation_id":"53846e1d-5893-4a9a-a9b3-f3639e2e5575","resolution":{"observed_at":"2026-08-12T14:14:08.943711Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.947350Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.947350Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:99c5e384f1fdb3d9ef862922d70adfcb118e1d7bd5bf4fa45e4c8595c16334a7","observation_id":"f8d5b14c-2b90-4df9-a3b1-0c3ded31e6c5","resolution":{"observed_at":"2026-08-12T14:14:08.947350Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.950892Z","title":null,"venue":null,"work_id":null,"year":2013},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.950892Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:0565db0688ad91fb4d1443638ab2cd0e0a3959c09a7db71685c250f59aeb7d80","observation_id":"1a2d0a2b-db78-4691-86c8-ab033f8a3697","resolution":{"observed_at":"2026-08-12T14:14:08.950892Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.954122Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.954122Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:f77bb4539a28bcfa146bbf8708ea8baf131aa5d634057f3c4be01cc917144561","observation_id":"329d69f7-9eb5-40b8-8364-ae8efd56136c","resolution":{"observed_at":"2026-08-12T14:14:08.954122Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.957274Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.957274Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:fbe93b4a41802897c25a59692b4c32f165dbd08f3a2305c996f09f0dc6d4f640","observation_id":"e01229e8-3aba-469b-b27b-12c6946860ea","resolution":{"observed_at":"2026-08-12T14:14:08.957274Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2210.16133","last_updated":"2022-11-30T09:15:10Z","snapshot_observed_at":"2026-08-12T23:53:19.450185Z","submitted_at":"2022-10-28T14:01:32Z","title":"Stop Measuring Calibration When Humans Disagree","version":2},"cited_work":{"arxiv_id":"2210.16133","doi":"10.48550/arxiv.2210.16133","metadata_source":"pith","pith_arxiv_id":"2210.16133","snapshot_observed_at":"2026-08-12T18:16:22.223912Z","title":"Stop Measuring Calibration When Humans Disagree","venue":"cs.CL","work_id":"8e16fe24-5664-484b-b081-ffd9aa2d7002","year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.959989Z"},"links":{"cited_paper":"/paper/2210.16133","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:ddc69e7b1185b7da23362ab6fbe5fe8f26822002eadddf574b9784a627883ff8","observation_id":"bb432597-0eef-421d-ad93-ebfc8bf5e1f9","resolution":{"observed_at":"2026-08-12T14:18:37.880314Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.16102","last_updated":"2024-02-25T15:00:13Z","snapshot_observed_at":"2026-08-13T05:53:56.705175Z","submitted_at":"2024-02-25T15:00:13Z","title":"Interpreting Predictive Probabilities: Model Confidence or Human Label Variation?","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.16102","snapshot_observed_at":"2026-08-12T14:14:08.963443Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.963443Z"},"links":{"cited_paper":"/paper/2402.16102","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:a1b31a30a218345d355f5ca22846cd069a0f27dbc87b131e6db1754ae67bad90","observation_id":"764eeefd-4ed4-4f10-b03e-bbedfd8cdc45","resolution":{"observed_at":"2026-08-12T14:14:08.963443Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.3386/w23478","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.878857Z","title":"Chin, Thomas J","venue":null,"work_id":"de926193-1b54-441c-818f-8f9be5599938","year":2017},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.966386Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:cc807cf50978290e0176415ac88a2debd591b290c98fd4893c7c497a570f07cf","observation_id":"06c0eb78-89be-4192-8748-e3b5d803c54a","resolution":{"observed_at":"2026-08-12T14:14:09.881615Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.969771Z","title":"Chin, Thomas J","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.969771Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:e4e501b257055ee77fcbb0fafd327904f5a0a06337cc7ee30d42e2c0855efd18","observation_id":"544bf104-fad7-4152-b825-8c2ef0b84b9f","resolution":{"observed_at":"2026-08-12T14:14:08.969771Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.973174Z","title":null,"venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.973174Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:8451478c5ddac231fe27f30cd34e3909f76c36c002ef3a665fea4ea2c5b80d77","observation_id":"0ea02359-f3e4-4344-b869-4a5d0e311b43","resolution":{"observed_at":"2026-08-12T14:14:08.973174Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.975460Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.975460Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:cf049cc42e979b8d53b67e00767b885e25ec7ae4e417f5be7dbee271785769b0","observation_id":"9fe81829-d64e-483d-8c6e-54a806e7bd72","resolution":{"observed_at":"2026-08-12T14:14:08.975460Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.977909Z","title":null,"venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.977909Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:ec9c5f92819bd21ab60c50c8f05bbeec7699e9eaf26155cbd679848e8ab3ceee","observation_id":"8aba37f7-ac55-4b65-8be1-a8e93c6fad9f","resolution":{"observed_at":"2026-08-12T14:14:08.977909Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.981168Z","title":"Williamson, and and Robert J","venue":null,"work_id":null,"year":2006},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.981168Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:3ac1350b70aa523cd7978ae7e8c7c030ed28ae55482a57df42af1996aa8cfb53","observation_id":"95e7f13e-cccc-4ce5-930f-b873d4a24225","resolution":{"observed_at":"2026-08-12T14:14:08.981168Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.983702Z","title":"Howcroft","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.983702Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:661f31e234cd8f95e5766ae41e93c2a9be1c1c2acc2e98d10518e39e7343992d","observation_id":"6090a068-df32-4fbd-a367-ae4f25d035a5","resolution":{"observed_at":"2026-08-12T14:14:08.983702Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.986305Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.986305Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:6920c8b2c6d09e3d9e720fb35f8da536661007831bd1ce7aa455ac6b5fc6a8c2","observation_id":"f31e83c6-1554-4910-93fe-bdd1a9fbc11d","resolution":{"observed_at":"2026-08-12T14:14:08.986305Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.20268","last_updated":"2025-04-28T11:50:42Z","snapshot_observed_at":"2026-08-12T22:14:34.277092Z","submitted_at":"2024-10-26T20:39:41Z","title":"Centaur: a foundation model of human cognition","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.20268","snapshot_observed_at":"2026-08-12T14:14:08.992453Z","title":"Eckstein, Noémi Éltető, Thomas L","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.992453Z"},"links":{"cited_paper":"/paper/2410.20268","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:5c924c6c8d0fa069a78f5c295e0451e24e1769375cc765e77f7651c52e5f84e3","observation_id":"2f2a0c7f-99c8-49f3-834f-f067e2594183","resolution":{"observed_at":"2026-08-12T14:14:08.992453Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2106.15590","last_updated":"2022-06-21T17:48:26Z","snapshot_observed_at":"2026-08-12T12:44:22.330348Z","submitted_at":"2021-06-29T17:24:14Z","title":"The Values Encoded in Machine Learning Research","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2106.15590","snapshot_observed_at":"2026-08-12T14:14:08.995399Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.995399Z"},"links":{"cited_paper":"/paper/2106.15590","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:59eeda5ddd3ca79a3905baa73a8cd576947644aa727552751144a5ed17766e6a","observation_id":"0e5030af-af46-4865-81d7-2c24413bdc18","resolution":{"observed_at":"2026-08-12T14:14:08.995399Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1162/edfp_a_00251","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.848827Z","title":null,"venue":null,"work_id":"de2a692d-4b97-4c3d-9f16-f30488b54112","year":2018},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.997769Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:982ba72cba7db79dff3a3d94e98eeb1fe8962fab332c5cde1fff6dcab5a11a2f","observation_id":"11072218-a1bd-4d8d-8c97-9e54122a8a3e","resolution":{"observed_at":"2026-08-12T14:14:09.851093Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:08.999694Z","title":"Charalambous, and Heather C","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:08.999694Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:1177042410a6ee41b07c6e94821e475592e34476d02af4d74ec302c8053bccf2","observation_id":"ac9939bb-bc79-4b56-89f7-b9e7e78073b1","resolution":{"observed_at":"2026-08-12T14:14:08.999694Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.001996Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.001996Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:35914f938b59704478c51e78a348760229aed4bb6cc5e96b5d031f30f8956c8d","observation_id":"8ccb3262-08df-45ad-994f-05809d624e0c","resolution":{"observed_at":"2026-08-12T14:14:09.001996Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.003948Z","title":null,"venue":null,"work_id":null,"year":2001},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.003948Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:9f80a406e85dee30993886841aac2412d41dc4cdec26fe3c57dbc651e2140aa0","observation_id":"0b6d44ed-e166-4446-8cc5-3c31dd5291a1","resolution":{"observed_at":"2026-08-12T14:14:09.003948Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1007/978-1-4757-3456-0_6","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-13T11:03:48.094449Z","title":null,"venue":null,"work_id":"e8b1fc36-81d1-4362-a2e3-b95953013841","year":2001},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.006003Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:c9f9aa08f5d6d9c311fe0b7e0ff802bc5c410b081b79d43f6c0cefb11ae75bb3","observation_id":"eeafe266-df5f-4a82-9800-ff37a02c99eb","resolution":{"observed_at":"2026-08-12T14:14:09.840105Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.008003Z","title":null,"venue":null,"work_id":null,"year":2013},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.008003Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:ec280fe9dc12dbd415158ae83967fa91087e0e215bda511eaf745282296de508","observation_id":"daecf482-9a0e-4fa4-9cce-f89fb007089c","resolution":{"observed_at":"2026-08-12T14:14:09.008003Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.010344Z","title":"Briggs and Mark Wilson","venue":null,"work_id":null,"year":2007},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.010344Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:b88c4c604333bea0a7865787b12b356a1a1ce3c2d5c6ab2ec7f1d42a3691f0b8","observation_id":"35fa43e3-5d0d-490f-8cf5-e92631b2b074","resolution":{"observed_at":"2026-08-12T14:14:09.010344Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1111/emip.12478","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.828774Z","title":"Casabianca","venue":null,"work_id":"6c317141-df30-4386-9968-c497cc59c3ba","year":2021},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":30,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.013050Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:c153c35a8c5fe4fcdbe05b2bb1f81ef970d4f612ace40f8585dd166ab1f2e938","observation_id":"af6b96f7-1b39-4ff2-8a68-0ac167d1df9f","resolution":{"observed_at":"2026-08-12T14:14:09.831925Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1177/0013164413486987","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.820594Z","title":"Casabianca, Daniel F","venue":null,"work_id":"5d337260-3686-41d8-b297-c6f803706488","year":2013},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.015935Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:a30414f8c8ff6ca601a5e260dda0edfdb6562c6639be8035ed06df4cc5395412","observation_id":"e611b204-bae3-4555-8638-3ed51b32f23b","resolution":{"observed_at":"2026-08-12T14:14:09.823640Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1163/9789004418875_014","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.811415Z","title":"Charalambous and Seán Delaney","venue":null,"work_id":"31388d5b-e8f9-4d78-9318-aa3e2adfe250","year":2019},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.019427Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:14e998b8a1faf7bb0c233f2aeb04e7bc1fbfbf203a022ef3cbb63ae378b55933","observation_id":"0b500ddc-90e0-437c-87a7-fbd377b50cbe","resolution":{"observed_at":"2026-08-12T14:14:09.814548Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1037/1082-989x.10.2.206","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.802711Z","title":null,"venue":null,"work_id":"7d09242a-db68-4d2c-9c2f-24a37b3664bb","year":2005},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.022707Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:9a1d2080350fd814dcdda11df5e09da71dcc4fc8edc8aa5a3fbc9b9e2508efa4","observation_id":"0c269d01-8519-4737-92ff-bd507f0a83ae","resolution":{"observed_at":"2026-08-12T14:14:09.806031Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1808.00023","last_updated":"2023-08-14T19:14:00Z","snapshot_observed_at":"2026-07-06T06:53:21.002757Z","submitted_at":"2018-07-31T18:38:04Z","title":"The Measure and Mismeasure of Fairness","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1808.00023","snapshot_observed_at":"2026-08-12T14:14:09.025189Z","title":"Gaebler, Hamed Nilforoshan, Ravi Shroff, and Sharad Goel","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.025189Z"},"links":{"cited_paper":"/paper/1808.00023","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:c5a81c731e17f0aecf58a52f7cd7ddfb7cc365dac6c4f9b0b50cdf3b694b0e89","observation_id":"44f0441b-d2b5-42aa-94a5-ca6bf6459fbe","resolution":{"observed_at":"2026-08-12T14:14:09.025189Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1007/s11336-024-09955-8","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.788085Z","title":null,"venue":null,"work_id":"7dde3b8f-200f-4b76-a6d8-badf7cd57103","year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.028229Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:2438e191336c8bd1de633f5d3916fa9965830000e9a72cc2170ceee717a584c2","observation_id":"50154bf5-db45-4aeb-b8e4-0a2f745a53e2","resolution":{"observed_at":"2026-08-12T14:14:09.790912Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2011.03395","last_updated":"2020-11-24T19:16:02Z","snapshot_observed_at":"2026-08-07T12:11:27.017871Z","submitted_at":"2020-11-06T14:53:13Z","title":"Underspecification Presents Challenges for Credibility in Modern Machine Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2011.03395","snapshot_observed_at":"2026-08-12T14:14:09.030940Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":36,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.030940Z"},"links":{"cited_paper":"/paper/2011.03395","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:2997c272e4adb4f50d0c3b0699bd031f76f6e89b2deb905a3470979baa1def6d","observation_id":"3ef5eb04-d820-4ac1-a0c4-69a7d3b40857","resolution":{"observed_at":"2026-08-12T14:14:09.030940Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.033901Z","title":null,"venue":null,"work_id":null,"year":2014},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":37,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.033901Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:9e18753e59540bf01b55c8ca11feec05b3e53c2ac8ebd1b7621aaec76aaee821","observation_id":"d3474244-fa02-47cf-aa99-4f3421691efa","resolution":{"observed_at":"2026-08-12T14:14:09.033901Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.036476Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":38,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.036476Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:37b8abc09c154faee056278bce171e1fe6ae13f338dbe876406be3784186d1f0","observation_id":"675e123f-daaa-46c0-97d6-75ee9988bfd6","resolution":{"observed_at":"2026-08-12T14:14:09.036476Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.3758/bf03195496","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.773933Z","title":null,"venue":null,"work_id":"eb7d17a3-3773-4d28-ac9d-6fb2d2e0b7bd","year":2003},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":39,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.039424Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:7b0e97887693de23db3d47044e09b1968651b0c70d2c0f30e1f4a44e8e29ae96","observation_id":"a76ef933-6240-48b3-aac6-b2e82fb86807","resolution":{"observed_at":"2026-08-12T14:14:09.776086Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.042096Z","title":null,"venue":null,"work_id":null,"year":2008},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":40,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.042096Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:a735458c84154c4be0529f23d1acf0261c36e6b4f9c9a75cfa2c14a358bb1015","observation_id":"0dbac9a9-8bd6-4c40-a56a-94c40a7ddccf","resolution":{"observed_at":"2026-08-12T14:14:09.042096Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1111/jedm.12358","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.767304Z","title":null,"venue":null,"work_id":"a57b84ee-488f-4431-afd4-65cf3f04972b","year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":41,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.044529Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:c57e820308a1d97dbfe0875ba40dcb9e3f6c2ab59d151bd03a499f45cb352db5","observation_id":"85e89cf4-d389-4670-b3f7-3787d03aa71d","resolution":{"observed_at":"2026-08-12T14:14:09.769712Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.047815Z","title":"DeCarlo, YoungKoung Kim, and Matthew S","venue":null,"work_id":null,"year":2011},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":42,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.047815Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:6b907c3d24fcc83fa6a3ff3ad89631b8895409ccb2e0e72265dcae7cee2803ad","observation_id":"18aa29b8-4c7d-4e2c-b8c1-9e6bdff76ac7","resolution":{"observed_at":"2026-08-12T14:14:09.047815Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.11772","last_updated":"2023-05-24T18:41:18Z","snapshot_observed_at":"2026-07-06T14:21:22.601555Z","submitted_at":"2022-11-21T19:00:01Z","title":"The NCTE Transcripts: A Dataset of Elementary Math Classroom Transcripts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.11772","snapshot_observed_at":"2026-08-12T14:14:09.050988Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":43,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.050988Z"},"links":{"cited_paper":"/paper/2211.11772","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:5d2aa2566a8951fa7c3cc4f793fee0ee573bd0cdc88caed8f038bd8a09a31e08","observation_id":"1a8a9b2c-826e-4a08-ad5b-82ac0974e7a1","resolution":{"observed_at":"2026-08-12T14:14:09.050988Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.053817Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":44,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.053817Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:aa793e6b62bc5c05514b7c2a22492420d277fcb1fadc0b127248d846404ed3c5","observation_id":"043c85a1-f15f-4c7e-b04a-0df0dc8c4bd6","resolution":{"observed_at":"2026-08-12T14:14:09.053817Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.060219Z","title":"Hill, Shyamoli Sanghi, and Ariel Chung","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":46,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.060219Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:7bc585dd07bf413888a0ab73ea374d272fdb576fa240446d605fa45c3312ac00","observation_id":"cdadb409-5722-431f-8152-f1b6b2736df6","resolution":{"observed_at":"2026-08-12T14:14:09.060219Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2106.03873","last_updated":"2021-06-07T18:00:06Z","snapshot_observed_at":"2026-08-12T18:35:17.230557Z","submitted_at":"2021-06-07T18:00:06Z","title":"Measuring Conversational Uptake: A Case Study on Student-Teacher Interactions","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2106.03873","snapshot_observed_at":"2026-08-12T14:14:09.062876Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":47,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.062876Z"},"links":{"cited_paper":"/paper/2106.03873","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:5d35471f6997722dda46a8bedba35c08b8ada90ba6c12525d567c16589bafa7b","observation_id":"174fb693-b55e-4da7-810f-f7010902b607","resolution":{"observed_at":"2026-08-12T14:14:09.062876Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.065992Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":48,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.065992Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:55f949a2544c2c5da27b54a9536c24f0d6c5d46a16ca9fa82e521357a8e2aa6e","observation_id":"79ab10e0-4033-4452-9be1-11ef46fb6c6c","resolution":{"observed_at":"2026-08-12T14:14:09.065992Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.068173Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":49,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.068173Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:1c3a0f84e4421ca5a56e232f63bb388b354a9ea2e8900f0a5d75c4bff25f72d4","observation_id":"e6ae7ae8-f87c-411a-9b37-803d6566e0ab","resolution":{"observed_at":"2026-08-12T14:14:09.068173Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2108.04884","last_updated":"2022-01-09T20:58:09Z","snapshot_observed_at":"2026-08-10T18:05:22.148964Z","submitted_at":"2021-08-10T19:19:41Z","title":"Retiring Adult: New Datasets for Fair Machine Learning","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2108.04884","snapshot_observed_at":"2026-08-12T14:14:09.070435Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":50,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.070435Z"},"links":{"cited_paper":"/paper/2108.04884","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:4b03d397679e90c39e048e14b67dca2aedb52890030b83c7710d6493df8edd7c","observation_id":"b1731622-ac80-4274-a663-ba213a473dc6","resolution":{"observed_at":"2026-08-12T14:14:09.070435Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.073853Z","title":"Donnelly, Nathaniel Blanchard, Andrew M","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":51,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.073853Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:6f27b68dd0c673b2ac26b60ac3f592aabae1be7689727ce53a27d9acc6d18bcd","observation_id":"6d2aae0d-b564-4a02-9410-a3000608a9bc","resolution":{"observed_at":"2026-08-12T14:14:09.073853Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.076972Z","title":null,"venue":null,"work_id":null,"year":2012},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":52,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.076972Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:5fb52488f084aaa28301b2cc6ffcdf434c1c3005bb35e12bc4f15560f5bc7225","observation_id":"ad7d84f9-f4bf-4777-8f70-fa5d852cffcf","resolution":{"observed_at":"2026-08-12T14:14:09.076972Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.080063Z","title":"Detecting Illusory Halo Effects in Rater - Mediated Assessment : A Mixture Rasch Facets Modeling Approach","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":53,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.080063Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:93e95dec0262b1730663c9b830585f61c7b1a095581bbfc00ed037286e6e80a1","observation_id":"b1b1208f-6afd-4e8e-a2db-05d3baa6a897","resolution":{"observed_at":"2026-08-12T14:14:09.080063Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.083613Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":54,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.083613Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:189f8d4aebc5b368ffdab7da193ebe65d5b89c1486f692bd12cb6bfa48f2f0d5","observation_id":"36885db8-7e67-4d6f-88de-de35f38a7b54","resolution":{"observed_at":"2026-08-12T14:14:09.083613Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.08818","last_updated":"2024-09-17T05:29:50Z","snapshot_observed_at":"2026-08-12T23:43:56.489990Z","submitted_at":"2024-06-13T05:20:42Z","title":"Linguistic Bias in ChatGPT: Language Models Reinforce Dialect Discrimination","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.08818","snapshot_observed_at":"2026-08-12T14:14:09.086235Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":55,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.086235Z"},"links":{"cited_paper":"/paper/2406.08818","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:e070629e917f2a191ba4984e55b80c2e79f69cf3a1f62c20a9b44eb3d916eed0","observation_id":"f14b809e-c568-4ddc-8322-15081224ee81","resolution":{"observed_at":"2026-08-12T14:14:09.086235Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.089727Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":56,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.089727Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:7d1eba4b1783b70570f228baf3923efef9c9e58fec5b3d0c388e8c87faf96ddb","observation_id":"7267ecc7-a057-4940-abe4-19e3b4c4c7c4","resolution":{"observed_at":"2026-08-12T14:14:09.089727Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2209.14375","last_updated":"2022-09-28T19:04:43Z","snapshot_observed_at":"2026-08-11T23:28:54.309888Z","submitted_at":"2022-09-28T19:04:43Z","title":"Improving alignment of dialogue agents via targeted human judgements","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2209.14375","snapshot_observed_at":"2026-08-12T14:14:09.092248Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":57,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.092248Z"},"links":{"cited_paper":"/paper/2209.14375","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:4d03b10db65f5c22f7489c3554be084c15edcdd4d0a1ce649aaa8e2f2eb13af4","observation_id":"b5bc1457-cec8-4ca7-9631-d3b204052af1","resolution":{"observed_at":"2026-08-12T14:14:09.092248Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.095272Z","title":"Gordon, Michelle S","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":58,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.095272Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:c33551ed66de29fd03c3d296ed0170b163be76caa47857aa36a937106ea76e0f","observation_id":"cd259315-3192-47fc-932e-d4ee47f352f6","resolution":{"observed_at":"2026-08-12T14:14:09.095272Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.097777Z","title":null,"venue":null,"work_id":null,"year":2013},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":59,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.097777Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:3b34d05b8b08b13b376f1a8490439633c4ddeafd74d4fbf7546c7419eb787efd","observation_id":"e88e1b8b-318b-4843-8643-967c012e8e04","resolution":{"observed_at":"2026-08-12T14:14:09.097777Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1007/bf02288892","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.717199Z","title":null,"venue":null,"work_id":"96df39f9-fed8-410d-b8d6-654fa976f943","year":1945},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":60,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.100749Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:4e52414e2d798da8f3426e70f4d93fe3885420261245853d1612f4d926afedaf","observation_id":"8be0001f-ad92-42a4-90e3-ff775b6a7cbf","resolution":{"observed_at":"2026-08-12T14:14:09.720638Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.104380Z","title":null,"venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":61,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.104380Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:8deea9744c5e84b2f629de3b8d30109c0e120cd164dd49baaae0fd8d969577e1","observation_id":"6cbb5cb8-a405-4ef9-a34e-9f69ae17a301","resolution":{"observed_at":"2026-08-12T14:14:09.104380Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1610.02413","last_updated":"2016-10-07T20:16:29Z","snapshot_observed_at":"2026-07-06T05:13:49.420787Z","submitted_at":"2016-10-07T20:16:29Z","title":"Equality of Opportunity in Supervised Learning","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1610.02413","snapshot_observed_at":"2026-08-12T14:14:09.107756Z","title":null,"venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":62,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.107756Z"},"links":{"cited_paper":"/paper/1610.02413","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:72216089bbfe67b436778f2fe2833008aaec7f9b31e99c06e7f06ae7e73e5d3f","observation_id":"d64aecd0-3008-4729-a79a-95ee69bcd88e","resolution":{"observed_at":"2026-08-12T14:14:09.107756Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2112.11973","last_updated":"2021-12-22T15:44:30Z","snapshot_observed_at":"2026-08-12T01:51:09.794653Z","submitted_at":"2021-12-22T15:44:30Z","title":"Toward Educator-focused Automated Scoring Systems for Reading and Writing","version":1},"cited_work":{"arxiv_id":"2112.11973","doi":"10.48550/arxiv.2112.11973","metadata_source":"pith","pith_arxiv_id":"2112.11973","snapshot_observed_at":"2026-08-12T18:16:22.223912Z","title":"Toward Educator-focused Automated Scoring Systems for Reading and Writing","venue":"cs.CL","work_id":"40e1d755-80cf-4160-bbb7-e4284ddf7522","year":2021},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":63,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.110605Z"},"links":{"cited_paper":"/paper/2112.11973","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:10616412c0cee0c4eddb03c61fca3187e520748008af121f644088a84fc2f4ec","observation_id":"dc0706ab-729d-4d53-8209-76c186711ea2","resolution":{"observed_at":"2026-08-12T14:14:09.705374Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.112901Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":64,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.112901Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:f97506b80be7d58d6147b5a9ac00b75784c5889b6c50c71dab500168f3a87ba1","observation_id":"f94444af-1cd2-4355-93f0-11241e196ab1","resolution":{"observed_at":"2026-08-12T14:14:09.112901Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.114895Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":65,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.114895Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:8b56a2d7fd75cc67f5a87364b9981cfa61cc6d14197c8262eea6ff8ecdb2f3f1","observation_id":"71f71d33-b8b1-4568-b4cd-17fb1fd6766b","resolution":{"observed_at":"2026-08-12T14:14:09.114895Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.14516","last_updated":"2025-03-28T15:40:49Z","snapshot_observed_at":"2026-08-12T22:20:02.504110Z","submitted_at":"2024-10-18T14:55:14Z","title":"Do LLMs \"know\" internally when they follow instructions?","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.14516","snapshot_observed_at":"2026-08-12T14:14:09.116865Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":66,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.116865Z"},"links":{"cited_paper":"/paper/2410.14516","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:078ba89c14f74970453e67626cd32fe9e7fa623a924404f914885d1ab19337df","observation_id":"19d33387-39af-4cdf-98fd-a09c0e9d5a32","resolution":{"observed_at":"2026-08-12T14:14:09.116865Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.119103Z","title":"Hill, Merrie L","venue":null,"work_id":null,"year":2008},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":67,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.119103Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:943f8d28b3296e5a79579c4796dd138b96bea63f15e5b6ad71413554cdddb8ac","observation_id":"17f6823b-b6f7-41f9-b09e-f106d60d0024","resolution":{"observed_at":"2026-08-12T14:14:09.119103Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.121664Z","title":"Hill, Charalambos Y","venue":null,"work_id":null,"year":2012},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":68,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.121664Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:b0b035880812a95b4d77f0e59c5521fd0336d4d21c8c5b99dc79f2ea8f48ea9a","observation_id":"e479c0b1-fa07-4a46-bfb0-5697962bacbb","resolution":{"observed_at":"2026-08-12T14:14:09.121664Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.3102/0013189x12437203","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.690695Z","title":"Hill, Charalambos Y","venue":null,"work_id":"a610c88f-0ed2-43c6-b149-5c90027a762f","year":2012},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":69,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.125222Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:af8e90ceb60430587660faa4d65c9e15ffee8c39c324720509944e3108447afc","observation_id":"a634179d-2636-452b-a7b5-9b5af09a1ec0","resolution":{"observed_at":"2026-08-12T14:14:09.693991Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.129009Z","title":"Ho and Thomas J","venue":null,"work_id":null,"year":2013},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":70,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.129009Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:3898cee5b0a82734f0f145aee9e767cf184f405e8e6ba90ff6d35d56c60671ed","observation_id":"89f67e84-f0ed-4f6b-94dd-02229a9e3d7c","resolution":{"observed_at":"2026-08-12T14:14:09.129009Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.131853Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":71,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.131853Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:bc0119d8aaa524e26de6e939c682f957653e30c21404ecad34268cecc2a7b656","observation_id":"4b9dc2b6-a12f-4325-bf96-3ec909ed363a","resolution":{"observed_at":"2026-08-12T14:14:09.131853Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.00742","last_updated":"2024-03-01T18:43:09Z","snapshot_observed_at":"2026-08-13T04:05:21.054158Z","submitted_at":"2024-03-01T18:43:09Z","title":"Dialect prejudice predicts AI decisions about people's character, employability, and criminality","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.00742","snapshot_observed_at":"2026-08-12T14:14:09.134401Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":72,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.134401Z"},"links":{"cited_paper":"/paper/2403.00742","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:067bd29349bc9a34ac3076401685ba0b0ab7deddaf58a84c222ba845c2f09d1e","observation_id":"64172633-dfc9-4e12-a695-d300d9c3410d","resolution":{"observed_at":"2026-08-12T14:14:09.134401Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.16349","last_updated":"2024-01-16T16:51:33Z","snapshot_observed_at":"2026-08-13T10:03:12.912726Z","submitted_at":"2023-09-28T11:18:20Z","title":"Human Feedback is not Gold Standard","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.16349","snapshot_observed_at":"2026-08-12T14:14:09.138144Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":73,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.138144Z"},"links":{"cited_paper":"/paper/2309.16349","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:6dce388b335ac3b6490f586e5d05870fee93454b6452b1c50ca49fda7ba60503","observation_id":"745caea2-c5bb-4f3f-9dfc-c3816e4c9293","resolution":{"observed_at":"2026-08-12T14:14:09.138144Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.140999Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":74,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.140999Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:5be712d2f31e1a05fe5d781fb6d28f4c1432526c9fa65057795e6aa788149820","observation_id":"c2598ee1-4bf0-4549-8c1e-d9b7d0475061","resolution":{"observed_at":"2026-08-12T14:14:09.140999Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.3758/s13428-023-02275-2","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.668456Z","title":null,"venue":null,"work_id":"51c9e345-6be0-4055-a7fe-c4e82bb5770b","year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":75,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.143596Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:7dddb8ea55cea2449575a7e7c914a1b7494f3fb3a12ddeb67e196e661a5a2ddd","observation_id":"9227160f-b29e-441c-be05-b3af01b69261","resolution":{"observed_at":"2026-08-12T14:14:09.670960Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.146120Z","title":"Jacobs, Ryan J","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":76,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.146120Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:3f0411ffda511d4f7c9af409a0ea2aae02c1d0c975f2afe541a58e21d3671e83","observation_id":"8b4f7385-7b87-4781-87f5-2e4352df86c3","resolution":{"observed_at":"2026-08-12T14:14:09.146120Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.14288/1.0437518","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.659135Z","title":null,"venue":null,"work_id":"b77bb800-1c77-42f4-b08d-5c1699ff1ef8","year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":77,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.148663Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:6c7fe2fe16be61fddbf6f9074c1e8267b4baad8a699d388cea9b62aa4f77686c","observation_id":"ff19faca-0d5e-4022-ab32-cddd4fc7742f","resolution":{"observed_at":"2026-08-12T14:14:09.663305Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.151878Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":78,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.151878Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:996c2535335529be56b515dade281ccd5144719a6d2bc3c979697d5e52dd8f17","observation_id":"84eac20c-2ad6-4870-9895-5ff169d9204e","resolution":{"observed_at":"2026-08-12T14:14:09.151878Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.3886/icpsr36095.v4","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.650979Z","title":null,"venue":null,"work_id":"f8436c10-3499-485a-b647-f689c32fd268","year":2015},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":79,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.154478Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:48ebace212ec094cdf2a89dc8af9fd357fc0e65a02d32dd644625acebe993976","observation_id":"976f21b2-d776-48b9-a956-4181e1006657","resolution":{"observed_at":"2026-08-12T14:14:09.654223Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.157097Z","title":"Kane, Daniel F","venue":null,"work_id":null,"year":2013},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":80,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.157097Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:23a655d11521e7784202c1577dd79a20e0f812d849191ff1114054af8421406c","observation_id":"a1ecae9d-d59b-44e0-a90e-83d6e767a5da","resolution":{"observed_at":"2026-08-12T14:14:09.157097Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.159929Z","title":"Kane and Douglas O","venue":null,"work_id":null,"year":2012},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":81,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.159929Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:9c59874d2753cda619dd8e02dec5dd29e233106b7d59b2c452f6bce3b6b65351","observation_id":"fecb4ad0-2c0b-40a8-b400-d0083e08f04f","resolution":{"observed_at":"2026-08-12T14:14:09.159929Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.161958Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":82,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.161958Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:bab16f830df8dbc609a13eb82bc0d177ca2b2c14110c32b198d723921779c982","observation_id":"86ab413f-a95b-41a5-b04b-913c8eea7065","resolution":{"observed_at":"2026-08-12T14:14:09.161958Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.164025Z","title":null,"venue":null,"work_id":null,"year":2013},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":83,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.164025Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:6cff78e6d5ab61237af35cfb3a097ce7cc7e5bea454accd178b460aa6664b332","observation_id":"d73ac0b4-0b64-4045-95cc-19ba4e1b3007","resolution":{"observed_at":"2026-08-12T14:14:09.164025Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.166358Z","title":"Olney, Patrick Donnelly, Martin Nystrand, and Sidney K","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":84,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.166358Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:b3b3f0ad211d1eafe4749be3be49a919cc60e8d66b035e9cb0a174a2e73a376d","observation_id":"e09eab0a-0f58-434e-8920-5153488f7288","resolution":{"observed_at":"2026-08-12T14:14:09.166358Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2104.14337","last_updated":"2021-04-07T17:49:17Z","snapshot_observed_at":"2026-08-03T23:24:34.241550Z","submitted_at":"2021-04-07T17:49:17Z","title":"Dynabench: Rethinking Benchmarking in NLP","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2104.14337","snapshot_observed_at":"2026-08-12T14:14:09.168262Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":85,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.168262Z"},"links":{"cited_paper":"/paper/2104.14337","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:a3a293fa2aa3d051f49b9910fb510cb25da6a477122c04f82bb5b50991a9905d","observation_id":"f15163c3-9e9c-41e5-a705-858294c34b73","resolution":{"observed_at":"2026-08-12T14:14:09.168262Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1711.11279","last_updated":"2018-06-07T04:33:27Z","snapshot_observed_at":"2026-08-04T20:10:18.590341Z","submitted_at":"2017-11-30T09:26:12Z","title":"Interpretability Beyond Feature Attribution: Quantitative Testing with Concept Activation Vectors (TCAV)","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1711.11279","snapshot_observed_at":"2026-08-12T14:14:09.171046Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":86,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.171046Z"},"links":{"cited_paper":"/paper/1711.11279","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:c1b11c16247a59e84b73ff54e81a3801b03ab9d44de5b6787311751b8e719a51","observation_id":"7e58f77d-5610-4cb9-b29c-100ba27d35c4","resolution":{"observed_at":"2026-08-12T14:14:09.171046Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1412.6980","last_updated":"2017-01-30T01:27:54Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2014-12-22T13:54:29Z","title":"Adam: A Method for Stochastic Optimization","version":9},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1412.6980","snapshot_observed_at":"2026-08-12T14:14:09.174083Z","title":"Kingma and Jimmy Ba","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":87,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.174083Z"},"links":{"cited_paper":"/paper/1412.6980","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:eec498fed29c56d7f13670b655fda02f413237b06ddf02428226f7a3a0c30b72","observation_id":"a8e4eb6a-89de-490e-8472-8011304fdcdc","resolution":{"observed_at":"2026-08-12T14:14:09.174083Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1073/pnas.1212738110","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.618029Z","title":null,"venue":null,"work_id":"52d87162-d004-49fd-94d7-05685d9dcf1c","year":2013},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":88,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.177851Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:b17a9b523462f4298eb74d201968e6a07b495d2ee1ee3f033749bc4798e9f7c5","observation_id":"aa4a5d4f-a25e-41e1-b5ec-df41ec7b7c4d","resolution":{"observed_at":"2026-08-12T14:14:09.620934Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.180739Z","title":"Kromrey, Robert H","venue":null,"work_id":null,"year":2008},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":89,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.180739Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:bfbc780b9a00e7fdc1ea249c7115f99977f3903c2af670d4bf38c6c6ffc84dcf","observation_id":"4089d83d-a529-4d28-b9d9-4da6d547d20b","resolution":{"observed_at":"2026-08-12T14:14:09.180739Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.183188Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":90,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.183188Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:f23682be63ab76b73377baaf54e01025e355506507124ef3e4add1ecfa931c22","observation_id":"0cd3a3ac-98c9-4a17-8eef-42a1b0e2d732","resolution":{"observed_at":"2026-08-12T14:14:09.183188Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.186148Z","title":null,"venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":91,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.186148Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:d0e409af61756fa9a6ff8f319f7179f19e1643c93fefd74fe717819bfcb0d84e","observation_id":"20bfe55c-f1af-43b3-b239-703504d6c0e7","resolution":{"observed_at":"2026-08-12T14:14:09.186148Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.03281","last_updated":"2023-08-07T03:52:59Z","snapshot_observed_at":"2026-08-04T23:10:23.964516Z","submitted_at":"2023-08-07T03:52:59Z","title":"Towards General Text Embeddings with Multi-stage Contrastive Learning","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.03281","snapshot_observed_at":"2026-08-12T14:14:09.188693Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":92,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.188693Z"},"links":{"cited_paper":"/paper/2308.03281","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:6dfa56bf5d96487efc5b387b5def9707c22458a6070d23315853f8e71e6b09c4","observation_id":"022f8911-eeff-48de-ab48-506de17adb63","resolution":{"observed_at":"2026-08-12T14:14:09.188693Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.191656Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":93,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.191656Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:ff37680713e60c6130bd5b1aa665031d54cff83a6831d70359d2abeb1beb5a47","observation_id":"fd527404-979f-4b42-9b1d-ef1def72db24","resolution":{"observed_at":"2026-08-12T14:14:09.191656Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.194277Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":94,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.194277Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:2345708596e420caaf9c381a777b51a7be4e9e2194a70371383ba549ff5fe116","observation_id":"fadda3b2-aac2-4ac7-91e6-75df02de3ce7","resolution":{"observed_at":"2026-08-12T14:14:09.194277Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2307.03172","last_updated":"2023-11-20T23:09:34Z","snapshot_observed_at":"2026-07-06T15:51:12.179086Z","submitted_at":"2023-07-06T17:54:11Z","title":"Lost in the Middle: How Language Models Use Long Contexts","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.03172","snapshot_observed_at":"2026-08-12T14:14:09.197109Z","title":"Liu, Kevin Lin, John Hewitt, Ashwin Paranjape, Michele Bevilacqua, Fabio Petroni, and Percy Liang","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":95,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.197109Z"},"links":{"cited_paper":"/paper/2307.03172","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:f2d96c4e6c355b19d62c533d10f33c8c5e69b08c94e8f8e3b3a27b5a561fafc3","observation_id":"8eb3a78a-58c6-4112-b410-6f179d23e31b","resolution":{"observed_at":"2026-08-12T14:14:09.197109Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.16634","last_updated":"2023-05-23T22:12:16Z","snapshot_observed_at":"2026-08-02T04:02:36.848064Z","submitted_at":"2023-03-29T12:46:54Z","title":"G-Eval: NLG Evaluation using GPT-4 with Better Human Alignment","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.16634","snapshot_observed_at":"2026-08-12T14:14:09.199888Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":96,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.199888Z"},"links":{"cited_paper":"/paper/2303.16634","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:b61fac6f935570e8f92b1b908ac32c702848203f17e1b656306836d3adc91ec4","observation_id":"5e9719f6-69ed-483f-a19d-ea072adccc45","resolution":{"observed_at":"2026-08-12T14:14:09.199888Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1907.11692","last_updated":"2019-07-26T17:48:29Z","snapshot_observed_at":"2026-07-31T22:31:37.910868Z","submitted_at":"2019-07-26T17:48:29Z","title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1907.11692","snapshot_observed_at":"2026-08-12T14:14:09.203006Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":97,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.203006Z"},"links":{"cited_paper":"/paper/1907.11692","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:1e41317f408382c3e03a9baf8f767f75993e8b8f9993f2d3f0b3480d49bdd7ea","observation_id":"14abb02f-0a96-43d5-af03-413d47d90daa","resolution":{"observed_at":"2026-08-12T14:14:09.203006Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1705.07874","last_updated":"2017-11-25T03:53:32Z","snapshot_observed_at":"2026-07-06T05:43:42.722222Z","submitted_at":"2017-05-22T17:38:10Z","title":"A Unified Approach to Interpreting Model Predictions","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1705.07874","snapshot_observed_at":"2026-08-12T14:14:09.206025Z","title":null,"venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":98,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.206025Z"},"links":{"cited_paper":"/paper/1705.07874","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:669cc73df78e02447edfdcd967b7c0a87d4b20d7f8ed0087ccfbd8d733545c82","observation_id":"1bf838de-3e95-4d69-84b1-583a98dc8054","resolution":{"observed_at":"2026-08-12T14:14:09.206025Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.208438Z","title":"French, and Helen Patrick","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":99,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.208438Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:be3b27a4a07daedddeb3281c30da9313f3164d01317dc4225903ae96af814759","observation_id":"fcd74786-a18c-4bce-8a27-429309a63ae3","resolution":{"observed_at":"2026-08-12T14:14:09.208438Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.3102/1076998606298033","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.577671Z","title":"Mariano and Brian W","venue":null,"work_id":"38116f54-7401-4817-99a3-80877d3e8c5c","year":2007},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":100,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.210718Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:d1dddaf7cb4529b54bae01cfba2a3f0c43b17c0bec73a46b7d4743846ae09597","observation_id":"f81256ce-2047-45b2-add8-e77e867f9d28","resolution":{"observed_at":"2026-08-12T14:14:09.580641Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2309.13638","last_updated":"2023-09-24T13:35:28Z","snapshot_observed_at":"2026-08-13T10:06:45.687474Z","submitted_at":"2023-09-24T13:35:28Z","title":"Embers of Autoregression: Understanding Large Language Models Through the Problem They are Trained to Solve","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.13638","snapshot_observed_at":"2026-08-12T14:14:09.212738Z","title":"Thomas McCoy, Shunyu Yao, Dan Friedman, Matthew Hardy, and Thomas L","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":101,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.212738Z"},"links":{"cited_paper":"/paper/2309.13638","citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:d03e598ba4672cdd52bae271c57782c1e5af7650d78bfbe5e2aa5a44733132ce","observation_id":"e1758391-6eef-4955-9203-fb5798cc9dd4","resolution":{"observed_at":"2026-08-12T14:14:09.212738Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:14:09.214957Z","title":null,"venue":null,"work_id":null,"year":1998},"citing_paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations","version":1},"reference_index":102,"source":"arxiv_source","source_observed_at":"2026-08-12T14:14:09.214957Z"},"links":{"citing_paper":"/paper/2411.15634"},"observation_digest":"sha256:39801d078355b0206d05b276fb319c3d1e6f68fde28f9a5a9a7162324769cb2f","observation_id":"224c9407-044d-446b-9281-7c88a40d4b72","resolution":{"observed_at":"2026-08-12T14:14:09.214957Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2411.15634","last_updated":"2024-11-23T19:18:08Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-13T11:31:52.601124Z","submitted_at":"2024-11-23T19:18:08Z","title":"\"All that Glitters\": Approaches to Evaluations with Unreliable Model and Human Annotations"},"reference_resolution":{"displayed":100,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":80,"verified_exact":20,"verified_fuzzy":0},"total_outbound_references":147},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"thesis":"As of 13 August 2026, this Paper Citation Record lists 100 of 147 outbound references and 0 inbound Pith citation observations for arXiv:2411.15634."}