{"as_of":"2026-08-23T00:46:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:a5004c120191e2ad3b11d1d1fa5e05d6e40bc340c592ff9471ae4ed2cd48528e","coverage":[{"denominator":48,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":48,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-12T15:58:47.648449Z","state":"measured"},{"denominator":48,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":48,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-22T06:32:14.747728+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2411.13760/citation-record","integrity":"/paper/2411.13760/integrity","json":"/paper/2411.13760/citation-record.json","paper":"/paper/2411.13760"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.833653Z","title":"DICES Dataset: Diversity in Conversational AI Evaluation for Safety","venue":null,"work_id":"deee1d94-7f46-4d40-ab07-38954a7328be","year":null},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.410920Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:67ba11024a12df32a29639a34d731e89a507cab8e3e494b6b6440657efcd6156","observation_id":"5c077c67-c093-4ebf-88d3-ecb29673ac03","resolution":{"observed_at":"2026-08-12T15:58:48.839258Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2210.16133","last_updated":"2022-11-30T09:15:10Z","snapshot_observed_at":"2026-08-20T23:20:12.146511Z","submitted_at":"2022-10-28T14:01:32Z","title":"Stop Measuring Calibration When Humans Disagree","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2210.16133","snapshot_observed_at":"2026-08-12T15:58:47.416548Z","title":"Stop Measuring Calibration When Humans Disagree, November 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.416548Z"},"links":{"cited_paper":"/paper/2210.16133","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:3aec0337967252a6a4203a8745220a78bc3bb92e60d5253e07a3bda5b1b2fc47","observation_id":"a57fd76d-a59b-4874-9ec1-70b47786c546","resolution":{"observed_at":"2026-08-12T15:58:47.416548Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.814957Z","title":"It’s the End of the Gold Standard as we Know it","venue":null,"work_id":"12bc3831-cebe-4f10-bad6-627305208a1d","year":null},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.422176Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:e6eb813745b667b6790efd44a0d07e9e4f96a329f97d72889826af670d6ee038","observation_id":"2b3da26a-e236-43ed-9e86-cac277deb601","resolution":{"observed_at":"2026-08-12T15:58:48.821568Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1707.01477","last_updated":"2017-07-05T17:19:45Z","snapshot_observed_at":"2026-08-18T19:03:03.383800Z","submitted_at":"2017-07-05T17:19:45Z","title":"Like trainer, like bot? Inheritance of bias in algorithmic content moderation","version":1},"cited_work":{"arxiv_id":"1707.01477","doi":null,"metadata_source":"pith","pith_arxiv_id":"1707.01477","snapshot_observed_at":"2026-08-12T15:58:48.513949Z","title":"Like trainer, like bot? Inheritance of bias in algorithmic content moderation","venue":"cs.CY","work_id":"6b8f95ff-d5bb-4e80-9c7f-d0f847d436d0","year":2017},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.427557Z"},"links":{"cited_paper":"/paper/1707.01477","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:a771cdb4b33b834bfc108cdc70600883aac7cf83f40bf921f9b02b4a92740b4e","observation_id":"8d37a0ce-5096-4610-a17a-d9fba6cdd5c2","resolution":{"observed_at":"2026-08-12T15:58:48.520699Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.797194Z","title":"Holistic evaluation of language models","venue":null,"work_id":"b893e7dc-0314-46ce-bfab-17271f7580b8","year":2023},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.433122Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:8c85c1698aea33c5f0210a68ad4240fb32acb5a099aa61c4615a40c2aa5b51ff","observation_id":"28c48695-e4b3-47dc-8a24-ac5d9e0ce8b6","resolution":{"observed_at":"2026-08-12T15:58:48.802595Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.438380Z","title":null,"venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.438380Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:1dec1502eba93fdaa7e4ab857b188de75abf09e9da2f94c5b24ce4ad5a6c752d","observation_id":"28671047-f62c-4e76-bf92-a379645b173b","resolution":{"observed_at":"2026-08-12T15:58:47.438380Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.447529Z","title":"Revolt: Collaborative Crowdsourcing for Labeling Machine Learning Datasets","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.447529Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:1aac7a6e5c203aa4227983ff2772842eb322fec526213a00f2fcc1849f0dd85c","observation_id":"00d71071-c431-4e74-8027-1c47e9925021","resolution":{"observed_at":"2026-08-12T15:58:47.447529Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.780952Z","title":"Judgment sieve: Reducing uncertainty in group judgments through interventions targeting ambiguity versus disagreement","venue":null,"work_id":"3dfa5c3f-9ec1-4407-871b-59c7cc02b7e8","year":2023},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.453452Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:45e98014722b9461445d2f8d7b5d5e8f21d594d687a650e57e494c29b85f5c92","observation_id":"c6af6ccc-9894-468b-b70a-b43f38dc4034","resolution":{"observed_at":"2026-08-12T15:58:48.786308Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2305.01615","last_updated":"2023-05-02T17:17:50Z","snapshot_observed_at":"2026-08-16T15:36:11.941235Z","submitted_at":"2023-05-02T17:17:50Z","title":"Judgment Sieve: Reducing Uncertainty in Group Judgments through Interventions Targeting Ambiguity versus Disagreement","version":1},"cited_work":{"arxiv_id":"2305.01615","doi":null,"metadata_source":"pith","pith_arxiv_id":"2305.01615","snapshot_observed_at":"2026-08-12T15:58:48.351605Z","title":"Judgment Sieve: Reducing Uncertainty in Group Judgments through Interventions Targeting Ambiguity versus Disagreement","venue":"cs.HC","work_id":"74725ac7-1a15-404f-af56-04e7f7557c2a","year":2023},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.459514Z"},"links":{"cited_paper":"/paper/2305.01615","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:534f5c34c96968cbfb52412d2e4f6d8fe6e8116150d1f67e1bbea0efa6ebd485","observation_id":"9a458d73-d10a-4790-80a3-159d996761a7","resolution":{"observed_at":"2026-08-12T15:58:48.356776Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1162/tacl_a_00449/","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.787401Z","title":"Dealing with Disagree- ments: Looking Beyond the Majority V ote in Subjective Annotations","venue":null,"work_id":"c16a0bc0-ad6b-46bb-80a2-4769600dcbbf","year":2022},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.465135Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:002cc4fb7a7a7fa80e5af14aa70e422d5ab9254ae3abfa4ca60eb4a82ea1e790","observation_id":"9b8ed86a-925b-4376-959a-e23aace9ef7d","resolution":{"observed_at":"2026-08-12T15:58:47.793619Z","resolver_source":"doi_truncated","status":"malformed_identifier"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2404.10857","last_updated":"2024-04-16T19:12:03Z","snapshot_observed_at":"2026-08-22T10:30:30.579332Z","submitted_at":"2024-04-16T19:12:03Z","title":"D3CODE: Disentangling Disagreements in Data across Cultures on Offensiveness Detection and Evaluation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.10857","snapshot_observed_at":"2026-08-12T15:58:47.470355Z","title":"D3CODE: Disentan- gling Disagreements in Data across Cultures on Offensiveness Detection and Evaluation, April 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.470355Z"},"links":{"cited_paper":"/paper/2404.10857","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:b310e4f24497542722a0262457d82ef683ed8b4e267764ef25b13bb6b61569de","observation_id":"ff854f86-582a-476c-b805-f332c7290525","resolution":{"observed_at":"2026-08-12T15:58:47.470355Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.15897","last_updated":"2024-08-27T18:23:44Z","snapshot_observed_at":"2026-08-18T14:41:00.471771Z","submitted_at":"2024-01-29T05:46:14Z","title":"Red-Teaming for Generative AI: Silver Bullet or Security Theater?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.15897","snapshot_observed_at":"2026-08-12T15:58:47.475898Z","title":"Red-teaming for generative ai: Silver bullet or security theater? arXiv preprint arXiv:2401.15897, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.475898Z"},"links":{"cited_paper":"/paper/2401.15897","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:83b05af6036fcbe58efac286ab6e7a5553d1c590970e1cb52376669a42bf554d","observation_id":"3ea58abb-d693-4056-aac1-58b585a5ad0a","resolution":{"observed_at":"2026-08-12T15:58:47.475898Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2007.03114","last_updated":"2021-02-02T06:29:04Z","snapshot_observed_at":"2026-08-07T09:30:47.028966Z","submitted_at":"2020-07-06T23:13:07Z","title":"Efficient Conformal Prediction via Cascaded Inference with Expanded Admission","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2007.03114","snapshot_observed_at":"2026-08-12T15:58:47.481439Z","title":"Efficient conformal prediction via cascaded inference with expanded admission","venue":null,"work_id":null,"year":2007},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.481439Z"},"links":{"cited_paper":"/paper/2007.03114","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:072cf60804bba8e6956ffc68f9c0aba7440d25f159c7d64852112d9fe1bd8693","observation_id":"8aed3702-9d7b-445e-aefd-781624d01b0f","resolution":{"observed_at":"2026-08-12T15:58:47.481439Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.05860","last_updated":"2024-05-09T15:48:07Z","snapshot_observed_at":"2026-08-16T18:14:12.748200Z","submitted_at":"2024-05-09T15:48:07Z","title":"The Perspectivist Paradigm Shift: Assumptions and Challenges of Capturing Human Labels","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.05860","snapshot_observed_at":"2026-08-12T15:58:47.486664Z","title":"The Perspectivist Paradigm Shift: Assumptions and Challenges of Capturing Human Labels, May 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.486664Z"},"links":{"cited_paper":"/paper/2405.05860","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:853c1098982f60606b9bc78f9bff4769cf93d979b84de252363879273ffc36b3","observation_id":"0ea16206-678b-402d-8ee8-379a7cd8505f","resolution":{"observed_at":"2026-08-12T15:58:47.486664Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2209.07858","last_updated":"2022-11-22T19:12:57Z","snapshot_observed_at":"2026-08-21T13:43:54.063053Z","submitted_at":"2022-08-23T23:37:14Z","title":"Red Teaming Language Models to Reduce Harms: Methods, Scaling Behaviors, and Lessons Learned","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2209.07858","snapshot_observed_at":"2026-08-12T15:58:47.491762Z","title":"Red teaming language models to reduce harms: Methods, scaling behaviors, and lessons learned","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.491762Z"},"links":{"cited_paper":"/paper/2209.07858","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:6a95ab3b92d2a740ee53aaf6472987df24bcd2730e46dd0885214e6afb1a3867","observation_id":"cb126527-a912-4f1a-a3e6-558a7fc321a7","resolution":{"observed_at":"2026-08-12T15:58:47.491762Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":"2017.26899","doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.234164Z","title":"Deep Label Distribution Learning With Label Ambiguity","venue":null,"work_id":"ea751607-5866-47a6-b26e-48f00a086360","year":2017},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.496224Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:e4377bcda64c51a7d3c0412bdfc2951d047b44e3992fe26e0bc2ba2227abb955","observation_id":"68160aa2-5850-4122-90bc-0f1ae0396ae7","resolution":{"observed_at":"2026-08-12T15:58:48.243981Z","resolver_source":"raw_fallback","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.500370Z","title":"Label Distribution Learning","venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.500370Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:3594c5590ed2c947db1981b838eaac7c4fe5b61d8b3da50c6204b0bc77358068","observation_id":"4809e15c-34b0-4d78-b633-a8e1ed9dc364","resolution":{"observed_at":"2026-08-12T15:58:47.500370Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.764392Z","title":"The disagreement deconvolution: Bringing machine learning performance metrics in line with reality","venue":null,"work_id":"950771a7-1b5e-4d6e-a50a-6f2505e18bca","year":2021},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.504721Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:ca9929342a6973b12a68991ad6b75c4c4c8a3ea89b68d94f26d85b3d47719994","observation_id":"210bc53a-7606-4252-9fbc-38252d34451d","resolution":{"observed_at":"2026-08-12T15:58:48.769904Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.508867Z","title":"Gordon, Kaitlyn Zhou, Kayur Patel, Tatsunori Hashimoto, and Michael S","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.508867Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:884edcfba4a6bcf264f4ba845db545d6f9cad634eb4b1c3c187e7fe77047a0ff","observation_id":"a7959e7b-177f-4f5d-a252-6e4a117e8c02","resolution":{"observed_at":"2026-08-12T15:58:47.508867Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.746148Z","title":"Jury learning: Integrating dissenting voices into machine learning models","venue":null,"work_id":"4b3b4df3-7f23-4390-95d1-e9bca9e08aca","year":2022},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.513299Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:c54d147301a0fa0c4a11bda7ad994f47eec779c84774105b73fb062bcbb0c0d7","observation_id":"a7d6d6bd-d8b1-45c2-9a75-a31e0516dfad","resolution":{"observed_at":"2026-08-12T15:58:48.752362Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.729244Z","title":"Is your toxicity my toxicity? exploring the impact of rater identity on toxicity annotation","venue":null,"work_id":"06ae60f1-8c81-4cbf-a6b6-547ee10b2434","year":2022},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.517490Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:8e67146ab2886e394394cd29df00f0ad3c3b3ed849fae1211c523befe75d66e7","observation_id":"9cbcbbe0-069c-473b-9568-f8cd5de417a1","resolution":{"observed_at":"2026-08-12T15:58:48.734640Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2009.03300","last_updated":"2021-01-12T18:57:11Z","snapshot_observed_at":"2026-08-13T20:44:28.824685Z","submitted_at":"2020-09-07T17:59:25Z","title":"Measuring Massive Multitask Language Understanding","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.03300","snapshot_observed_at":"2026-08-12T15:58:47.522165Z","title":"Measuring massive multitask language understanding","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.522165Z"},"links":{"cited_paper":"/paper/2009.03300","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:08db79b64a3c6b4960b06c9bb58adbfbd723dfa5980227ea0a8e0b9f8a4ecd61","observation_id":"50d372cd-4d44-4a5d-aa16-ca30d7b7bb7b","resolution":{"observed_at":"2026-08-12T15:58:47.522165Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.711218Z","title":"Intersectionality in ai safety: Using multilevel models to understand diverse perceptions of safety in conversational ai","venue":null,"work_id":"d5117602-5332-4a1a-a918-c8c6a380e8de","year":2024},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.527335Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:27569602600cbefc4336c54634e6a143fb47cb49961e889dd9d9ed01893dc1e2","observation_id":"28b35d4a-d867-48bc-9e37-4ffe9a43f1d8","resolution":{"observed_at":"2026-08-12T15:58:48.717943Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.531722Z","title":"Culturally Aware Natural Language Inference","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.531722Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:eee41e067a1708924872f20468997af9d2f6a60fc195c7ace570497fb5cda121","observation_id":"1bf6244c-5633-479d-990b-713b16189533","resolution":{"observed_at":"2026-08-12T15:58:47.531722Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.694012Z","title":"Chaithanya Manam, Dwarakanath Jampani, Mariam Zaim, Meng-Han Wu, and Alexander J","venue":null,"work_id":"81e3c1c8-c4ee-44d6-8be1-60e290fdd1ad","year":2019},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.536697Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:123c5a35418c1b5a4d4c69b6ed9bfc624af18f862e70ed2c6f91177b714dea84","observation_id":"77dd0d86-8374-4649-9493-e5226e7d39dd","resolution":{"observed_at":"2026-08-12T15:58:48.699453Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.541649Z","title":"Annotation Error Detection: Analyzing the Past and Present for a More Coherent Future","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.541649Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:715866519f4694ccf399a07358f078ef4722fe4cd98bf57aeb9dd72cdb2e0f3f","observation_id":"a874b2e7-e4c3-487c-88ed-ec1d04b97ef8","resolution":{"observed_at":"2026-08-12T15:58:47.541649Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.546651Z","title":"A Bayesian Framework for Modeling Human Evaluations","venue":null,"work_id":null,"year":2015},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.546651Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:184efdeff38bf0df4dbaae0126563114a408f95c3ea54cf6992a943e17898a9e","observation_id":"d383fbc1-768b-4684-90d8-75a83dca527e","resolution":{"observed_at":"2026-08-12T15:58:47.546651Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.672946Z","title":"Reconsidering Annotator Disagreement about Racist Language: Noise or Signal?","venue":null,"work_id":"eb19a687-cd6d-4c49-b6ea-a5877b6f57fe","year":null},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.552442Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:5e34245ec0b2cdf6437b8156e1078f2b60ecf7c25801d470a2885595bd578480","observation_id":"d3ef8992-3d96-46b0-ad4e-d464487125d0","resolution":{"observed_at":"2026-08-12T15:58:48.679755Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.655723Z","title":"Learning to predict population-level label distributions","venue":null,"work_id":"7e31b0be-9973-417e-9a45-7218315032e0","year":2019},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.557193Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:a46b9b04f831604c7db21ab8be459c8187751bf30c297aab21ee12019a9f8aaa","observation_id":"c16a2fd9-fa2f-488f-aada-166b24f23f4d","resolution":{"observed_at":"2026-08-12T15:58:48.661299Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.04893","last_updated":"2024-03-07T20:55:08Z","snapshot_observed_at":"2026-08-16T14:11:48.806169Z","submitted_at":"2024-03-07T20:55:08Z","title":"A Safe Harbor for AI Evaluation and Red Teaming","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.04893","snapshot_observed_at":"2026-08-12T15:58:47.561742Z","title":"A safe harbor for ai evaluation and red teaming","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.561742Z"},"links":{"cited_paper":"/paper/2403.04893","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:9c1268c92faff6b5e916dc910844bb6d72540bb77c5fe8193dc7bca384f2c28e","observation_id":"f5c68c08-3d94-4613-8b4f-72dae82cb0c8","resolution":{"observed_at":"2026-08-12T15:58:47.561742Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.17750","last_updated":"2023-10-26T19:45:06Z","snapshot_observed_at":"2026-08-20T07:04:36.874800Z","submitted_at":"2023-10-26T19:45:06Z","title":"A Framework for Automated Measurement of Responsible AI Harms in Generative AI Applications","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.17750","snapshot_observed_at":"2026-08-12T15:58:47.566997Z","title":"A framework for automated measurement of responsible ai harms in generative ai applications","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.566997Z"},"links":{"cited_paper":"/paper/2310.17750","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:05a3c021e167e459d8f199bb280751fb181c0be49723268cd2e1ed0a6029ae48","observation_id":"b032cac8-7e8e-4f0f-ac4b-c349810d2c66","resolution":{"observed_at":"2026-08-12T15:58:47.566997Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.1609/hcomp.v6i1.13338","metadata_source":"doi_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.708020Z","title":null,"venue":null,"work_id":"32302c9f-78da-4ed9-967e-146df77b1f90","year":2018},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.572311Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:556e39c174440becae6117add11784fbd1e9a229b4255492ec34c1d40d758789","observation_id":"e9a420e2-695e-46aa-97db-9bd5713d93c7","resolution":{"observed_at":"2026-08-12T15:58:47.716100Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2004.09456","last_updated":"2020-04-20T17:14:33Z","snapshot_observed_at":"2026-08-18T18:59:00.636649Z","submitted_at":"2020-04-20T17:14:33Z","title":"StereoSet: Measuring stereotypical bias in pretrained language models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2004.09456","snapshot_observed_at":"2026-08-12T15:58:47.577191Z","title":"Stereoset: Measuring stereotypical bias in pretrained language models","venue":null,"work_id":null,"year":2004},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.577191Z"},"links":{"cited_paper":"/paper/2004.09456","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:64ba0d269558b15d5e73fb59f515ecb32d50439cf82b88f695c16313e6d393bb","observation_id":"630a4f9b-ddfe-4f13-bd99-180ec7bc36f4","resolution":{"observed_at":"2026-08-12T15:58:47.577191Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.638907Z","title":"Diversity-aware annotation for conversational ai safety","venue":null,"work_id":"b178a89e-dd2c-4e14-b400-0273f87dfa42","year":2024},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.581750Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:9fdd8597330b477c8ffcb740eec2c27748b41cc23524f415480d6b5454f76d5f","observation_id":"6657a046-e9a1-4318-a733-e6ca4ee0f37e","resolution":{"observed_at":"2026-08-12T15:58:48.645028Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.622287Z","title":"Inherent disagreements in human textual inferences","venue":null,"work_id":"97c0e4f2-b05f-4896-ae43-fd1f51b5a1fc","year":2019},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.586220Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:7c8e1a2be1d467b184ac95dd85bc830d676b616023a2b3783b019e114da5f32b","observation_id":"a6e9d212-c5aa-486e-ac2b-6bb52305a3f3","resolution":{"observed_at":"2026-08-12T15:58:48.627575Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.591125Z","title":"Inherent Disagreements in Human Textual Inferences","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.591125Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:dc23409fd4ff0ef47d0ebf2c2cde5b46451449e2b4f7dbc3321511998bfb157d","observation_id":"4540f88d-e957-4faf-8bf6-61f13e4b7019","resolution":{"observed_at":"2026-08-12T15:58:47.591125Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.605626Z","title":"Human uncertainty makes classification more robust","venue":null,"work_id":"f3b31b87-2b79-402e-bb28-c75d0de90932","year":2019},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.595776Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:9f071574c70ac8be2450cd44d5a5920068830b5419939920f581ec8016ffed79","observation_id":"ea6b370c-3186-416d-83f1-dc46594621fe","resolution":{"observed_at":"2026-08-12T15:58:48.610984Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2211.02570","last_updated":"2022-11-04T16:38:09Z","snapshot_observed_at":"2026-08-16T21:40:11.527720Z","submitted_at":"2022-11-04T16:38:09Z","title":"The 'Problem' of Human Label Variation: On Ground Truth in Data, Modeling and Evaluation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.02570","snapshot_observed_at":"2026-08-12T15:58:47.600485Z","title":"The ’Problem’ of Human Label Variation: On Ground Truth in Data, Modeling and Evaluation, November 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.600485Z"},"links":{"cited_paper":"/paper/2211.02570","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:01d88b8cbf215c14e17b2bb739fd6578023302104d2d7adfca7ac9b20da88860","observation_id":"8c67e5e5-81c0-44dc-87a4-d46a433706d4","resolution":{"observed_at":"2026-08-12T15:58:47.600485Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2110.05699","last_updated":"2021-10-12T02:35:45Z","snapshot_observed_at":"2026-08-20T16:11:37.872473Z","submitted_at":"2021-10-12T02:35:45Z","title":"On Releasing Annotator-Level Labels and Information in Datasets","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2110.05699","snapshot_observed_at":"2026-08-12T15:58:47.605264Z","title":"On Releasing Annotator- Level Labels and Information in Datasets, October 2021","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.605264Z"},"links":{"cited_paper":"/paper/2110.05699","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:06df5d6438442288ccde610e2dc55e110572b500117da0c7ee45ede9edbdfba6","observation_id":"272987e6-d92a-44e6-a2a4-c316535e7c9a","resolution":{"observed_at":"2026-08-12T15:58:47.605264Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2106.01254","last_updated":"2026-04-22T19:47:51Z","snapshot_observed_at":"2026-07-06T11:15:17.300144Z","submitted_at":"2021-06-02T16:07:32Z","title":"Principled Evaluation with Human Labels: One Rater at a Time and Rater Equivalence","version":3},"cited_work":{"arxiv_id":"2106.01254","doi":null,"metadata_source":"pith","pith_arxiv_id":"2106.01254","snapshot_observed_at":"2026-08-12T15:58:47.865516Z","title":"Principled Evaluation with Human Labels: One Rater at a Time and Rater Equivalence","venue":"cs.LG","work_id":"b1e344dc-7442-45f9-bfbe-9bd477c4fa46","year":2021},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.610350Z"},"links":{"cited_paper":"/paper/2106.01254","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:68c8725a5776324e52fee491e0d0e97493b0d0d3c533d131c5314f1d63dc224b","observation_id":"f1e7394c-d3f0-43e8-b506-d192bc39d250","resolution":{"observed_at":"2026-08-12T15:58:47.871195Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2111.07997","last_updated":"2022-05-09T23:58:07Z","snapshot_observed_at":"2026-08-16T17:41:40.563990Z","submitted_at":"2021-11-15T18:58:20Z","title":"Annotators with Attitudes: How Annotator Beliefs And Identities Bias Toxic Language Detection","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2111.07997","snapshot_observed_at":"2026-08-12T15:58:47.615404Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.615404Z"},"links":{"cited_paper":"/paper/2111.07997","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:bb81907be020e02e2c832bb1855b9980a3d9172049fa53164ad07288193afa37","observation_id":"602c5eaf-4359-4f96-8eaa-e3f5f29ae945","resolution":{"observed_at":"2026-08-12T15:58:47.615404Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.12272","last_updated":"2024-04-18T15:45:27Z","snapshot_observed_at":"2026-08-20T21:06:18.098294Z","submitted_at":"2024-04-18T15:45:27Z","title":"Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.12272","snapshot_observed_at":"2026-08-12T15:58:47.620627Z","title":"Who validates the validators? aligning llm-assisted evaluation of llm outputs with human preferences","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.620627Z"},"links":{"cited_paper":"/paper/2404.12272","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:150c5b2374f4c9c2ec806529b06270924a467dac14b95d68f49af8da340d20aa","observation_id":"09088a7d-bbd7-4f5d-ac12-f8a906a710b3","resolution":{"observed_at":"2026-08-12T15:58:47.620627Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.625408Z","title":"A case for soft loss functions","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.625408Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:d8b691af51d78829765be4ba965387f8413bf6bdd55f037db69e79b205ccb9d1","observation_id":"83efe431-4f75-4d3f-a57a-9139d956405b","resolution":{"observed_at":"2026-08-12T15:58:47.625408Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1804.07461","last_updated":"2019-02-22T23:53:34Z","snapshot_observed_at":"2026-08-16T09:50:11.319379Z","submitted_at":"2018-04-20T06:35:04Z","title":"GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1804.07461","snapshot_observed_at":"2026-08-12T15:58:47.630144Z","title":"Glue: A multi-task benchmark and analysis platform for natural language understanding","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.630144Z"},"links":{"cited_paper":"/paper/1804.07461","citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:861b6fbaab03d67ad75142fd95d735fb1be61d007e50d8df853cd1dfe0019850","observation_id":"e5c52f95-88a5-46ce-9d23-698238ed1636","resolution":{"observed_at":"2026-08-12T15:58:47.630144Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.577470Z","title":"gold data","venue":null,"work_id":"626d2fbc-5cb6-4c10-ba16-e614c3ce753b","year":2024},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.634597Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:f99821445945be7fd3d0453b4edb3bd87fa1afc81793f3c6bc279471b96d0303","observation_id":"affa9a7a-e8f2-410a-a2c9-bcd63e415c00","resolution":{"observed_at":"2026-08-12T15:58:48.582824Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.638882Z","title":"Super- naturalinstructions:generalization via declarative instructions on 1600+ tasks","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.638882Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:4f30969cd61db14b60e502b40f64d6df99b966703b51c455a619b61608cbd02a","observation_id":"993a6c94-0d98-48a6-ac30-d1adeb6b00b7","resolution":{"observed_at":"2026-08-12T15:58:47.638882Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:47.643852Z","title":"Disagreement Matters: Preserving Label Diversity by Jointly Modeling Item and Anno- tator Label Distributions with DisCo","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.643852Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:ca699175f7854a2302dd15242ba3c2b47bcf88fbe11342b4148ef9755da30206","observation_id":"86907fa7-da6a-4695-adfa-7f5c27498499","resolution":{"observed_at":"2026-08-12T15:58:47.643852Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T15:58:48.551171Z","title":"Many islands, many problems: An empirical examination of online safety behaviors in the caribbean","venue":null,"work_id":"f4945d3c-e9e7-4803-acd3-954af7aa0cda","year":2022},"citing_paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-12T15:58:47.648449Z"},"links":{"citing_paper":"/paper/2411.13760"},"observation_digest":"sha256:88ab208994420df0e6df1b4ac69bd953955854424b3a41fc3119c2f536866bfb","observation_id":"35c8acda-de24-46ca-a0e7-f120e657f2f7","resolution":{"observed_at":"2026-08-12T15:58:48.555782Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2411.13760","last_updated":"2024-11-21T00:15:44Z","latest_version":1,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-18T19:00:41.370788Z","submitted_at":"2024-11-21T00:15:44Z","title":"A Framework for Evaluating LLMs Under Task Indeterminacy"},"reference_resolution":{"displayed":48,"state_counts":{"malformed_identifier":3,"metadata_mismatch":3,"parse_uncertain":0,"unresolved":24,"verified_exact":2,"verified_fuzzy":16},"total_outbound_references":48},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"thesis":"As of 23 August 2026, this Paper Citation Record lists 48 of 48 outbound references and 0 inbound Pith citation observations for arXiv:2411.13760."}