{"as_of":"2026-08-09T17:45:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:b71c6cf6f1104d4edf79e3430fad0aa70f127db84cba6d34ad4cef186f6d462d","coverage":[{"denominator":21,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":21,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-04T20:11:41.723677Z","state":"measured"},{"denominator":21,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":21,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-09T06:31:02.800959+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2509.08809/citation-record","integrity":"/paper/2509.08809/integrity","json":"/paper/2509.08809/citation-record.json","paper":"/paper/2509.08809"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T20:11:41.593705Z","title":"Language models are few-shot learners.Advances in neural information processing systems, 33:1877–1901,","venue":null,"work_id":null,"year":1901},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.593705Z"},"links":{"citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:3d817edde67311db79fa380e338f2029663e0d4417030b390f3205787d850684","observation_id":"a1155b71-e107-4bbe-aeae-b27a94747926","resolution":{"observed_at":"2026-08-04T20:11:41.593705Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T20:11:41.606910Z","title":"Self-teaching prompting for multi-intent learning with limited super- vision","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.606910Z"},"links":{"citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:3933b6aa934f91c244f64eb8a267458a0c999c63d91745c0181e27ff3576bffc","observation_id":"780f3e7d-73e0-4609-81b2-53a1f4c6ece5","resolution":{"observed_at":"2026-08-04T20:11:41.606910Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1909.02027","last_updated":"2019-09-04T18:04:56Z","snapshot_observed_at":"2026-08-04T07:50:24.317079Z","submitted_at":"2019-09-04T18:04:56Z","title":"An Evaluation Dataset for Intent Classification and Out-of-Scope Prediction","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1909.02027","snapshot_observed_at":"2026-08-04T20:11:41.630126Z","title":"An evaluation dataset for intent classification and out-of-scope prediction.arXiv preprint arXiv:1909.02027,","venue":null,"work_id":null,"year":1909},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.630126Z"},"links":{"cited_paper":"/paper/1909.02027","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:e3ab785437f04a50e9a8ad97824a706402e7ef35c693cddba8e5d4fbd302eb4b","observation_id":"fbd2193e-118d-4d9c-bd11-fc22c1d5cc00","resolution":{"observed_at":"2026-08-04T20:11:41.630126Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.08291","last_updated":"2023-05-15T01:18:23Z","snapshot_observed_at":"2026-07-06T15:27:02.376191Z","submitted_at":"2023-05-15T01:18:23Z","title":"Large Language Model Guided Tree-of-Thought","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.08291","snapshot_observed_at":"2026-08-04T20:11:41.647592Z","title":"Large language model guided tree-of-thought.arXiv preprint arXiv:2305.08291,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.647592Z"},"links":{"cited_paper":"/paper/2305.08291","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:8757f7191b55726291067f64957ccd113eaebbcb4afdb455d8ebc10e753c1bfd","observation_id":"ec452bde-1b91-4f30-835a-199a972eb0e2","resolution":{"observed_at":"2026-08-04T20:11:41.647592Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.13446","last_updated":"2024-12-02T20:55:15Z","snapshot_observed_at":"2026-08-09T02:48:32.516752Z","submitted_at":"2024-02-21T00:44:04Z","title":"Large Language Models for Data Annotation and Synthesis: A Survey","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.13446","snapshot_observed_at":"2026-08-04T20:11:41.653118Z","title":"Large language models for data annotation: A survey.arXiv preprint arXiv:2402.13446,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.653118Z"},"links":{"cited_paper":"/paper/2402.13446","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:f1a248c3b118e43bc63f59fcc599992ac9ed1e4ef69d0a6e11401f0b60242aab","observation_id":"92527c59-d016-460c-89fa-24f1b07f8594","resolution":{"observed_at":"2026-08-04T20:11:41.653118Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.05875","last_updated":"2024-04-08T21:15:36Z","snapshot_observed_at":"2026-08-07T12:08:06.802905Z","submitted_at":"2024-04-08T21:15:36Z","title":"CodecLM: Aligning Language Models with Tailored Synthetic Data","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.05875","snapshot_observed_at":"2026-08-04T20:11:41.660463Z","title":"Codeclm: Aligning language models with tailored synthetic data.arXiv preprint arXiv:2404.05875,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.660463Z"},"links":{"cited_paper":"/paper/2404.05875","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:f3f43fab3ef66e497655ef0aedc271a6ed1b4d8d2d580ff3b7904b256af50349","observation_id":"8b125de1-3aa7-49cb-8222-c159da1a5b8f","resolution":{"observed_at":"2026-08-04T20:11:41.660463Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2109.01652","last_updated":"2022-02-08T20:26:45Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2021-09-03T17:55:52Z","title":"Finetuned Language Models Are Zero-Shot Learners","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2109.01652","snapshot_observed_at":"2026-08-04T20:11:41.668846Z","title":"Finetuned language models are zero-shot learners.arXiv preprint arXiv:2109.01652,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.668846Z"},"links":{"cited_paper":"/paper/2109.01652","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:323e2dbc5a51ba3229a7d2ac7a7f0e087315cb9ace467ac353600a3a137a447b","observation_id":"1f8d1606-cbe6-497b-9bbe-dbc658520440","resolution":{"observed_at":"2026-08-04T20:11:41.668846Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T20:11:41.676658Z","title":"Unigen: A unified framework for textual dataset generation using large language models.arXiv preprint arXiv:2406.18966,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.676658Z"},"links":{"citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:6a30069c6030edd39750b48d31439c25ff288ab58798d16ba079b892a94ab931","observation_id":"eaf3db79-79cd-464d-983f-d6e95b491a94","resolution":{"observed_at":"2026-08-04T20:11:41.676658Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2306.13063","last_updated":"2024-03-17T04:38:48Z","snapshot_observed_at":"2026-07-06T15:45:39.649725Z","submitted_at":"2023-06-22T17:31:44Z","title":"Can LLMs Express Their Uncertainty? An Empirical Evaluation of Confidence Elicitation in LLMs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.13063","snapshot_observed_at":"2026-08-04T20:11:41.683557Z","title":"Can llms express their uncertainty? an empirical evaluation of confidence elicitation in llms.arXiv preprint arXiv:2306.13063,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.683557Z"},"links":{"cited_paper":"/paper/2306.13063","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:84302fbec74c1e066b32266b6aebe0a84f90f6ee354831bd6b6dba4e358e2d51","observation_id":"2e045291-eb2a-472f-b5bc-c7a70f28a214","resolution":{"observed_at":"2026-08-04T20:11:41.683557Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2210.03629","last_updated":"2023-03-10T01:00:17Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-10-06T01:00:32Z","title":"ReAct: Synergizing Reasoning and Acting in Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2210.03629","snapshot_observed_at":"2026-08-04T20:11:41.691968Z","title":"React: Synergizing reasoning and acting in language models.arXiv preprint arXiv:2210.03629,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.691968Z"},"links":{"cited_paper":"/paper/2210.03629","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:13727f2b3bbd44a8ebed863137b9268173c6a4e3501de0a58a391cabf2c28491","observation_id":"338a3504-808b-4418-8baf-c4d8a489ba21","resolution":{"observed_at":"2026-08-04T20:11:41.691968Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2202.07922","last_updated":"2022-10-22T01:32:03Z","snapshot_observed_at":"2026-08-09T02:50:43.323418Z","submitted_at":"2022-02-16T08:18:02Z","title":"ZeroGen: Efficient Zero-shot Learning via Dataset Generation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2202.07922","snapshot_observed_at":"2026-08-04T20:11:41.698533Z","title":"Zerogen: Efficient zero-shot learning via dataset generation.arXiv preprint arXiv:2202.07922,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.698533Z"},"links":{"cited_paper":"/paper/2202.07922","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:594126a118aea9661c2c1a839f5f05be98fdfe75a4510d1b370ebbdc808bd76a","observation_id":"30e50ffc-9452-48bb-ac5d-c14f7fac065b","resolution":{"observed_at":"2026-08-04T20:11:41.698533Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T20:11:41.711658Z","title":"Clusterllm: Large language models as a guide for text clustering","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.711658Z"},"links":{"citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:47a5fdb9242185a2867c08ea3ddd064fbee263495241b4536293c58f59fef8d9","observation_id":"07457aea-99d2-4fe8-a7a1-b60ed49e0470","resolution":{"observed_at":"2026-08-04T20:11:41.711658Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06730","last_updated":"2024-07-09T23:53:06Z","snapshot_observed_at":"2026-07-06T17:14:53.706063Z","submitted_at":"2024-01-12T18:03:30Z","title":"Relying on the Unreliable: The Impact of Language Models' Reluctance to Express Uncertainty","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.06730","snapshot_observed_at":"2026-08-04T20:11:41.717627Z","title":"Relying on the unreliable: The impact of language models’ reluctance to express uncertainty.arXiv preprint arXiv:2401.06730,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.717627Z"},"links":{"cited_paper":"/paper/2401.06730","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:e4fb97f39a18141bededa5069466134881971bde20fa2dec6cfb759006e24064","observation_id":"e8ead15a-d86e-4033-a881-23c70738845c","resolution":{"observed_at":"2026-08-04T20:11:41.717627Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T20:11:41.723677Z","title":"yi denotes the LLM annotation accuracies","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.723677Z"},"links":{"citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:d32a323a405f88eb695f1c6b9e7377cb8eeeae719b86a6d1515eac12b3286a31","observation_id":"7c32b9ed-be89-420c-b367-3de21fab166e","resolution":{"observed_at":"2026-08-04T20:11:41.723677Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2210.11610","last_updated":"2022-10-25T17:45:17Z","snapshot_observed_at":"2026-08-09T02:27:34.152063Z","submitted_at":"2022-10-20T21:53:54Z","title":"Large Language Models Can Self-Improve","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2210.11610","snapshot_observed_at":"2026-08-04T20:11:41.623904Z","title":"Large language models can self-improve.arXiv preprint arXiv:2210.11610,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":2018,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.623904Z"},"links":{"cited_paper":"/paper/2210.11610","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:2178b932b6ab3e25e66e453c0d1f517414cb4a0719281b93f85aac170537bf0e","observation_id":"70bfafd6-52d9-4e8e-a797-b91e80301d73","resolution":{"observed_at":"2026-08-04T20:11:41.623904Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2008.09335","last_updated":"2021-01-27T03:36:21Z","snapshot_observed_at":"2026-07-06T09:49:00.420420Z","submitted_at":"2020-08-21T07:02:11Z","title":"MTOP: A Comprehensive Multilingual Task-Oriented Semantic Parsing Benchmark","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2008.09335","snapshot_observed_at":"2026-08-04T20:11:41.635830Z","title":"Mtop: A comprehensive multilingual task-oriented semantic parsing benchmark.arXiv preprint arXiv:2008.09335,","venue":null,"work_id":null,"year":2008},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":2019,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.635830Z"},"links":{"cited_paper":"/paper/2008.09335","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:195fc7a96259bd1a1c98e4e8a6420da835881b4edece7a80971af161535ef4f5","observation_id":"8ddb5ba5-2b9b-4000-8a18-fbe2b55a60cb","resolution":{"observed_at":"2026-08-04T20:11:41.635830Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2003.04807","last_updated":"2020-03-10T15:33:54Z","snapshot_observed_at":"2026-08-06T01:22:03.305472Z","submitted_at":"2020-03-10T15:33:54Z","title":"Efficient Intent Detection with Dual Sentence Encoders","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2003.04807","snapshot_observed_at":"2026-08-04T20:11:41.601157Z","title":"Efficient intent detection with dual sentence encoders.arXiv preprint arXiv:2003.04807,","venue":null,"work_id":null,"year":2003},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":2020,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.601157Z"},"links":{"cited_paper":"/paper/2003.04807","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:18c28f51cea3822d80b90e9586e635eb881303a1cdc4fb7cc6e3ae80f6df7444","observation_id":"7c8b7f60-0c63-4701-888c-dcdcc7e57049","resolution":{"observed_at":"2026-08-04T20:11:41.601157Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2205.12914","last_updated":"2025-04-06T23:25:50Z","snapshot_observed_at":"2026-08-01T16:37:38.483173Z","submitted_at":"2022-05-25T17:07:25Z","title":"New Intent Discovery with Pre-training and Contrastive Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2205.12914","snapshot_observed_at":"2026-08-04T20:11:41.704280Z","title":"New intent discovery with pre-training and contrastive learning.arXiv preprint arXiv:2205.12914,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":2021,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.704280Z"},"links":{"cited_paper":"/paper/2205.12914","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:2d9616b0e9ea782f615df6b2ea1fc11a6e0c3c54d34ec0e1c9438db426a70127","observation_id":"4ace878f-ff85-441e-ae45-d2f847d49ad1","resolution":{"observed_at":"2026-08-04T20:11:41.704280Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2104.07081","last_updated":"2021-09-16T10:52:59Z","snapshot_observed_at":"2026-08-09T01:13:21.874425Z","submitted_at":"2021-04-14T19:06:11Z","title":"TWEAC: Transformer with Extendable QA Agent Classifiers","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2104.07081","snapshot_observed_at":"2026-08-04T20:11:41.612448Z","title":"Tweac: transformer with extendable qa agent classifiers.arXiv preprint arXiv:2104.07081,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":2022,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.612448Z"},"links":{"cited_paper":"/paper/2104.07081","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:25bf8b1a5cf065187c5fa34631c4fea9b5e3d35991801b6b558b0c7b28dfca88","observation_id":"71e14911-340e-4b2c-bdec-063d2428ef9e","resolution":{"observed_at":"2026-08-04T20:11:41.612448Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1810.10147","last_updated":"2018-10-26T21:41:46Z","snapshot_observed_at":"2026-07-06T07:10:12.570041Z","submitted_at":"2018-10-24T01:18:08Z","title":"FewRel: A Large-Scale Supervised Few-Shot Relation Classification Dataset with State-of-the-Art Evaluation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1810.10147","snapshot_observed_at":"2026-08-04T20:11:41.618338Z","title":"Fewrel: A large-scale supervised few-shot relation classification dataset with state-of-the-art evaluation","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":2023,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.618338Z"},"links":{"cited_paper":"/paper/1810.10147","citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:65562645dd842e284cf73043ec9188664ba2dda27392a6d2a6c57c387efee6b2","observation_id":"8d39323f-e64d-429f-abf4-1048cb2d4f7d","resolution":{"observed_at":"2026-08-04T20:11:41.618338Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-04T20:11:41.641752Z","title":"Graphprompt: Unifying pre-training and downstream tasks for graph neural networks","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals","version":1},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-04T20:11:41.641752Z"},"links":{"citing_paper":"/paper/2509.08809"},"observation_digest":"sha256:9229be223181a5a5f358bd268ab1f238b2ca657bf467123d70ff0d45406c40ea","observation_id":"b97cea59-8b75-4c4d-b098-1dd48215103d","resolution":{"observed_at":"2026-08-04T20:11:41.641752Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2509.08809","last_updated":"2025-09-10T17:42:41Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-09T14:35:58.129278Z","submitted_at":"2025-09-10T17:42:41Z","title":"Evaluating LLMs Without Oracle Feedback: Agentic Annotation Evaluation Through Unsupervised Consistency Signals"},"reference_resolution":{"displayed":21,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":21,"verified_exact":0,"verified_fuzzy":0},"total_outbound_references":21},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"thesis":"As of 9 August 2026, this Paper Citation Record lists 21 of 21 outbound references and 0 inbound Pith citation observations for arXiv:2509.08809."}