{"as_of":"2026-08-18T06:31:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:3efa4caf446bd0ec75491b22a1d0ebc55d1c1e3a9a6f6cff3c08d0b03a96beb9","coverage":[{"denominator":52,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":52,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-05T21:58:03.190450Z","state":"measured"},{"denominator":54,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":54,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-17T06:30:58.91139+00:00","state":"measured"},{"denominator":2,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":2,"source":"paper_references, paper_reference_links","source_observed_at":"2026-06-29T21:36:22.389047Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-06-29T21:43:59.674035Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"cited_work":{"arxiv_id":"2508.07827","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2508.07827","snapshot_observed_at":"2026-06-29T21:43:59.674035Z","title":"Evaluating large language models as expert annotators","venue":null,"work_id":"af83fa78-1993-4075-9f13-820b6784adf6","year":2025},"citing_paper":{"arxiv_id":"2605.03808","last_updated":"2026-05-05T14:35:47Z","snapshot_observed_at":"2026-08-15T04:58:20.105540Z","submitted_at":"2026-05-05T14:35:47Z","title":"Agentic-imodels: Evolving agentic interpretability tools via autoresearch","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-05-07T16:37:43.371592Z"},"links":{"cited_paper":"/paper/2508.07827","citing_paper":"/paper/2605.03808"},"observation_digest":"sha256:f282c11d028fdf1b1d79c3112bb895ba9014e4e250bb35cbca96ed23eebe3b6c","observation_id":"c8a8e6be-c643-4a74-8401-59d26806e229","resolution":{"observed_at":"2026-05-11T23:36:36.282145Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"cited_work":{"arxiv_id":"2508.07827","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2508.07827","snapshot_observed_at":"2026-06-29T21:43:59.674035Z","title":"Evaluating large language models as expert annotators","venue":null,"work_id":"af83fa78-1993-4075-9f13-820b6784adf6","year":2025},"citing_paper":{"arxiv_id":"2605.25781","last_updated":"2026-05-25T12:29:30Z","snapshot_observed_at":"2026-08-16T11:03:33.184803Z","submitted_at":"2026-05-25T12:29:30Z","title":"Double Triangle Annotation: A Scalable Human-in-the-Loop Framework for High-Precision Historical Document Annotation","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-06-29T21:36:22.389047Z"},"links":{"cited_paper":"/paper/2508.07827","citing_paper":"/paper/2605.25781"},"observation_digest":"sha256:159eaceb93da8175934b6e76732be8bdf08fd7958637a66e90879939bc313893","observation_id":"7de7f1b9-5adf-4e10-b065-2fa6ed303ab6","resolution":{"observed_at":"2026-06-29T21:43:59.676021Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2508.07827/citation-record","integrity":"/paper/2508.07827/integrity","json":"/paper/2508.07827/citation-record.json","paper":"/paper/2508.07827"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:57:58.492704Z","title":"write newline","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-05T21:57:58.492704Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:c53c2fc7ee606a9305e705dea08783f3b7c72045a33f48350658d20343745732","observation_id":"6b936667-020d-4e5e-a448-86eb1033055c","resolution":{"observed_at":"2026-08-05T21:57:58.492704Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-17T09:58:46.058102Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-08-05T21:57:58.634448Z","title":"Gpt-4 technical report","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-05T21:57:58.634448Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:8d41aba0a408ef76d529e399aa7990c74f6e7bee3c3532132eac0acd943693d5","observation_id":"b4c63d06-2427-4046-be84-810b6c61a8cf","resolution":{"observed_at":"2026-08-05T21:57:58.634448Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2307.02179","last_updated":"2024-05-29T12:29:08Z","snapshot_observed_at":"2026-08-16T15:18:36.767174Z","submitted_at":"2023-07-05T10:15:07Z","title":"Open-Source LLMs for Text Annotation: A Practical Guide for Model Setting and Fine-Tuning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.02179","snapshot_observed_at":"2026-08-05T21:57:58.829185Z","title":"Open-source large language models outperform crowd workers and approach chatgpt in text-annotation tasks","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-05T21:57:58.829185Z"},"links":{"cited_paper":"/paper/2307.02179","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:162b991707f1878919f4775a4258709695be7bb25449c50f1e59af3d519d0d59","observation_id":"1d58986c-7e14-4e38-8c82-ee71a1672364","resolution":{"observed_at":"2026-08-05T21:57:58.829185Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:57:58.988358Z","title":"The claude 3 model family: Opus, sonnet, haiku","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-05T21:57:58.988358Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:7249691921e330d1671eb3e030b18256b8d9d37e09a973eb3086fd780deb43ac","observation_id":"b0c71ad9-7577-4825-b493-a4d9c14aa089","resolution":{"observed_at":"2026-08-05T21:57:58.988358Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:05.433164Z","title":"Claude 3.7 sonnet and claude code","venue":null,"work_id":"9810002b-b8f7-499e-bc73-506d8d66fba2","year":2025},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-05T21:57:59.152154Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:e83384ca09e8fbae583ec8dee9bad04cde9a73d545a9568eb1e0c3dcdcd2a9f9","observation_id":"529cf28d-af15-4939-80ac-bf7d8d69c3d7","resolution":{"observed_at":"2026-08-05T21:58:05.436139Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2306.15766","last_updated":"2023-06-27T19:29:55Z","snapshot_observed_at":"2026-08-16T15:20:37.660751Z","submitted_at":"2023-06-27T19:29:55Z","title":"Large Language Models as Annotators: Enhancing Generalization of NLP Models at Minimal Cost","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.15766","snapshot_observed_at":"2026-08-05T21:57:59.306767Z","title":"Large language models as annotators: Enhancing generalization of nlp models at minimal cost","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-05T21:57:59.306767Z"},"links":{"cited_paper":"/paper/2306.15766","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:5fe1fdbe93da805c6268822bad7549618480d6334216181190e83560056b6587","observation_id":"fb27085f-bb3b-4086-a198-57b7f2157896","resolution":{"observed_at":"2026-08-05T21:57:59.306767Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:57:59.405114Z","title":"Must read: A systematic survey of computational persuasion","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-05T21:57:59.405114Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:d9de173e622b4f93f094f2de1ae3e8ebbdb2fc6c7dfe55997a26c69dab19207f","observation_id":"f06ed9c3-bca1-4c83-b6eb-db6b4708b7c8","resolution":{"observed_at":"2026-08-05T21:57:59.405114Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.08678","last_updated":"2023-10-12T19:28:57Z","snapshot_observed_at":"2026-08-16T14:52:33.254570Z","submitted_at":"2023-10-12T19:28:57Z","title":"Can GPT models be Financial Analysts? An Evaluation of ChatGPT and GPT-4 on mock CFA Exams","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.08678","snapshot_observed_at":"2026-08-05T21:57:59.514351Z","title":"Can gpt models be financial analysts? an evaluation of chatgpt and gpt-4 on mock cfa exams","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-05T21:57:59.514351Z"},"links":{"cited_paper":"/paper/2310.08678","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:66f21323cb8348278c2eb8cd42c7088a95128c4323a3ad2ee9aafcfc0766ec6b","observation_id":"bae1414c-9156-4b55-b678-7dd155b038f4","resolution":{"observed_at":"2026-08-05T21:57:59.514351Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.13007","last_updated":"2024-06-21T19:34:27Z","snapshot_observed_at":"2026-08-16T14:58:22.593686Z","submitted_at":"2023-09-22T17:12:45Z","title":"ReConcile: Round-Table Conference Improves Reasoning via Consensus among Diverse LLMs","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.13007","snapshot_observed_at":"2026-08-05T21:57:59.640281Z","title":"Reconcile: Round-table conference improves reasoning via consensus among diverse llms","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-05T21:57:59.640281Z"},"links":{"cited_paper":"/paper/2309.13007","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:dba2409734c84e80152a857b62f6a95982e7f5d2cfbf50268fb0d17ceb9b2d05","observation_id":"1a55427f-961a-4075-a876-0c55903314d1","resolution":{"observed_at":"2026-08-05T21:57:59.640281Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-08T11:58:24.516369Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-05T21:57:59.744177Z","title":"Evaluating large language models trained on code","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-05T21:57:59.744177Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:d57657861daa17ae09c0d41fb17c0852a86b53061b7b4241487542971c59a244","observation_id":"5ae76b19-00d9-4629-9bc3-94eaa1c256df","resolution":{"observed_at":"2026-08-05T21:57:59.744177Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:05.423850Z","title":null,"venue":null,"work_id":"8633ae51-e8f7-481a-b17e-80de51bd6b49","year":2023},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-05T21:57:59.875292Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:66997b2cf09f76c83ef1bad1a172f9bfbfd344c6f4f05de9ddab13caca3812e5","observation_id":"517ec177-3791-4aa5-8cf0-ef478a09746b","resolution":{"observed_at":"2026-08-05T21:58:05.427175Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:05.414743Z","title":"Chatgpt goes to law school","venue":null,"work_id":"9dcf0460-59da-4ffa-a362-039d28b3b656","year":2021},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-05T21:57:59.945208Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:2801031e420703c855c4b7378835190e3e2626d80260fc7056c8e3d90e872fe1","observation_id":"b4fce70c-4c5d-4064-8b83-7f70e29dbcd3","resolution":{"observed_at":"2026-08-05T21:58:05.417901Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.05512","last_updated":"2024-02-08T09:44:02Z","snapshot_observed_at":"2026-08-16T14:20:18.508384Z","submitted_at":"2024-02-08T09:44:02Z","title":"GPTs Are Multilingual Annotators for Sequence Generation Tasks","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.05512","snapshot_observed_at":"2026-08-05T21:58:00.004961Z","title":"Gpts are multilingual annotators for sequence generation tasks","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:00.004961Z"},"links":{"cited_paper":"/paper/2402.05512","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:698f50404165f1c1483f3b032a9131df3b253c9b781733cad2142cef932d2f99","observation_id":"6872d044-9693-4b34-a660-6621fcd6be4b","resolution":{"observed_at":"2026-08-05T21:58:00.004961Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:05.404862Z","title":"Is gpt-3 a good data annotator? In Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp.\\ 11173--11195, 2023","venue":null,"work_id":"1312fb37-943b-4df9-8d13-88f5411b1c3b","year":2023},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:00.061528Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:2f6bfaa3e12b650e8a60176599426f29250764d02faa9fc7bd9cee4d67b2ac65","observation_id":"f02c00c8-f313-47e0-b984-c949688251ec","resolution":{"observed_at":"2026-08-05T21:58:05.408117Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2305.14325","last_updated":"2023-05-23T17:55:11Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-05-23T17:55:11Z","title":"Improving Factuality and Reasoning in Language Models through Multiagent Debate","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.14325","snapshot_observed_at":"2026-08-05T21:58:00.167703Z","title":"Improving factuality and reasoning in language models through multiagent debate","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:00.167703Z"},"links":{"cited_paper":"/paper/2305.14325","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:075d56fcca25459a87bada410c8b57f2ac9960caf1940ace8e8c331b0a6e2be7","observation_id":"a7c5c73a-8d72-48e5-9517-8f6c6df3b268","resolution":{"observed_at":"2026-08-05T21:58:00.167703Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:00.256478Z","title":"Measuring the persuasiveness of language models, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:00.256478Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:e3fab751670dc04a839ee6010f8beaaac24782aad3be774ef857ff97ae138ec4","observation_id":"17132b9e-4e9c-4631-bf10-a58a9008b6d5","resolution":{"observed_at":"2026-08-05T21:58:00.256478Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:05.389267Z","title":"Measuring nominal scale agreement among many raters","venue":null,"work_id":"50c933ae-3063-4b78-bcee-1f23c79f3201","year":1971},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:00.323738Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:9ee3b9f90abfd7c347d162a88b0f34ab1e806179a07dd0ee0dd9cbfa1c56c35a","observation_id":"de7121dc-276f-4728-ab52-9717e23e1123","resolution":{"observed_at":"2026-08-05T21:58:05.392774Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:05.379793Z","title":"Chatgpt outperforms crowd workers for text-annotation tasks","venue":null,"work_id":"bc397644-3a94-4808-98d4-0681c8c5fb3b","year":2023},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:00.395214Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:c17592cf0b8df837739343dc110d5e9cf082a2c6909012aee4fd50b18175fea3","observation_id":"e9c6e715-aa56-4a68-95f0-99378478ba55","resolution":{"observed_at":"2026-08-05T21:58:05.383309Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:05.369780Z","title":"Introducing gemini 2.0: our new ai model for the agentic era","venue":null,"work_id":"a8a74f50-924a-480d-86e8-7113743cf246","year":2024},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:00.441674Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:ccfdf961320b47099e672fca5ef8277843e7b98417f1381bdac0617342cce010","observation_id":"56a7c1a1-538c-4428-95bd-b23114d5e49b","resolution":{"observed_at":"2026-08-05T21:58:05.373126Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:05.360329Z","title":"Legalbench: A collaboratively built benchmark for measuring legal reasoning in large language models","venue":null,"work_id":"66cbae20-5e2c-4176-9cdf-0a9e4e0690a6","year":2024},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:00.556464Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:84297be45474322a5fae4595be686ec2d47e375dea515f2ab7f4b94c82b92271","observation_id":"986d6739-07a7-47d7-b125-516761e6fd95","resolution":{"observed_at":"2026-08-05T21:58:05.363839Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.01680","last_updated":"2024-04-19T01:15:16Z","snapshot_observed_at":"2026-08-10T13:10:07.804621Z","submitted_at":"2024-01-21T23:36:14Z","title":"Large Language Model based Multi-Agents: A Survey of Progress and Challenges","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.01680","snapshot_observed_at":"2026-08-05T21:58:00.622649Z","title":"Large language model based multi-agents: A survey of progress and challenges","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:00.622649Z"},"links":{"cited_paper":"/paper/2402.01680","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:3d5ec97df447cef96763762dd362daa976c0f1be38024c979fff270cb970584a","observation_id":"b75d865b-3672-4cb4-b829-b76d31677352","resolution":{"observed_at":"2026-08-05T21:58:00.622649Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.16854","last_updated":"2024-04-05T15:19:19Z","snapshot_observed_at":"2026-08-16T15:44:19.759726Z","submitted_at":"2023-03-29T17:03:21Z","title":"AnnoLLM: Making Large Language Models to Be Better Crowdsourced Annotators","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.16854","snapshot_observed_at":"2026-08-05T21:58:00.717175Z","title":"Annollm: Making large language models to be better crowdsourced annotators","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:00.717175Z"},"links":{"cited_paper":"/paper/2303.16854","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:0acaaebb085fd2fb1495b7ed02abbe98de916a40a6b0b98ac8e58a35229b9fa5","observation_id":"b214e203-5069-48d5-a24d-1bd72469d499","resolution":{"observed_at":"2026-08-05T21:58:00.717175Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:00.790500Z","title":"Measuring massive multitask language understanding","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:00.790500Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:e3e1649f4bb82cab5ef7f778e6a3a914436086c5b540a038ff5998f08bef3a74","observation_id":"de5df38a-bde8-4855-9660-b973c203d59e","resolution":{"observed_at":"2026-08-05T21:58:00.790500Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.06268","last_updated":"2021-11-08T21:23:22Z","snapshot_observed_at":"2026-08-16T18:39:49.468256Z","submitted_at":"2021-03-10T18:59:34Z","title":"CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.06268","snapshot_observed_at":"2026-08-05T21:58:00.863174Z","title":"Cuad: An expert-annotated nlp dataset for legal contract review","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:00.863174Z"},"links":{"cited_paper":"/paper/2103.06268","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:8411971ba1df0592eb5b62eb7a9aee9452648fcc7271f79080149e236d1af3bb","observation_id":"4c4a376f-c68b-4e41-8805-cc10a7c880d8","resolution":{"observed_at":"2026-08-05T21:58:00.863174Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:05.287692Z","title":"Coda-19: Using a non-expert crowd to annotate research aspects on 10,000+ abstracts in the covid-19 open research dataset","venue":null,"work_id":"d20adb75-44ea-40f7-8d48-06b00da6433e","year":2020},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:00.938390Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:d095b0d4dd21e05eb517a4cc68fdbb371b7fd5a4afb1563c25c682589f46190d","observation_id":"4f533cea-5aac-4b70-9d42-58f50de4ed87","resolution":{"observed_at":"2026-08-05T21:58:05.347992Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:05.157255Z","title":"Pubmedqa: A dataset for biomedical research question answering","venue":null,"work_id":"4adabee4-0edc-425c-a42a-246e72b6884d","year":2019},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:01.028600Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:1b3a1610d815482db46199f0a45bef7a59d432b47335e402bb71688836a40440","observation_id":"fad2b1ea-507f-4d2e-a830-d3b9e44fa91f","resolution":{"observed_at":"2026-08-05T21:58:05.234433Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:05.036044Z","title":"Gpt-4 passes the bar exam","venue":null,"work_id":"32cb16e6-070b-4f56-80a2-f154371b97db","year":2024},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:01.127299Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:c34d7512cabd76ade13d203cf0496e40571a8ec78a0f6f8974247bc7286260b4","observation_id":"b92f1980-6828-457c-bc87-e8a9b0266c36","resolution":{"observed_at":"2026-08-05T21:58:05.085182Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:04.919622Z","title":"Refind: Relation extraction financial dataset","venue":null,"work_id":"bf36bde8-f86c-44fc-a6b8-afd41b18288f","year":2023},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:01.208891Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:86fae6b6f5354f435d47327f15869c3763c1b0eda26d4eb587bae0fdd248efd7","observation_id":"f50825cc-d97b-4807-baac-a61adcee6c86","resolution":{"observed_at":"2026-08-05T21:58:04.966746Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:01.287975Z","title":"Large language models are zero-shot reasoners","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:01.287975Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:95d911a7b5bcf2982602f85b8cfd4ff3248bb0ae27db1f04fcf2580907043e17","observation_id":"48a1eac5-5a42-4160-a50b-fd833586a3fb","resolution":{"observed_at":"2026-08-05T21:58:01.287975Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.19118","last_updated":"2024-10-09T02:41:21Z","snapshot_observed_at":"2026-08-15T13:46:19.286791Z","submitted_at":"2023-05-30T15:25:45Z","title":"Encouraging Divergent Thinking in Large Language Models through Multi-Agent Debate","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.19118","snapshot_observed_at":"2026-08-05T21:58:01.346989Z","title":"Encouraging divergent thinking in large language models through multi-agent debate","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":30,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:01.346989Z"},"links":{"cited_paper":"/paper/2305.19118","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:853f2159a285621035b3f7887e31a6be46a4be6bd3f7241cdf9e1dd1d1f7c13b","observation_id":"75c933e7-86b6-4513-8c8e-6d0ebec54055","resolution":{"observed_at":"2026-08-05T21:58:01.346989Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:01.425203Z","title":"Self-refine: Iterative refinement with self-feedback","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:01.425203Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:2251c4cd7fe8263a9638cbc14f4bc8f4fdd2e311ff46e4189c52eca440aa696a","observation_id":"c6e66b29-847b-4193-bdea-3249e44aa8c9","resolution":{"observed_at":"2026-08-05T21:58:01.425203Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:01.502147Z","title":"Note on the sampling error of the difference between correlated proportions or percentages","venue":null,"work_id":null,"year":1947},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:01.502147Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:165e2e4dc23715bb15d498134476f9ba2ee54be78d8afc0a5c75d845d5b7ec84","observation_id":"ac125cd0-dd4a-4c59-9341-46d9049aed29","resolution":{"observed_at":"2026-08-05T21:58:01.502147Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:04.745134Z","title":"Hello gpt4-o","venue":null,"work_id":"22eef432-8f23-4019-84ad-631a7484fa12","year":2024},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:01.606341Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:e4c0749e91e240fc906b79025680eb6066c671ef2928d7d30f74d36f83689fe6","observation_id":"015ce650-1ee1-475e-ab60-f723fe4f1b30","resolution":{"observed_at":"2026-08-05T21:58:04.825842Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:04.635900Z","title":"Openai o3-mini","venue":null,"work_id":"deba5ada-4b4e-46a3-a7dd-7406830ec0e4","year":2025},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:01.663633Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:be2a7349d9970b9e0b4f2c5ecc916ac76fb2dec8579362675eb574ee9f40b39d","observation_id":"6380658b-753d-4b7e-a4a2-483069c51e97","resolution":{"observed_at":"2026-08-05T21:58:04.688878Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.05530","last_updated":"2024-12-16T17:39:39Z","snapshot_observed_at":"2026-08-14T18:15:53.516440Z","submitted_at":"2024-03-08T18:54:20Z","title":"Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.05530","snapshot_observed_at":"2026-08-05T21:58:01.771395Z","title":"Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:01.771395Z"},"links":{"cited_paper":"/paper/2403.05530","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:1ab3e7fd42d35c509f98d37ea3a420c5498966ffd9ac73ff3954bf065b460b78","observation_id":"bd27a646-139c-455a-a90f-9e1b16a8886b","resolution":{"observed_at":"2026-08-05T21:58:01.771395Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.12022","last_updated":"2023-11-20T18:57:34Z","snapshot_observed_at":"2026-08-16T07:06:07.822225Z","submitted_at":"2023-11-20T18:57:34Z","title":"GPQA: A Graduate-Level Google-Proof Q&A Benchmark","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.12022","snapshot_observed_at":"2026-08-05T21:58:01.848446Z","title":"Gpqa: A graduate-level google-proof q&a benchmark","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":36,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:01.848446Z"},"links":{"cited_paper":"/paper/2311.12022","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:6bed5879953424ad5c944be104f6d7e3a66e66dd1783c9ad35e8b61e938b1a16","observation_id":"39af5a28-0b05-4d68-a5d0-524cab284565","resolution":{"observed_at":"2026-08-05T21:58:01.848446Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:04.473945Z","title":"Trillion dollar words: A new financial dataset, task & market analysis","venue":null,"work_id":"83ab845f-5c95-46bb-a1e5-4f81c09f7659","year":2023},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":37,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:01.919674Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:85932c888981c840510bea5bd5de1ac79511b5fe179d4772358b37415ad2d916","observation_id":"5f4f6826-db1a-4fe3-bf6e-c0ffabb20674","resolution":{"observed_at":"2026-08-05T21:58:04.560419Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:04.299670Z","title":"Large language models encode clinical knowledge","venue":null,"work_id":"6d73ee99-0fad-4c1b-acee-f57bc1f64ff9","year":2023},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":38,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:02.006829Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:e8324a54013f2189d7bfb83778cdfd670dc1ee9f9675a9d042953fc2cba71617","observation_id":"8d3760b4-75fb-4393-b1ec-f8a4ad59da4e","resolution":{"observed_at":"2026-08-05T21:58:04.399509Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2305.09617","last_updated":"2023-05-16T17:11:29Z","snapshot_observed_at":"2026-08-16T14:30:19.311365Z","submitted_at":"2023-05-16T17:11:29Z","title":"Towards Expert-Level Medical Question Answering with Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.09617","snapshot_observed_at":"2026-08-05T21:58:02.067890Z","title":"Towards expert-level medical question answering with large language models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":39,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:02.067890Z"},"links":{"cited_paper":"/paper/2305.09617","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:fc37949d88cd994a6a6dd95249cda66e14d8d12171c75036550b64a8b826a987","observation_id":"e6f28250-193d-4583-8c0e-52a1c449e99c","resolution":{"observed_at":"2026-08-05T21:58:02.067890Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.13446","last_updated":"2024-12-02T20:55:15Z","snapshot_observed_at":"2026-08-16T14:16:46.810700Z","submitted_at":"2024-02-21T00:44:04Z","title":"Large Language Models for Data Annotation and Synthesis: A Survey","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.13446","snapshot_observed_at":"2026-08-05T21:58:02.162770Z","title":"Large language models for data annotation: A survey","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":40,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:02.162770Z"},"links":{"cited_paper":"/paper/2402.13446","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:d26380079ac2d587c913783fce01d945be9ec36c209e5f73e152a079792a5131","observation_id":"b23d2c8a-924e-4b8e-b24b-ea4a1e70e351","resolution":{"observed_at":"2026-08-05T21:58:02.162770Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.03254","last_updated":"2024-10-04T09:17:09Z","snapshot_observed_at":"2026-08-16T13:12:39.850936Z","submitted_at":"2024-10-04T09:17:09Z","title":"Are Expert-Level Language Models Expert-Level Annotators?","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.03254","snapshot_observed_at":"2026-08-05T21:58:02.244277Z","title":"Are expert-level language models expert-level annotators? arXiv preprint arXiv:2410.03254, 2024 a","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":41,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:02.244277Z"},"links":{"cited_paper":"/paper/2410.03254","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:b13f85ea2df92918760dcedd40f94ed976d3bee7e13001e3a043fe76c5abeb35","observation_id":"ae8488ec-0c4b-4283-b3ae-f7adc6aeb2bd","resolution":{"observed_at":"2026-08-05T21:58:02.244277Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.01171","last_updated":"2024-10-05T04:29:12Z","snapshot_observed_at":"2026-08-16T13:46:48.971030Z","submitted_at":"2024-06-03T10:08:23Z","title":"Two Tales of Persona in LLMs: A Survey of Role-Playing and Personalization","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.01171","snapshot_observed_at":"2026-08-05T21:58:02.325509Z","title":"Two tales of persona in llms: A survey of role-playing and personalization","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":42,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:02.325509Z"},"links":{"cited_paper":"/paper/2406.01171","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:1925e567fdfa7db46ed72e00d23b45b5a62d3799e150e877b380383b6ed47318","observation_id":"6bf80663-a3d0-4615-ab02-2226a7e484fe","resolution":{"observed_at":"2026-08-05T21:58:02.325509Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:04.092823Z","title":"Foundational autoraters: Taming large language models for better automatic evaluation","venue":null,"work_id":"e52c515b-f29f-4822-8313-23264b823efb","year":2024},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":43,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:02.380263Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:08d611288aa73447fe5b36dd6cdd88fbc71f19cd0f85228efcda93f351f543b4","observation_id":"666ab889-8119-4572-9afa-ea7a8261fe26","resolution":{"observed_at":"2026-08-05T21:58:04.187402Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:03.866184Z","title":"Cord-19: The covid-19 open research dataset","venue":null,"work_id":"e4efb023-95bc-429b-937d-38f0b3c79eef","year":2020},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":44,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:02.438362Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:930bc0885a6c0addf822d2edd4fbfe3689aecf6007765e4d946f3db5961fc356","observation_id":"dab46878-2ba9-4c4a-a4cc-256cdbdafad0","resolution":{"observed_at":"2026-08-05T21:58:03.971639Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:02.534703Z","title":"Self-consistency improves chain of thought reasoning in language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":45,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:02.534703Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:2a26f5049d00fbc93c396d8c637570f2f04696a47633cbf05792e1a0f4688716","observation_id":"1d12a2d4-d2c7-4b0b-ba50-5b488092f484","resolution":{"observed_at":"2026-08-05T21:58:02.534703Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:02.595024Z","title":"Chain-of-thought prompting elicits reasoning in large language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":46,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:02.595024Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:63db9d8ff66aed91b404afd16484cae8b2185f9f59d78e140404ed630fb6da44","observation_id":"70c0cfab-0624-4c8e-896e-26cd8cd03869","resolution":{"observed_at":"2026-08-05T21:58:02.595024Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:03.725809Z","title":"The rise and potential of large language model based agents: A survey","venue":null,"work_id":"1ce0e3c8-e085-47dd-beab-19144659abdb","year":2025},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":47,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:02.612999Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:3b7f380dabfed26ddcd31805d9ff6fc1945c713f49f9d9a9b6a74651cdae35ae","observation_id":"e80120f2-9e6a-4fc6-a9ad-4e0b6f294264","resolution":{"observed_at":"2026-08-05T21:58:03.823825Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-17T06:30:58.91139+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2310.19596","last_updated":"2023-10-31T08:19:52Z","snapshot_observed_at":"2026-08-16T14:47:37.092167Z","submitted_at":"2023-10-30T14:54:15Z","title":"LLMaAA: Making Large Language Models as Active Annotators","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.19596","snapshot_observed_at":"2026-08-05T21:58:02.659226Z","title":"Llmaaa: Making large language models as active annotators","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":48,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:02.659226Z"},"links":{"cited_paper":"/paper/2310.19596","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:b6fa20e0c5bfdf7992ca6cc7d7fd4d836d3facab7ccfb00e1b54407807727551","observation_id":"f22bc1e0-d7fa-4bd8-9ba3-941f36e36b57","resolution":{"observed_at":"2026-08-05T21:58:02.659226Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2304.10145","last_updated":"2023-04-22T08:55:33Z","snapshot_observed_at":"2026-08-17T14:45:26.211282Z","submitted_at":"2023-04-20T08:08:12Z","title":"Can ChatGPT Reproduce Human-Generated Labels? A Study of Social Computing Tasks","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2304.10145","snapshot_observed_at":"2026-08-05T21:58:02.738693Z","title":"Can chatgpt reproduce human-generated labels? a study of social computing tasks","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":49,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:02.738693Z"},"links":{"cited_paper":"/paper/2304.10145","citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:514d5487aae34b4b0706f9ac1c82a80acdc3a204af295ee486de0675757355db","observation_id":"687ed8e3-6ff3-4836-bc6f-f9702943e27c","resolution":{"observed_at":"2026-08-05T21:58:02.738693Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:02.878965Z","title":"@esa (Ref","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":50,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:02.878965Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:0f77e503ae1f47d7f0caeed53c204d45c1aa35f41d01b120a9cf2ea3faa705f0","observation_id":"cc5ebd11-3889-42b7-a92d-ea7c3b9c517f","resolution":{"observed_at":"2026-08-05T21:58:02.878965Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:03.021696Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":51,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:03.021696Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:fb17acd1672513638e066ecf7ebe97a12fec12b85ec0ddadf44f075b06ea6ec4","observation_id":"97540c71-885f-451f-a6d0-8d9139f62244","resolution":{"observed_at":"2026-08-05T21:58:03.021696Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T21:58:03.190450Z","title":"By leveraging additional inference-time compute, we explore whether individual LLMs can serve as a direct alternative to expert data annotators","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators","version":1},"reference_index":52,"source":"arxiv_source","source_observed_at":"2026-08-05T21:58:03.190450Z"},"links":{"citing_paper":"/paper/2508.07827"},"observation_digest":"sha256:3b9670a78126af00e582fe0042dc54444b629c241a6695c9f30574a1ea075872","observation_id":"5e82cdd5-a4b0-4c2c-8998-871c749e6f46","resolution":{"observed_at":"2026-08-05T21:58:03.190450Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2508.07827","last_updated":"2025-08-11T10:19:10Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-13T07:59:45.021221Z","submitted_at":"2025-08-11T10:19:10Z","title":"Evaluating Large Language Models as Expert Annotators"},"reference_resolution":{"displayed":52,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":34,"verified_exact":0,"verified_fuzzy":18},"total_outbound_references":52},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-17T06:30:58.91139+00:00","source":"crossref"},{"observed_at":"2026-08-17T06:30:54.323127+00:00","source":"retraction_watch"}],"thesis":"As of 18 August 2026, this Paper Citation Record lists 52 of 52 outbound references and 2 inbound Pith citation observations for arXiv:2508.07827."}