{"as_of":"2026-08-21T14:46:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:dc89b06472bbc19bc759b94e471e0d0b84e2ad8b61af8f2a9f23e1efdf69efd2","coverage":[{"denominator":35,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":35,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-07T04:34:13.658964Z","state":"measured"},{"denominator":38,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":38,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-21T06:32:19.484+00:00","state":"measured"},{"denominator":3,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":3,"source":"paper_references, paper_reference_links","source_observed_at":"2026-07-30T15:14:05.529138Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":1,"source":"arxiv_reference","source_observed_at":"2026-08-05T02:28:24.338817Z","state":"measured"}],"external_citation_measurements":[{"count":0,"observed_at":"2026-08-05T02:28:24.338817Z","source":"arxiv_reference"}],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"cited_work":{"arxiv_id":"2506.10403","doi":"10.48550/arxiv.2506.10403","metadata_source":"arxiv_reference","pith_arxiv_id":"2506.10403","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Time to impeach LLM -as-a-judge: Programs are the future of evaluation","venue":"ArXiv.org","work_id":"2cf4719c-2fb2-4b3b-aefd-e074a3157869","year":2025},"citing_paper":{"arxiv_id":"2604.04532","last_updated":"2026-07-02T12:30:40Z","snapshot_observed_at":"2026-08-08T10:17:25.391274Z","submitted_at":"2026-04-06T08:54:16Z","title":"Multilingual Prompt Localization for Agent-as-a-Judge: Language and Backbone Sensitivity in Requirement-Level Evaluation","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-05-10T20:02:40.980538Z"},"links":{"cited_paper":"/paper/2506.10403","citing_paper":"/paper/2604.04532"},"observation_digest":"sha256:c092650a341932220d07be6df28ababe37287c6f90391226897e3d8c47273617","observation_id":"0f017e17-0894-4340-9915-1d2ef361703d","resolution":{"observed_at":"2026-05-10T22:15:50.709981Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"cited_work":{"arxiv_id":"2506.10403","doi":"10.48550/arxiv.2506.10403","metadata_source":"arxiv_reference","pith_arxiv_id":"2506.10403","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Time to impeach LLM -as-a-judge: Programs are the future of evaluation","venue":"ArXiv.org","work_id":"2cf4719c-2fb2-4b3b-aefd-e074a3157869","year":2025},"citing_paper":{"arxiv_id":"2604.05083","last_updated":"2026-04-06T18:36:54Z","snapshot_observed_at":"2026-08-08T12:41:02.824259Z","submitted_at":"2026-04-06T18:36:54Z","title":"Beyond LLM-as-a-Judge: Deterministic Metrics for Multilingual Generative Text Evaluation","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-05-10T19:15:37.744461Z"},"links":{"cited_paper":"/paper/2506.10403","citing_paper":"/paper/2604.05083"},"observation_digest":"sha256:6b1e9b811bc6ed1246aac2830c80a2a898ab782be53263d5187d0ff92523f32f","observation_id":"286f4207-5d0a-40c4-b6ab-05c718a80828","resolution":{"observed_at":"2026-05-10T19:15:44.221307Z","resolver_source":"arxiv_id","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2506.10403","snapshot_observed_at":"2026-07-30T15:14:05.529138Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.26977","last_updated":"2026-08-10T02:07:10Z","snapshot_observed_at":"2026-08-13T23:29:21.404865Z","submitted_at":"2026-07-29T14:35:29Z","title":"TREK: A Travel Reasoning and Evaluation Kit for LLM Agents in Complex Trip Planning","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-07-30T15:14:05.529138Z"},"links":{"cited_paper":"/paper/2506.10403","citing_paper":"/paper/2607.26977"},"observation_digest":"sha256:0d13c1cadf219756a0b0a491eae741b731d6c8ed5ca5ab34e7278ef007767e58","observation_id":"b7d92186-e247-4ba5-ae91-d64872b60041","resolution":{"observed_at":"2026-07-30T15:14:05.529138Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2506.10403/citation-record","integrity":"/paper/2506.10403/integrity","json":"/paper/2506.10403/citation-record.json","paper":"/paper/2506.10403"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2408.13006","last_updated":"2025-03-30T17:59:47Z","snapshot_observed_at":"2026-08-18T13:17:17.701561Z","submitted_at":"2024-08-23T11:49:01Z","title":"Systematic Evaluation of LLM-as-a-Judge in LLM Alignment Tasks: Explainable Metrics and Diverse Prompt Templates","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.13006","snapshot_observed_at":"2026-08-07T04:34:08.584381Z","title":"Systematic evaluation of llm-as-a-judge in llm alignment tasks: Explainable metrics and diverse prompt templates","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:08.584381Z"},"links":{"cited_paper":"/paper/2408.13006","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:089f150637313545b1abdfcddaa60f0396a4064e080bd79a6a9d939eddf2b79a","observation_id":"ef315790-9b74-4f37-80c1-6a150431efaa","resolution":{"observed_at":"2026-08-07T04:34:08.584381Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:18.770228Z","title":"N.; Li, T.; Li, D.; Zhu, B.; Zhang, H.; Jordan, M.; 5 Gonzalez, J","venue":null,"work_id":"118a7b46-a156-4f25-8c2c-926017753486","year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:08.700690Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:01b3d990b6fb915b47c2a3045b74c723b9dc233ca9cab0233354ea1ec2aab81d","observation_id":"6acd8fde-133d-4d07-9f32-eedf388fa975","resolution":{"observed_at":"2026-08-07T04:34:18.882859Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:18.569236Z","title":"Advances in Neural Information Processing Systems 2023, 36, 46595–46623","venue":null,"work_id":"2136df04-b880-423c-aa1e-5f1f21f093f8","year":2023},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:08.819739Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:f82c6f4848c913531f5e2cca75a8511d73f3dd497938bfcda6251a9ac41fe94f","observation_id":"113822d6-e3d9-4b00-a9c9-05185fb377f3","resolution":{"observed_at":"2026-08-07T04:34:18.658458Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.10934","last_updated":"2024-10-16T17:54:12Z","snapshot_observed_at":"2026-08-20T14:40:50.372602Z","submitted_at":"2024-10-14T17:57:02Z","title":"Agent-as-a-Judge: Evaluate Agents with Agents","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.10934","snapshot_observed_at":"2026-08-07T04:34:08.958962Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:08.958962Z"},"links":{"cited_paper":"/paper/2410.10934","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:3ff8989d6d59b16c10c9875de5b162dc69e9a02769f288a7a797b9c819f72c7a","observation_id":"a029ec1d-50a9-4bef-9736-cd09ca5b1c94","resolution":{"observed_at":"2026-08-07T04:34:08.958962Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.09739","last_updated":"2024-07-17T20:50:11Z","snapshot_observed_at":"2026-08-16T14:18:25.426536Z","submitted_at":"2024-02-15T06:36:07Z","title":"QuRating: Selecting High-Quality Data for Training Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.09739","snapshot_observed_at":"2026-08-07T04:34:09.115749Z","title":"Qurating: Selecting high-quality data for training language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:09.115749Z"},"links":{"cited_paper":"/paper/2402.09739","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:aa9f35e0fccf7f7a5ecc13244ba7d2715a0b1d3bfb7d185e63ae1a9016032489","observation_id":"290c994f-6f21-4700-9db7-cf29e9336f1d","resolution":{"observed_at":"2026-08-07T04:34:09.115749Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:18.351340Z","title":"F.; Leike, J.; Brown, T.; Martic, M.; Legg, S.; Amodei, D","venue":null,"work_id":"1cc1e2c8-7f44-4840-96af-be2848ac8ba3","year":2017},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:09.307806Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:4ac519660d3f9026b8530b29ca9be14626a2be3da0208a4e9cd60f6a103e34a3","observation_id":"a2b0b47b-a7e8-4a48-85b5-8d71ab66010a","resolution":{"observed_at":"2026-08-07T04:34:18.485017Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:18.132651Z","title":null,"venue":null,"work_id":"8f769747-2851-44e9-a701-d6df131fd790","year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:09.434801Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:f16b3975c0df10c99313fb27492c66a4dc6cee9351ac52fb713377d3845c8c35","observation_id":"a74c5cd5-db0d-4001-9307-d494038856ad","resolution":{"observed_at":"2026-08-07T04:34:18.243036Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2310.17631","last_updated":"2025-03-01T17:06:43Z","snapshot_observed_at":"2026-08-18T04:53:42.954410Z","submitted_at":"2023-10-26T17:48:58Z","title":"JudgeLM: Fine-tuned Large Language Models are Scalable Judges","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.17631","snapshot_observed_at":"2026-08-07T04:34:09.627591Z","title":"Judgelm: Fine-tuned large language models are scalable judges.arXiv preprint arXiv:2310.17631 2023,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:09.627591Z"},"links":{"cited_paper":"/paper/2310.17631","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:6642d79131c934ec670ef9c24bc02fde7dc2d348501a411aa4dd7143bdd23a61","observation_id":"f980c253-bbe6-46ce-83bc-1e9c8ad4a062","resolution":{"observed_at":"2026-08-07T04:34:09.627591Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.11874","last_updated":"2025-02-25T11:04:02Z","snapshot_observed_at":"2026-08-19T17:53:42.307231Z","submitted_at":"2024-05-20T08:30:13Z","title":"xFinder: Large Language Models as Automated Evaluators for Reliable Evaluation","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.11874","snapshot_observed_at":"2026-08-07T04:34:09.757567Z","title":"xfinder: Robust and pinpoint answer extraction for large language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:09.757567Z"},"links":{"cited_paper":"/paper/2405.11874","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:7a5b7c8567afb96336be9629c19cbab23dc85e57f805f1a4723b8735965c0c62","observation_id":"5d5285b6-e5ed-4824-bb79-f08bef70f2f6","resolution":{"observed_at":"2026-08-07T04:34:09.757567Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.03631","last_updated":"2024-06-05T21:29:09Z","snapshot_observed_at":"2026-08-17T22:49:19.665780Z","submitted_at":"2024-06-05T21:29:09Z","title":"Discovering Bias in Latent Space: An Unsupervised Debiasing Approach","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.03631","snapshot_observed_at":"2026-08-07T04:34:09.900260Z","title":"Discovering bias in latent space: An unsupervised debiasing approach","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:09.900260Z"},"links":{"cited_paper":"/paper/2406.03631","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:e53bf9dd25808e0383263e68ac2b606f85b64bca90a52c8ef05717e0175f94b1","observation_id":"19dac899-d440-48c8-8b57-7f5e6991fc71","resolution":{"observed_at":"2026-08-07T04:34:09.900260Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:17.909289Z","title":"H.; Chen, S.; Liu, Z.; Jiang, F.; Wang, B","venue":null,"work_id":"377b1f24-dbf5-4cfe-9bad-390440c377b6","year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:10.023607Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:58c4aa787a4df0abfdc8934f86504b274c0dfdd6b193556861f52c0375db4026","observation_id":"d5d4530c-185d-4291-8809-faa66fc8ed31","resolution":{"observed_at":"2026-08-07T04:34:18.013754Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.02736","last_updated":"2024-10-04T03:57:47Z","snapshot_observed_at":"2026-08-01T08:21:19.528254Z","submitted_at":"2024-10-03T17:53:30Z","title":"Justice or Prejudice? Quantifying Biases in LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.02736","snapshot_observed_at":"2026-08-07T04:34:10.196137Z","title":"arXiv preprint arXiv:2410.02736 2024,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:10.196137Z"},"links":{"cited_paper":"/paper/2410.02736","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:4567a99d1126aa9d1453a12f546b0cf6bfdf2ca3ba513185e584d22dc243a16f","observation_id":"8f607aac-142f-4e00-b349-0551a68d1b97","resolution":{"observed_at":"2026-08-07T04:34:10.196137Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:17.649292Z","title":"J.; De Sa, C","venue":null,"work_id":"e68d4ff2-50e2-44c5-918b-659009881cd8","year":2016},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:10.307887Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:1695ac4867bcdf144ac9e1458eddbc82af69c6ba0ebe6e4b93b8d97983cdb4a0","observation_id":"c07bf243-e9b2-49b9-9f36-fdb29e17b4f4","resolution":{"observed_at":"2026-08-07T04:34:17.805382Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:17.437628Z","title":"H.; Ehrenberg, H.; Fries, J.; Wu, S.; Ré, C","venue":null,"work_id":"d1e79aa5-6b5b-4c18-bd97-c66df08d907a","year":2017},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:10.487327Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:85c34b9044c4089f39321851b5816482b44564bbe015c5bb3a7950e8e81b2349","observation_id":"e8c7f540-ff3c-4302-b88c-0909217293e3","resolution":{"observed_at":"2026-08-07T04:34:17.526067Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:17.202174Z","title":"Training complex models with multi-task weak supervision","venue":null,"work_id":"fdfb6c58-37dc-44af-95de-3050dd0f9688","year":2019},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:10.653766Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:dea5498543e975fe5d9f9ddb841f81a9ba9930999476b40e9d8baaf57107de84","observation_id":"ca798256-8249-4e82-8760-09a04738b7a4","resolution":{"observed_at":"2026-08-07T04:34:17.359182Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:17.042926Z","title":"Fast and three-rious: Speeding up weak supervision with triplet methods","venue":null,"work_id":"edb46c26-845c-487e-b8c3-f2356f4a4b67","year":2020},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:10.819345Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:8ee907a242dd95789ece85078fb365cbcdb13e67b51ee2e3a0f96d77941e666d","observation_id":"affb8cf4-53ab-49ed-bdea-b30ec53d5f5a","resolution":{"observed_at":"2026-08-07T04:34:17.089083Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2403.13787","last_updated":"2024-06-08T16:40:12Z","snapshot_observed_at":"2026-08-19T21:41:44.622958Z","submitted_at":"2024-03-20T17:49:54Z","title":"RewardBench: Evaluating Reward Models for Language Modeling","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.13787","snapshot_observed_at":"2026-08-07T04:34:10.968389Z","title":"Y .; Chandu, K.; Dziri, N.; Kumar, S.; Zick, T.; Choi, Y .; others Rewardbench: Evaluating reward models for language modeling.arXiv preprint arXiv:2403.13787 2024,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:10.968389Z"},"links":{"cited_paper":"/paper/2403.13787","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:b961a9396bd06502e3f9d857429befe88f67c0e81daeae6f5f9534be70b29f07","observation_id":"d07bcf6c-d883-4dbd-83f3-8cbd8345e7f8","resolution":{"observed_at":"2026-08-07T04:34:10.968389Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:16.918586Z","title":"The Twelfth International Conference on Learning Representations","venue":null,"work_id":"fabd5da8-9a09-48cb-88dc-fc41f207fe79","year":2023},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:11.147019Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:d2df2e42b01d0f56dc38db0818192c0496d4a5e77b973106ba4c88be960ea583","observation_id":"0e7e9cc8-1810-4351-af7f-ae67533c8332","resolution":{"observed_at":"2026-08-07T04:34:16.956616Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.21276","last_updated":"2024-10-25T17:43:01Z","snapshot_observed_at":"2026-08-15T14:02:47.366139Z","submitted_at":"2024-10-25T17:43:01Z","title":"GPT-4o System Card","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.21276","snapshot_observed_at":"2026-08-07T04:34:11.322747Z","title":"P.; Perelman, A.; Ramesh, A.; Clark, A.; Ostrow, A.; Welihinda, A.; Hayes, A.; Radford, A.; others Gpt-4o system card","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:11.322747Z"},"links":{"cited_paper":"/paper/2410.21276","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:3fb03fdc8cc3adb9622a40cfeef00decc6497dc687572c634a813ad702e9c4ea","observation_id":"08063b97-5a06-43dc-b46d-d612811d8fcc","resolution":{"observed_at":"2026-08-07T04:34:11.322747Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:16.733816Z","title":null,"venue":null,"work_id":"77167843-7a2f-45d1-9c74-01ad916689d0","year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:11.471540Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:c70a1f5095308aa19f82202162048cdd5bd41dc0059b15404aadbc25f7b8dc46","observation_id":"7849f027-4f10-4708-9357-5ee5d7c19757","resolution":{"observed_at":"2026-08-07T04:34:16.873686Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:16.443020Z","title":"P.; Fishburne Jr, R","venue":null,"work_id":"d7015ea2-d4cd-472c-b0fc-c9c7001349ad","year":1975},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:11.632589Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:f60606a19035e6b386459b7afe30e28e1a7b060853c36abc46ca606c58dadc7e","observation_id":"292117a4-594a-45ad-826b-945007f1d7f5","resolution":{"observed_at":"2026-08-07T04:34:16.587401Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:16.181056Z","title":"FactKB: Generalizable Factuality Evaluation using Language Models Enhanced with Factual Knowledge","venue":null,"work_id":"1e75fc73-32d4-4fec-b68e-8501673fb244","year":2023},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:11.812496Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:155a73f91c8420c1632ef397f6ce302ee3494b0d139540cf02b7072adfbf1de5","observation_id":"89546b02-b98c-41dd-a339-c03f10bba31b","resolution":{"observed_at":"2026-08-07T04:34:16.299829Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:15.829438Z","title":"Learning from the Worst: Dynamically Generated Datasets to Improve Online Hate Detection","venue":null,"work_id":"b47aab82-4d9e-4ff0-b532-c253c4ca59c8","year":2021},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:11.939726Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:c5a75e63a7f348cde18db5cba401693a1acbaec2fece12bc393264bcdc5a4af0","observation_id":"8562dfc4-bc17-4ef9-bcd6-9bcc0311d637","resolution":{"observed_at":"2026-08-07T04:34:16.018586Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:15.541517Z","title":"Universalizing weak supervision","venue":null,"work_id":"141c3c27-27f7-4c76-99ab-bfb310cdf8a9","year":2022},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:12.071052Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:957ba1d475c88738edd19b1013095de44b9abe98e111fa561779c115170542aa","observation_id":"6bdd328f-b03a-4208-8ac1-24a3eca19819","resolution":{"observed_at":"2026-08-07T04:34:15.672364Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:15.225376Z","title":"Weak-to-Strong Generalization Through the Data-Centric Lens","venue":null,"work_id":"6272b063-3e2e-4be7-91f9-376b1c675704","year":2025},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:12.237890Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:c3f69c0c3464a7a8daba31fde550f10a247dc0dea2d4233d765019380af85dd0","observation_id":"5711fb54-8843-4ca0-8982-a659fa51253a","resolution":{"observed_at":"2026-08-07T04:34:15.353380Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:14.907405Z","title":"PandaLM: An Automatic Evaluation Benchmark for LLM Instruction Tuning Optimization","venue":null,"work_id":"2575edf2-e97b-413e-a8fa-f8adb6d4ac85","year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:12.346391Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:916c804a94314c42ffc07cb4b02188dbae0b161c8d0c43db78e442143900f1d1","observation_id":"f6833448-268e-4a5a-8a56-f5b92c2d2589","resolution":{"observed_at":"2026-08-07T04:34:15.060847Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-17T09:58:46.058102Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-08-07T04:34:12.492714Z","title":"L.; Almeida, D.; Altenschmidt, J.; Altman, S.; Anadkat, S.; others Gpt-4 technical report","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:12.492714Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:20c68db9b3cb335201e7133985944ff5282ff70f664cc84e3d89fecbbacabdc2","observation_id":"bb382b29-30e9-41dc-8491-3df9c32420b7","resolution":{"observed_at":"2026-08-07T04:34:12.492714Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.00118","last_updated":"2024-10-02T15:22:49Z","snapshot_observed_at":"2026-08-21T05:15:28.840279Z","submitted_at":"2024-07-31T19:13:07Z","title":"Gemma 2: Improving Open Language Models at a Practical Size","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.00118","snapshot_observed_at":"2026-08-07T04:34:12.651746Z","title":"G.; Hardin, C.; Bhupatiraju, S.; Hussenot, L.; Mesnard, T.; Shahriari, B.; Ramé, A.; others Gemma 2: Improving open language models at a practical size","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:12.651746Z"},"links":{"cited_paper":"/paper/2408.00118","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:a7dfa16e3724be13f1c504d72c98e9fe82bae7a6efe4a3d58d1cde8eefc9a0b2","observation_id":"1a7646b5-cff3-46c1-b0d7-07c9a51d592e","resolution":{"observed_at":"2026-08-07T04:34:12.651746Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.05579","last_updated":"2024-12-10T05:49:12Z","snapshot_observed_at":"2026-08-17T10:18:05.650518Z","submitted_at":"2024-12-07T08:07:24Z","title":"LLMs-as-Judges: A Comprehensive Survey on LLM-based Evaluation Methods","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.05579","snapshot_observed_at":"2026-08-07T04:34:12.759896Z","title":"Llms-as-judges: a comprehensive survey on llm-based evaluation methods","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:12.759896Z"},"links":{"cited_paper":"/paper/2412.05579","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:5ddd146ccadcac9375758661b09f1c2330e7e3328385d3e3f5e45805fe4b968b","observation_id":"733cb78b-219d-4772-a6d9-9a4b30d992f6","resolution":{"observed_at":"2026-08-07T04:34:12.759896Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:12.934810Z","title":"Reference-Guided Verdict: LLMs-as-Judges in Automatic Evaluation of Free-Form Text","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:12.934810Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:7f2aa828bbddbb8d7755c5f8942afe83dab14f93cb73700747bfeb145860d64f","observation_id":"86e8dac7-4bd3-4ca3-bfa5-b16cfe8053b8","resolution":{"observed_at":"2026-08-07T04:34:12.934810Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:14.730238Z","title":"The ALCHEmist: Automated Labeling 500x CHEaper than LLM Data Annotators","venue":null,"work_id":"5aca053f-0b79-4c8b-99d6-9c0f0a3a3def","year":2024},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:13.041107Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:621ca14a63c75c7fbc43e7a93909dbd0fede0cab6d0e260da16a6189c90645b4","observation_id":"4c2ad085-1fa2-444a-9242-9cd64b26d309","resolution":{"observed_at":"2026-08-07T04:34:14.817946Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2502.12366","last_updated":"2025-02-17T23:07:14Z","snapshot_observed_at":"2026-08-16T12:57:32.061158Z","submitted_at":"2025-02-17T23:07:14Z","title":"ScriptoriumWS: A Code Generation Assistant for Weak Supervision","version":1},"cited_work":{"arxiv_id":"2502.12366","doi":null,"metadata_source":"pith","pith_arxiv_id":"2502.12366","snapshot_observed_at":"2026-08-07T04:34:13.805026Z","title":"ScriptoriumWS: A Code Generation Assistant for Weak Supervision","venue":"cs.LG","work_id":"47e56f7a-f2d5-41c7-bf93-dc7de2d64e19","year":2025},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:13.235686Z"},"links":{"cited_paper":"/paper/2502.12366","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:832df42a14ff71d4997f3d585190935baa39c20a188e53b4195e0afa067033f6","observation_id":"abc0dae9-433c-4b1f-96ed-5fa5cfe702e3","resolution":{"observed_at":"2026-08-07T04:34:13.922531Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:14.444699Z","title":"Autows-bench-101: Benchmarking automated weak supervision with 100 labels","venue":null,"work_id":"0d1051be-95e7-471f-963c-0c4608e36532","year":2022},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:13.376326Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:ff6285340ee61b4553b676cfb3fc74f5d8435092a2de406250e735253e02f3af","observation_id":"5e2df8e1-3ce8-4888-8d33-589c4ba98afa","resolution":{"observed_at":"2026-08-07T04:34:14.616136Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2501.06708","last_updated":"2026-05-25T22:00:48Z","snapshot_observed_at":"2026-08-17T03:31:02.136201Z","submitted_at":"2025-01-12T04:28:14Z","title":"Evaluating Sample Utility for Efficient Data Selection by Mimicking Model Weights","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.06708","snapshot_observed_at":"2026-08-07T04:34:13.508340Z","title":"Evaluating Sample Utility for Data Selection by Mimicking Model Weights","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:13.508340Z"},"links":{"cited_paper":"/paper/2501.06708","citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:9f4a4b3816229f57cae4bfd602434a620acc777b9f26aa9752f86e7d8e20f1fc","observation_id":"cc286cef-bb27-4d85-80b2-35448f585fb9","resolution":{"observed_at":"2026-08-07T04:34:13.508340Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-07T04:34:14.261642Z","title":"\"\"Calculate readability metrics for response","venue":null,"work_id":"0ca0aba9-0783-4bfa-a807-b7f4b20f8779","year":2022},"citing_paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-07T04:34:13.658964Z"},"links":{"citing_paper":"/paper/2506.10403"},"observation_digest":"sha256:70ead90af3f06e76ee1e45756a6c869d4fcd8a866afe67b46525a73f50a12cd3","observation_id":"997eca7d-7afe-4568-bec9-8d04b8b2df7c","resolution":{"observed_at":"2026-08-07T04:34:14.331717Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2506.10403","last_updated":"2025-06-12T06:53:22Z","latest_version":1,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-21T12:19:49.684291Z","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation"},"reference_resolution":{"displayed":35,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":16,"verified_exact":1,"verified_fuzzy":18},"total_outbound_references":35},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"thesis":"As of 21 August 2026, this Paper Citation Record lists 35 of 35 outbound references and 3 inbound Pith citation observations for arXiv:2506.10403."}