{"as_of":"2026-08-19T15:24:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:1dbd31eac298b3d64f844e71ad0c765ff6ab3671e688f8922cf8dbd64b583da7","coverage":[{"denominator":75,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":75,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-02T12:49:17.313990Z","state":"measured"},{"denominator":75,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":75,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-19T06:32:44.657259+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2607.22561/citation-record","integrity":"/paper/2607.22561/integrity","json":"/paper/2607.22561/citation-record.json","paper":"/paper/2607.22561"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:08.649159Z","title":"The Innovation2024,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:08.649159Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:37be490f76173851117d26d0c7172484342b067a769fa77aae43a955b1a1243f","observation_id":"86e1de7f-a778-4f37-b864-25f0a17054cb","resolution":{"observed_at":"2026-08-02T12:49:08.649159Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.05579","last_updated":"2024-12-10T05:49:12Z","snapshot_observed_at":"2026-08-17T10:18:05.650518Z","submitted_at":"2024-12-07T08:07:24Z","title":"LLMs-as-Judges: A Comprehensive Survey on LLM-based Evaluation Methods","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.05579","snapshot_observed_at":"2026-08-02T12:49:08.695963Z","title":"Llms-as-judges: a comprehensive survey on llm-based evaluation methods.arXiv preprint arXiv:2412.055792024,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:08.695963Z"},"links":{"cited_paper":"/paper/2412.05579","citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:127609ac71a7aad75fff5c1ce7d5d9acc638a0d42a4b00508356a7d943629785","observation_id":"01558392-8d9f-4d4f-8b29-ad399e7bec6c","resolution":{"observed_at":"2026-08-02T12:49:08.695963Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:08.778979Z","title":"JudgeLM: Fine-tuned Large Language Models are Scalable Judges","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:08.778979Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:64c2006bae0793f7722001684f95c6914ae1a23df467a01663169f9b6d147ea3","observation_id":"ecf22b85-92cc-40fe-b100-5e6b4c305e73","resolution":{"observed_at":"2026-08-02T12:49:08.778979Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:08.848395Z","title":"Prometheus: Inducing Fine-Grained Evaluation Capability in Language Models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:08.848395Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:b2fe66ea1bc4d70f186d5521a86aaf63673e20c7090ff2de4cedd2784df2b9bd","observation_id":"982707aa-4572-4e05-8f3c-0c82a4e48702","resolution":{"observed_at":"2026-08-02T12:49:08.848395Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:08.936162Z","title":"PandaLM: An Automatic Evaluation Benchmark for LLM Instruction Tuning Optimization","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:08.936162Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:811b7ecddb22029094681c5d4a8539a0e3089f281e23161529b56cc266ccd8dd","observation_id":"d5dff37d-8461-4dd7-9a71-42399f72f72a","resolution":{"observed_at":"2026-08-02T12:49:08.936162Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:09.014344Z","title":"Learning LLM-as-a-Judge for Preference Alignment","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:09.014344Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:39996308ccf9eec16abf9a268d99966c681be4e19b90fcc808f1d0fdbfc81cd9","observation_id":"f2322a79-8fa5-4ede-aa5f-3755fba4a169","resolution":{"observed_at":"2026-08-02T12:49:09.014344Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:09.154738Z","title":"E.; Sukhbaatar, S","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:09.154738Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:4145c69f53b70305fa130f86c1a2ada71b35f1dcaea3a6c3cd8c7114fcd97bd4","observation_id":"fa6b12f0-4025-49d1-bd16-51766b628baa","resolution":{"observed_at":"2026-08-02T12:49:09.154738Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.02666","last_updated":"2024-08-08T17:09:58Z","snapshot_observed_at":"2026-08-18T20:05:45.096677Z","submitted_at":"2024-08-05T17:57:02Z","title":"Self-Taught Evaluators","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.02666","snapshot_observed_at":"2026-08-02T12:49:09.307551Z","title":"Y .; Fazel-Zarandi, M.; Weston, J.; Li, X","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:09.307551Z"},"links":{"cited_paper":"/paper/2408.02666","citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:72fd5ef4e8d54e3ba775eaeeb1dcf2514b7e979ae4b80d458cb6bba9a402be66","observation_id":"551babe2-fe27-43d0-a6b4-11cc905ba28b","resolution":{"observed_at":"2026-08-02T12:49:09.307551Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:09.465680Z","title":null,"venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:09.465680Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:4dbef0c541db337a43e2fff44ba356042e7f1ee12ea9fb812e489694abbf3e18","observation_id":"a3e219fa-4939-48a4-9944-7c4e242e9ca4","resolution":{"observed_at":"2026-08-02T12:49:09.465680Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:09.566172Z","title":"RubiCap: Rubric-Guided Reinforcement Learning for Dense Image Captioning.arXiv preprint arXiv:2603.091602026,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:09.566172Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:3fc48332bd677a0da78272fe12fa4b089fc27d6bef824fdd2eae591f1c1c057d","observation_id":"692c4f01-7ed5-4e87-a5f3-152978951f54","resolution":{"observed_at":"2026-08-02T12:49:09.566172Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2511.19399","last_updated":"2026-05-15T01:32:52Z","snapshot_observed_at":"2026-08-14T13:19:21.831179Z","submitted_at":"2025-11-24T18:35:54Z","title":"DR Tulu: Reinforcement Learning with Evolving Rubrics for Deep Research","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2511.19399","snapshot_observed_at":"2026-08-02T12:49:09.660219Z","title":"Z.; Ivison, H.; Kishore, V .; Zhuo, J.; Zhao, X.; Park, M.; Finlayson, S","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:09.660219Z"},"links":{"cited_paper":"/paper/2511.19399","citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:bc60bc014fdd657e0b022f9deb1f30bca2d2d8a56968dec857efc2851e4bc981","observation_id":"2e4da422-f945-49b9-97d0-d9073f02948c","resolution":{"observed_at":"2026-08-02T12:49:09.660219Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2601.03267","last_updated":"2026-05-01T23:55:43Z","snapshot_observed_at":"2026-08-15T00:17:32.875866Z","submitted_at":"2025-12-19T07:05:38Z","title":"OpenAI GPT-5 System Card","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2601.03267","snapshot_observed_at":"2026-08-02T12:49:09.743492Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:09.743492Z"},"links":{"cited_paper":"/paper/2601.03267","citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:20bd56f5b65a309244cfa5a701bfc72b1a22db10fb2260122cc67b502441913a","observation_id":"1bb26942-3e47-4c4e-9ad7-9c275163ca29","resolution":{"observed_at":"2026-08-02T12:49:09.743492Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:10.029536Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:10.029536Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:86b64724511991c492f80e0e6454563450f377a8b0109bb76402f3304c406bf6","observation_id":"79f67867-9df5-455f-8c47-902e03dca5a8","resolution":{"observed_at":"2026-08-02T12:49:10.029536Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:10.144752Z","title":"Tuning LLM Judge Design Decisions for 1/1000 of the Cost","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:10.144752Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:b7d5fca30de61836a75642cc43031980c91dc6ebeec1e70daa93d095fa81db52","observation_id":"ac4a0ddb-416b-4301-9931-24de8facd88e","resolution":{"observed_at":"2026-08-02T12:49:10.144752Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2507.08794","last_updated":"2026-06-11T04:21:36Z","snapshot_observed_at":"2026-08-17T18:32:55.983199Z","submitted_at":"2025-07-11T17:55:22Z","title":"One Token to Fool LLM-as-a-Judge","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2507.08794","snapshot_observed_at":"2026-08-02T12:49:10.192392Z","title":"One token to fool llm-as-a-judge.arXiv preprint arXiv:2507.087942025,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:10.192392Z"},"links":{"cited_paper":"/paper/2507.08794","citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:ea7a8342f9e7889b2ec4c54c0be09313c95bc5901182e44c058d4d4f4046c759","observation_id":"ee393540-d4c3-4fec-88af-3568e6d6f7fc","resolution":{"observed_at":"2026-08-02T12:49:10.192392Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:10.318374Z","title":"Investigating the Vulnerability of LLM-as-a-Judge Architectures to Prompt-Injection Attacks.International Journal of Open Information Technologies2025,13, 1–6","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:10.318374Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:4a3e0d5eb46564f99b8daae2d1961701970e85b77e45c5a5afd516022e464b2d","observation_id":"2d5f8df9-50ee-452d-9a34-259c34739a6d","resolution":{"observed_at":"2026-08-02T12:49:10.318374Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:10.402306Z","title":"H.; Chen, S.; Liu, Z.; Jiang, F.; Wang, B","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:10.402306Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:3c3e3ecd0b6e4d8cf95c5168cd142b2abd0df6dd5341858db2219c18d224b013","observation_id":"8470c192-28f4-48fe-8de0-cb5deaf10e2e","resolution":{"observed_at":"2026-08-02T12:49:10.402306Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:10.534828Z","title":"V .; Zhang, X","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:10.534828Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:088bc8019e5c902e6faaffcfeac1d4b7a4ce12870ab89758f0204f7087b86d61","observation_id":"3492a5f2-1299-4b1c-8cbd-1596e2d581e3","resolution":{"observed_at":"2026-08-02T12:49:10.534828Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:10.661038Z","title":"Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:10.661038Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:406a30d79e68b307ee05cca2d96fd1c8c7c8c7e31023e65480a51900a57a398d","observation_id":"c79fa538-6edf-4dfa-bf87-390c4602cf7e","resolution":{"observed_at":"2026-08-02T12:49:10.661038Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.12509","last_updated":"2025-02-18T15:24:25Z","snapshot_observed_at":"2026-08-16T21:16:12.321849Z","submitted_at":"2024-12-17T03:37:31Z","title":"Can You Trust LLM Judgments? Reliability of LLM-as-a-Judge","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.12509","snapshot_observed_at":"2026-08-02T12:49:10.833195Z","title":"Can you trust llm judgments? reliability of llm-as-a-judge.arXiv preprint arXiv:2412.125092024,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:10.833195Z"},"links":{"cited_paper":"/paper/2412.12509","citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:49ce8c6a45f60ee2e8bd48bd5516f8e0ecbfb5bcb6462113a5856fb752bef425","observation_id":"98c7724a-5157-4c4d-a965-2f801e888550","resolution":{"observed_at":"2026-08-02T12:49:10.833195Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:10.976810Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:10.976810Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:1a0b40d3afa9d1a2924fce7814b809fadd8fed5789aa8a198022f2691eb03c80","observation_id":"067c43fb-443f-48de-9d0d-33839da46a04","resolution":{"observed_at":"2026-08-02T12:49:10.976810Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:11.205985Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:11.205985Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:4be412a819db4a1cd3d7f105103744e8f77d7f95c46fee639544d92fa9f6fcde","observation_id":"14251072-9c7e-4146-a569-207157793ddb","resolution":{"observed_at":"2026-08-02T12:49:11.205985Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.15115","last_updated":"2025-01-03T02:18:21Z","snapshot_observed_at":"2026-08-17T18:50:07.059564Z","submitted_at":"2024-12-19T17:56:09Z","title":"Qwen2.5 Technical Report","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.15115","snapshot_observed_at":"2026-08-02T12:49:11.369244Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:11.369244Z"},"links":{"cited_paper":"/paper/2412.15115","citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:f6684016649f2b68eb7458ff7bb697fe0fb9cf851305acf0a775490e779517c7","observation_id":"5f203a7a-6cf1-46db-9670-f99a231fe8f8","resolution":{"observed_at":"2026-08-02T12:49:11.369244Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:11.475810Z","title":"Y .; Chandu, K.; Dziri, N.; Kumar, S.; Zick, T.; Choi, Y .; Smith, N","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:11.475810Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:500107d90bd8d9e93a9ff95f0d22100a759f5d26b629038eb2e1c3de979f4923","observation_id":"f98e9cd4-c47e-4fdc-ac47-45109f789985","resolution":{"observed_at":"2026-08-02T12:49:11.475810Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:11.552215Z","title":"Can Large Language Models Be an Alternative to Human Evaluations? Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (V olume 1: Long Papers)","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:11.552215Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:2eeb0f4e3dd0a055a942c481484feedd4048ad4d4aab46ac4e908ce8a7121efe","observation_id":"fd0e2a7d-54ca-45d1-9fa0-67158dde2430","resolution":{"observed_at":"2026-08-02T12:49:11.552215Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:11.717579Z","title":"Length-Controlled AlpacaEval: A Simple Debiasing of Automatic Evaluators","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:11.717579Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:5c8bed51c2679834c512e0783c6ecdaacc9a1b4276934e9e218bbcbcaed677dc","observation_id":"ccf2ed11-0870-4836-a005-5c59b7508932","resolution":{"observed_at":"2026-08-02T12:49:11.717579Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:11.908400Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:11.908400Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:2c6cc250a89d53ceb3fb528cdbf4940ea35bd73250522c5fa2efc81585226c7d","observation_id":"4137f46c-b3bd-48d6-9f55-1b4777ede575","resolution":{"observed_at":"2026-08-02T12:49:11.908400Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:12.026141Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:12.026141Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:480557f2349d2aab87d33b0b3347c3fa6a4d0390e0c12710c4342920be10b213","observation_id":"931a5c6f-6440-43c1-b0f0-36d76d60e6c4","resolution":{"observed_at":"2026-08-02T12:49:12.026141Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2204.05862","last_updated":"2022-04-12T15:02:38Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-04-12T15:02:38Z","title":"Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2204.05862","snapshot_observed_at":"2026-08-02T12:49:12.179514Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:12.179514Z"},"links":{"cited_paper":"/paper/2204.05862","citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:6522a679ea9b7c399363d78b2b021d0f3042354a9925a71b41efc94457c826c7","observation_id":"47b2055a-3bec-4df3-822a-77ad7dc25a64","resolution":{"observed_at":"2026-08-02T12:49:12.179514Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:12.289782Z","title":"J.; Choi, E","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:12.289782Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:5414a35df2363db7bc6dccaa6d94fc98ba4a1835d5b69eade84ff88828d893fd","observation_id":"e5e194c0-6c27-454a-b7d0-83c4cbfdddd6","resolution":{"observed_at":"2026-08-02T12:49:12.289782Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:12.401722Z","title":"H.; Ehrenberg, H.; Fries, J.; Wu, S.; Ré, C","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:12.401722Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:48463e4f9689dad558c26aa8939c4982589482f1938b77c517f09ac4597e9860","observation_id":"7c207e63-7eca-40eb-a82e-2dfde7aa19cc","resolution":{"observed_at":"2026-08-02T12:49:12.401722Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:12.506541Z","title":"J.; De Sa, C","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:12.506541Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:9d679da2832a4012009d31f603b82f10bd6e7aa4d1cb93bb697523ba3bd86621","observation_id":"b1f9d5c9-47a4-4d32-9247-8748132fa2a1","resolution":{"observed_at":"2026-08-02T12:49:12.506541Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:12.652292Z","title":"Training complex models with multi-task weak supervision","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:12.652292Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:9c98bdbe07e87e4b0df50cb1cd308da46ee7f456474545b96c0b89678f39093a","observation_id":"be779dee-688c-42c5-8bd7-ab74b7bac719","resolution":{"observed_at":"2026-08-02T12:49:12.652292Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:12.710884Z","title":"C.; Sala, F","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:12.710884Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:41cb5e9c761cba4c09bbd4a5d086b9be3df5fa9ed9de9685c471edb232fa3b04","observation_id":"e4cdfcc3-750c-40c4-978f-517366be634c","resolution":{"observed_at":"2026-08-02T12:49:12.710884Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:12.797978Z","title":"Snorkel metal: Weak supervision for multi-task learning","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:12.797978Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:67fda1521cceea49b316f442efb7949020aaf432cfcdac59fed694b7ce5a2d59","observation_id":"6b9539bb-38ba-468a-9a4a-aecae3af7576","resolution":{"observed_at":"2026-08-02T12:49:12.797978Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:12.871922Z","title":"The ALCHEmist: Automated Labeling 500x CHEaper than LLM Data Annotators","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:12.871922Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:f6071178626fbf9d1c9354b6b4b7595703ae3821a30089e8bf52939ad900ca64","observation_id":"69688967-3854-4867-b05d-ef24a6b0b7e7","resolution":{"observed_at":"2026-08-02T12:49:12.871922Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.12366","last_updated":"2025-02-17T23:07:14Z","snapshot_observed_at":"2026-08-16T12:57:32.061158Z","submitted_at":"2025-02-17T23:07:14Z","title":"ScriptoriumWS: A Code Generation Assistant for Weak Supervision","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.12366","snapshot_observed_at":"2026-08-02T12:49:12.955916Z","title":"Scriptoriumws: A code generation assistant for weak supervision.arXiv preprint arXiv:2502.123662025,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:12.955916Z"},"links":{"cited_paper":"/paper/2502.12366","citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:eaffb8dc6edf7701bbce1d45c16bbeaa1241f76257d6cf1f02d204141f2fb82e","observation_id":"9074d841-9b15-4eaa-afda-0103ef88c704","resolution":{"observed_at":"2026-08-02T12:49:12.955916Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:13.104765Z","title":"Autows-bench-101: Benchmarking automated weak supervision with 100 labels.Advances in Neural Information Processing Systems2022,35, 8912–8925","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:13.104765Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:b1029e9f0c1376a4fc2df5cf83a5a9424b339114a18810fdf762e38aa0b0ea15","observation_id":"0cbb58af-0106-4b1f-8f1a-46171ab88dfb","resolution":{"observed_at":"2026-08-02T12:49:13.104765Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.12225","last_updated":"2024-01-05T08:40:06Z","snapshot_observed_at":"2026-08-17T03:53:16.467544Z","submitted_at":"2024-01-05T08:40:06Z","title":"Multimodal Data Curation via Object Detection and Filter Ensembles","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.12225","snapshot_observed_at":"2026-08-02T12:49:13.266271Z","title":"J.; Adila, D.; Sala, F","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:13.266271Z"},"links":{"cited_paper":"/paper/2401.12225","citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:fc9f4861791dc522679d45cc14e33e6a2e1a4808dbc5c1fcae65c3dd61b0ef1e","observation_id":"52b68521-9fa9-43d3-a7b5-acdabd9cc56f","resolution":{"observed_at":"2026-08-02T12:49:13.266271Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:13.421063Z","title":"H.; Kellman, P.; Xue, H.; Sala, F.; Langlotz, C.; Re, C","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:13.421063Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:2ce2b215bbff68d2e4dfad88880419bd3a376eb500be665deab0f4fcc55e239f","observation_id":"0f091e68-a15d-4e97-a867-0803e419f75a","resolution":{"observed_at":"2026-08-02T12:49:13.421063Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:13.584975Z","title":"F.; Orr, L.; Guha, N.; Bhatia, K.; Chami, I.; Re, C","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:13.584975Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:411b200893a9ed3252bfbc39732b4fb3313f14583397bc62611b85ad1e24549b","observation_id":"cb36c387-e8d9-46d5-bfe6-cb19d39bc201","resolution":{"observed_at":"2026-08-02T12:49:13.584975Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:13.667192Z","title":"K.; Chen, M","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:13.667192Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:7b6562afeeec8459e5ac3b142002e9f4585a503218ccd28c85225febe078946e","observation_id":"c742bd23-a3c9-4cec-8981-1e09c21872ce","resolution":{"observed_at":"2026-08-02T12:49:13.667192Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.06708","last_updated":"2026-05-25T22:00:48Z","snapshot_observed_at":"2026-08-17T03:31:02.136201Z","submitted_at":"2025-01-12T04:28:14Z","title":"Evaluating Sample Utility for Efficient Data Selection by Mimicking Model Weights","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.06708","snapshot_observed_at":"2026-08-02T12:49:13.711886Z","title":"Evaluating Sample Utility for Efficient Data Selection by Mimicking Model Weights.arXiv preprint arXiv:2501.067082025,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:13.711886Z"},"links":{"cited_paper":"/paper/2501.06708","citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:d36ee4a9f5ed23cbce184b74279fcc08734faeef32164f962dc1a94864d2d62b","observation_id":"2ded89f0-b699-425e-82ca-d639e3dde50c","resolution":{"observed_at":"2026-08-02T12:49:13.711886Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:13.847612Z","title":"Lifting weak supervision to structured prediction.Advances in Neural Information Processing Systems2022,35, 37563–37574","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:13.847612Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:650a9fb9f70c270107965ec2a642e0d74e0fef61fa9e23e051f8169895cec089","observation_id":"ebe747bd-5e49-4240-b4b2-87c993747339","resolution":{"observed_at":"2026-08-02T12:49:13.847612Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:13.897344Z","title":"E.; Kadous, M","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:13.897344Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:e1180a1f599bae2b5be32b87e56d1746bad2001b4340312493398ee11e54b870","observation_id":"7f9cd947-6aaa-4bf4-a86e-f97b5e211a91","resolution":{"observed_at":"2026-08-02T12:49:13.897344Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:14.064491Z","title":"J.; Bieker, J.; Li, X.; Jiang, N.; Keigwin, B.; Ranganath, G.; Keutzer, K.; Upadhyay, S","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:14.064491Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:dedcc33c485f924b8fb71ca34f8e31a5588a86f699d5c465521b6759a542cbc1","observation_id":"47a729ab-78e8-48ca-8e24-396f0b71bbfd","resolution":{"observed_at":"2026-08-02T12:49:14.064491Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:14.217591Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:14.217591Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:38dca8098e27d25a3700773e1c55dc4fb245de82dd411e7956a04240cc64ba06","observation_id":"1191ef7a-8406-489d-9ec4-ed2fc6cb1a96","resolution":{"observed_at":"2026-08-02T12:49:14.217591Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:14.333230Z","title":"S.; Juneja, J.; Wang, C.; Wang, Z.; Go, A.; Lee, C.-Y .; Shenoy, P.; Panigrahy, R.; Menon, A","venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:14.333230Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:bb6fd99718fb436f3db492281160f5827e96a0f49b38ced148fbd6f375a2e558","observation_id":"3108bc72-b255-446c-81ab-ef06443dacdb","resolution":{"observed_at":"2026-08-02T12:49:14.333230Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.15789","last_updated":"2023-09-27T17:08:40Z","snapshot_observed_at":"2026-08-16T14:57:08.067521Z","submitted_at":"2023-09-27T17:08:40Z","title":"Large Language Model Routing with Benchmark Datasets","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.15789","snapshot_observed_at":"2026-08-02T12:49:14.475806Z","title":"Large language model routing with benchmark datasets.arXiv preprint arXiv:2309.157892023,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:14.475806Z"},"links":{"cited_paper":"/paper/2309.15789","citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:75187167196ec38236a20d702d7ab998169a5d24d22ce60bd0a9dbe9deadf16a","observation_id":"ec40c102-8d13-4c0a-9139-057f0720bb22","resolution":{"observed_at":"2026-08-02T12:49:14.475806Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.11601","last_updated":"2023-08-23T17:34:17Z","snapshot_observed_at":"2026-08-18T06:46:01.047725Z","submitted_at":"2023-08-22T17:48:24Z","title":"Tryage: Real-time, intelligent Routing of User Prompts to Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.11601","snapshot_observed_at":"2026-08-02T12:49:14.576579Z","title":"N.; Thomson, M","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:14.576579Z"},"links":{"cited_paper":"/paper/2308.11601","citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:3f3023f98fb3436bb6058011e1b3da7cda39d5e76e3e59b2d95c879347f9ad62","observation_id":"02d9e9dc-bb67-449c-9c5d-15c508457a90","resolution":{"observed_at":"2026-08-02T12:49:14.576579Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:14.636858Z","title":"Routing to the expert: Efficient reward-guided ensemble of large language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:14.636858Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:47ba0305066953bf153864bbc027bf7865329692b1aae2c0328eb521a8ece43d","observation_id":"47568622-746b-47b1-905d-6e57b949d26a","resolution":{"observed_at":"2026-08-02T12:49:14.636858Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.07863","last_updated":"2024-11-12T11:18:43Z","snapshot_observed_at":"2026-08-15T18:13:42.319653Z","submitted_at":"2024-05-13T15:50:39Z","title":"RLHF Workflow: From Reward Modeling to Online RLHF","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.07863","snapshot_observed_at":"2026-08-02T12:49:14.702968Z","title":"RLHF Workflow: From Reward Modeling to Online RLHF.arXiv preprint arXiv:2405.078632024,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:14.702968Z"},"links":{"cited_paper":"/paper/2405.07863","citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:5377ef04d03ee9634d4bc7dd872a9c55a5db795d12bb5e059bc33562332b52ae","observation_id":"88c1bd85-3293-420c-80d1-31d6fc73e99c","resolution":{"observed_at":"2026-08-02T12:49:14.702968Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:14.760278Z","title":null,"venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:14.760278Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:6eb230f83b4dd3b5ac64dd21442c07d2f32d8243cfad5034f9f3ba5ccd198b9b","observation_id":"fb930b25-575e-4929-a44f-a4ad4280e1be","resolution":{"observed_at":"2026-08-02T12:49:14.760278Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-17T09:58:46.058102Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-08-02T12:49:14.827234Z","title":"L.; Almeida, D.; Altenschmidt, J.; Altman, S.; Anadkat, S.; others Gpt-4 technical report.arXiv preprint arXiv:2303.087742023,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:14.827234Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:5c7cb0c697cead4095f74c71f3bc5b12fc406b97965e968ab87e2fa26d4c1734","observation_id":"98619d9d-e035-4a24-add6-8a15912df9cd","resolution":{"observed_at":"2026-08-02T12:49:14.827234Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2503.19786","last_updated":"2025-03-25T15:52:34Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2025-03-25T15:52:34Z","title":"Gemma 3 Technical Report","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.19786","snapshot_observed_at":"2026-08-02T12:49:14.884047Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:14.884047Z"},"links":{"cited_paper":"/paper/2503.19786","citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:ab0923efcb5760657725ba6167324d4728bf082c02802aa7de67de988c176146","observation_id":"6e13ec76-17d1-4044-9b42-bd60f501f8fd","resolution":{"observed_at":"2026-08-02T12:49:14.884047Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:14.961390Z","title":"good” vs “bad","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:14.961390Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:3efce6bb4ac4dc56052931eca1fee5df09edc0258e71d3580202efdab99d039b","observation_id":"5415b1cf-b0a0-4ac7-bbd4-dd7e73dccdfe","resolution":{"observed_at":"2026-08-02T12:49:14.961390Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:15.124540Z","title":"Evaluate how semantically relevant the response is to the question asked","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:15.124540Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:e6d8668ead1fae5571a4d27297ddd9f8a117bceb912d81dba3b6146be290ee14","observation_id":"60d6cea8-3c47-4cdb-bb81-302f12f4c89f","resolution":{"observed_at":"2026-08-02T12:49:15.124540Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:15.280292Z","title":"Evaluate language quality and readability","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:15.280292Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:3703bdd485237f55c365cc4e04fb359af231c52de6081df3cfb877f5ece40fb8","observation_id":"1246e7e4-982a-4b90-aab7-0220ec51c94a","resolution":{"observed_at":"2026-08-02T12:49:15.280292Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:15.367431Z","title":"Evaluate completeness and thoroughness of the answer","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":59,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:15.367431Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:a2c3e492786199bdb1e15ecd2a677e396fcae9c0b462adc530424ad2b50cef5a","observation_id":"299a4bd9-bc21-4af5-af5d-b9c4ac7377a2","resolution":{"observed_at":"2026-08-02T12:49:15.367431Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:15.439853Z","title":"Evaluate indicators of factual reliability","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:15.439853Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:c25164db72db7c9df280e6ebfbd69894c6ad9a9314b1333992feb2c41a59ed31","observation_id":"e2b18d21-5906-44df-9df0-d57c288b464a","resolution":{"observed_at":"2026-08-02T12:49:15.439853Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:15.496430Z","title":"Evaluate logical coherence","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":61,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:15.496430Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:32892d3bab32d9963c8b12efb4385a8762763015c84d0e768ccea5c266d73475","observation_id":"a063c678-6285-43f2-aa9e-5db083abcf03","resolution":{"observed_at":"2026-08-02T12:49:15.496430Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:15.566520Z","title":"Evaluate clarity and conciseness","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":62,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:15.566520Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:b21f31e3ce69fb43c7308c032cdf3e873af1071ab24d22f81a2a61fad73e9ddf","observation_id":"a6fd832a-e985-4d7f-8647-ed91a818dcf6","resolution":{"observed_at":"2026-08-02T12:49:15.566520Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:15.670868Z","title":"Evaluate how transparently the response shows its reasoning process","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":63,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:15.670868Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:b30832c934842c18e33ee39005d32e92ff43f90b5490ddf2b77cdab274bc96f1","observation_id":"658a2b09-1910-4d29-aaee-a1197dc2c780","resolution":{"observed_at":"2026-08-02T12:49:15.670868Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:15.769133Z","title":"likely”, “research suggests","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":64,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:15.769133Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:7a96191464c65f697b54d479464c983c3d29b1fdce4467fc5faa6d65ab83a2a7","observation_id":"721afa0a-df22-4097-af73-4fd4ad3bc272","resolution":{"observed_at":"2026-08-02T12:49:15.769133Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:15.895168Z","title":"Evaluate the structural organization of the response","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":65,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:15.895168Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:59d6bfe5a659944a50cf9044d2a42ff89518f3ca37da38ee82c8f9a5551441ff","observation_id":"e998f238-1b0d-4448-b982-d584ee52b18d","resolution":{"observed_at":"2026-08-02T12:49:15.895168Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:16.057537Z","title":"many people think","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":66,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:16.057537Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:4ab28516b706d893118701a019603799bbb22ca0eb6e372346ad95cffc96eabe","observation_id":"2cc68719-540e-4b66-8942-9309a55c9f26","resolution":{"observed_at":"2026-08-02T12:49:16.057537Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:16.134604Z","title":"One answer is rewritten from a male-only perspective","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":67,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:16.134604Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:c0215804d64a8d017a0bf07fe6cc8361e0104cab6298a849b8512d50546bbf3f","observation_id":"8c6bc9f8-7b6c-4d4f-9c9d-da8eb1160fa0","resolution":{"observed_at":"2026-08-02T12:49:16.134604Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:16.294752Z","title":"One answer is reformatted with markdown, headers, bullet points, and emojis, while preserving the same substantive content","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":68,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:16.294752Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:9b7b966d25ae477c30d96b9000e93823a3743a7e141b110e7f0d3f28fba0d9ee","observation_id":"c268a055-77ab-40ae-964f-5a079011a096","resolution":{"observed_at":"2026-08-02T12:49:16.294752Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:16.507514Z","title":"One answer is augmented with fake or irrelevant citations","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":69,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:16.507514Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:52718901a4b500fb36d0f55b8e0d4670092bb378d672ac2f2afdaa8070f530e6","observation_id":"f9ea5a41-3f31-4e0d-87b6-7fac20b20396","resolution":{"observed_at":"2026-08-02T12:49:16.507514Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:16.721390Z","title":"One answer is padded with filler text to make it longer","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":70,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:16.721390Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:6699d5d0e08c8a3bf6e67788b319e525b8567203c5285cabfbdb6cbe56fb164c","observation_id":"8dbe153d-bc2f-4005-a641-8b8c17b3ec5d","resolution":{"observed_at":"2026-08-02T12:49:16.721390Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:16.904883Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":71,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:16.904883Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:a8c0417d8b28dc01f3ff51e18e9484834175fb5cdbc04e387f5cbbe8fde6e48d","observation_id":"b9c32b16-3864-4585-bec3-4c7f878e784b","resolution":{"observed_at":"2026-08-02T12:49:16.904883Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:16.994935Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":72,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:16.994935Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:8c80e88792c138dcb223bad59dfbcfe209a5f1c48b37294bca9fffecebce1daf","observation_id":"ecffc8bb-77f9-4ca8-964b-072ec958fafa","resolution":{"observed_at":"2026-08-02T12:49:16.994935Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:17.137920Z","title":"18 Table 3: Dataset statistics after filtering, along with the source of ground-truth preference labels","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":73,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:17.137920Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:ddb8ff788c80459d7c8677f4cb7add8b55b21c9e42571d1bdf76b557972cf561","observation_id":"f6a09cdb-fb88-4015-b582-97b1c5a6728c","resolution":{"observed_at":"2026-08-02T12:49:17.137920Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:17.255854Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":74,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:17.255854Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:7ed8c122a9bbb74a3b82554fefa74c0d0bbbf5978a4047cc4702dfb8e687e2c7","observation_id":"27ce8947-14df-4858-84ab-7e3de9e68a81","resolution":{"observed_at":"2026-08-02T12:49:17.255854Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T12:49:17.313990Z","title":"Data Filtering.To construct our evaluation set, we retain only samples with reliable preference signals and discardambiguous, tied, or low-confidence cases","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation","version":1},"reference_index":75,"source":"pdf_text","source_observed_at":"2026-08-02T12:49:17.313990Z"},"links":{"citing_paper":"/paper/2607.22561"},"observation_digest":"sha256:5b5e0cf0de20ebbc7914915899b3e83b7b34140b659738bd245bb16231137aad","observation_id":"33dc076d-9319-4d7f-bdb8-4e6975f454d6","resolution":{"observed_at":"2026-08-02T12:49:17.313990Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2607.22561","last_updated":"2026-05-29T03:13:24Z","latest_version":1,"primary_category":"cs.AI","snapshot_observed_at":"2026-08-18T08:50:06.635971Z","submitted_at":"2026-05-29T03:13:24Z","title":"Codifying the Judge: Scalable Evaluation via Program Distillation"},"reference_resolution":{"displayed":75,"state_counts":{"malformed_identifier":1,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":74,"verified_exact":0,"verified_fuzzy":0},"total_outbound_references":75},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"thesis":"As of 19 August 2026, this Paper Citation Record lists 75 of 75 outbound references and 0 inbound Pith citation observations for arXiv:2607.22561."}