{"as_of":"2026-08-14T11:29:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:9aec4677e9c879630e8d71536a27d1dfdc40dbf2845fe41d0100fcc75eb1b80f","coverage":[{"denominator":39,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":39,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-12T14:27:57.472919Z","state":"measured"},{"denominator":40,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":40,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-14T06:32:32.682623+00:00","state":"measured"},{"denominator":1,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":1,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-06T23:12:43.460215Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"pith","source_observed_at":"2026-08-06T23:12:44.210234Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"cited_work":{"arxiv_id":"2411.15387","doi":null,"metadata_source":"pith","pith_arxiv_id":"2411.15387","snapshot_observed_at":"2026-08-06T23:12:44.210234Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","venue":"cs.CL","work_id":"27ba8a91-80f1-43fd-bf6b-bd2eb8045158","year":2024},"citing_paper":{"arxiv_id":"2506.19571","last_updated":"2025-06-24T12:35:00Z","snapshot_observed_at":"2026-08-14T09:05:02.162450Z","submitted_at":"2025-06-24T12:35:00Z","title":"Has Machine Translation Evaluation Achieved Human Parity? The Human Reference and the Limits of Progress","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-06T23:12:43.460215Z"},"links":{"cited_paper":"/paper/2411.15387","citing_paper":"/paper/2506.19571"},"observation_digest":"sha256:28081cf960e61afca42d814084d0184faf8ebb47b28f7f358dd50f20536a4b45","observation_id":"07ce0915-a39c-4b1a-ade1-b61e9c2304cd","resolution":{"observed_at":"2026-08-06T23:12:44.214904Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2411.15387/citation-record","integrity":"/paper/2411.15387/integrity","json":"/paper/2411.15387/citation-record.json","paper":"/paper/2411.15387"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:27:57.295402Z","title":"write newline","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.295402Z"},"links":{"citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:c332dd8bdb18d7450b5f1c85a32c2c931f4b7b2e916f6234e3f0badb80a4090b","observation_id":"c2370a0a-672d-497f-8ea2-e4fd8fa89166","resolution":{"observed_at":"2026-08-12T14:27:57.295402Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-07T07:30:12.213965Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-08-12T14:27:57.301443Z","title":"L., Almeida, D., Altenschmidt, J., Altman, S., Anadkat, S., et al","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.301443Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:2e2df2e0dbd83fa435310b03593a8900d6fa0458535d9fbec102fbfa1d29bec0","observation_id":"aa36a654-e550-43e4-9213-79c5aad0fa78","resolution":{"observed_at":"2026-08-12T14:27:57.301443Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2212.08073","last_updated":"2022-12-15T06:19:23Z","snapshot_observed_at":"2026-08-02T04:53:58.766070Z","submitted_at":"2022-12-15T06:19:23Z","title":"Constitutional AI: Harmlessness from AI Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2212.08073","snapshot_observed_at":"2026-08-12T14:27:57.306616Z","title":"Constitutional ai: Harmlessness from ai feedback","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.306616Z"},"links":{"cited_paper":"/paper/2212.08073","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:870a0bbbbd2874542148f5c001e8653c4d15187f26d6276338adb4e982932da8","observation_id":"765fb702-624b-4d61-8a61-ddbaaa0a1c59","resolution":{"observed_at":"2026-08-12T14:27:57.306616Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:27:58.199434Z","title":"M., Kanojia, D., de Souza, J","venue":null,"work_id":"5aca4ff5-3e13-4c7a-9521-0f7571ccc406","year":2023},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.311576Z"},"links":{"citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:fc72810f4d852f45a2fa92ace9fa6ef5448a81f6a80e6e80ba36035ab020ec44","observation_id":"3dcade8a-0a5c-43b3-8cef-98894326297d","resolution":{"observed_at":"2026-08-12T14:27:58.204094Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:27:58.183271Z","title":null,"venue":null,"work_id":"1043f025-3069-4c94-aa10-ba1eb9a8ab3f","year":2008},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.316049Z"},"links":{"citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:cb7c10c86c81f94b1ede005b00ce1ffc991969913e51b7e6542f936992a51d14","observation_id":"4b980d68-92df-4fd5-9dfb-96c22b49e959","resolution":{"observed_at":"2026-08-12T14:27:58.188254Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.17600","last_updated":"2024-10-04T15:22:29Z","snapshot_observed_at":"2026-08-14T07:22:27.964077Z","submitted_at":"2024-06-25T14:42:17Z","title":"\"Seeing the Big through the Small\": Can LLMs Approximate Human Judgment Distributions on NLI from a Few Explanations?","version":2},"cited_work":{"arxiv_id":"2406.17600","doi":null,"metadata_source":"pith","pith_arxiv_id":"2406.17600","snapshot_observed_at":"2026-08-12T14:27:58.036417Z","title":"\"Seeing the Big through the Small\": Can LLMs Approximate Human Judgment Distributions on NLI from a Few Explanations?","venue":"cs.CL","work_id":"57adcbb8-d56c-4e3b-9909-0cfc89c5da19","year":2024},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.320397Z"},"links":{"cited_paper":"/paper/2406.17600","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:1eb1c141fc3e3e572e07918aebfd1e321e21ce1c04981fe9bbbbc22721f53d20","observation_id":"2b2bf236-a994-413c-a1eb-17e9eb7896a0","resolution":{"observed_at":"2026-08-12T14:27:58.041465Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2305.14324","last_updated":"2023-10-17T16:33:33Z","snapshot_observed_at":"2026-08-13T11:35:16.464029Z","submitted_at":"2023-05-23T17:54:57Z","title":"Ties Matter: Meta-Evaluating Modern Metrics with Pairwise Accuracy and Tie Calibration","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.14324","snapshot_observed_at":"2026-08-12T14:27:57.324998Z","title":"Ties matter: Meta-evaluating modern metrics with pairwise accuracy and tie calibration","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.324998Z"},"links":{"cited_paper":"/paper/2305.14324","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:26be62179aa496b9b20746564e07208436589fc94944850e8c11876e48dfa53f","observation_id":"39a3ad6b-604d-4c7a-8f84-3b60f3ed1628","resolution":{"observed_at":"2026-08-12T14:27:57.324998Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.07286","last_updated":"2023-08-14T17:17:21Z","snapshot_observed_at":"2026-08-13T10:34:45.715282Z","submitted_at":"2023-08-14T17:17:21Z","title":"The Devil is in the Errors: Leveraging Large Language Models for Fine-grained Machine Translation Evaluation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.07286","snapshot_observed_at":"2026-08-12T14:27:57.330246Z","title":"F., Neubig, G., Garg, A., Clark, J","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.330246Z"},"links":{"cited_paper":"/paper/2308.07286","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:24ccd749dc78e373052f79057883e3f180536b2f87fa4f2f6036227b06056266","observation_id":"dd017cac-cb1f-4e0f-bfb7-b32a7ec95626","resolution":{"observed_at":"2026-08-12T14:27:57.330246Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:27:58.167310Z","title":"Experts, errors, and context: A large-scale study of human evaluation for machine translation","venue":null,"work_id":"09d318f2-3c50-4fb3-9696-93073c18a819","year":2021},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.334819Z"},"links":{"citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:3efaa8ccce67d34ac90b9feb33521308fdfd34548b6c7d740146f754ec938bb8","observation_id":"73e525a6-2169-4b4e-bf02-4281df5ba786","resolution":{"observed_at":"2026-08-12T14:27:58.172302Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:27:58.152579Z","title":"Results of wmt23 metrics shared task: Metrics might be guilty but references are not innocent","venue":null,"work_id":"1fd27e2e-7bc4-44b3-b618-983fe1f584ce","year":2023},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.339072Z"},"links":{"citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:3ddf9c40671abff8c7c2f5c997641448e6fae94ddf31684ca8cf1abd539c0931","observation_id":"80c61444-66a1-4897-8179-48de0c385e17","resolution":{"observed_at":"2026-08-12T14:27:58.157513Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:27:58.137419Z","title":"Are llms breaking mt metrics? results of the wmt24 metrics shared task","venue":null,"work_id":"179e89f4-b2d0-4482-8913-eda711919c59","year":2024},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.343580Z"},"links":{"citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:cbab8fbde9b2b40763d85fa12efe8f804d7342971d131fad615c970499c67d85","observation_id":"3a98697c-e771-411f-9872-735787c7add2","resolution":{"observed_at":"2026-08-12T14:27:58.142333Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2312.11805","last_updated":"2025-05-09T21:04:06Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-12-19T02:39:27Z","title":"Gemini: A Family of Highly Capable Multimodal Models","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2312.11805","snapshot_observed_at":"2026-08-12T14:27:57.347977Z","title":"Gemini: A family of highly capable multimodal models, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.347977Z"},"links":{"cited_paper":"/paper/2312.11805","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:0ec6ee70963cbaa866e11f4a6182bc12e99e5f1cc0a2e75fece12aefd3ddc6fb","observation_id":"162cdb0d-e881-4c64-8604-04fcfeee94d6","resolution":{"observed_at":"2026-08-12T14:27:57.347977Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1908.07898","last_updated":"2019-08-28T15:55:37Z","snapshot_observed_at":"2026-08-14T11:09:22.671093Z","submitted_at":"2019-08-21T14:48:01Z","title":"Are We Modeling the Task or the Annotator? An Investigation of Annotator Bias in Natural Language Understanding Datasets","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1908.07898","snapshot_observed_at":"2026-08-12T14:27:57.352788Z","title":"Are we modeling the task or the annotator? an investigation of annotator bias in natural language understanding datasets","venue":null,"work_id":null,"year":1908},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.352788Z"},"links":{"cited_paper":"/paper/1908.07898","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:bf517bfcb32520f67ee063ba0a9efd8aa324c5a081058941c391dd560cee39c9","observation_id":"3d97bd7f-daf8-4e7e-8dd3-0d97340ee51e","resolution":{"observed_at":"2026-08-12T14:27:57.352788Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.14101","last_updated":"2024-09-05T16:21:56Z","snapshot_observed_at":"2026-08-13T04:13:24.634894Z","submitted_at":"2024-02-21T19:53:36Z","title":"Cost-Efficient Subjective Task Annotation and Modeling through Few-Shot Annotator Adaptation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.14101","snapshot_observed_at":"2026-08-12T14:27:57.357217Z","title":"S., Omrani, A., and Dehghani, M","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.357217Z"},"links":{"cited_paper":"/paper/2402.14101","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:3b310d789fb11a12c4b674009a423abb3d83158907210bd82254ccfe88090b19","observation_id":"e99e2b29-2927-4043-87dc-330d4cace675","resolution":{"observed_at":"2026-08-12T14:27:57.357217Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.10482","last_updated":"2023-10-16T15:03:14Z","snapshot_observed_at":"2026-08-13T05:48:22.001226Z","submitted_at":"2023-10-16T15:03:14Z","title":"xCOMET: Transparent Machine Translation Evaluation through Fine-grained Error Detection","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.10482","snapshot_observed_at":"2026-08-12T14:27:57.361653Z","title":"M., Rei, R., van Stigt, D., Coheur, L., Colombo, P., and Martins, A","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.361653Z"},"links":{"cited_paper":"/paper/2310.10482","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:9cc87ccf623672e543f90e8467a493832d099d38ddb6ec03ede5e5a8dc2cfa62","observation_id":"41501d35-36a2-46ca-a0ae-7dbdd4ab5b33","resolution":{"observed_at":"2026-08-12T14:27:57.361653Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2009.03300","last_updated":"2021-01-12T18:57:11Z","snapshot_observed_at":"2026-08-13T20:44:28.824685Z","submitted_at":"2020-09-07T17:59:25Z","title":"Measuring Massive Multitask Language Understanding","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.03300","snapshot_observed_at":"2026-08-12T14:27:57.366287Z","title":"Measuring massive multitask language understanding","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.366287Z"},"links":{"cited_paper":"/paper/2009.03300","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:d23cf8870120f073b0b0e09d940bb8330ce48393451996b765d13ea755866f67","observation_id":"8f166c86-3490-4477-ab0e-aed8fbd52560","resolution":{"observed_at":"2026-08-12T14:27:57.366287Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.03983","last_updated":"2024-10-04T23:52:28Z","snapshot_observed_at":"2026-08-12T22:30:19.413552Z","submitted_at":"2024-10-04T23:52:28Z","title":"MetricX-24: The Google Submission to the WMT 2024 Metrics Shared Task","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.03983","snapshot_observed_at":"2026-08-12T14:27:57.370889Z","title":"Metricx-24: The google submission to the wmt 2024 metrics shared task","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.370889Z"},"links":{"cited_paper":"/paper/2410.03983","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:ab84b252cd5c827f355f10f01406bb8a6b1e3b9b786a35db603d5e604719e8ae","observation_id":"0f58b68c-837e-4e60-a890-89530f504d13","resolution":{"observed_at":"2026-08-12T14:27:57.370889Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.03602","last_updated":"2024-07-27T22:01:50Z","snapshot_observed_at":"2026-08-13T00:37:33.676177Z","submitted_at":"2024-04-04T17:19:47Z","title":"Evaluating LLMs at Detecting Errors in LLM Responses","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.03602","snapshot_observed_at":"2026-08-12T14:27:57.375768Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.375768Z"},"links":{"cited_paper":"/paper/2404.03602","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:2680b36257fc312ce82793d4e14e9c2a76b351abf29aa1868579b7115dd160e3","observation_id":"9d47ae83-cd20-4f87-866c-0c52ca05b969","resolution":{"observed_at":"2026-08-12T14:27:57.375768Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2109.06835","last_updated":"2021-09-14T17:20:30Z","snapshot_observed_at":"2026-08-13T18:11:58.420988Z","submitted_at":"2021-09-14T17:20:30Z","title":"The Perils of Using Mechanical Turk to Evaluate Open-Ended Text Generation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2109.06835","snapshot_observed_at":"2026-08-12T14:27:57.380621Z","title":"The perils of using mechanical turk to evaluate open-ended text generation","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.380621Z"},"links":{"cited_paper":"/paper/2109.06835","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:7640986b7e32ea413cddc359a09135c4d9fe346940666e6d678fd352cbb5aad8","observation_id":"62cbb40d-b465-49bf-969c-f7f871e7ca8b","resolution":{"observed_at":"2026-08-12T14:27:57.380621Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:27:58.122595Z","title":"Prometheus: Inducing fine-grained evaluation capability in language models","venue":null,"work_id":"c46004bd-6580-4593-9988-ff0e7e7d7dd9","year":2023},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.386232Z"},"links":{"citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:26c40ab8c2e33d1587c7bd629129df7db14c86dd468a8a36f9606dcc6c29a69e","observation_id":"7b5fb24d-e0af-416f-ba39-48c523efc5fe","resolution":{"observed_at":"2026-08-12T14:27:58.127272Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2405.01535","last_updated":"2024-12-04T19:23:17Z","snapshot_observed_at":"2026-08-13T05:47:51.554813Z","submitted_at":"2024-05-02T17:59:35Z","title":"Prometheus 2: An Open Source Language Model Specialized in Evaluating Other Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.01535","snapshot_observed_at":"2026-08-12T14:27:57.390500Z","title":"Y., Shin, J., Welleck, S., Neubig, G., Lee, M., Lee, K., and Seo, M","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.390500Z"},"links":{"cited_paper":"/paper/2405.01535","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:f477734824786ad48e47ac9e3479772f4167732a81785eede61b28270b41edb9","observation_id":"e798bd68-e6a6-4124-8cd8-31a2e27dc89b","resolution":{"observed_at":"2026-08-12T14:27:57.390500Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.13988","last_updated":"2023-10-21T12:30:33Z","snapshot_observed_at":"2026-08-13T05:44:08.053028Z","submitted_at":"2023-10-21T12:30:33Z","title":"GEMBA-MQM: Detecting Translation Quality Error Spans with GPT-4","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.13988","snapshot_observed_at":"2026-08-12T14:27:57.394882Z","title":"and Federmann, C","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.394882Z"},"links":{"cited_paper":"/paper/2310.13988","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:73994ebd32cee66dc8c279f4bd0ff8bc3456f4ed0290d29a6540ab8f6f6033c8","observation_id":"56011669-a2a1-470e-9f09-bd69fa38be1a","resolution":{"observed_at":"2026-08-12T14:27:57.394882Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2302.14520","last_updated":"2023-05-31T19:51:51Z","snapshot_observed_at":"2026-08-13T12:35:22.929236Z","submitted_at":"2023-02-28T12:23:48Z","title":"Large Language Models Are State-of-the-Art Evaluators of Translation Quality","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2302.14520","snapshot_observed_at":"2026-08-12T14:27:57.399246Z","title":"and Federmann, C","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.399246Z"},"links":{"cited_paper":"/paper/2302.14520","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:831ac8f78ba12d2788b7f9f4ce1e8ee6eb416a2d1116d7b5a309160cd20ba030","observation_id":"da9dd46f-6bce-454b-b4fd-afeb106f7cc9","resolution":{"observed_at":"2026-08-12T14:27:57.399246Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2301.13298","last_updated":"2023-01-30T21:31:48Z","snapshot_observed_at":"2026-08-13T12:54:47.654946Z","submitted_at":"2023-01-30T21:31:48Z","title":"LongEval: Guidelines for Human Evaluation of Faithfulness in Long-form Summarization","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2301.13298","snapshot_observed_at":"2026-08-12T14:27:57.403790Z","title":"Longeval: Guidelines for human evaluation of faithfulness in long-form summarization","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.403790Z"},"links":{"cited_paper":"/paper/2301.13298","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:631d316ac4c3588fe45dfb3f67f185cd72d6f5ee86f800a98e69b747e0717965","observation_id":"7e24ccdd-f903-4822-aaaf-b41255033d4f","resolution":{"observed_at":"2026-08-12T14:27:57.403790Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.05470","last_updated":"2023-12-07T08:48:36Z","snapshot_observed_at":"2026-08-13T05:54:10.811176Z","submitted_at":"2023-10-09T07:27:15Z","title":"Generative Judge for Evaluating Alignment","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.05470","snapshot_observed_at":"2026-08-12T14:27:57.408571Z","title":"Generative judge for evaluating alignment","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.408571Z"},"links":{"cited_paper":"/paper/2310.05470","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:3f436c5a64c07175305dcabf6f30751722fd3d27ba194a887b9d0fa456a68e0d","observation_id":"af837742-f587-423a-9371-0a94188de7f7","resolution":{"observed_at":"2026-08-12T14:27:57.408571Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2211.09110","last_updated":"2023-10-01T21:44:23Z","snapshot_observed_at":"2026-08-10T23:10:13.900680Z","submitted_at":"2022-11-16T18:51:34Z","title":"Holistic Evaluation of Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2211.09110","snapshot_observed_at":"2026-08-12T14:27:57.413062Z","title":"Holistic evaluation of language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.413062Z"},"links":{"cited_paper":"/paper/2211.09110","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:0b66a77f4e45c44eb46ad5de77f2338a00b4c936274e85700aeb00eb0138b7c9","observation_id":"dbf6de75-1d22-4c99-acec-3c2b57599080","resolution":{"observed_at":"2026-08-12T14:27:57.413062Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:27:58.107381Z","title":"Multidimensional quality metrics (mqm): A framework for declaring and describing translation quality metrics","venue":null,"work_id":"7705df2b-db77-427c-8f80-f9929ab22886","year":2014},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.417856Z"},"links":{"citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:dc9d62667295ec2f43552d3e50df9046668cc5ae546170fcc8eb92c8a1ac75d1","observation_id":"79d59b41-a065-4226-a157-4df6031187b6","resolution":{"observed_at":"2026-08-12T14:27:58.112367Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:27:57.422180Z","title":"Training language models to follow instructions with human feedback","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.422180Z"},"links":{"citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:d3e439d67c95cfffe5ce295880fb8e06d769220fb06b785be307d7815c2c4332","observation_id":"aac3a4ad-0c5a-45fa-8caf-873e72cc8048","resolution":{"observed_at":"2026-08-12T14:27:57.422180Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2009.09025","last_updated":"2020-10-19T14:10:10Z","snapshot_observed_at":"2026-08-04T14:24:24.611946Z","submitted_at":"2020-09-18T18:54:15Z","title":"COMET: A Neural Framework for MT Evaluation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.09025","snapshot_observed_at":"2026-08-12T14:27:57.426608Z","title":"C., and Lavie, A","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.426608Z"},"links":{"cited_paper":"/paper/2009.09025","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:6c9c879e947a5ad514cda12171c0480d374eae52570942395b6d7a54fd7b8810","observation_id":"ee483ebd-aa30-4ff4-8e25-2c27482c604c","resolution":{"observed_at":"2026-08-12T14:27:57.426608Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.01474","last_updated":"2024-04-01T20:50:13Z","snapshot_observed_at":"2026-08-13T00:39:46.894992Z","submitted_at":"2024-04-01T20:50:13Z","title":"Finding Replicable Human Evaluations via Stable Ranking Probability","version":1},"cited_work":{"arxiv_id":"2404.01474","doi":null,"metadata_source":"pith","pith_arxiv_id":"2404.01474","snapshot_observed_at":"2026-08-12T14:27:57.765843Z","title":"Finding Replicable Human Evaluations via Stable Ranking Probability","venue":"cs.CL","work_id":"0d9aa036-ba07-40da-9813-ec5cff32f55b","year":2024},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":30,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.430960Z"},"links":{"cited_paper":"/paper/2404.01474","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:9f18e3814035cb69cdcc19ba894e50ec970bce3b883136892d9cba5884858b8a","observation_id":"2be859e2-4de5-4729-930f-44b2ad56413d","resolution":{"observed_at":"2026-08-12T14:27:57.772867Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2004.04696","last_updated":"2020-05-21T16:53:47Z","snapshot_observed_at":"2026-08-10T19:14:01.245716Z","submitted_at":"2020-04-09T17:26:52Z","title":"BLEURT: Learning Robust Metrics for Text Generation","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2004.04696","snapshot_observed_at":"2026-08-12T14:27:57.435735Z","title":null,"venue":null,"work_id":null,"year":2004},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.435735Z"},"links":{"cited_paper":"/paper/2004.04696","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:ae64286582339334c3bed635ca8caf6721e1aa114866df388de35255307454ca","observation_id":"d831d68e-f4cf-4eec-8d3f-f8c3b811329e","resolution":{"observed_at":"2026-08-12T14:27:57.435735Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.16575","last_updated":"2024-02-09T13:55:53Z","snapshot_observed_at":"2026-08-13T10:02:57.931243Z","submitted_at":"2023-09-28T16:32:28Z","title":"A Benchmark for Learning to Translate a New Language from One Grammar Book","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.16575","snapshot_observed_at":"2026-08-12T14:27:57.440359Z","title":"A benchmark for learning to translate a new language from one grammar book","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.440359Z"},"links":{"cited_paper":"/paper/2309.16575","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:51025a8dce30a322a0ca0f111abbb446a3d2014fa8fcd85b993b3a9afc1f04e0","observation_id":"64af8eac-5edb-4c99-9728-25da842f435b","resolution":{"observed_at":"2026-08-12T14:27:57.440359Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.10817","last_updated":"2024-07-15T15:33:45Z","snapshot_observed_at":"2026-08-12T23:21:55.244752Z","submitted_at":"2024-07-15T15:33:45Z","title":"Foundational Autoraters: Taming Large Language Models for Better Automatic Evaluation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.10817","snapshot_observed_at":"2026-08-12T14:27:57.444884Z","title":"Foundational autoraters: Taming large language models for better automatic evaluation","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.444884Z"},"links":{"cited_paper":"/paper/2407.10817","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:161d61b0be8c9ade0df52059626a8b5e34279847b50f9564de379b0e38b6f6c4","observation_id":"82323832-097c-4040-9c37-f788075d9b54","resolution":{"observed_at":"2026-08-12T14:27:57.444884Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:27:57.449578Z","title":"Y., Li, L., and Freitag, M","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.449578Z"},"links":{"citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:455c5780e173356275406f53e78befcf9a7277c4824f57fc85a8cdbc2583cf16","observation_id":"bb60349e-0bc1-4a53-b4dc-57677226650d","resolution":{"observed_at":"2026-08-12T14:27:57.449578Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.00297","last_updated":"2024-02-21T09:21:52Z","snapshot_observed_at":"2026-08-13T10:01:00.126581Z","submitted_at":"2023-09-30T08:13:49Z","title":"Understanding In-Context Learning from Repetitions","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.00297","snapshot_observed_at":"2026-08-12T14:27:57.454353Z","title":"Understanding in-context learning from repetitions","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.454353Z"},"links":{"cited_paper":"/paper/2310.00297","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:a76321b3597fa94ffba288070f481cc2e9115437874e09806fae2e131602aa60","observation_id":"8c0a4f2c-9491-4ecb-83cc-ce198d6d09b0","resolution":{"observed_at":"2026-08-12T14:27:57.454353Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.16509","last_updated":"2024-10-21T21:00:47Z","snapshot_observed_at":"2026-08-12T22:18:06.068423Z","submitted_at":"2024-10-21T21:00:47Z","title":"Learning from others' mistakes: Finetuning machine translation models with span-level error annotations","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.16509","snapshot_observed_at":"2026-08-12T14:27:57.459572Z","title":"H., Dadkhahi, H., Finkelstein, M., Trabelsi, F., Luo, J., and Freitag, M","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":36,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.459572Z"},"links":{"cited_paper":"/paper/2410.16509","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:f17cb197d34473d3bd529b8899717ae845d4b7105a48a1475e30db08322ff1a0","observation_id":"49bb4659-453f-4f01-a427-8747bd00d0f2","resolution":{"observed_at":"2026-08-12T14:27:57.459572Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:27:57.463831Z","title":"J., Wang, Z., Hwang, J","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":37,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.463831Z"},"links":{"citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:efd38954a23bae0962bbb159f99149d7c9b20bc536d9eb3b48bb6b56735a81be","observation_id":"177f4ef1-ec04-471b-8832-8d477b046815","resolution":{"observed_at":"2026-08-12T14:27:57.463831Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.11998","last_updated":"2024-03-10T19:34:57Z","snapshot_observed_at":"2026-08-13T10:08:28.290453Z","submitted_at":"2023-09-21T12:13:55Z","title":"LMSYS-Chat-1M: A Large-Scale Real-World LLM Conversation Dataset","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.11998","snapshot_observed_at":"2026-08-12T14:27:57.468381Z","title":"P., et al","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":38,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.468381Z"},"links":{"cited_paper":"/paper/2309.11998","citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:3cde43e21e892e5566378b3e0a5579ed02ba85edb571f00c1a73007c1ddc434c","observation_id":"1412af09-f169-42d1-9937-190f21bf84d6","resolution":{"observed_at":"2026-08-12T14:27:57.468381Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T14:27:58.082919Z","title":"Judging llm-as-a-judge with mt-bench and chatbot arena","venue":null,"work_id":"2bf82fff-ffd7-4cb1-a3a9-d959277fb73d","year":2023},"citing_paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set","version":2},"reference_index":39,"source":"arxiv_source","source_observed_at":"2026-08-12T14:27:57.472919Z"},"links":{"citing_paper":"/paper/2411.15387"},"observation_digest":"sha256:63a09859d4245f2395404c5750e271f5779a4f2b1f66ff4f1be138541b54c478","observation_id":"0500f9c6-192d-415a-89e4-b9e6981223a0","resolution":{"observed_at":"2026-08-12T14:27:58.088220Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-14T06:32:32.682623+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2411.15387","last_updated":"2024-12-11T23:00:55Z","latest_version":2,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-14T09:04:15.598600Z","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set"},"reference_resolution":{"displayed":39,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":30,"verified_exact":2,"verified_fuzzy":7},"total_outbound_references":39},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-14T06:32:32.682623+00:00","source":"crossref"},{"observed_at":"2026-08-14T06:32:18.44784+00:00","source":"retraction_watch"}],"thesis":"As of 14 August 2026, this Paper Citation Record lists 39 of 39 outbound references and 1 inbound Pith citation observation for arXiv:2411.15387."}